add assertion rules

README.md

@@ -46,6 +46,10 @@ requests:
       status: 200  
       values:
         comment: This is my comment! # the JSON response `comment` field must match this value
+        date_posted:
+          exists: true # assertion rules can be used instead of a simple string value
+        num_replies:
+          gt: 0 # assert that num_replies is greater than 0
 ```
 
 #### Test spec properties
@@ -86,6 +90,13 @@ requests:
     * `values`: key/value pairs 
     * `strict`: use `strict: true` to require expect & response type to be exactly the same (e.g. the integer `10` is not equal to the string "10"). Default is `false`.
 
+Keys defined under `values` can use a basic comparison syntax (e.g. `type: Pepperoni`) or use an object block to add assertion rules:
+
+  * gt, lt: greater than, less than
+  * ge, le: greater than or equal to, less than or equal to
+  * equals: equality check. Note: not a strict comparison, so 123 is equivalent to "123" despite the type difference)
+  * exists: simple check that the key exists in the response body (use: `exists: true`). Note: "exists: false" currently not supported.
+
 ```yaml
 requests:
   - name: Get pizza
@@ -95,9 +106,10 @@ requests:
       status: 200
       values:
         size: Large
-        type: Pepperoni
-        quantity: 10
-      strict: true # quantity must be a number 10, not a string "10".  Use false if not important.
+        type:
+          equals: Pepperoni # assertion rule syntax. Can also simply use "type: Pepperoni" for basic equality comparisons
+        quantity:
+          gt: 10 # greater than
 ```
 
   * `set`: a list of env variables to set from the response. Each item should have a `var` (the variable to be set) and `from` (a field in the response). This will be helpful for capturing the ID of a created resource to use in a later request.

assertions.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"fmt"
+	"strconv"
+)
+
+// checkAssertions takes the rules provided in the test spec and iterates over them,
+// returning an error if at any point a comparison is false.
+// value comparisons:
+// "equals"
+// "lt" (less than)
+// "gt" (greater than)
+// "le" (less than or equal to)
+// "ge" (greater than or equal to)
+// "exists" (key exists in the JSON response body)
+func checkAssertions(value interface{}, rules map[string]interface{}) error {
+	for k, comparisonValue := range rules {
+		switch k {
+		case "equals":
+			// compare values using string formatting. This is equivalent to "strict=false"
+			// e.g.:
+			// 123 == 123 > true
+			// 123 == "123" > true
+
+			if !equals(value, comparisonValue) {
+				return fmt.Errorf("expected: %v received: %v", comparisonValue, value)
+			}
+		case "lt":
+			// convert to floats, returning an error if that's not possible (bad input)
+			val1, val2, err := asFloat(value, comparisonValue)
+			if err != nil {
+				return err
+			}
+
+			// perform comparison
+			if val1 >= val2 {
+				return fmt.Errorf("expected %v less than %v", value, comparisonValue)
+			}
+		case "gt":
+			val1, val2, err := asFloat(value, comparisonValue)
+			if err != nil {
+				return err
+			}
+
+			if val1 <= val2 {
+				return fmt.Errorf("expected %v greater than %v", value, comparisonValue)
+			}
+		case "le":
+			val1, val2, err := asFloat(value, comparisonValue)
+			if err != nil {
+				return err
+			}
+
+			if val1 > val2 {
+				return fmt.Errorf("expected %v less than or equal to %v", value, comparisonValue)
+			}
+		case "ge":
+			val1, val2, err := asFloat(value, comparisonValue)
+			if err != nil {
+				return err
+			}
+
+			if val1 < val2 {
+				return fmt.Errorf("expected %v greater than or equal to %v", value, comparisonValue)
+			}
+		case "exists":
+			// check whether this key was received as part of the body, even if null.
+			// not elegant, but due to the jq parsing in checkJSONResponse(), this api test case will
+			// fail earlier in checkJSONResponse if the key doesn't exist. Therefore, if the test case
+			// gets this far, we already know the key exists. This is here to provide a means to check
+			// the "exists" case without having to provide a comparison value. In the future, I need
+			// to refactor to allow `exists: false`.
+		default:
+			// invalid rule (not defined above)
+			return fmt.Errorf("invalid rule: %s", k)
+		}
+	}
+	return nil
+}
+
+// equals returns true if two values are equal, when cast to strings.
+// this means that 123 == "123" (int vs string).
+func equals(val interface{}, comparison interface{}) bool {
+	return fmt.Sprintf("%v", val) == fmt.Sprintf("%v", comparison)
+}
+
+// asFloat converts two values (received value and comparison value) to floats.
+// returns an error if not possible.
+func asFloat(val1 interface{}, val2 interface{}) (float64, float64, error) {
+	float1, err := strconv.ParseFloat(fmt.Sprintf("%v", val1), 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("unable to parse %v as a float", val1)
+	}
+	float2, err := strconv.ParseFloat(fmt.Sprintf("%v", val2), 64)
+	if err != nil {
+		return 0, 0, fmt.Errorf("unable to parse %v as a float", val2)
+	}
+	return float1, float2, nil
+}

assertions_test.go

@@ -0,0 +1,122 @@
+package main
+
+import "testing"
+
+func TestEqualsAssertion(t *testing.T) {
+	type testCase struct {
+		Value      interface{}
+		Comparison interface{}
+		Expected   bool
+	}
+
+	cases := []testCase{
+		testCase{Value: "123", Comparison: 123, Expected: true},
+		testCase{Value: "123", Comparison: 124, Expected: false},
+		testCase{Value: 123, Comparison: 123, Expected: true},
+		testCase{Value: "the quick brown fox", Comparison: "the quick brown fox", Expected: true},
+		testCase{Value: "quick", Comparison: 421421, Expected: false},
+		testCase{Value: 123.4, Comparison: 123.4, Expected: true},
+		testCase{Value: 123.4, Comparison: 123.5, Expected: false},
+	}
+
+	for _, c := range cases {
+		err := checkAssertions(c.Value, map[string]interface{}{"equals": c.Comparison})
+		if (err == nil) != c.Expected {
+			t.Errorf("failed: expected %v == %v to have been %v; %v", c.Value, c.Comparison, c.Expected, err)
+		}
+	}
+}
+
+// TestLTAssertion tests the "less than" comparison rule
+func TestLTAssertion(t *testing.T) {
+	type testCase struct {
+		Value      interface{}
+		Comparison interface{}
+		Expected   bool
+	}
+
+	cases := []testCase{
+		testCase{Value: "123", Comparison: 123, Expected: false},
+		testCase{Value: "123", Comparison: 124, Expected: true},
+		testCase{Value: 123, Comparison: 124, Expected: true},
+		testCase{Value: 123, Comparison: 122, Expected: false},
+	}
+
+	for _, c := range cases {
+		err := checkAssertions(c.Value, map[string]interface{}{"lt": c.Comparison})
+		if (err == nil) != c.Expected {
+			t.Errorf("failed: expected %v < %v to have been %v; %v", c.Value, c.Comparison, c.Expected, err)
+		}
+	}
+}
+
+// TestGTAssertion tests the "greater than" comparison rule
+func TestGTAssertion(t *testing.T) {
+	type testCase struct {
+		Value      interface{}
+		Comparison interface{}
+		Expected   bool
+	}
+
+	cases := []testCase{
+		testCase{Value: "123", Comparison: 123, Expected: false},
+		testCase{Value: "123", Comparison: 124, Expected: false},
+		testCase{Value: 123, Comparison: 124, Expected: false},
+		testCase{Value: 123, Comparison: 122, Expected: true},
+	}
+
+	for _, c := range cases {
+		err := checkAssertions(c.Value, map[string]interface{}{"gt": c.Comparison})
+		if (err == nil) != c.Expected {
+			t.Errorf("failed: expected %v > %v to have been %v; %v", c.Value, c.Comparison, c.Expected, err)
+		}
+	}
+}
+
+// TestLEAssertion tests the "less than or equal to" comparison rule
+func TestLEAssertion(t *testing.T) {
+	type testCase struct {
+		Value      interface{}
+		Comparison interface{}
+		Expected   bool
+	}
+
+	cases := []testCase{
+		testCase{Value: "123", Comparison: 123, Expected: true},
+		testCase{Value: "123", Comparison: 124, Expected: true},
+		testCase{Value: 123, Comparison: 124, Expected: true},
+		testCase{Value: 123, Comparison: 122, Expected: false},
+		testCase{Value: 123, Comparison: 123, Expected: true},
+	}
+
+	for _, c := range cases {
+		err := checkAssertions(c.Value, map[string]interface{}{"le": c.Comparison})
+		if (err == nil) != c.Expected {
+			t.Errorf("failed: expected %v <= %v to have been %v; %v", c.Value, c.Comparison, c.Expected, err)
+		}
+	}
+}
+
+// TestGEAssertion tests the "greater than or equal to" comparison rule
+func TestGEAssertion(t *testing.T) {
+	type testCase struct {
+		Value      interface{}
+		Comparison interface{}
+		Expected   bool
+	}
+
+	cases := []testCase{
+		testCase{Value: "123", Comparison: 123, Expected: true},
+		testCase{Value: "123", Comparison: 124, Expected: false},
+		testCase{Value: 123, Comparison: 124, Expected: false},
+		testCase{Value: 123, Comparison: 122, Expected: true},
+		testCase{Value: 123, Comparison: 123, Expected: true},
+	}
+
+	for _, c := range cases {
+		err := checkAssertions(c.Value, map[string]interface{}{"ge": c.Comparison})
+		if (err == nil) != c.Expected {
+			t.Errorf("failed: expected %v >= %v to have been %v; %v", c.Value, c.Comparison, c.Expected, err)
+		}
+	}
+}

main.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	flag "github.com/spf13/pflag"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // TestSet is a set of requests and assertions

requests.go

@@ -321,14 +321,22 @@ func checkJSONResponse(body []byte, selector string, expectedValue interface{},
 		return fmt.Errorf("could not decode value from key %s", selector)
 	}
 
-	sValue := fmt.Sprintf("%v", iValue)
-	sExpected := fmt.Sprintf("%v", expectedValue)
+	switch expectedValue.(type) {
+	case map[string]interface{}:
+		// if expectedValue is a map instead of a string, check
+		// for assertion rules.  we expect an error return, or nil (meaning assertion check passed).
+		return checkAssertions(iValue, expectedValue.(map[string]interface{}))
+	default:
+		sValue := fmt.Sprintf("%v", iValue)
+		sExpected := fmt.Sprintf("%v", expectedValue)
+
+		if sValue != sExpected {
+			return fmt.Errorf("expected: %v received: %v", sExpected, sValue)
+		}
 
-	if sValue != sExpected {
-		return fmt.Errorf("expected: %v received: %v", sExpected, sValue)
+		return nil
 	}
 
-	return nil
 }
 
 // contains is a helper function to check if a slice of strings contains a particular string.

requests_test.go

@@ -16,12 +16,14 @@ func basicRequestHandler(w http.ResponseWriter, req *http.Request) {
 		ID          int    `json:"id"`
 		Title       string `json:"title"`
 		Description string `json:"description"`
+		NumTasks    int    `json:"num_tasks"`
 	}
 
 	data := todo{
 		ID:          1,
 		Title:       "delectus aut autem",
 		Description: "something to do",
+		NumTasks:    2,
 	}
 
 	switch req.Method {

test/test.yaml

@@ -12,9 +12,18 @@ requests:
       status: 200
       values:
         id: 1
-        title: delectus aut autem
+        title:
+          equals: delectus aut autem
+        num_tasks:
+          gt: 1
+          lt: 3
   - name: Create a todo item
     url: "{{host}}/todos"
     method: post
     expect:
       status: 201
+      values:
+        title:
+          exists: true
+        id:
+          equals: 1