Merge pull request #5 from kevinjqiu/more-descriptive-test-names

Friendlier test names

README.md

@@ -45,9 +45,9 @@ Top level attributes
 --------------------
 
 * `name` - The name of the test case
-* [`rules`](#Rules) - The rule definitions that are under test
-* [`fixtures`](#Fixtures) - The fixture setup for the tests
-* [`assertions`](#Assertions) - The test assertions
+* [`rules`](#rules) - The rule definitions that are under test
+* [`fixtures`](#fixtures) - The fixture setup for the tests
+* [`assertions`](#assertions) - The test assertions
 
 Rules
 -----
@@ -198,9 +198,10 @@ assertions:
 Run the test:
 
 ```bash
-$ ./pat test/test.yaml
-=== RUN   Test_HTTP_Requests_too_low_alert_0
---- PASS: Test_HTTP_Requests_too_low_alert_0 (0.00s)
-=== RUN   Test_HTTP_Requests_too_low_alert_1
---- PASS: Test_HTTP_Requests_too_low_alert_1 (0.00s)
+$ ./pat examples/test.yaml
+=== RUN   Test_HTTP_Requests_too_low_alert_at_0m
+--- PASS: Test_HTTP_Requests_too_low_alert_at_0m (0.00s)
+=== RUN   Test_HTTP_Requests_too_low_alert_at_5m
+--- PASS: Test_HTTP_Requests_too_low_alert_at_5m (0.00s)
+PASS
 ```

examples/test.yaml

@@ -21,6 +21,8 @@ assertions:
         instance: "1"
         job: app-server
         severity: critical
+    comment: |-
+      At 0m, the alerts met the threshold but has not met the duration requirement. Expect the alert to be pending
   - at: 5m
     expected:
       - alertname: HTTPRequestRateLow
@@ -35,3 +37,5 @@ assertions:
         instance: "1"
         job: app-server
         severity: critical
+    comment: |-
+      At 5m, the alerts should be firing because the duration requirement is met.

pkg/ruleloader.go

@@ -44,7 +44,7 @@ func (ruleLoader RuleLoader) Load() ([]*rules.Group, error) {
 		return nil, fmt.Errorf("Must specify a rule loading strategy")
 	}
 
-	retval := []*rules.Group{}
+	var retval []*rules.Group
 	for _, rg := range ruleGroups.Groups {
 		rls := make([]rules.Rule, 0, len(rg.Rules))
 		for _, r := range rg.Rules {

pkg/ruletest.go

@@ -80,7 +80,8 @@ func (prt PromRuleTest) getTestCaseName(assertionIdx int) string {
 	if err != nil {
 		log.Fatal(err)
 	}
-	return fmt.Sprintf("%s_%d", reg.ReplaceAllString(prt.Name, "_"), assertionIdx)
+	assertion := prt.Assertions[assertionIdx]
+	return fmt.Sprintf("%s_at_%s", reg.ReplaceAllString(prt.Name, "_"), assertion.At)
 }
 
 func (prt PromRuleTest) newTestCase(idx int, assertion Assertion, resultAlerts []map[string]string) TestCase {

pkg/ruletest_test.go

@@ -8,8 +8,19 @@ import (
 )
 
 func TestGetTestCaseName(t *testing.T) {
-	prt := PromRuleTest{Name: "Test HTTP Requests too low alert"}
-	assert.Equal(t, "Test_HTTP_Requests_too_low_alert_1", prt.getTestCaseName(1))
+	prt := PromRuleTest{
+		Name: "Test HTTP Requests too low alert",
+		Assertions: []Assertion{
+			{
+				At: "5m",
+			},
+			{
+				At: "10m",
+			},
+		},
+	}
+	assert.Equal(t, "Test_HTTP_Requests_too_low_alert_at_5m", prt.getTestCaseName(0))
+	assert.Equal(t, "Test_HTTP_Requests_too_low_alert_at_10m", prt.getTestCaseName(1))
 }
 
 func TestNewPromRuleTestFromFile(t *testing.T) {

pkg/schema/schema.json

@@ -3,17 +3,25 @@
   "additionalProperties": false,
   "definitions": {
     "Assertion": {
+      "additionalProperties": false,
       "properties": {
         "at": {
           "$ref": "#/definitions/Duration"
         },
+        "comment": {
+          "type": "string"
+        },
         "expected": {
           "items": {
             "$ref": "#/definitions/Expectation"
           },
           "type": "array"
         }
       },
+      "required": [
+        "at",
+        "expected"
+      ],
       "type": "object"
     },
     "Duration": {

pkg/schema/schema.yaml

@@ -54,6 +54,12 @@ definitions:
         type: array
         items:
           $ref: "#/definitions/Expectation"
+      comment:
+        type: string
+    required:
+      - at
+      - expected
+    additionalProperties: false
   Duration:
     type: string
     pattern: "^\\d+(ns|us|ms|s|m|h)$"