refactor: improved error handling

README.md

@@ -5,10 +5,10 @@
 
 **HTTP smoke testing made easy.**
 
-Simply declare a list of URLs and their expected response values, and Emberfall will test, compare, and report which URLs fail along with details of what expectations were not met.
+Simply declare a list of URLs and their expected response values, and Emberfall will test, compare, and report which URLs fail along with details of what expectations were not met. Tests are merely a list of request objects, each with a url, method, headers to be sent, and an expects field. With the `expects` field you can define the status code, body contents (as a string), and any headers (as strings) that should be present in the response. If anything expected is not present or not equal to the defined value `emberfall` will exit with a non-zero code.
+
 
 ## Configuring Tests
-Tests are merely a list of request objects, each with a url, method, headers to be sent, and an expects field. With the `expects` field you can define the status code, body contents (as a string), and any headers (as strings) that should be present in the response. If anything expected is not present or not equal to the defined value `emberfall` will exit with a non-zero code.
 
 The YAML tests config can be provided in two ways:
 - as a file: `emberfall --config path/to/config.yaml`

cmd/root.go

@@ -16,7 +16,7 @@ import (
 var configPath string
 
 var rootCmd = &cobra.Command{
-	Use:   "gatling",
+	Use:   "emberfall",
 	Short: "Declarative API Testing",
 	Run: func(cmd *cobra.Command, args []string) {
 		configPath = strings.TrimSpace(configPath)
@@ -25,7 +25,10 @@ var rootCmd = &cobra.Command{
 			fmt.Println(err)
 			os.Exit(1)
 		}
-		engine.Run(conf)
+
+		if !engine.Run(conf) {
+			os.Exit(2)
+		}
 	},
 }
 

internal/engine/config.go

@@ -25,13 +25,11 @@ func LoadConfig(configPath string) (*config, error) {
 	if configPath == "-" {
 		stat, err = os.Stdin.Stat()
 		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
+			return nil, err
 		}
 
 		if stat.Size() < 1 {
-			fmt.Println("no config provided")
-			os.Exit(1)
+			return nil, fmt.Errorf("no config provided")
 		}
 
 		b, err = io.ReadAll(os.Stdin)

internal/engine/engine.go

@@ -5,12 +5,13 @@ import (
 	"net/http"
 )
 
-func Run(cfg *config) {
+func Run(cfg *config) (success bool) {
 	var (
-		client *http.Client = &http.Client{}
-		req    *http.Request
-		res    *http.Response
-		err    error
+		client   *http.Client = &http.Client{}
+		req      *http.Request
+		res      *http.Response
+		err      error
+		failures int
 	)
 
 	for _, test := range cfg.Tests {
@@ -19,6 +20,7 @@ func Run(cfg *config) {
 
 		if err != nil {
 			fmt.Println(err)
+			failures++
 			continue
 		}
 
@@ -35,11 +37,21 @@ func Run(cfg *config) {
 		res, err = client.Do(req)
 		if err != nil {
 			fmt.Println(err)
+			failures++
 			continue
 		}
 
-		test.report(res)
+		if !test.report(res) {
+			failures++
+		}
+	}
+
+	if failures == 0 {
+		success = true
 	}
+
+	fmt.Printf("\n Ran %d tests with %d failures\n", len(cfg.Tests), failures)
+	return
 }
 
 func noRedirect(req *http.Request, via []*http.Request) error {

internal/engine/test.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"io"
 	"net/http"
-	"os"
 	"strings"
 )
 
@@ -20,9 +19,13 @@ type test struct {
 	}
 }
 
-func (t *test) report(res *http.Response) {
-	var errors []string
-	result := "PASS"
+func (t *test) report(res *http.Response) (success bool) {
+	errors := []string{}
+
+	result := map[bool]string{
+		true:  "PASS",
+		false: "FAIL",
+	}
 
 	if t.Expect.Status != res.StatusCode {
 		errors = append(errors, fmt.Sprintf("expected status == %d got %d", t.Expect.Status, res.StatusCode))
@@ -48,19 +51,16 @@ func (t *test) report(res *http.Response) {
 		}
 	}
 
-	if len(errors) > 0 {
-		result = "FAIL"
+	if len(errors) == 0 {
+		success = true
 	}
 
-	fmt.Printf("%s : %s %s\n", result, t.Method, t.Url)
+	fmt.Printf("%s : %s %s\n", result[success], t.Method, t.Url)
 	if len(errors) > 0 {
 		for _, e := range errors {
 			fmt.Printf("  %s\n", e)
 		}
-		code := len(errors)
-		if code > 125 {
-			code = 125
-		}
-		os.Exit(code)
 	}
+
+	return
 }

tests/cli.bats

@@ -29,11 +29,17 @@ setup() {
   assert_output --partial 'PASS'
 }
 
-
 @test "SHOULD FAIL with missing headers" {
-  run ./emberfall --config ./tests/pass-missing-headers.yml
+  run ./emberfall --config ./tests/fail-missing-headers.yml
   assert_failure
   assert_output --partial 'FAIL'
   assert_output --partial 'expected header x-no-exist was missing'
 }
 
+
+@test "SHOULD FAIL with bad url" {
+  run ./emberfall --config ./tests/fail-bad-url.yml
+  assert_failure
+  assert_output --partial 'no such host'
+}
+

tests/fail-bad-url.yml

@@ -0,0 +1,5 @@
+tests:
+  - url: https://dfsdfkshfawsladfkhj.com
+    method: GET
+    expect:
+      status: 200 # should fail with != 301