Merge pull request #1856 from loadimpact/feature/HTTPReqFailures

Add a way to mark http requests as failed or not(passed)

core/engine.go

@@ -108,6 +108,20 @@ func NewEngine(
 		e.submetrics[parent] = append(e.submetrics[parent], sm)
 	}
 
+	// TODO: refactor this out of here when https://github.com/loadimpact/k6/issues/1832 lands and
+	// there is a better way to enable a metric with tag
+	if opts.SystemTags.Has(stats.TagExpectedResponse) {
+		for _, name := range []string{
+			"http_req_duration{expected_response:true}",
+		} {
+			if _, ok := e.thresholds[name]; ok {
+				continue
+			}
+			parent, sm := stats.NewSubmetric(name)
+			e.submetrics[parent] = append(e.submetrics[parent], sm)
+		}
+	}
+
 	return e, nil
 }
 

core/local/local_test.go

@@ -329,12 +329,13 @@ func TestExecutionSchedulerSystemTags(t *testing.T) {
 	}()
 
 	expCommonTrailTags := stats.IntoSampleTags(&map[string]string{
-		"group":  "",
-		"method": "GET",
-		"name":   sr("HTTPBIN_IP_URL/"),
-		"url":    sr("HTTPBIN_IP_URL/"),
-		"proto":  "HTTP/1.1",
-		"status": "200",
+		"group":             "",
+		"method":            "GET",
+		"name":              sr("HTTPBIN_IP_URL/"),
+		"url":               sr("HTTPBIN_IP_URL/"),
+		"proto":             "HTTP/1.1",
+		"status":            "200",
+		"expected_response": "true",
 	})
 	expTrailPVUTagsRaw := expCommonTrailTags.CloneTags()
 	expTrailPVUTagsRaw["scenario"] = "per_vu_test"

js/internal/modules/modules.go

@@ -35,7 +35,18 @@ var (
 func Get(name string) interface{} {
 	mx.RLock()
 	defer mx.RUnlock()
-	return modules[name]
+	mod := modules[name]
+	if i, ok := mod.(HasModuleInstancePerVU); ok {
+		return i.NewModuleInstancePerVU()
+	}
+	return mod
+}
+
+// HasModuleInstancePerVU should be implemented by all native Golang modules that
+// would require per-VU state. k6 will call their NewModuleInstancePerVU() methods
+// every time a VU imports the module and use its result as the returned object.
+type HasModuleInstancePerVU interface {
+	NewModuleInstancePerVU() interface{}
 }
 
 // Register the given mod as a JavaScript module, available

js/modules/k6/http/file_test.go

@@ -80,7 +80,7 @@ func TestHTTPFile(t *testing.T) {
 					assert.Equal(t, tc.expErr, fmt.Sprintf("%s", val["value"]))
 				}()
 			}
-			h := New()
+			h := new(GlobalHTTP).NewModuleInstancePerVU().(*HTTP)
 			ctx := common.WithRuntime(context.Background(), rt)
 			out := h.File(ctx, tc.input, tc.args...)
 			assert.Equal(t, tc.expected, out)

js/modules/k6/http/http.go

@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-	modules.Register("k6/http", New())
+	modules.Register("k6/http", new(GlobalHTTP))
 }
 
 const (
@@ -46,6 +46,38 @@ const (
 // ErrJarForbiddenInInitContext is used when a cookie jar was made in the init context
 var ErrJarForbiddenInInitContext = common.NewInitContextError("Making cookie jars in the init context is not supported")
 
+// GlobalHTTP is a global HTTP module for a k6 instance/test run
+type GlobalHTTP struct{}
+
+var _ modules.HasModuleInstancePerVU = new(GlobalHTTP)
+
+// NewModuleInstancePerVU returns an HTTP instance for each VU
+func (g *GlobalHTTP) NewModuleInstancePerVU() interface{} { // this here needs to return interface{}
+	return &HTTP{ // change the below fields to be not writable or not fields
+		SSL_3_0:                            netext.SSL_3_0,
+		TLS_1_0:                            netext.TLS_1_0,
+		TLS_1_1:                            netext.TLS_1_1,
+		TLS_1_2:                            netext.TLS_1_2,
+		TLS_1_3:                            netext.TLS_1_3,
+		OCSP_STATUS_GOOD:                   netext.OCSP_STATUS_GOOD,
+		OCSP_STATUS_REVOKED:                netext.OCSP_STATUS_REVOKED,
+		OCSP_STATUS_SERVER_FAILED:          netext.OCSP_STATUS_SERVER_FAILED,
+		OCSP_STATUS_UNKNOWN:                netext.OCSP_STATUS_UNKNOWN,
+		OCSP_REASON_UNSPECIFIED:            netext.OCSP_REASON_UNSPECIFIED,
+		OCSP_REASON_KEY_COMPROMISE:         netext.OCSP_REASON_KEY_COMPROMISE,
+		OCSP_REASON_CA_COMPROMISE:          netext.OCSP_REASON_CA_COMPROMISE,
+		OCSP_REASON_AFFILIATION_CHANGED:    netext.OCSP_REASON_AFFILIATION_CHANGED,
+		OCSP_REASON_SUPERSEDED:             netext.OCSP_REASON_SUPERSEDED,
+		OCSP_REASON_CESSATION_OF_OPERATION: netext.OCSP_REASON_CESSATION_OF_OPERATION,
+		OCSP_REASON_CERTIFICATE_HOLD:       netext.OCSP_REASON_CERTIFICATE_HOLD,
+		OCSP_REASON_REMOVE_FROM_CRL:        netext.OCSP_REASON_REMOVE_FROM_CRL,
+		OCSP_REASON_PRIVILEGE_WITHDRAWN:    netext.OCSP_REASON_PRIVILEGE_WITHDRAWN,
+		OCSP_REASON_AA_COMPROMISE:          netext.OCSP_REASON_AA_COMPROMISE,
+
+		responseCallback: defaultExpectedStatuses.match,
+	}
+}
+
 //nolint: golint
 type HTTP struct {
 	SSL_3_0                            string `js:"SSL_3_0"`
@@ -67,31 +99,8 @@ type HTTP struct {
 	OCSP_REASON_REMOVE_FROM_CRL        string `js:"OCSP_REASON_REMOVE_FROM_CRL"`
 	OCSP_REASON_PRIVILEGE_WITHDRAWN    string `js:"OCSP_REASON_PRIVILEGE_WITHDRAWN"`
 	OCSP_REASON_AA_COMPROMISE          string `js:"OCSP_REASON_AA_COMPROMISE"`
-}
 
-func New() *HTTP {
-	//TODO: move this as an anonymous struct somewhere...
-	return &HTTP{
-		SSL_3_0:                            netext.SSL_3_0,
-		TLS_1_0:                            netext.TLS_1_0,
-		TLS_1_1:                            netext.TLS_1_1,
-		TLS_1_2:                            netext.TLS_1_2,
-		TLS_1_3:                            netext.TLS_1_3,
-		OCSP_STATUS_GOOD:                   netext.OCSP_STATUS_GOOD,
-		OCSP_STATUS_REVOKED:                netext.OCSP_STATUS_REVOKED,
-		OCSP_STATUS_SERVER_FAILED:          netext.OCSP_STATUS_SERVER_FAILED,
-		OCSP_STATUS_UNKNOWN:                netext.OCSP_STATUS_UNKNOWN,
-		OCSP_REASON_UNSPECIFIED:            netext.OCSP_REASON_UNSPECIFIED,
-		OCSP_REASON_KEY_COMPROMISE:         netext.OCSP_REASON_KEY_COMPROMISE,
-		OCSP_REASON_CA_COMPROMISE:          netext.OCSP_REASON_CA_COMPROMISE,
-		OCSP_REASON_AFFILIATION_CHANGED:    netext.OCSP_REASON_AFFILIATION_CHANGED,
-		OCSP_REASON_SUPERSEDED:             netext.OCSP_REASON_SUPERSEDED,
-		OCSP_REASON_CESSATION_OF_OPERATION: netext.OCSP_REASON_CESSATION_OF_OPERATION,
-		OCSP_REASON_CERTIFICATE_HOLD:       netext.OCSP_REASON_CERTIFICATE_HOLD,
-		OCSP_REASON_REMOVE_FROM_CRL:        netext.OCSP_REASON_REMOVE_FROM_CRL,
-		OCSP_REASON_PRIVILEGE_WITHDRAWN:    netext.OCSP_REASON_PRIVILEGE_WITHDRAWN,
-		OCSP_REASON_AA_COMPROMISE:          netext.OCSP_REASON_AA_COMPROMISE,
-	}
+	responseCallback func(int) bool
 }
 
 func (*HTTP) XCookieJar(ctx *context.Context) *HTTPCookieJar {

js/modules/k6/http/http_test.go

@@ -34,7 +34,7 @@ import (
 func TestTagURL(t *testing.T) {
 	rt := goja.New()
 	rt.SetFieldNameMapper(common.FieldNameMapper{})
-	rt.Set("http", common.Bind(rt, New(), nil))
+	rt.Set("http", common.Bind(rt, new(GlobalHTTP).NewModuleInstancePerVU(), nil))
 
 	testdata := map[string]struct{ u, n string }{
 		`http://localhost/anything/`:               {"http://localhost/anything/", "http://localhost/anything/"},

js/modules/k6/http/request.go

@@ -113,7 +113,7 @@ func (h *HTTP) Request(ctx context.Context, method string, url goja.Value, args
 	if err != nil {
 		return nil, err
 	}
-	return responseFromHttpext(resp), nil
+	return h.responseFromHttpext(resp), nil
 }
 
 //TODO break this function up
@@ -134,12 +134,14 @@ func (h *HTTP) parseRequest(
 			URL:    reqURL.GetURL(),
 			Header: make(http.Header),
 		},
-		Timeout:   60 * time.Second,
-		Throw:     state.Options.Throw.Bool,
-		Redirects: state.Options.MaxRedirects,
-		Cookies:   make(map[string]*httpext.HTTPRequestCookie),
-		Tags:      make(map[string]string),
+		Timeout:          60 * time.Second,
+		Throw:            state.Options.Throw.Bool,
+		Redirects:        state.Options.MaxRedirects,
+		Cookies:          make(map[string]*httpext.HTTPRequestCookie),
+		Tags:             make(map[string]string),
+		ResponseCallback: h.responseCallback,
 	}
+
 	if state.Options.DiscardResponseBodies.Bool {
 		result.ResponseType = httpext.ResponseTypeNone
 	} else {
@@ -349,6 +351,15 @@ func (h *HTTP) parseRequest(
 					return nil, err
 				}
 				result.ResponseType = responseType
+			case "responseCallback":
+				v := params.Get(k).Export()
+				if v == nil {
+					result.ResponseCallback = nil
+				} else if c, ok := v.(*expectedStatuses); ok {
+					result.ResponseCallback = c.match
+				} else {
+					return nil, fmt.Errorf("unsupported responseCallback")
+				}
 			}
 		}
 	}
@@ -377,7 +388,7 @@ func (h *HTTP) prepareBatchArray(
 			ParsedHTTPRequest: parsedReq,
 			Response:          response,
 		}
-		results[i] = &Response{response}
+		results[i] = h.responseFromHttpext(response)
 	}
 
 	return batchReqs, results, nil
@@ -401,7 +412,7 @@ func (h *HTTP) prepareBatchObject(
 			ParsedHTTPRequest: parsedReq,
 			Response:          response,
 		}
-		results[key] = &Response{response}
+		results[key] = h.responseFromHttpext(response)
 		i++
 	}
 

js/modules/k6/http/request_test.go

@@ -81,6 +81,7 @@ func TestRunES6String(t *testing.T) {
 	})
 }
 
+// TODO replace this with the Single version
 func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleContainer, method, url, name string, status int, group string) {
 	if name == "" {
 		name = url
@@ -130,6 +131,29 @@ func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleCo
 	assert.True(t, seenReceiving, "url %s didn't emit Receiving", url)
 }
 
+func assertRequestMetricsEmittedSingle(t *testing.T, sampleContainer stats.SampleContainer, expectedTags map[string]string, metrics []*stats.Metric, callback func(sample stats.Sample)) {
+	t.Helper()
+
+	metricMap := make(map[string]bool, len(metrics))
+	for _, m := range metrics {
+		metricMap[m.Name] = false
+	}
+	for _, sample := range sampleContainer.GetSamples() {
+		tags := sample.Tags.CloneTags()
+		v, ok := metricMap[sample.Metric.Name]
+		assert.True(t, ok, "unexpected metric %s", sample.Metric.Name)
+		assert.False(t, v, "second metric %s", sample.Metric.Name)
+		metricMap[sample.Metric.Name] = true
+		assert.EqualValues(t, expectedTags, tags, "%s", tags)
+		if callback != nil {
+			callback(sample)
+		}
+	}
+	for k, v := range metricMap {
+		assert.True(t, v, "didn't emit %s", k)
+	}
+}
+
 func newRuntime(
 	t testing.TB,
 ) (*httpmultibin.HTTPMultiBin, *lib.State, chan stats.SampleContainer, *goja.Runtime, *context.Context) {
@@ -169,7 +193,7 @@ func newRuntime(
 	ctx := new(context.Context)
 	*ctx = lib.WithState(tb.Context, state)
 	*ctx = common.WithRuntime(*ctx, rt)
-	rt.Set("http", common.Bind(rt, New(), ctx))
+	rt.Set("http", common.Bind(rt, new(GlobalHTTP).NewModuleInstancePerVU(), ctx))
 
 	return tb, state, samples, rt, ctx
 }
@@ -1945,26 +1969,28 @@ func TestRedirectMetricTags(t *testing.T) {
 
 	checkTags := func(sc stats.SampleContainer, expTags map[string]string) {
 		allSamples := sc.GetSamples()
-		assert.Len(t, allSamples, 8)
+		assert.Len(t, allSamples, 9)
 		for _, s := range allSamples {
 			assert.Equal(t, expTags, s.Tags.CloneTags())
 		}
 	}
 	expPOSTtags := map[string]string{
-		"group":  "",
-		"method": "POST",
-		"url":    sr("HTTPBIN_URL/redirect/post"),
-		"name":   sr("HTTPBIN_URL/redirect/post"),
-		"status": "301",
-		"proto":  "HTTP/1.1",
+		"group":             "",
+		"method":            "POST",
+		"url":               sr("HTTPBIN_URL/redirect/post"),
+		"name":              sr("HTTPBIN_URL/redirect/post"),
+		"status":            "301",
+		"proto":             "HTTP/1.1",
+		"expected_response": "true",
 	}
 	expGETtags := map[string]string{
-		"group":  "",
-		"method": "GET",
-		"url":    sr("HTTPBIN_URL/get"),
-		"name":   sr("HTTPBIN_URL/get"),
-		"status": "200",
-		"proto":  "HTTP/1.1",
+		"group":             "",
+		"method":            "GET",
+		"url":               sr("HTTPBIN_URL/get"),
+		"name":              sr("HTTPBIN_URL/get"),
+		"status":            "200",
+		"proto":             "HTTP/1.1",
+		"expected_response": "true",
 	}
 	checkTags(<-samples, expPOSTtags)
 	checkTags(<-samples, expGETtags)

js/modules/k6/http/response.go

@@ -36,11 +36,11 @@ import (
 // Response is a representation of an HTTP response to be returned to the goja VM
 type Response struct {
 	*httpext.Response `js:"-"`
+	h                 *HTTP
 }
 
-func responseFromHttpext(resp *httpext.Response) *Response {
-	res := Response{resp}
-	return &res
+func (h *HTTP) responseFromHttpext(resp *httpext.Response) *Response {
+	return &Response{Response: resp, h: h}
 }
 
 // JSON parses the body of a response as json and returns it to the goja VM
@@ -159,9 +159,9 @@ func (res *Response) SubmitForm(args ...goja.Value) (*Response, error) {
 			q.Add(k, v.String())
 		}
 		requestURL.RawQuery = q.Encode()
-		return New().Request(res.GetCtx(), requestMethod, rt.ToValue(requestURL.String()), goja.Null(), requestParams)
+		return res.h.Request(res.GetCtx(), requestMethod, rt.ToValue(requestURL.String()), goja.Null(), requestParams)
 	}
-	return New().Request(res.GetCtx(), requestMethod, rt.ToValue(requestURL.String()), rt.ToValue(values), requestParams)
+	return res.h.Request(res.GetCtx(), requestMethod, rt.ToValue(requestURL.String()), rt.ToValue(values), requestParams)
 }
 
 // ClickLink parses the body as an html, looks for a link and than makes a request as if the link was
@@ -202,5 +202,5 @@ func (res *Response) ClickLink(args ...goja.Value) (*Response, error) {
 	}
 	requestURL := responseURL.ResolveReference(hrefURL)
 
-	return New().Get(res.GetCtx(), rt.ToValue(requestURL.String()), requestParams)
+	return res.h.Get(res.GetCtx(), rt.ToValue(requestURL.String()), requestParams)
 }