From c03368574f6b92f557addd8a6c9732be37d91562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9o=20Crevon?= Date: Wed, 12 Jan 2022 14:14:29 +0100 Subject: [PATCH 1/4] Merge pull request #2251 from grafana/fix/1443_remove_the_js_runtime_from_threshold_calcultations Fix/1443 remove the JS runtime from threshold calculations --- core/engine_test.go | 29 ++- stats/thresholds.go | 181 ++++++++------ stats/thresholds_parser.go | 208 ++++++++++++++++ stats/thresholds_parser_test.go | 361 ++++++++++++++++++++++++++++ stats/thresholds_test.go | 411 ++++++++++++++++++++++++-------- 5 files changed, 1010 insertions(+), 180 deletions(-) create mode 100644 stats/thresholds_parser.go create mode 100644 stats/thresholds_parser_test.go diff --git a/core/engine_test.go b/core/engine_test.go index ecd81b5f773..e9f3296cca3 100644 --- a/core/engine_test.go +++ b/core/engine_test.go @@ -257,7 +257,7 @@ func TestEngine_processSamples(t *testing.T) { }) t.Run("submetric", func(t *testing.T) { t.Parallel() - ths, err := stats.NewThresholds([]string{`1+1==2`}) + ths, err := stats.NewThresholds([]string{`value<2`}) assert.NoError(t, err) e, _, wait := newTestEngine(t, nil, nil, nil, lib.Options{ @@ -285,7 +285,10 @@ func TestEngineThresholdsWillAbort(t *testing.T) { t.Parallel() metric := stats.New("my_metric", stats.Gauge) - ths, err := stats.NewThresholds([]string{"1+1==3"}) + // The incoming samples for the metric set it to 1.25. Considering + // the metric is of type Gauge, value > 1.25 should always fail, and + // trigger an abort. + ths, err := stats.NewThresholds([]string{"value>1.25"}) assert.NoError(t, err) ths.Thresholds[0].AbortOnFail = true @@ -304,7 +307,11 @@ func TestEngineAbortedByThresholds(t *testing.T) { t.Parallel() metric := stats.New("my_metric", stats.Gauge) - ths, err := stats.NewThresholds([]string{"1+1==3"}) + // The MiniRunner sets the value of the metric to 1.25. Considering + // the metric is of type Gauge, value > 1.25 should always fail, and + // trigger an abort. + // **N.B**: a threshold returning an error, won't trigger an abort. + ths, err := stats.NewThresholds([]string{"value>1.25"}) assert.NoError(t, err) ths.Thresholds[0].AbortOnFail = true @@ -342,14 +349,14 @@ func TestEngine_processThresholds(t *testing.T) { ths map[string][]string abort bool }{ - "passing": {true, map[string][]string{"my_metric": {"1+1==2"}}, false}, - "failing": {false, map[string][]string{"my_metric": {"1+1==3"}}, false}, - "aborting": {false, map[string][]string{"my_metric": {"1+1==3"}}, true}, - - "submetric,match,passing": {true, map[string][]string{"my_metric{a:1}": {"1+1==2"}}, false}, - "submetric,match,failing": {false, map[string][]string{"my_metric{a:1}": {"1+1==3"}}, false}, - "submetric,nomatch,passing": {true, map[string][]string{"my_metric{a:2}": {"1+1==2"}}, false}, - "submetric,nomatch,failing": {true, map[string][]string{"my_metric{a:2}": {"1+1==3"}}, false}, + "passing": {true, map[string][]string{"my_metric": {"value<2"}}, false}, + "failing": {false, map[string][]string{"my_metric": {"value>1.25"}}, false}, + "aborting": {false, map[string][]string{"my_metric": {"value>1.25"}}, true}, + + "submetric,match,passing": {true, map[string][]string{"my_metric{a:1}": {"value<2"}}, false}, + "submetric,match,failing": {false, map[string][]string{"my_metric{a:1}": {"value>1.25"}}, false}, + "submetric,nomatch,passing": {true, map[string][]string{"my_metric{a:2}": {"value<2"}}, false}, + "submetric,nomatch,failing": {true, map[string][]string{"my_metric{a:2}": {"value>1.25"}}, false}, } for name, data := range testdata { diff --git a/stats/thresholds.go b/stats/thresholds.go index bb3d58c4ebc..1b7cbaea53f 100644 --- a/stats/thresholds.go +++ b/stats/thresholds.go @@ -17,54 +17,35 @@ * along with this program. If not, see . * */ - package stats import ( "bytes" "encoding/json" "fmt" + "strings" "time" - "github.com/dop251/goja" - "go.k6.io/k6/lib/types" ) -const jsEnvSrc = ` -function p(pct) { - return __sink__.P(pct/100.0); -}; -` - -var jsEnv *goja.Program - -func init() { - pgm, err := goja.Compile("__env__", jsEnvSrc, true) - if err != nil { - panic(err) - } - jsEnv = pgm -} - // Threshold is a representation of a single threshold for a single metric type Threshold struct { // Source is the text based source of the threshold Source string - // LastFailed is a makrer if the last testing of this threshold failed + // LastFailed is a marker if the last testing of this threshold failed LastFailed bool // AbortOnFail marks if a given threshold fails that the whole test should be aborted AbortOnFail bool // AbortGracePeriod is a the minimum amount of time a test should be running before a failing // this threshold will abort the test AbortGracePeriod types.NullDuration - - pgm *goja.Program - rt *goja.Runtime + // parsed is the threshold expression parsed from the Source + parsed *thresholdExpression } -func newThreshold(src string, newThreshold *goja.Runtime, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) { - pgm, err := goja.Compile("__threshold__", src, true) +func newThreshold(src string, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) { + parsedExpression, err := parseThresholdExpression(src) if err != nil { return nil, err } @@ -73,23 +54,57 @@ func newThreshold(src string, newThreshold *goja.Runtime, abortOnFail bool, grac Source: src, AbortOnFail: abortOnFail, AbortGracePeriod: gracePeriod, - pgm: pgm, - rt: newThreshold, + parsed: parsedExpression, }, nil } -func (t Threshold) runNoTaint() (bool, error) { - v, err := t.rt.RunProgram(t.pgm) - if err != nil { - return false, err +func (t *Threshold) runNoTaint(sinks map[string]float64) (bool, error) { + // Extract the sink value for the aggregation method used in the threshold + // expression + lhs, ok := sinks[t.parsed.AggregationMethod] + if !ok { + return false, fmt.Errorf("unable to apply threshold %s over metrics; reason: "+ + "no metric supporting the %s aggregation method found", + t.Source, + t.parsed.AggregationMethod) } - return v.ToBoolean(), nil + + // Apply the threshold expression operator to the left and + // right hand side values + var passes bool + switch t.parsed.Operator { + case ">": + passes = lhs > t.parsed.Value + case ">=": + passes = lhs >= t.parsed.Value + case "<=": + passes = lhs <= t.parsed.Value + case "<": + passes = lhs < t.parsed.Value + case "==", "===": + // Considering a sink always maps to float64 values, + // strictly equal is equivalent to loosely equal + passes = lhs == t.parsed.Value + case "!=": + passes = lhs != t.parsed.Value + default: + // The parseThresholdExpression function should ensure that no invalid + // operator gets through, but let's protect our future selves anyhow. + return false, fmt.Errorf("unable to apply threshold %s over metrics; "+ + "reason: %s is an invalid operator", + t.Source, + t.parsed.Operator, + ) + } + + // Perform the actual threshold verification + return passes, nil } -func (t *Threshold) run() (bool, error) { - b, err := t.runNoTaint() - t.LastFailed = !b - return b, err +func (t *Threshold) run(sinks map[string]float64) (bool, error) { + passes, err := t.runNoTaint(sinks) + t.LastFailed = !passes + return passes, err } type thresholdConfig struct { @@ -98,11 +113,11 @@ type thresholdConfig struct { AbortGracePeriod types.NullDuration `json:"delayAbortEval"` } -//used internally for JSON marshalling +// used internally for JSON marshalling type rawThresholdConfig thresholdConfig func (tc *thresholdConfig) UnmarshalJSON(data []byte) error { - //shortcircuit unmarshalling for simple string format + // shortcircuit unmarshalling for simple string format if err := json.Unmarshal(data, &tc.Threshold); err == nil { return nil } @@ -122,9 +137,9 @@ func (tc thresholdConfig) MarshalJSON() ([]byte, error) { // Thresholds is the combination of all Thresholds for a given metric type Thresholds struct { - Runtime *goja.Runtime Thresholds []*Threshold Abort bool + sinked map[string]float64 } // NewThresholds returns Thresholds objects representing the provided source strings @@ -138,60 +153,88 @@ func NewThresholds(sources []string) (Thresholds, error) { } func newThresholdsWithConfig(configs []thresholdConfig) (Thresholds, error) { - rt := goja.New() - if _, err := rt.RunProgram(jsEnv); err != nil { - return Thresholds{}, fmt.Errorf("threshold builtin error: %w", err) - } + thresholds := make([]*Threshold, len(configs)) + sinked := make(map[string]float64) - ts := make([]*Threshold, len(configs)) for i, config := range configs { - t, err := newThreshold(config.Threshold, rt, config.AbortOnFail, config.AbortGracePeriod) + t, err := newThreshold(config.Threshold, config.AbortOnFail, config.AbortGracePeriod) if err != nil { return Thresholds{}, fmt.Errorf("threshold %d error: %w", i, err) } - ts[i] = t + thresholds[i] = t } - return Thresholds{rt, ts, false}, nil + return Thresholds{thresholds, false, sinked}, nil } -func (ts *Thresholds) updateVM(sink Sink, t time.Duration) error { - ts.Runtime.Set("__sink__", sink) - f := sink.Format(t) - for k, v := range f { - ts.Runtime.Set(k, v) - } - return nil -} - -func (ts *Thresholds) runAll(t time.Duration) (bool, error) { - succ := true - for i, th := range ts.Thresholds { - b, err := th.run() +func (ts *Thresholds) runAll(timeSpentInTest time.Duration) (bool, error) { + succeeded := true + for i, threshold := range ts.Thresholds { + b, err := threshold.run(ts.sinked) if err != nil { return false, fmt.Errorf("threshold %d run error: %w", i, err) } + if !b { - succ = false + succeeded = false - if ts.Abort || !th.AbortOnFail { + if ts.Abort || !threshold.AbortOnFail { continue } - ts.Abort = !th.AbortGracePeriod.Valid || - th.AbortGracePeriod.Duration < types.Duration(t) + ts.Abort = !threshold.AbortGracePeriod.Valid || + threshold.AbortGracePeriod.Duration < types.Duration(timeSpentInTest) } } - return succ, nil + + return succeeded, nil } // Run processes all the thresholds with the provided Sink at the provided time and returns if any // of them fails -func (ts *Thresholds) Run(sink Sink, t time.Duration) (bool, error) { - if err := ts.updateVM(sink, t); err != nil { - return false, err +func (ts *Thresholds) Run(sink Sink, duration time.Duration) (bool, error) { + // Initialize the sinks store + ts.sinked = make(map[string]float64) + + // FIXME: Remove this comment as soon as the stats.Sink does not expose Format anymore. + // + // As of December 2021, this block reproduces the behavior of the + // stats.Sink.Format behavior. As we intend to try to get away from it, + // we instead implement the behavior directly here. + // + // For more details, see https://github.com/grafana/k6/issues/2320 + switch sinkImpl := sink.(type) { + case *CounterSink: + ts.sinked["count"] = sinkImpl.Value + ts.sinked["rate"] = sinkImpl.Value / (float64(duration) / float64(time.Second)) + case *GaugeSink: + ts.sinked["value"] = sinkImpl.Value + case *TrendSink: + ts.sinked["min"] = sinkImpl.Min + ts.sinked["max"] = sinkImpl.Max + ts.sinked["avg"] = sinkImpl.Avg + ts.sinked["med"] = sinkImpl.Med + + // Parse the percentile thresholds and insert them in + // the sinks mapping. + for _, threshold := range ts.Thresholds { + if !strings.HasPrefix(threshold.parsed.AggregationMethod, "p(") { + continue + } + + ts.sinked[threshold.parsed.AggregationMethod] = sinkImpl.P(threshold.parsed.AggregationValue.Float64 / 100) + } + case *RateSink: + ts.sinked["rate"] = float64(sinkImpl.Trues) / float64(sinkImpl.Total) + case DummySink: + for k, v := range sinkImpl { + ts.sinked[k] = v + } + default: + return false, fmt.Errorf("unable to run Thresholds; reason: unknown sink type") } - return ts.runAll(t) + + return ts.runAll(duration) } // UnmarshalJSON is implementation of json.Unmarshaler diff --git a/stats/thresholds_parser.go b/stats/thresholds_parser.go new file mode 100644 index 00000000000..72b0f82e2a7 --- /dev/null +++ b/stats/thresholds_parser.go @@ -0,0 +1,208 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package stats + +import ( + "fmt" + "strconv" + "strings" + + "gopkg.in/guregu/null.v3" +) + +// thresholdExpression holds the parsed result of a threshold expression, +// as described in: https://k6.io/docs/using-k6/thresholds/#threshold-syntax +type thresholdExpression struct { + // AggregationMethod holds the aggregation method parsed + // from the threshold expression. Possible values are described + // by `aggregationMethodTokens`. + AggregationMethod string + + // AggregationValue will hold the aggregation method's pivot value + // in the event it is a percentile. For instance: an expression of the form p(99.9) < 200, + // would result in AggregationValue to be set to 99.9. + AggregationValue null.Float + + // Operator holds the operator parsed from the threshold expression. + // Possible values are described by `operatorTokens`. + Operator string + + // Value holds the value parsed from the threshold expression. + Value float64 +} + +// parseThresholdAssertion parses a threshold condition expression, +// as defined in a JS script (for instance p(95)<1000), into a thresholdExpression +// instance. +// +// It is expected to be of the form: `aggregation_method operator value`. +// As defined by the following BNF: +// ``` +// assertion -> aggregation_method whitespace* operator whitespace* float +// aggregation_method -> trend | rate | gauge | counter +// counter -> "count" | "rate" +// gauge -> "value" +// rate -> "rate" +// trend -> "avg" | "min" | "max" | "med" | percentile +// percentile -> "p(" float ")" +// operator -> ">" | ">=" | "<=" | "<" | "==" | "===" | "!=" +// float -> digit+ ("." digit+)? +// digit -> "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" +// whitespace -> " " +// ``` +func parseThresholdExpression(input string) (*thresholdExpression, error) { + // Scanning makes no assumption on the underlying values, and only + // checks that the expression has the right format. + method, operator, value, err := scanThresholdExpression(input) + if err != nil { + return nil, fmt.Errorf("failed parsing threshold expression; reason: %w", err) + } + + parsedMethod, parsedMethodValue, err := parseThresholdAggregationMethod(method) + if err != nil { + return nil, fmt.Errorf("failed parsing threshold expression's left hand side; reason: %w", err) + } + + parsedValue, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil, fmt.Errorf("failed parsing threshold expresion's right hand side; reason: %w", err) + } + + condition := &thresholdExpression{ + AggregationMethod: parsedMethod, + AggregationValue: parsedMethodValue, + Operator: operator, + Value: parsedValue, + } + + return condition, nil +} + +// Define accepted threshold expression operators tokens +const ( + tokenLessEqual = "<=" + tokenLess = "<" + tokenGreaterEqual = ">=" + tokenGreater = ">" + tokenStrictlyEqual = "===" + tokenLooselyEqual = "==" + tokenBangEqual = "!=" +) + +// operatorTokens defines the list of operator-related tokens +// used in threshold expressions parsing. +// +// It is meant to be used during the scan of threshold expressions. +// Although declared as a `var`, being an array, it is effectively +// immutable and can be considered constant. +// +// Note that because scanning uses a substring parser, and will match +// the smallest common substring, the actual slice order matters. +// Longer tokens with symbols in common with shorter ones must appear +// first in the list in order to be effectively matched. +var operatorTokens = [7]string{ // nolint:gochecknoglobals + tokenLessEqual, + tokenLess, + tokenGreaterEqual, + tokenGreater, + tokenStrictlyEqual, + tokenLooselyEqual, + tokenBangEqual, +} + +// scanThresholdExpression scans a threshold condition expression of the +// form: `aggregation_method operator value`. An invalid or unknown operator +// will produce an error. However, no assertions regarding +// either the left-hand side aggregation method nor the right-hand +// side value will be made: they will be returned as is, only trimmed from +// their spaces. +func scanThresholdExpression(input string) (string, string, string, error) { + for _, op := range operatorTokens { + substrings := strings.SplitN(input, op, 2) + if len(substrings) == 2 { + return strings.TrimSpace(substrings[0]), op, strings.TrimSpace(substrings[1]), nil + } + } + + return "", "", "", fmt.Errorf("malformed threshold expression") +} + +// Define accepted threshold expression aggregation tokens +// Percentile token `p(..)` is accepted too but handled separately. +const ( + tokenValue = "value" + tokenCount = "count" + tokenRate = "rate" + tokenAvg = "avg" + tokenMin = "min" + tokenMed = "med" + tokenMax = "max" + tokenPercentile = "p" +) + +// aggregationMethodTokens defines the list of aggregation method +// used in the parsing of threshold expressions. +// +// It is meant to be used during the parsing of threshold expressions. +// Although declared as a `var`, being an array, it is effectively +// immutable and can be considered constant. +var aggregationMethodTokens = [8]string{ // nolint:gochecknoglobals + tokenValue, + tokenCount, + tokenRate, + tokenAvg, + tokenMin, + tokenMed, + tokenMax, + tokenPercentile, +} + +// parseThresholdMethod will parse a threshold condition expression's method. +// It assumes the provided input argument is already trimmed and cleaned up. +// If it encounters a percentile method, it will parse it and verify it +// boils down to an expression of the form: `p(float64)`, but will return +// it verbatim, as a string. +func parseThresholdAggregationMethod(input string) (string, null.Float, error) { + // Is the input one of the methods keywords? + for _, m := range aggregationMethodTokens { + // Percentile expressions being of the form p(value), + // they won't be matched here. + if m == input { + return input, null.Float{}, nil + } + } + + // Otherwise, attempt to parse a percentile expression + if strings.HasPrefix(input, tokenPercentile+"(") && strings.HasSuffix(input, ")") { + aggregationValue, err := strconv.ParseFloat(trimDelimited("p(", input, ")"), 64) + if err != nil { + return "", null.Float{}, fmt.Errorf("malformed percentile value; reason: %w", err) + } + + return input, null.FloatFrom(aggregationValue), nil + } + + return "", null.Float{}, fmt.Errorf("failed parsing method from expression") +} + +func trimDelimited(prefix, input, suffix string) string { + return strings.TrimSuffix(strings.TrimPrefix(input, prefix), suffix) +} diff --git a/stats/thresholds_parser_test.go b/stats/thresholds_parser_test.go new file mode 100644 index 00000000000..25120684d6e --- /dev/null +++ b/stats/thresholds_parser_test.go @@ -0,0 +1,361 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package stats + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/guregu/null.v3" +) + +func TestParseThresholdExpression(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantExpression *thresholdExpression + wantErr bool + }{ + { + name: "unknown expression's operator fails", + input: "count!20", + wantExpression: nil, + wantErr: true, + }, + { + name: "unknown expression's method fails", + input: "foo>20", + wantExpression: nil, + wantErr: true, + }, + { + name: "non numerical expression's value fails", + input: "count>abc", + wantExpression: nil, + wantErr: true, + }, + { + name: "valid threshold expression syntax", + input: "count>20", + wantExpression: &thresholdExpression{AggregationMethod: "count", Operator: ">", Value: 20}, + wantErr: false, + }, + } + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + gotExpression, gotErr := parseThresholdExpression(testCase.input) + + assert.Equal(t, + testCase.wantErr, + gotErr != nil, + "parseThresholdExpression() error = %v, wantErr %v", gotErr, testCase.wantErr, + ) + + assert.Equal(t, + testCase.wantExpression, + gotExpression, + "parseThresholdExpression() gotExpression = %v, want %v", gotExpression, testCase.wantExpression, + ) + }) + } +} + +func BenchmarkParseThresholdExpression(b *testing.B) { + for i := 0; i < b.N; i++ { + parseThresholdExpression("count>20") // nolint + } +} + +func TestParseThresholdAggregationMethod(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantMethod string + wantMethodValue null.Float + wantErr bool + }{ + { + name: "count method is parsed", + input: "count", + wantMethod: "count", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "rate method is parsed", + input: "rate", + wantMethod: "rate", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "value method is parsed", + input: "value", + wantMethod: "value", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "avg method is parsed", + input: "avg", + wantMethod: "avg", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "min method is parsed", + input: "min", + wantMethod: "min", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "max method is parsed", + input: "max", + wantMethod: "max", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "med method is parsed", + input: "med", + wantMethod: "med", + wantMethodValue: null.Float{}, + wantErr: false, + }, + { + name: "percentile method with integer value is parsed", + input: "p(99)", + wantMethod: "p(99)", + wantMethodValue: null.FloatFrom(99), + wantErr: false, + }, + { + name: "percentile method with floating point value is parsed", + input: "p(99.9)", + wantMethod: "p(99.9)", + wantMethodValue: null.FloatFrom(99.9), + wantErr: false, + }, + { + name: "parsing invalid method fails", + input: "foo", + wantMethod: "", + wantMethodValue: null.Float{}, + wantErr: true, + }, + { + name: "parsing empty percentile expression fails", + input: "p()", + wantMethod: "", + wantMethodValue: null.Float{}, + wantErr: true, + }, + { + name: "parsing incomplete percentile expression fails", + input: "p(99", + wantMethod: "", + wantMethodValue: null.Float{}, + wantErr: true, + }, + { + name: "parsing non-numerical percentile value fails", + input: "p(foo)", + wantMethod: "", + wantMethodValue: null.Float{}, + wantErr: true, + }, + } + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + gotMethod, gotMethodValue, gotErr := parseThresholdAggregationMethod(testCase.input) + + assert.Equal(t, + testCase.wantErr, + gotErr != nil, + "parseThresholdAggregationMethod() error = %v, wantErr %v", gotErr, testCase.wantErr, + ) + + assert.Equal(t, + testCase.wantMethod, + gotMethod, + "parseThresholdAggregationMethod() gotMethod = %v, want %v", gotMethod, testCase.wantMethod, + ) + + assert.Equal(t, + testCase.wantMethodValue, + gotMethodValue, + "parseThresholdAggregationMethod() gotMethodValue = %v, want %v", gotMethodValue, testCase.wantMethodValue, + ) + }) + } +} + +func BenchmarkParseThresholdAggregationMethod(b *testing.B) { + for i := 0; i < b.N; i++ { + parseThresholdAggregationMethod("p(99.9)") // nolint + } +} + +func TestScanThresholdExpression(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantMethod string + wantOperator string + wantValue string + wantErr bool + }{ + { + name: "expression with <= operator is scanned", + input: "foo<=bar", + wantMethod: "foo", + wantOperator: "<=", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with < operator is scanned", + input: "foo= operator is scanned", + input: "foo>=bar", + wantMethod: "foo", + wantOperator: ">=", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with > operator is scanned", + input: "foo>bar", + wantMethod: "foo", + wantOperator: ">", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with === operator is scanned", + input: "foo===bar", + wantMethod: "foo", + wantOperator: "===", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with == operator is scanned", + input: "foo==bar", + wantMethod: "foo", + wantOperator: "==", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with != operator is scanned", + input: "foo!=bar", + wantMethod: "foo", + wantOperator: "!=", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression's method is trimmed", + input: " foo <=bar", + wantMethod: "foo", + wantOperator: "<=", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression's value is trimmed", + input: "foo<= bar ", + wantMethod: "foo", + wantOperator: "<=", + wantValue: "bar", + wantErr: false, + }, + { + name: "expression with unknown operator fails", + input: "foo!bar", + wantMethod: "", + wantOperator: "", + wantValue: "", + wantErr: true, + }, + } + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + gotMethod, gotOperator, gotValue, gotErr := scanThresholdExpression(testCase.input) + + assert.Equal(t, + testCase.wantErr, + gotErr != nil, + "scanThresholdExpression() error = %v, wantErr %v", gotErr, testCase.wantErr, + ) + + assert.Equal(t, + testCase.wantMethod, + gotMethod, + "scanThresholdExpression() gotMethod = %v, want %v", gotMethod, testCase.wantMethod, + ) + + assert.Equal(t, + testCase.wantOperator, + gotOperator, + "scanThresholdExpression() gotOperator = %v, want %v", gotOperator, testCase.wantOperator, + ) + + assert.Equal(t, + testCase.wantValue, + gotValue, + "scanThresholdExpression() gotValue = %v, want %v", gotValue, testCase.wantValue, + ) + }) + } +} + +func BenchmarkScanThresholdExpression(b *testing.B) { + for i := 0; i < b.N; i++ { + scanThresholdExpression("foo<=bar") // nolint + } +} diff --git a/stats/thresholds_test.go b/stats/thresholds_test.go index 4d06dd0f05f..9e381bf75fb 100644 --- a/stats/thresholds_test.go +++ b/stats/thresholds_test.go @@ -25,76 +25,257 @@ import ( "testing" "time" - "github.com/dop251/goja" "github.com/stretchr/testify/assert" - + "github.com/stretchr/testify/require" "go.k6.io/k6/lib/types" + "gopkg.in/guregu/null.v3" ) func TestNewThreshold(t *testing.T) { - src := `1+1==2` - rt := goja.New() + t.Parallel() + + src := `rate<0.01` abortOnFail := false gracePeriod := types.NullDurationFrom(2 * time.Second) - th, err := newThreshold(src, rt, abortOnFail, gracePeriod) + wantParsed := &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01} + + gotThreshold, err := newThreshold(src, abortOnFail, gracePeriod) + assert.NoError(t, err) + assert.Equal(t, src, gotThreshold.Source) + assert.False(t, gotThreshold.LastFailed) + assert.Equal(t, abortOnFail, gotThreshold.AbortOnFail) + assert.Equal(t, gracePeriod, gotThreshold.AbortGracePeriod) + assert.Equal(t, wantParsed, gotThreshold.parsed) +} + +func TestNewThreshold_InvalidThresholdConditionExpression(t *testing.T) { + t.Parallel() + + src := "1+1==2" + abortOnFail := false + gracePeriod := types.NullDurationFrom(2 * time.Second) - assert.Equal(t, src, th.Source) - assert.False(t, th.LastFailed) - assert.NotNil(t, th.pgm) - assert.Equal(t, rt, th.rt) - assert.Equal(t, abortOnFail, th.AbortOnFail) - assert.Equal(t, gracePeriod, th.AbortGracePeriod) + gotThreshold, err := newThreshold(src, abortOnFail, gracePeriod) + + assert.Error(t, err, "instantiating a threshold with an invalid expression should fail") + assert.Nil(t, gotThreshold, "instantiating a threshold with an invalid expression should return a nil Threshold") +} + +func TestThreshold_runNoTaint(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + parsed *thresholdExpression + abortGracePeriod types.NullDuration + sinks map[string]float64 + wantOk bool + wantErr bool + }{ + { + name: "valid expression using the > operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 1}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the > operator over passing threshold and defined abort grace period", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(2 * time.Second), + sinks: map[string]float64{"rate": 1}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the >= operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreaterEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the <= operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLessEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the < operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the == operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLooselyEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using the === operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenStrictlyEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.01}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression using != operator over passing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenBangEqual, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.02}, + wantOk: true, + wantErr: false, + }, + { + name: "valid expression over failing threshold", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: false, + wantErr: false, + }, + { + name: "valid expression over non-existing sink", + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"med": 27.2}, + wantOk: false, + wantErr: true, + }, + { + // The ParseThresholdCondition constructor should ensure that no invalid + // operator gets through, but let's protect our future selves anyhow. + name: "invalid expression operator", + parsed: &thresholdExpression{tokenRate, null.Float{}, "&", 0.01}, + abortGracePeriod: types.NullDurationFrom(0 * time.Second), + sinks: map[string]float64{"rate": 0.00001}, + wantOk: false, + wantErr: true, + }, + } + for _, testCase := range tests { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + threshold := &Threshold{ + LastFailed: false, + AbortOnFail: false, + AbortGracePeriod: testCase.abortGracePeriod, + parsed: testCase.parsed, + } + + gotOk, gotErr := threshold.runNoTaint(testCase.sinks) + + assert.Equal(t, + testCase.wantErr, + gotErr != nil, + "Threshold.runNoTaint() error = %v, wantErr %v", gotErr, testCase.wantErr, + ) + + assert.Equal(t, + testCase.wantOk, + gotOk, + "Threshold.runNoTaint() gotOk = %v, want %v", gotOk, testCase.wantOk, + ) + }) + } +} + +func BenchmarkRunNoTaint(b *testing.B) { + threshold := &Threshold{ + Source: "rate>0.01", + LastFailed: false, + AbortOnFail: false, + AbortGracePeriod: types.NullDurationFrom(2 * time.Second), + parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + } + + sinks := map[string]float64{"rate": 1} + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + threshold.runNoTaint(sinks) // nolint + } } func TestThresholdRun(t *testing.T) { + t.Parallel() + t.Run("true", func(t *testing.T) { - th, err := newThreshold(`1+1==2`, goja.New(), false, types.NullDuration{}) + t.Parallel() + + sinks := map[string]float64{"rate": 0.0001} + threshold, err := newThreshold(`rate<0.01`, false, types.NullDuration{}) assert.NoError(t, err) t.Run("no taint", func(t *testing.T) { - b, err := th.runNoTaint() + b, err := threshold.runNoTaint(sinks) assert.NoError(t, err) assert.True(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) t.Run("taint", func(t *testing.T) { - b, err := th.run() + t.Parallel() + + b, err := threshold.run(sinks) assert.NoError(t, err) assert.True(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) }) t.Run("false", func(t *testing.T) { - th, err := newThreshold(`1+1==4`, goja.New(), false, types.NullDuration{}) + t.Parallel() + + sinks := map[string]float64{"rate": 1} + threshold, err := newThreshold(`rate<0.01`, false, types.NullDuration{}) assert.NoError(t, err) t.Run("no taint", func(t *testing.T) { - b, err := th.runNoTaint() + b, err := threshold.runNoTaint(sinks) assert.NoError(t, err) assert.False(t, b) - assert.False(t, th.LastFailed) + assert.False(t, threshold.LastFailed) }) t.Run("taint", func(t *testing.T) { - b, err := th.run() + b, err := threshold.run(sinks) assert.NoError(t, err) assert.False(t, b) - assert.True(t, th.LastFailed) + assert.True(t, threshold.LastFailed) }) }) } func TestNewThresholds(t *testing.T) { + t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + ts, err := NewThresholds([]string{}) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 0) }) t.Run("two", func(t *testing.T) { - sources := []string{`1+1==2`, `1+1==4`} + t.Parallel() + + sources := []string{`rate<0.01`, `p(95)<200`} ts, err := NewThresholds(sources) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 2) @@ -102,22 +283,26 @@ func TestNewThresholds(t *testing.T) { assert.Equal(t, sources[i], th.Source) assert.False(t, th.LastFailed) assert.False(t, th.AbortOnFail) - assert.NotNil(t, th.pgm) - assert.Equal(t, ts.Runtime, th.rt) } }) } func TestNewThresholdsWithConfig(t *testing.T) { + t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + ts, err := NewThresholds([]string{}) assert.NoError(t, err) assert.Len(t, ts.Thresholds, 0) }) t.Run("two", func(t *testing.T) { + t.Parallel() + configs := []thresholdConfig{ - {`1+1==2`, false, types.NullDuration{}}, - {`1+1==4`, true, types.NullDuration{}}, + {`rate<0.01`, false, types.NullDuration{}}, + {`p(95)<200`, true, types.NullDuration{}}, } ts, err := newThresholdsWithConfig(configs) assert.NoError(t, err) @@ -126,53 +311,47 @@ func TestNewThresholdsWithConfig(t *testing.T) { assert.Equal(t, configs[i].Threshold, th.Source) assert.False(t, th.LastFailed) assert.Equal(t, configs[i].AbortOnFail, th.AbortOnFail) - assert.NotNil(t, th.pgm) - assert.Equal(t, ts.Runtime, th.rt) } }) } -func TestThresholdsUpdateVM(t *testing.T) { - ts, err := NewThresholds(nil) - assert.NoError(t, err) - assert.NoError(t, ts.updateVM(DummySink{"a": 1234.5}, 0)) - assert.Equal(t, 1234.5, ts.Runtime.Get("a").ToFloat()) -} - func TestThresholdsRunAll(t *testing.T) { + t.Parallel() + zero := types.NullDuration{} oneSec := types.NullDurationFrom(time.Second) twoSec := types.NullDurationFrom(2 * time.Second) testdata := map[string]struct { - succ bool - err bool - abort bool - grace types.NullDuration - srcs []string + succeeded bool + err bool + abort bool + grace types.NullDuration + sources []string }{ - "one passing": {true, false, false, zero, []string{`1+1==2`}}, - "one failing": {false, false, false, zero, []string{`1+1==4`}}, - "two passing": {true, false, false, zero, []string{`1+1==2`, `2+2==4`}}, - "two failing": {false, false, false, zero, []string{`1+1==4`, `2+2==2`}}, - "two mixed": {false, false, false, zero, []string{`1+1==2`, `1+1==4`}}, - "one erroring": {false, true, false, zero, []string{`throw new Error('?!');`}}, - "one aborting": {false, false, true, zero, []string{`1+1==4`}}, - "abort with grace period": {false, false, true, oneSec, []string{`1+1==4`}}, - "no abort with grace period": {false, false, true, twoSec, []string{`1+1==4`}}, + "one passing": {true, false, false, zero, []string{`rate<0.01`}}, + "one failing": {false, false, false, zero, []string{`p(95)<200`}}, + "two passing": {true, false, false, zero, []string{`rate<0.1`, `rate<0.01`}}, + "two failing": {false, false, false, zero, []string{`p(95)<200`, `rate<0.1`}}, + "two mixed": {false, false, false, zero, []string{`rate<0.01`, `p(95)<200`}}, + "one aborting": {false, false, true, zero, []string{`p(95)<200`}}, + "abort with grace period": {false, false, true, oneSec, []string{`p(95)<200`}}, + "no abort with grace period": {false, false, true, twoSec, []string{`p(95)<200`}}, } for name, data := range testdata { t.Run(name, func(t *testing.T) { - ts, err := NewThresholds(data.srcs) - assert.Nil(t, err) - ts.Thresholds[0].AbortOnFail = data.abort - ts.Thresholds[0].AbortGracePeriod = data.grace + t.Parallel() + + thresholds, err := NewThresholds(data.sources) + thresholds.sinked = map[string]float64{"rate": 0.0001, "p(95)": 500} + thresholds.Thresholds[0].AbortOnFail = data.abort + thresholds.Thresholds[0].AbortGracePeriod = data.grace runDuration := 1500 * time.Millisecond assert.NoError(t, err) - b, err := ts.runAll(runDuration) + succeeded, err := thresholds.runAll(runDuration) if data.err { assert.Error(t, err) @@ -180,48 +359,74 @@ func TestThresholdsRunAll(t *testing.T) { assert.NoError(t, err) } - if data.succ { - assert.True(t, b) + if data.succeeded { + assert.True(t, succeeded) } else { - assert.False(t, b) + assert.False(t, succeeded) } if data.abort && data.grace.Duration < types.Duration(runDuration) { - assert.True(t, ts.Abort) + assert.True(t, thresholds.Abort) } else { - assert.False(t, ts.Abort) + assert.False(t, thresholds.Abort) } }) } } -func TestThresholdsRun(t *testing.T) { - ts, err := NewThresholds([]string{"a>0"}) - assert.NoError(t, err) +func TestThresholds_Run(t *testing.T) { + t.Parallel() - t.Run("error", func(t *testing.T) { - b, err := ts.Run(DummySink{}, 0) - assert.Error(t, err) - assert.False(t, b) - }) + type args struct { + sink Sink + duration time.Duration + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + "Running thresholds of existing sink", + args{DummySink{"p(95)": 1234.5}, 0}, + true, + false, + }, + { + "Running thresholds of existing sink but failing threshold", + args{DummySink{"p(95)": 3000}, 0}, + false, + false, + }, + { + "Running threshold on non existing sink fails", + args{DummySink{"dummy": 0}, 0}, + false, + true, + }, + } + for _, testCase := range tests { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() - t.Run("pass", func(t *testing.T) { - b, err := ts.Run(DummySink{"a": 1234.5}, 0) - assert.NoError(t, err) - assert.True(t, b) - }) + thresholds, err := NewThresholds([]string{"p(95)<2000"}) + require.NoError(t, err, "Initializing new thresholds should not fail") - t.Run("fail", func(t *testing.T) { - b, err := ts.Run(DummySink{"a": 0}, 0) - assert.NoError(t, err) - assert.False(t, b) - }) + gotOk, gotErr := thresholds.Run(testCase.args.sink, testCase.args.duration) + assert.Equal(t, gotErr != nil, testCase.wantErr, "Thresholds.Run() error = %v, wantErr %v", gotErr, testCase.wantErr) + assert.Equal(t, gotOk, testCase.want, "Thresholds.Run() = %v, want %v", gotOk, testCase.want) + }) + } } func TestThresholdsJSON(t *testing.T) { - var testdata = []struct { + t.Parallel() + + testdata := []struct { JSON string - srcs []string + sources []string abortOnFail bool gracePeriod types.NullDuration outputJSON string @@ -234,8 +439,8 @@ func TestThresholdsJSON(t *testing.T) { "", }, { - `["1+1==2"]`, - []string{"1+1==2"}, + `["rate<0.01"]`, + []string{"rate<0.01"}, false, types.NullDuration{}, "", @@ -248,55 +453,59 @@ func TestThresholdsJSON(t *testing.T) { `["rate<0.01"]`, }, { - `["1+1==2","1+1==3"]`, - []string{"1+1==2", "1+1==3"}, + `["rate<0.01","p(95)<200"]`, + []string{"rate<0.01", "p(95)<200"}, false, types.NullDuration{}, "", }, { - `[{"threshold":"1+1==2"}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01"}]`, + []string{"rate<0.01"}, false, types.NullDuration{}, - `["1+1==2"]`, + `["rate<0.01"]`, }, { - `[{"threshold":"1+1==2","abortOnFail":true,"delayAbortEval":null}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":true,"delayAbortEval":null}]`, + []string{"rate<0.01"}, true, types.NullDuration{}, "", }, { - `[{"threshold":"1+1==2","abortOnFail":true,"delayAbortEval":"2s"}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":true,"delayAbortEval":"2s"}]`, + []string{"rate<0.01"}, true, types.NullDurationFrom(2 * time.Second), "", }, { - `[{"threshold":"1+1==2","abortOnFail":false}]`, - []string{"1+1==2"}, + `[{"threshold":"rate<0.01","abortOnFail":false}]`, + []string{"rate<0.01"}, false, types.NullDuration{}, - `["1+1==2"]`, + `["rate<0.01"]`, }, { - `[{"threshold":"1+1==2"}, "1+1==3"]`, - []string{"1+1==2", "1+1==3"}, + `[{"threshold":"rate<0.01"}, "p(95)<200"]`, + []string{"rate<0.01", "p(95)<200"}, false, types.NullDuration{}, - `["1+1==2","1+1==3"]`, + `["rate<0.01","p(95)<200"]`, }, } for _, data := range testdata { + data := data + t.Run(data.JSON, func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.NoError(t, json.Unmarshal([]byte(data.JSON), &ts)) - assert.Equal(t, len(data.srcs), len(ts.Thresholds)) - for i, src := range data.srcs { + assert.Equal(t, len(data.sources), len(ts.Thresholds)) + for i, src := range data.sources { assert.Equal(t, src, ts.Thresholds[i].Source) assert.Equal(t, data.abortOnFail, ts.Thresholds[i].AbortOnFail) assert.Equal(t, data.gracePeriod, ts.Thresholds[i].AbortGracePeriod) @@ -315,18 +524,20 @@ func TestThresholdsJSON(t *testing.T) { } t.Run("bad JSON", func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.Error(t, json.Unmarshal([]byte("42"), &ts)) assert.Nil(t, ts.Thresholds) - assert.Nil(t, ts.Runtime) assert.False(t, ts.Abort) }) t.Run("bad source", func(t *testing.T) { + t.Parallel() + var ts Thresholds assert.Error(t, json.Unmarshal([]byte(`["="]`), &ts)) assert.Nil(t, ts.Thresholds) - assert.Nil(t, ts.Runtime) assert.False(t, ts.Abort) }) } From a52e48d693d74f0d413cf4a03656180abdcc304d Mon Sep 17 00:00:00 2001 From: oleiade Date: Wed, 26 Jan 2022 13:59:07 +0100 Subject: [PATCH 2/4] Expose stats' relevant threshold expression symbols publicly This commit makes some minor modifications to the `stats` package API. Namely, it makes `stats.ThresholdExpression` and `stats.token*` symbols public. It also makes `stats.Threshold.parsed` public. These changes are made in order to facilitate validation of thresholds from outside the `stats` package. Having access to both the parsed Threshold, and the aggregation methods symbols will allow comparing them and asserting their meaningfulness in a context where we have typed metrics available. ref #2330 --- stats/thresholds.go | 32 ++++---- stats/thresholds_parser.go | 140 ++++++++++++++++++++++---------- stats/thresholds_parser_test.go | 22 ++--- stats/thresholds_test.go | 32 ++++---- 4 files changed, 141 insertions(+), 85 deletions(-) diff --git a/stats/thresholds.go b/stats/thresholds.go index 1b7cbaea53f..0dac52ff55a 100644 --- a/stats/thresholds.go +++ b/stats/thresholds.go @@ -23,7 +23,6 @@ import ( "bytes" "encoding/json" "fmt" - "strings" "time" "go.k6.io/k6/lib/types" @@ -40,8 +39,8 @@ type Threshold struct { // AbortGracePeriod is a the minimum amount of time a test should be running before a failing // this threshold will abort the test AbortGracePeriod types.NullDuration - // parsed is the threshold expression parsed from the Source - parsed *thresholdExpression + // Parsed is the threshold expression Parsed from the Source + Parsed *ThresholdExpression } func newThreshold(src string, abortOnFail bool, gracePeriod types.NullDuration) (*Threshold, error) { @@ -54,46 +53,46 @@ func newThreshold(src string, abortOnFail bool, gracePeriod types.NullDuration) Source: src, AbortOnFail: abortOnFail, AbortGracePeriod: gracePeriod, - parsed: parsedExpression, + Parsed: parsedExpression, }, nil } func (t *Threshold) runNoTaint(sinks map[string]float64) (bool, error) { // Extract the sink value for the aggregation method used in the threshold // expression - lhs, ok := sinks[t.parsed.AggregationMethod] + lhs, ok := sinks[t.Parsed.SinkKey()] if !ok { return false, fmt.Errorf("unable to apply threshold %s over metrics; reason: "+ "no metric supporting the %s aggregation method found", t.Source, - t.parsed.AggregationMethod) + t.Parsed.AggregationMethod) } // Apply the threshold expression operator to the left and // right hand side values var passes bool - switch t.parsed.Operator { + switch t.Parsed.Operator { case ">": - passes = lhs > t.parsed.Value + passes = lhs > t.Parsed.Value case ">=": - passes = lhs >= t.parsed.Value + passes = lhs >= t.Parsed.Value case "<=": - passes = lhs <= t.parsed.Value + passes = lhs <= t.Parsed.Value case "<": - passes = lhs < t.parsed.Value + passes = lhs < t.Parsed.Value case "==", "===": // Considering a sink always maps to float64 values, // strictly equal is equivalent to loosely equal - passes = lhs == t.parsed.Value + passes = lhs == t.Parsed.Value case "!=": - passes = lhs != t.parsed.Value + passes = lhs != t.Parsed.Value default: // The parseThresholdExpression function should ensure that no invalid // operator gets through, but let's protect our future selves anyhow. return false, fmt.Errorf("unable to apply threshold %s over metrics; "+ "reason: %s is an invalid operator", t.Source, - t.parsed.Operator, + t.Parsed.Operator, ) } @@ -218,11 +217,12 @@ func (ts *Thresholds) Run(sink Sink, duration time.Duration) (bool, error) { // Parse the percentile thresholds and insert them in // the sinks mapping. for _, threshold := range ts.Thresholds { - if !strings.HasPrefix(threshold.parsed.AggregationMethod, "p(") { + if threshold.Parsed.AggregationMethod != TokenPercentile { continue } - ts.sinked[threshold.parsed.AggregationMethod] = sinkImpl.P(threshold.parsed.AggregationValue.Float64 / 100) + key := fmt.Sprintf("p(%g)", threshold.Parsed.AggregationValue.Float64) + ts.sinked[key] = sinkImpl.P(threshold.Parsed.AggregationValue.Float64 / 100) } case *RateSink: ts.sinked["rate"] = float64(sinkImpl.Trues) / float64(sinkImpl.Total) diff --git a/stats/thresholds_parser.go b/stats/thresholds_parser.go index 72b0f82e2a7..aaa7f670e51 100644 --- a/stats/thresholds_parser.go +++ b/stats/thresholds_parser.go @@ -21,6 +21,7 @@ package stats import ( + "errors" "fmt" "strconv" "strings" @@ -28,9 +29,12 @@ import ( "gopkg.in/guregu/null.v3" ) -// thresholdExpression holds the parsed result of a threshold expression, +// ErrThresholdParsing is returned by failing threshold parsing operations. +var ErrThresholdParsing = errors.New("parsing threshold expression failed") + +// ThresholdExpression holds the parsed result of a threshold expression, // as described in: https://k6.io/docs/using-k6/thresholds/#threshold-syntax -type thresholdExpression struct { +type ThresholdExpression struct { // AggregationMethod holds the aggregation method parsed // from the threshold expression. Possible values are described // by `aggregationMethodTokens`. @@ -49,6 +53,49 @@ type thresholdExpression struct { Value float64 } +// NewThresholdExpression instantiates a new ThresholdExpression +func NewThresholdExpression( + aggregationMethod string, + aggregationValue null.Float, + operator string, + value float64, +) *ThresholdExpression { + return &ThresholdExpression{ + AggregationMethod: aggregationMethod, + AggregationValue: aggregationValue, + Operator: operator, + Value: value, + } +} + +// NewThresholdExpressionFrom parses a threshold expression from an input string +// and returns it as a ThresholdExpression pointer. +func NewThresholdExpressionFrom(input string) (*ThresholdExpression, error) { + return parseThresholdExpression(input) +} + +// SinkKey computes the key used to index a ThresholdExpression in the engine's sinks. +// +// During execution, the engine "sinks" metrics into a internal mapping, so that +// thresholds can be tried against them. This method is a helper to normalize the +// sink the threshold expression should be applied to. +// +// Because a theshold expression's aggregation method can either be +// a static keyword ("count", "rate", etc...), or a parametric +// expression ("p(somefloatingpointvalue)"), we need to handle this +// case specifically. If we encounter the percentile aggregation method token, +// we recompute the whole "p(value)" expression in order to look for it in the +// sinks. +func (te *ThresholdExpression) SinkKey() string { + // + sinkKey := te.AggregationMethod + if te.AggregationMethod == TokenPercentile { + sinkKey = fmt.Sprintf("%s(%g)", TokenPercentile, te.AggregationValue.Float64) + } + + return sinkKey +} + // parseThresholdAssertion parses a threshold condition expression, // as defined in a JS script (for instance p(95)<1000), into a thresholdExpression // instance. @@ -68,25 +115,31 @@ type thresholdExpression struct { // digit -> "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" // whitespace -> " " // ``` -func parseThresholdExpression(input string) (*thresholdExpression, error) { +func parseThresholdExpression(input string) (*ThresholdExpression, error) { // Scanning makes no assumption on the underlying values, and only // checks that the expression has the right format. method, operator, value, err := scanThresholdExpression(input) if err != nil { - return nil, fmt.Errorf("failed parsing threshold expression; reason: %w", err) + return nil, fmt.Errorf("%w '%s'; reason: %v", ErrThresholdParsing, input, err) } parsedMethod, parsedMethodValue, err := parseThresholdAggregationMethod(method) if err != nil { - return nil, fmt.Errorf("failed parsing threshold expression's left hand side; reason: %w", err) + return nil, fmt.Errorf("%w '%s'; reason: %v", ErrThresholdParsing, input, err) } parsedValue, err := strconv.ParseFloat(value, 64) if err != nil { - return nil, fmt.Errorf("failed parsing threshold expresion's right hand side; reason: %w", err) + return nil, fmt.Errorf( + "%w '%s', right hand side could not be parsed as a "+ + "64-bit precision floating point value; reason: %v", + ErrThresholdParsing, + input, + err, + ) } - condition := &thresholdExpression{ + condition := &ThresholdExpression{ AggregationMethod: parsedMethod, AggregationValue: parsedMethodValue, Operator: operator, @@ -98,13 +151,13 @@ func parseThresholdExpression(input string) (*thresholdExpression, error) { // Define accepted threshold expression operators tokens const ( - tokenLessEqual = "<=" - tokenLess = "<" - tokenGreaterEqual = ">=" - tokenGreater = ">" - tokenStrictlyEqual = "===" - tokenLooselyEqual = "==" - tokenBangEqual = "!=" + TokenLessEqual = "<=" + TokenLess = "<" + TokenGreaterEqual = ">=" + TokenGreater = ">" + TokenStrictlyEqual = "===" + TokenLooselyEqual = "==" + TokenBangEqual = "!=" ) // operatorTokens defines the list of operator-related tokens @@ -119,13 +172,13 @@ const ( // Longer tokens with symbols in common with shorter ones must appear // first in the list in order to be effectively matched. var operatorTokens = [7]string{ // nolint:gochecknoglobals - tokenLessEqual, - tokenLess, - tokenGreaterEqual, - tokenGreater, - tokenStrictlyEqual, - tokenLooselyEqual, - tokenBangEqual, + TokenLessEqual, + TokenLess, + TokenGreaterEqual, + TokenGreater, + TokenStrictlyEqual, + TokenLooselyEqual, + TokenBangEqual, } // scanThresholdExpression scans a threshold condition expression of the @@ -142,20 +195,23 @@ func scanThresholdExpression(input string) (string, string, string, error) { } } - return "", "", "", fmt.Errorf("malformed threshold expression") + return "", "", "", fmt.Errorf( + "no valid operator found in the threshold expression. " + + "valid operators are: <, <=, >, >=, ==, !=, ===", + ) } // Define accepted threshold expression aggregation tokens // Percentile token `p(..)` is accepted too but handled separately. const ( - tokenValue = "value" - tokenCount = "count" - tokenRate = "rate" - tokenAvg = "avg" - tokenMin = "min" - tokenMed = "med" - tokenMax = "max" - tokenPercentile = "p" + TokenValue = "value" + TokenCount = "count" + TokenRate = "rate" + TokenAvg = "avg" + TokenMin = "min" + TokenMed = "med" + TokenMax = "max" + TokenPercentile = "p" ) // aggregationMethodTokens defines the list of aggregation method @@ -165,14 +221,14 @@ const ( // Although declared as a `var`, being an array, it is effectively // immutable and can be considered constant. var aggregationMethodTokens = [8]string{ // nolint:gochecknoglobals - tokenValue, - tokenCount, - tokenRate, - tokenAvg, - tokenMin, - tokenMed, - tokenMax, - tokenPercentile, + TokenValue, + TokenCount, + TokenRate, + TokenAvg, + TokenMin, + TokenMed, + TokenMax, + TokenPercentile, } // parseThresholdMethod will parse a threshold condition expression's method. @@ -186,21 +242,21 @@ func parseThresholdAggregationMethod(input string) (string, null.Float, error) { // Percentile expressions being of the form p(value), // they won't be matched here. if m == input { - return input, null.Float{}, nil + return m, null.Float{}, nil } } // Otherwise, attempt to parse a percentile expression - if strings.HasPrefix(input, tokenPercentile+"(") && strings.HasSuffix(input, ")") { + if strings.HasPrefix(input, TokenPercentile+"(") && strings.HasSuffix(input, ")") { aggregationValue, err := strconv.ParseFloat(trimDelimited("p(", input, ")"), 64) if err != nil { return "", null.Float{}, fmt.Errorf("malformed percentile value; reason: %w", err) } - return input, null.FloatFrom(aggregationValue), nil + return TokenPercentile, null.FloatFrom(aggregationValue), nil } - return "", null.Float{}, fmt.Errorf("failed parsing method from expression") + return "", null.Float{}, fmt.Errorf("no valid aggregation method found in the threshold expression") } func trimDelimited(prefix, input, suffix string) string { diff --git a/stats/thresholds_parser_test.go b/stats/thresholds_parser_test.go index 25120684d6e..d9673b50d2d 100644 --- a/stats/thresholds_parser_test.go +++ b/stats/thresholds_parser_test.go @@ -33,7 +33,7 @@ func TestParseThresholdExpression(t *testing.T) { tests := []struct { name string input string - wantExpression *thresholdExpression + wantExpression *ThresholdExpression wantErr bool }{ { @@ -57,7 +57,7 @@ func TestParseThresholdExpression(t *testing.T) { { name: "valid threshold expression syntax", input: "count>20", - wantExpression: &thresholdExpression{AggregationMethod: "count", Operator: ">", Value: 20}, + wantExpression: &ThresholdExpression{AggregationMethod: "count", Operator: ">", Value: 20}, wantErr: false, }, } @@ -103,63 +103,63 @@ func TestParseThresholdAggregationMethod(t *testing.T) { { name: "count method is parsed", input: "count", - wantMethod: "count", + wantMethod: TokenCount, wantMethodValue: null.Float{}, wantErr: false, }, { name: "rate method is parsed", input: "rate", - wantMethod: "rate", + wantMethod: TokenRate, wantMethodValue: null.Float{}, wantErr: false, }, { name: "value method is parsed", input: "value", - wantMethod: "value", + wantMethod: TokenValue, wantMethodValue: null.Float{}, wantErr: false, }, { name: "avg method is parsed", input: "avg", - wantMethod: "avg", + wantMethod: TokenAvg, wantMethodValue: null.Float{}, wantErr: false, }, { name: "min method is parsed", input: "min", - wantMethod: "min", + wantMethod: TokenMin, wantMethodValue: null.Float{}, wantErr: false, }, { name: "max method is parsed", input: "max", - wantMethod: "max", + wantMethod: TokenMax, wantMethodValue: null.Float{}, wantErr: false, }, { name: "med method is parsed", input: "med", - wantMethod: "med", + wantMethod: TokenMed, wantMethodValue: null.Float{}, wantErr: false, }, { name: "percentile method with integer value is parsed", input: "p(99)", - wantMethod: "p(99)", + wantMethod: TokenPercentile, wantMethodValue: null.FloatFrom(99), wantErr: false, }, { name: "percentile method with floating point value is parsed", input: "p(99.9)", - wantMethod: "p(99.9)", + wantMethod: TokenPercentile, wantMethodValue: null.FloatFrom(99.9), wantErr: false, }, diff --git a/stats/thresholds_test.go b/stats/thresholds_test.go index 9e381bf75fb..878aef5020b 100644 --- a/stats/thresholds_test.go +++ b/stats/thresholds_test.go @@ -37,7 +37,7 @@ func TestNewThreshold(t *testing.T) { src := `rate<0.01` abortOnFail := false gracePeriod := types.NullDurationFrom(2 * time.Second) - wantParsed := &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01} + wantParsed := &ThresholdExpression{TokenRate, null.Float{}, TokenLess, 0.01} gotThreshold, err := newThreshold(src, abortOnFail, gracePeriod) @@ -46,7 +46,7 @@ func TestNewThreshold(t *testing.T) { assert.False(t, gotThreshold.LastFailed) assert.Equal(t, abortOnFail, gotThreshold.AbortOnFail) assert.Equal(t, gracePeriod, gotThreshold.AbortGracePeriod) - assert.Equal(t, wantParsed, gotThreshold.parsed) + assert.Equal(t, wantParsed, gotThreshold.Parsed) } func TestNewThreshold_InvalidThresholdConditionExpression(t *testing.T) { @@ -67,7 +67,7 @@ func TestThreshold_runNoTaint(t *testing.T) { tests := []struct { name string - parsed *thresholdExpression + parsed *ThresholdExpression abortGracePeriod types.NullDuration sinks map[string]float64 wantOk bool @@ -75,7 +75,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }{ { name: "valid expression using the > operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreater, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 1}, wantOk: true, @@ -83,7 +83,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the > operator over passing threshold and defined abort grace period", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreater, 0.01}, abortGracePeriod: types.NullDurationFrom(2 * time.Second), sinks: map[string]float64{"rate": 1}, wantOk: true, @@ -91,7 +91,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the >= operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreaterEqual, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreaterEqual, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.01}, wantOk: true, @@ -99,7 +99,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the <= operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLessEqual, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenLessEqual, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.01}, wantOk: true, @@ -107,7 +107,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the < operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLess, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenLess, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.00001}, wantOk: true, @@ -115,7 +115,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the == operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenLooselyEqual, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenLooselyEqual, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.01}, wantOk: true, @@ -123,7 +123,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using the === operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenStrictlyEqual, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenStrictlyEqual, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.01}, wantOk: true, @@ -131,7 +131,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression using != operator over passing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenBangEqual, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenBangEqual, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.02}, wantOk: true, @@ -139,7 +139,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression over failing threshold", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreater, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.00001}, wantOk: false, @@ -147,7 +147,7 @@ func TestThreshold_runNoTaint(t *testing.T) { }, { name: "valid expression over non-existing sink", - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreater, 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"med": 27.2}, wantOk: false, @@ -157,7 +157,7 @@ func TestThreshold_runNoTaint(t *testing.T) { // The ParseThresholdCondition constructor should ensure that no invalid // operator gets through, but let's protect our future selves anyhow. name: "invalid expression operator", - parsed: &thresholdExpression{tokenRate, null.Float{}, "&", 0.01}, + parsed: &ThresholdExpression{TokenRate, null.Float{}, "&", 0.01}, abortGracePeriod: types.NullDurationFrom(0 * time.Second), sinks: map[string]float64{"rate": 0.00001}, wantOk: false, @@ -174,7 +174,7 @@ func TestThreshold_runNoTaint(t *testing.T) { LastFailed: false, AbortOnFail: false, AbortGracePeriod: testCase.abortGracePeriod, - parsed: testCase.parsed, + Parsed: testCase.parsed, } gotOk, gotErr := threshold.runNoTaint(testCase.sinks) @@ -200,7 +200,7 @@ func BenchmarkRunNoTaint(b *testing.B) { LastFailed: false, AbortOnFail: false, AbortGracePeriod: types.NullDurationFrom(2 * time.Second), - parsed: &thresholdExpression{tokenRate, null.Float{}, tokenGreater, 0.01}, + Parsed: &ThresholdExpression{TokenRate, null.Float{}, TokenGreater, 0.01}, } sinks := map[string]float64{"rate": 1} From f2c171e1c3d6bdbf2694813df24d47b1573117bf Mon Sep 17 00:00:00 2001 From: oleiade Date: Wed, 26 Jan 2022 15:58:13 +0100 Subject: [PATCH 3/4] Add threshold validation to cmd/config This commit adds a `validateThresholdConfig` function to `cmd/config`, and integrates it as part of the `validateConfig` operations. From now on, `validateConfig` takes a `metrics.Registry` as input, and validates that thresholds defined in the config apply to existing metrics, and use methods that are valid for the metric they apply to. As a side effect, this commit adds a `Get` method to `metrics.Registry` in order to be able to query registered metrics, regardless of whether they are custom or builtin metrics. As another side effect, this commit introduces a `lib.Contains` helper function allowing to check if a slice of strings contains a given string. This is used to simplify the matching of supported aggregation methods on metrics in the `validateThresholdConfig` function. ref #2330 --- cmd/config.go | 80 +++++++- cmd/config_test.go | 408 +++++++++++++++++++++++++++++++++++++++- cmd/run_test.go | 45 +++++ lib/metrics/registry.go | 8 + lib/util.go | 11 ++ 5 files changed, 546 insertions(+), 6 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 2dd0462dabc..711e1d4837f 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -38,6 +38,7 @@ import ( "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/lib" "go.k6.io/k6/lib/executor" + "go.k6.io/k6/lib/metrics" "go.k6.io/k6/lib/types" "go.k6.io/k6/stats" ) @@ -235,17 +236,21 @@ func applyDefault(conf Config) Config { } func deriveAndValidateConfig( - conf Config, isExecutable func(string) bool, logger logrus.FieldLogger, + conf Config, + registry *metrics.Registry, + isExecutable func(string) bool, + logger logrus.FieldLogger, ) (result Config, err error) { result = conf result.Options, err = executor.DeriveScenariosFromShortcuts(conf.Options, logger) if err == nil { - err = validateConfig(result, isExecutable) + err = validateConfig(result, registry, isExecutable) } + return result, errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig) } -func validateConfig(conf Config, isExecutable func(string) bool) error { +func validateConfig(conf Config, registry *metrics.Registry, isExecutable func(string) bool) error { errList := conf.Validate() for _, ec := range conf.Scenarios { @@ -254,9 +259,78 @@ func validateConfig(conf Config, isExecutable func(string) bool) error { } } + // If there are thresholds to validate, the registry paramater is not allowed to be nil. + // Note that the reason for passing it as a pointer in the first place is + // because it holds a Mutex, which effectively forbids passing it by value. + if conf.Thresholds != nil && len(conf.Thresholds) > 0 && registry == nil { + err := fmt.Errorf( + "unable to validate thresholds configuration; " + + "reason: provided registry is nil", + ) + errList = append(errList, err) + return consolidateErrorMessage(errList, "there were problems while validating the specified script configuration: ") + } + + for thresholdName, thresholds := range conf.Thresholds { + // Fetch metric matching the threshold's name + metric, ok := registry.Get(thresholdName) + if !ok { + // The defined threshold applies to a non-existing metrics + err := fmt.Errorf("invalid threshold defined on %s; reason: no metric named %s found", thresholdName, thresholdName) + return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig) + } + + // Validate the threshold definition against its matching + // metric. + err := validateThresholdsConfig(thresholdName, thresholds, metric) + if err != nil { + return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig) + } + } + return consolidateErrorMessage(errList, "There were problems with the specified script configuration:") } +// validateThresholdsConfig validates a threshold definition is consistent with the metric it applies to. +// Given a threshold name, expressions and a metric to apply the expressions too, validateThresholdConfig will +// assert that each expression uses an aggregation method that's supported by the provided metric. It returns +// an error otherwise. Note that this function expects the passed in thresholds to have been parsed already, and +// have their Parsed (ThresholdExpression) field already filled. +func validateThresholdsConfig(thresholdName string, expressions stats.Thresholds, metric *stats.Metric) error { + var supportedMethods []string + + switch metric.Type { + case stats.Counter: + supportedMethods = []string{stats.TokenCount, stats.TokenRate} + case stats.Gauge: + supportedMethods = []string{stats.TokenValue} + case stats.Rate: + supportedMethods = []string{stats.TokenRate} + case stats.Trend: + supportedMethods = []string{ + stats.TokenAvg, + stats.TokenMin, + stats.TokenMax, + stats.TokenMed, + stats.TokenPercentile, + } + } + + for _, expression := range expressions.Thresholds { + if !lib.Contains(supportedMethods, expression.Parsed.AggregationMethod) { + return fmt.Errorf( + "invalid threshold expression %s: '%s'; "+ + "reason: invalid aggregation method '%s' applied to the '%s' metric. "+ + "%s is a metric of type %s, did you mean to use the any of the "+ + "'count' or 'rate' aggregation methods instead?", + thresholdName, expression.Source, expression.Parsed.AggregationMethod, thresholdName, thresholdName, metric.Type, + ) + } + } + + return nil +} + func consolidateErrorMessage(errList []error, title string) error { if len(errList) == 0 { return nil diff --git a/cmd/config_test.go b/cmd/config_test.go index ec1eb7ecebc..1061b8d5d08 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -26,13 +26,15 @@ import ( "github.com/mstoykov/envconfig" "github.com/stretchr/testify/assert" - "gopkg.in/guregu/null.v3" - "go.k6.io/k6/errext" "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/lib" "go.k6.io/k6/lib/executor" + "go.k6.io/k6/lib/metrics" + "go.k6.io/k6/lib/testutils" "go.k6.io/k6/lib/types" + "go.k6.io/k6/stats" + "gopkg.in/guregu/null.v3" ) type testCmdData struct { @@ -202,7 +204,7 @@ func TestDeriveAndValidateConfig(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { t.Parallel() - _, err := deriveAndValidateConfig(tc.conf, + _, err := deriveAndValidateConfig(tc.conf, metrics.NewRegistry(), func(_ string) bool { return tc.isExec }, nil) if tc.err != "" { var ecerr errext.HasExitCode @@ -215,3 +217,403 @@ func TestDeriveAndValidateConfig(t *testing.T) { }) } } + +func TestValidateConfig(t *testing.T) { + t.Parallel() + + // A registry filled with builtin metrics + builtinMetricsRegistry := metrics.NewRegistry() + metrics.RegisterBuiltinMetrics(builtinMetricsRegistry) + + // A registry filled with a custom metric + customMetricsRegistry := metrics.NewRegistry() + customMetricsRegistry.MustNewMetric("counter_ok", stats.Counter) + + testCases := []struct { + name string + conf Config + registry *metrics.Registry + wantErr bool + wantExitCode bool + }{ + { + name: "config applying a threshold over an existing builtin metric succeeds", + conf: Config{Options: lib.Options{Thresholds: map[string]stats.Thresholds{ + metrics.HTTPReqsName: {Thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }}, + }}}, + registry: builtinMetricsRegistry, + wantErr: false, + wantExitCode: false, + }, + { + name: "config applying a threshold over an existing custom metric succeeds", + conf: Config{Options: lib.Options{Thresholds: map[string]stats.Thresholds{ + "counter_ok": {Thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }}, + }}}, + registry: customMetricsRegistry, + wantErr: false, + wantExitCode: false, + }, + { + name: "passes when no thresholds are defined, and the provided registry is nil", + conf: Config{}, + registry: nil, + wantErr: false, + wantExitCode: false, + }, + { + name: "fails when thresholds are defined, and the provided registry is nil", + conf: Config{Options: lib.Options{Thresholds: map[string]stats.Thresholds{ + "counter_ok": {Thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }}, + }}}, + registry: nil, + wantErr: true, + wantExitCode: false, + }, + { + name: "config applying a threshold to a non-existing metric fails", + conf: Config{Options: lib.Options{Thresholds: map[string]stats.Thresholds{ + "nonexisting": {Thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }}, + }}}, + registry: builtinMetricsRegistry, + wantErr: true, + wantExitCode: true, + }, + { + name: "config applying a threshold to an existing metric not supporting its aggregation method", + conf: Config{Options: lib.Options{Thresholds: map[string]stats.Thresholds{ + metrics.HTTPReqFailedName: {Thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenPercentile, null.NewFloat(99, true), stats.TokenGreater, 1)}, + }}, + }}}, + registry: builtinMetricsRegistry, + wantErr: true, + wantExitCode: true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + // TODO: when/if scenario validation is tested, the isExecutable should be part of testCase + gotErr := validateConfig(testCase.conf, testCase.registry, func(string) bool { return true }) + + assert.Equal(t, testCase.wantErr, gotErr != nil) + if testCase.wantErr == true && testCase.wantExitCode == true { + var ecerr errext.HasExitCode + assert.ErrorAs(t, gotErr, &ecerr) + assert.Equal(t, exitcodes.InvalidConfig, ecerr.ExitCode()) + } + }) + } +} + +func TestValidateThresholdsConfig(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + thresholds []*stats.Threshold + metric *stats.Metric + wantErr bool + }{ + { + name: "threshold using count method over Counter metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Counter}, + wantErr: false, + }, + { + name: "threshold using rate method over Counter metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenRate, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Counter}, + wantErr: false, + }, + { + name: "threshold using unsupported method over Counter metric is invalid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenValue, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Counter}, + wantErr: true, + }, + { + name: "mixed threshold using supported/unsupported method over Counter metric is invalid", + thresholds: []*stats.Threshold{ + { + // valid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + { + // invalid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenValue, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + }, + metric: &stats.Metric{Type: stats.Counter}, + wantErr: true, + }, + { + name: "threshold using value method over Gauge metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenValue, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Gauge}, + wantErr: false, + }, + { + name: "threshold using unsupported method over Gauge metric is invalid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenRate, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Gauge}, + wantErr: true, + }, + { + name: "mixed threshold using supported/unsupported method over Gauge metric is invalid", + thresholds: []*stats.Threshold{ + { + // valid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenValue, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + { + // invalid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenRate, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + }, + metric: &stats.Metric{Type: stats.Gauge}, + wantErr: true, + }, + { + name: "threshold using rate method over Rate metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenRate, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Rate}, + wantErr: false, + }, + { + name: "threshold using unsupported method over Rate metric is invalid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenAvg, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Rate}, + wantErr: true, + }, + { + name: "mixed threshold using supported/unsupported method over Rate metric is invalid", + thresholds: []*stats.Threshold{ + { + // valid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenRate, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + { + // invalid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenAvg, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + }, + metric: &stats.Metric{Type: stats.Gauge}, + wantErr: true, + }, + + { + name: "threshold using avg method over Trend metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenAvg, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: false, + }, + { + name: "threshold using min method over Trend metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenMin, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: false, + }, + { + name: "threshold using med method over Trend metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenMed, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: false, + }, + { + name: "threshold using p(N) method over Trend metric is valid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenPercentile, null.NewFloat(99, true), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: false, + }, + { + name: "threshold using unsupported method over Trend metric is invalid", + thresholds: []*stats.Threshold{ + {Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1)}, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: true, + }, + { + name: "mixed threshold using supported/unsupported method over Trend metric is invalid", + thresholds: []*stats.Threshold{ + { + // valid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenAvg, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + { + // invalid method over counter metric + Parsed: stats.NewThresholdExpression(stats.TokenCount, null.FloatFromPtr(nil), stats.TokenGreater, 1), + }, + }, + metric: &stats.Metric{Type: stats.Trend}, + wantErr: true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + wrappedThresholds := stats.Thresholds{Thresholds: testCase.thresholds} + + gotErr := validateThresholdsConfig("ignoreme", wrappedThresholds, testCase.metric) + + assert.Equal(t, testCase.wantErr, gotErr != nil) + }) + } +} + +// FIXME: Remove these... +// func TestValidateThresholdsConfigWithNilRegistry(t *testing.T) { +// t.Parallel() +// var registry *metrics.Registry +// config := Config{} +// var wantErrType errext.HasExitCode + +// gotErr := validateThresholdsConfig(config, registry) + +// assert.Error(t, gotErr, "validateThresholdsConfig should fail when passed registry is nil") +// assert.ErrorAs(t, gotErr, &wantErrType, "validateThresholdsConfig error should be an instance of errext.HasExitCode") +// } + +// func TestValidateThresholdsConfigAppliesToBuiltinMetrics(t *testing.T) { +// t.Parallel() +// // Prepare a registry loaded with builtin metrics +// registry := metrics.NewRegistry() +// metrics.RegisterBuiltinMetrics(registry) + +// // Assuming builtin metrics are indeed registered, and +// // thresholds parsing works as expected, we prepare +// // thresholds for a counter builting metric; namely http_reqs +// HTTPReqsThresholds, err := stats.NewThresholds([]string{"count>0", "rate>1"}) +// require.NoError(t, err, "instantiating Thresholds with expression 'count>0' should not fail") +// options := lib.Options{ +// Thresholds: map[string]stats.Thresholds{ +// metrics.HTTPReqsName: HTTPReqsThresholds, +// }, +// } +// config := Config{Options: options} + +// gotErr := validateThresholdsConfig(config, registry) + +// assert.NoError(t, gotErr, "validateThresholdsConfig should not fail against builtin metrics") +// } + +// func TestValidateThresholdsConfigAppliesToCustomMetrics(t *testing.T) { +// t.Parallel() + +// // Prepare a registry loaded with both builtin metrics, +// // and a custom counter metric. +// testCounterMetricName := "testcounter" +// registry := metrics.NewRegistry() +// metrics.RegisterBuiltinMetrics(registry) +// _, err := registry.NewMetric(testCounterMetricName, stats.Counter) +// require.NoError(t, err, "registering custom counter metric should not fail") + +// // Prepare a configuration containing a Threshold +// counterThresholds, err := stats.NewThresholds([]string{"count>0", "rate>1"}) +// require.NoError(t, err, "instantiating Thresholds with expression 'count>0' should not fail") +// options := lib.Options{ +// Thresholds: map[string]stats.Thresholds{ +// testCounterMetricName: counterThresholds, +// }, +// } +// config := Config{Options: options} + +// gotErr := validateThresholdsConfig(config, registry) + +// // Assert +// assert.NoError(t, gotErr, "validateThresholdsConfig should not fail against existing and valid custom metric") +// } + +// func TestValidateThresholdsConfigFailsOnNonExistingMetric(t *testing.T) { +// t.Parallel() + +// // Prepare a registry loaded with builtin metrics only +// registry := metrics.NewRegistry() +// metrics.RegisterBuiltinMetrics(registry) + +// // Prepare a configuration containing a Threshold applying to +// // a non-existing metric +// counterThresholds, err := stats.NewThresholds([]string{"count>0", "rate>1"}) +// require.NoError(t, err, "instantiating Thresholds with expression 'count>0' should not fail") +// options := lib.Options{ +// Thresholds: map[string]stats.Thresholds{ +// "nonexisting": counterThresholds, +// }, +// } +// config := Config{Options: options} +// var wantErrType errext.HasExitCode + +// gotErr := validateThresholdsConfig(config, registry) + +// // Assert +// assert.Error(t, gotErr, "validateThresholdsConfig should fail on thresholds applied to a non-existing metric") +// assert.ErrorAs(t, gotErr, &wantErrType, "validateThresholdsConfig error should be an instance of errext.HasExitCode") +// } + +// func TestValidateThresholdsConfigFailsOnThresholdInvalidMetricType(t *testing.T) { +// t.Parallel() + +// // Prepare a registry loaded with builtin metrics only +// registry := metrics.NewRegistry() +// metrics.RegisterBuiltinMetrics(registry) + +// // Prepare a configuration containing a Threshold using a Counter metric +// // specific aggregation method, against a metric of type Gauge: which doesn't support +// // that method. +// VUsThresholds, err := stats.NewThresholds([]string{"count>0"}) +// require.NoError(t, err, "instantiating Thresholds with expression 'count>0' should not fail") +// options := lib.Options{ +// Thresholds: map[string]stats.Thresholds{ +// metrics.VUsName: VUsThresholds, +// }, +// } +// config := Config{Options: options} +// var wantErrType errext.HasExitCode + +// gotErr := validateThresholdsConfig(config, registry) + +// // Assert +// assert.Error(t, gotErr, "validateThresholdsConfig should fail applying the count method to a Gauge metric") +// assert.ErrorAs(t, gotErr, &wantErrType, "validateThresholdsConfig error should be an instance of errext.HasExitCode") +// } diff --git a/cmd/run_test.go b/cmd/run_test.go index 0cd9fde24e0..e61ae1583b6 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -211,3 +211,48 @@ func TestInitErrExitCode(t *testing.T) { "Status code must be %d", exitcodes.ScriptException) assert.Contains(t, err.Error(), "ReferenceError: someUndefinedVar is not defined") } + +func TestInvalidOptionsThresholdErrExitCode(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + testFilename string + }{ + { + "run should fail with exit status 104 on a malformed threshold expression", + "testdata/thresholds/malformed_expression.js", + }, + { + "run should fail with exit status 104 on a threshold applied to a non existing metric", + "testdata/thresholds/non_existing_metric.js", + }, + { + "run should fail with exit status 104 on a threshold method being unsupported by the metric", + "testdata/thresholds/unsupported_aggregation_method.js", + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cmd := getRunCmd(ctx, testutils.NewLogger(t), newCommandFlags()) + a, err := filepath.Abs(testCase.testFilename) + require.NoError(t, err) + cmd.SetArgs([]string{a}) + wantExitCode := exitcodes.InvalidConfig + + var gotErrExt errext.HasExitCode + gotErr := cmd.Execute() + + require.ErrorAs(t, gotErr, &gotErrExt) + assert.Equalf(t, wantExitCode, gotErrExt.ExitCode(), + "status code must be %d", wantExitCode, + ) + }) + } +} diff --git a/lib/metrics/registry.go b/lib/metrics/registry.go index fc0a8c17a42..b7d2075851e 100644 --- a/lib/metrics/registry.go +++ b/lib/metrics/registry.go @@ -85,3 +85,11 @@ func (r *Registry) MustNewMetric(name string, typ stats.MetricType, t ...stats.V } return m } + +// Get returns the metric with the provided name. +// Similar to maps, if the metric with the provided name was found, +// the second return value will be true, false otherwise. +func (r *Registry) Get(name string) (*stats.Metric, bool) { + m, ok := r.metrics[name] + return m, ok +} diff --git a/lib/util.go b/lib/util.go index 2be0b770c5b..b7c22212495 100644 --- a/lib/util.go +++ b/lib/util.go @@ -66,3 +66,14 @@ func Min(a, b int64) int64 { } return b } + +// Contains checks if a string is present in a slice +func Contains(slice []string, str string) bool { + for _, element := range slice { + if element == str { + return true + } + } + + return false +} From c510664ce0db795946225d7e8715cefb89225e8e Mon Sep 17 00:00:00 2001 From: oleiade Date: Wed, 26 Jan 2022 16:33:47 +0100 Subject: [PATCH 4/4] Validate threshold configuration before starting the execution This commit makes sure that the threshold configuration (as passed in the script exported options for instance) is valid, before starting the execution. A valid threshold must pass the following assertion: - Its expression is syntaxically correct and is parsable - It applies to a metrics that's known to k6, either builtin or custom - Its expression's aggregation method is valid for the metric it applies to Threshold validation will be made in the context of the `run`, `cloud`, `archive`, `inspect`, and `archive` commands. If a threshold definition is invalid, the k6 program will exit with a status code of 104. ref #2330 --- cmd/archive.go | 2 +- cmd/cloud.go | 2 +- cmd/config_test.go | 1 - cmd/inspect.go | 2 +- cmd/run.go | 7 ++++++- cmd/testdata/thresholds/malformed_expression.js | 11 +++++++++++ cmd/testdata/thresholds/non_existing_metric.js | 13 +++++++++++++ .../thresholds/unsupported_aggregation_method.js | 15 +++++++++++++++ 8 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 cmd/testdata/thresholds/malformed_expression.js create mode 100644 cmd/testdata/thresholds/non_existing_metric.js create mode 100644 cmd/testdata/thresholds/unsupported_aggregation_method.js diff --git a/cmd/archive.go b/cmd/archive.go index c3bb61e3503..814d3c95779 100644 --- a/cmd/archive.go +++ b/cmd/archive.go @@ -75,7 +75,7 @@ An archive is a fully self-contained test run, and can be executed identically e return err } - _, err = deriveAndValidateConfig(conf, r.IsExecutable, logger) + _, err = deriveAndValidateConfig(conf, registry, r.IsExecutable, logger) if err != nil { return err } diff --git a/cmd/cloud.go b/cmd/cloud.go index fac936ce076..ad394dd502f 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -114,7 +114,7 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth return err } - derivedConf, err := deriveAndValidateConfig(conf, r.IsExecutable, logger) + derivedConf, err := deriveAndValidateConfig(conf, registry, r.IsExecutable, logger) if err != nil { return err } diff --git a/cmd/config_test.go b/cmd/config_test.go index 1061b8d5d08..432a6828835 100644 --- a/cmd/config_test.go +++ b/cmd/config_test.go @@ -31,7 +31,6 @@ import ( "go.k6.io/k6/lib" "go.k6.io/k6/lib/executor" "go.k6.io/k6/lib/metrics" - "go.k6.io/k6/lib/testutils" "go.k6.io/k6/lib/types" "go.k6.io/k6/stats" "gopkg.in/guregu/null.v3" diff --git a/cmd/inspect.go b/cmd/inspect.go index 76d411fde16..c3c300d8d65 100644 --- a/cmd/inspect.go +++ b/cmd/inspect.go @@ -129,7 +129,7 @@ func addExecRequirements(b *js.Bundle, return nil, err } - conf, err = deriveAndValidateConfig(conf, runner.IsExecutable, logger) + conf, err = deriveAndValidateConfig(conf, registry, runner.IsExecutable, logger) if err != nil { return nil, err } diff --git a/cmd/run.go b/cmd/run.go index 95885724809..46addcda97a 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -51,6 +51,7 @@ import ( "go.k6.io/k6/lib/consts" "go.k6.io/k6/lib/metrics" "go.k6.io/k6/loader" + "go.k6.io/k6/stats" "go.k6.io/k6/ui/pb" ) @@ -110,6 +111,10 @@ a commandline interface for interacting with it.`, builtinMetrics := metrics.RegisterBuiltinMetrics(registry) initRunner, err := newRunner(logger, src, globalFlags.runType, filesystems, runtimeOptions, builtinMetrics, registry) if err != nil { + if errors.Is(err, stats.ErrThresholdParsing) { + return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig) + } + return common.UnwrapGojaInterruptedError(err) } @@ -125,7 +130,7 @@ a commandline interface for interacting with it.`, return err } - conf, err = deriveAndValidateConfig(conf, initRunner.IsExecutable, logger) + conf, err = deriveAndValidateConfig(conf, registry, initRunner.IsExecutable, logger) if err != nil { return err } diff --git a/cmd/testdata/thresholds/malformed_expression.js b/cmd/testdata/thresholds/malformed_expression.js new file mode 100644 index 00000000000..59a3a452f5f --- /dev/null +++ b/cmd/testdata/thresholds/malformed_expression.js @@ -0,0 +1,11 @@ +export const options = { + thresholds: { + http_reqs: ["foo>0"], // Counter + }, +}; + +export default function () { + console.log( + "asserting that a malformed threshold fails with exit code 104 (Invalid config)" + ); +} diff --git a/cmd/testdata/thresholds/non_existing_metric.js b/cmd/testdata/thresholds/non_existing_metric.js new file mode 100644 index 00000000000..e4cdc4cfd7b --- /dev/null +++ b/cmd/testdata/thresholds/non_existing_metric.js @@ -0,0 +1,13 @@ +export const options = { + thresholds: { + // non existing is neither registered, nor a builtin metric. + // k6 should catch that. + "non existing": ["rate>0"], + }, +}; + +export default function () { + console.log( + "asserting that a threshold over a non-existing metric fails with exit code 104 (Invalid config)" + ); +} diff --git a/cmd/testdata/thresholds/unsupported_aggregation_method.js b/cmd/testdata/thresholds/unsupported_aggregation_method.js new file mode 100644 index 00000000000..4c2038edfd5 --- /dev/null +++ b/cmd/testdata/thresholds/unsupported_aggregation_method.js @@ -0,0 +1,15 @@ +export const options = { + thresholds: { + // http_reqs is a Counter metric. As such, it supports + // only the 'count' and 'rate' operations. Thus, 'value' + // being a Gauge's metric aggregation method, the threshold + // configuration evaluation should fail. + http_reqs: ["value>0"], + }, +}; + +export default function () { + console.log( + "asserting that a threshold applying a method over a metric not supporting it fails with exit code 104 (Invalid config)" + ); +}