From c03595577adf58b281cc5f5623460a871fa1b125 Mon Sep 17 00:00:00 2001 From: John Ryan Date: Sun, 11 Sep 2022 19:31:18 -0700 Subject: [PATCH] Reformat validation violation messages ... to improve readability. Usability testing (#707) made clear that violation messages on a single line are quite difficult to visually parse, making such error messages less readable. To support the reformatting, context is gathered at the time a violation is detected. The message is formatted once all violations have been collected. Also includes minor improvements to "named" rule messages to improve readability. --- pkg/cmd/template/schema_consumer_test.go | 94 +++++++++---- .../filetests/all-rules-are-run.tpltest | 18 ++- ...nvalid-when-value-greater-than-max.tpltest | 6 +- .../rejects-non-comparable-constraint.tpltest | 6 +- .../max=/rejects-non-comparable-value.tpltest | 6 +- ...valid-when-length-greater-than-max.tpltest | 6 +- .../max_len=/works-with-int-and-float | 12 +- .../invalid-when-value-less-than-min.tpltest | 6 +- .../rejects-non-comparable-constraint.tpltest | 6 +- .../min=/rejects-non-comparable-value.tpltest | 6 +- ...valid-when-length-is-less-than-min.tpltest | 6 +- .../min_len=/works-with-int-and-float.tpltest | 12 +- .../invalid-when-value-is-null.tpltest | 6 +- ...is-null-short-circuits-other-rules.tpltest | 6 +- ...lid-when-more-than-one-is-not-null.tpltest | 6 +- .../invalid-when-value-not-in-enum.tpltest | 6 +- .../value-is-null--skips-rules.tpltest} | 0 ...-is-null-with-not_null--runs-rules.tpltest | 11 ++ .../value-not-null--runs-rules.tpltest | 11 ++ .../when=/can-use-parent-value.tpltest | 6 +- .../when=/can-use-root-value.tpltest | 6 +- .../rejects-callable-arg-failing.tpltest | 2 +- ...ects-callable-arg-returns-non-bool.tpltest | 2 +- .../filetests/when=/true--runs-rules.tpltest | 6 +- .../true--and-not_null--runs-rules.tpltest | 7 - ...ue--and-value-not-null--runs-rules.tpltest | 7 - pkg/validations/validate.go | 123 ++++++++++++------ pkg/validations/validations_test.go | 4 +- pkg/workspace/library_execution.go | 10 +- pkg/yamlmeta/walk.go | 13 +- .../assert/max/value-fails-assertion.tpltest | 2 +- .../max_len/value-fails-assertion.tpltest | 2 +- .../assert/min/value-fails-assertion.tpltest | 2 +- .../min_len/value-fails-assertion.tpltest | 2 +- .../assert/one_not_null/err-in-check.tpltest | 2 +- .../assert/one_not_null/main.tpltest | 2 +- .../one_not_null/works-with-structs.tpltest | 2 +- .../ytt-library/assert/one_of/main.tpltest | 2 +- pkg/yttlibrary/assert.go | 22 ++-- 39 files changed, 316 insertions(+), 138 deletions(-) rename pkg/validations/filetests/{when_null_skip=/true--and-value-is-null--skips-rules.tpltest => when-null-skips/value-is-null--skips-rules.tpltest} (100%) create mode 100644 pkg/validations/filetests/when-null-skips/value-is-null-with-not_null--runs-rules.tpltest create mode 100644 pkg/validations/filetests/when-null-skips/value-not-null--runs-rules.tpltest delete mode 100644 pkg/validations/filetests/when_null_skip=/true--and-not_null--runs-rules.tpltest delete mode 100644 pkg/validations/filetests/when_null_skip=/true--and-value-not-null--runs-rules.tpltest diff --git a/pkg/cmd/template/schema_consumer_test.go b/pkg/cmd/template/schema_consumer_test.go index 2b9b46ec..354073a4 100644 --- a/pkg/cmd/template/schema_consumer_test.go +++ b/pkg/cmd/template/schema_consumer_test.go @@ -1536,8 +1536,11 @@ foo: 0 ` valuesYAML := `foo: 1` - expectedErrMsg := `One or more data values were invalid: -- (schema.yaml:3) requires "foo > 2" (by schema.yaml:2) + expectedErrMsg := `Validating final data values: + (document) + from: schema.yaml:3 + - must be: foo > 2 (by: schema.yaml:2) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1549,8 +1552,11 @@ foo: 0 ` valuesYAML := `foo: 1` - expectedErrMsg := `One or more data values were invalid: -- .foo (values.yaml:1) requires "foo > 2" (by schema.yaml:3) + expectedErrMsg := `Validating final data values: + foo + from: values.yaml:1 + - must be: foo > 2 (by: schema.yaml:3) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1566,9 +1572,15 @@ foo: - 1 ` - expectedErrMsg := `One or more data values were invalid: -- .foo[0] (values.yaml:2) requires "foo > 2" (by schema.yaml:4) -- .foo[1] (values.yaml:3) requires "foo > 2" (by schema.yaml:4) + expectedErrMsg := `Validating final data values: + foo[0] + from: values.yaml:2 + - must be: foo > 2 (by: schema.yaml:4) + + foo[1] + from: values.yaml:3 + - must be: foo > 2 (by: schema.yaml:4) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1585,8 +1597,12 @@ bar: 0 ` valuesYAML := `` - expectedErrMsg := `One or more data values were invalid: -- .bar (schema.yaml:8) requires "not null"; fail: value is null (by schema.yaml:7) + expectedErrMsg := `Validating final data values: + bar + from: schema.yaml:8 + - must be: not null (by: schema.yaml:7) + found: value is null + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1600,8 +1616,11 @@ foo: 0 ` valuesYAML := `foo: 1` - expectedErrMsg := `One or more data values were invalid: -- .foo (values.yaml:1) requires "foo > 2" (by schema.yaml:4) + expectedErrMsg := `Validating final data values: + foo + from: values.yaml:1 + - must be: foo > 2 (by: schema.yaml:4) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1622,8 +1641,11 @@ new: "" ` valuesYAML := `existing: foo` - expectedErrMsg := `One or more data values were invalid: -- .new (schema.yaml:11) requires "non-empty" (by schema.yaml:10) + expectedErrMsg := `Validating final data values: + new + from: schema.yaml:11 + - must be: non-empty (by: schema.yaml:10) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1643,8 +1665,11 @@ existing: "" existing: foo ` - expectedErrMsg := `One or more data values were invalid: -- .existing (values.yaml:2) requires "a long string" (by schema.yaml:4) + expectedErrMsg := `Validating final data values: + existing + from: values.yaml:2 + - must be: a long string (by: schema.yaml:4) + ` assertFailsWithSchemaAndDataValues(t, schemaYAML, valuesYAML, expectedErrMsg) }) @@ -1673,10 +1698,19 @@ bar: ` // none of the rules are from values.yml: - expectedErr := `One or more data values were invalid: -- (schema.yml:3) requires "has 3 map items" (by schema.yml:2) -- .foo (values.yml:5) requires "non-zero" (by schema.yml:4) -- .bar (schema.yml:7) requires "has 2 items" (by schema.yml:6) + expectedErr := `Validating final data values: + (document) + from: schema.yml:3 + - must be: has 3 map items (by: schema.yml:2) + + foo + from: values.yml:5 + - must be: non-zero (by: schema.yml:4) + + bar + from: schema.yml:7 + - must be: has 2 items (by: schema.yml:6) + ` opts := &cmdtpl.Options{} @@ -1702,11 +1736,23 @@ foo: - -1 ` - expectedErr := `One or more data values were invalid: -- .foo[0] (values.yml:5) requires "non-zero" (by schema.yml:4) -- .foo[0] (values.yml:5) requires "be odd" (by values.yml:4) -- .foo[1] (values.yml:7) requires "non-zero" (by schema.yml:4) -- .foo[1] (values.yml:7) requires "be even" (by values.yml:6) + expectedErr := `Validating final data values: + foo[0] + from: values.yml:5 + - must be: non-zero (by: schema.yml:4) + + foo[0] + from: values.yml:5 + - must be: be odd (by: values.yml:4) + + foo[1] + from: values.yml:7 + - must be: non-zero (by: schema.yml:4) + + foo[1] + from: values.yml:7 + - must be: be even (by: values.yml:6) + ` opts := &cmdtpl.Options{} diff --git a/pkg/validations/filetests/all-rules-are-run.tpltest b/pkg/validations/filetests/all-rules-are-run.tpltest index 508fa6d4..77684cdd 100644 --- a/pkg/validations/filetests/all-rules-are-run.tpltest +++ b/pkg/validations/filetests/all-rules-are-run.tpltest @@ -6,8 +6,16 @@ config: [1,1] +++ ERR: -- .config (stdin:4) requires "custom rule"; fail: fails (by stdin:3) -- .config (stdin:4) requires "length greater or equal to 10"; fail: length of 2 is less than 10 (by stdin:3) -- .config (stdin:4) requires "length less than or equal to 1"; fail: length of 2 is more than 1 (by stdin:3) -- .config (stdin:4) requires "a value greater or equal to [2, 2]"; fail: value is less than [2, 2] (by stdin:3) -- .config (stdin:4) requires "a value less than or equal to [0, 0]"; fail: value is more than [0, 0] (by stdin:3) + config + from: stdin:4 + - must be: custom rule (by: stdin:3) + found: fails + - must be: length >= 10 (by: stdin:3) + found: length = 2 + - must be: length <= 1 (by: stdin:3) + found: length = 2 + - must be: a value >= [2, 2] (by: stdin:3) + found: value < [2, 2] + - must be: a value <= [0, 0] (by: stdin:3) + found: value > [0, 0] + diff --git a/pkg/validations/filetests/max=/invalid-when-value-greater-than-max.tpltest b/pkg/validations/filetests/max=/invalid-when-value-greater-than-max.tpltest index c695d7bc..2905f601 100644 --- a/pkg/validations/filetests/max=/invalid-when-value-greater-than-max.tpltest +++ b/pkg/validations/filetests/max=/invalid-when-value-greater-than-max.tpltest @@ -4,4 +4,8 @@ foo: 11 +++ ERR: -- .foo (stdin:2) requires "a value less than or equal to 10"; fail: value is more than 10 (by stdin:1) + foo + from: stdin:2 + - must be: a value <= 10 (by: stdin:1) + found: value > 10 + diff --git a/pkg/validations/filetests/max=/rejects-non-comparable-constraint.tpltest b/pkg/validations/filetests/max=/rejects-non-comparable-constraint.tpltest index f8559009..2342ae06 100644 --- a/pkg/validations/filetests/max=/rejects-non-comparable-constraint.tpltest +++ b/pkg/validations/filetests/max=/rejects-non-comparable-constraint.tpltest @@ -5,4 +5,8 @@ value: +++ ERR: -- .value (stdin:2) requires "a value less than or equal to {\"foo\": True}"; dict <= dict not implemented (by stdin:1) + value + from: stdin:2 + - must be: a value <= {"foo": True} (by: stdin:1) + found: dict <= dict not implemented + diff --git a/pkg/validations/filetests/max=/rejects-non-comparable-value.tpltest b/pkg/validations/filetests/max=/rejects-non-comparable-value.tpltest index 1555040b..e692dc1d 100644 --- a/pkg/validations/filetests/max=/rejects-non-comparable-value.tpltest +++ b/pkg/validations/filetests/max=/rejects-non-comparable-value.tpltest @@ -6,4 +6,8 @@ value: +++ ERR: -- .value (stdin:2) requires "a value less than or equal to 4"; dict <= int not implemented (by stdin:1) + value + from: stdin:2 + - must be: a value <= 4 (by: stdin:1) + found: dict <= int not implemented + diff --git a/pkg/validations/filetests/max_len=/invalid-when-length-greater-than-max.tpltest b/pkg/validations/filetests/max_len=/invalid-when-length-greater-than-max.tpltest index e737f934..c21e3990 100644 --- a/pkg/validations/filetests/max_len=/invalid-when-length-greater-than-max.tpltest +++ b/pkg/validations/filetests/max_len=/invalid-when-length-greater-than-max.tpltest @@ -4,4 +4,8 @@ value: "123456" +++ ERR: -- .value (stdin:2) requires "length less than or equal to 5"; fail: length of 6 is more than 5 (by stdin:1) + value + from: stdin:2 + - must be: length <= 5 (by: stdin:1) + found: length = 6 + diff --git a/pkg/validations/filetests/max_len=/works-with-int-and-float b/pkg/validations/filetests/max_len=/works-with-int-and-float index b8b9e4d9..cd566ba6 100644 --- a/pkg/validations/filetests/max_len=/works-with-int-and-float +++ b/pkg/validations/filetests/max_len=/works-with-int-and-float @@ -6,5 +6,13 @@ bar: "longer than 10" +++ ERR: -- .foo (stdin:2) requires "length less than or equal to 10"; fail: length of 14 is more than 10 (by stdin:1) -- .bar (stdin:4) requires "length less than or equal to 10"; fail: length of 14 is more than 10 (by stdin:3) + foo + from: stdin:2 + - must be: length <= 10 (by: stdin:1) + found: length = 14 + + bar + from: stdin:4 + - must be: length <= 10 (by: stdin:3) + found: length = 14 + diff --git a/pkg/validations/filetests/min=/invalid-when-value-less-than-min.tpltest b/pkg/validations/filetests/min=/invalid-when-value-less-than-min.tpltest index 5abf2402..aeed0cf6 100644 --- a/pkg/validations/filetests/min=/invalid-when-value-less-than-min.tpltest +++ b/pkg/validations/filetests/min=/invalid-when-value-less-than-min.tpltest @@ -4,4 +4,8 @@ value: 9 +++ ERR: -- .value (stdin:2) requires "a value greater or equal to 10"; fail: value is less than 10 (by stdin:1) + value + from: stdin:2 + - must be: a value >= 10 (by: stdin:1) + found: value < 10 + diff --git a/pkg/validations/filetests/min=/rejects-non-comparable-constraint.tpltest b/pkg/validations/filetests/min=/rejects-non-comparable-constraint.tpltest index ebe0b0ac..b1317a3d 100644 --- a/pkg/validations/filetests/min=/rejects-non-comparable-constraint.tpltest +++ b/pkg/validations/filetests/min=/rejects-non-comparable-constraint.tpltest @@ -5,4 +5,8 @@ value: +++ ERR: -- .value (stdin:2) requires "a value greater or equal to {\"foo\": True}"; dict >= dict not implemented (by stdin:1) + value + from: stdin:2 + - must be: a value >= {"foo": True} (by: stdin:1) + found: dict >= dict not implemented + diff --git a/pkg/validations/filetests/min=/rejects-non-comparable-value.tpltest b/pkg/validations/filetests/min=/rejects-non-comparable-value.tpltest index c2672b06..08716557 100644 --- a/pkg/validations/filetests/min=/rejects-non-comparable-value.tpltest +++ b/pkg/validations/filetests/min=/rejects-non-comparable-value.tpltest @@ -6,4 +6,8 @@ value: +++ ERR: -- .value (stdin:2) requires "a value greater or equal to 4"; dict >= int not implemented (by stdin:1) + value + from: stdin:2 + - must be: a value >= 4 (by: stdin:1) + found: dict >= int not implemented + diff --git a/pkg/validations/filetests/min_len=/invalid-when-length-is-less-than-min.tpltest b/pkg/validations/filetests/min_len=/invalid-when-length-is-less-than-min.tpltest index f4a2b083..b175a0ae 100644 --- a/pkg/validations/filetests/min_len=/invalid-when-length-is-less-than-min.tpltest +++ b/pkg/validations/filetests/min_len=/invalid-when-length-is-less-than-min.tpltest @@ -4,4 +4,8 @@ value: "1234" +++ ERR: -- .value (stdin:2) requires "length greater or equal to 5"; fail: length of 4 is less than 5 (by stdin:1) + value + from: stdin:2 + - must be: length >= 5 (by: stdin:1) + found: length = 4 + diff --git a/pkg/validations/filetests/min_len=/works-with-int-and-float.tpltest b/pkg/validations/filetests/min_len=/works-with-int-and-float.tpltest index 3bce4f85..f8800904 100644 --- a/pkg/validations/filetests/min_len=/works-with-int-and-float.tpltest +++ b/pkg/validations/filetests/min_len=/works-with-int-and-float.tpltest @@ -6,5 +6,13 @@ bar: "shorter than 20" +++ ERR: -- .foo (stdin:2) requires "length greater or equal to 20"; fail: length of 15 is less than 20 (by stdin:1) -- .bar (stdin:4) requires "length greater or equal to 20"; fail: length of 15 is less than 20 (by stdin:3) + foo + from: stdin:2 + - must be: length >= 20 (by: stdin:1) + found: length = 15 + + bar + from: stdin:4 + - must be: length >= 20 (by: stdin:3) + found: length = 15 + diff --git a/pkg/validations/filetests/not_null=/invalid-when-value-is-null.tpltest b/pkg/validations/filetests/not_null=/invalid-when-value-is-null.tpltest index 11877469..6c13899c 100644 --- a/pkg/validations/filetests/not_null=/invalid-when-value-is-null.tpltest +++ b/pkg/validations/filetests/not_null=/invalid-when-value-is-null.tpltest @@ -4,4 +4,8 @@ value: null +++ ERR: -- .value (stdin:2) requires "not null"; fail: value is null (by stdin:1) + value + from: stdin:2 + - must be: not null (by: stdin:1) + found: value is null + diff --git a/pkg/validations/filetests/not_null=/when-value-is-null-short-circuits-other-rules.tpltest b/pkg/validations/filetests/not_null=/when-value-is-null-short-circuits-other-rules.tpltest index 83511eeb..3d62055e 100644 --- a/pkg/validations/filetests/not_null=/when-value-is-null-short-circuits-other-rules.tpltest +++ b/pkg/validations/filetests/not_null=/when-value-is-null-short-circuits-other-rules.tpltest @@ -4,4 +4,8 @@ value: null +++ ERR: -- .value (stdin:2) requires "not null"; fail: value is null (by stdin:1) + value + from: stdin:2 + - must be: not null (by: stdin:1) + found: value is null + diff --git a/pkg/validations/filetests/one_not_null=/invalid-when-more-than-one-is-not-null.tpltest b/pkg/validations/filetests/one_not_null=/invalid-when-more-than-one-is-not-null.tpltest index b68e1523..ae8f78bb 100644 --- a/pkg/validations/filetests/one_not_null=/invalid-when-more-than-one-is-not-null.tpltest +++ b/pkg/validations/filetests/one_not_null=/invalid-when-more-than-one-is-not-null.tpltest @@ -6,4 +6,8 @@ config: +++ ERR: -- .config (stdin:2) requires "exactly one child not null"; check: multiple values are not null ["foo" "bar"] (by stdin:1) + config + from: stdin:2 + - must be: exactly one of all children to be not null (by: stdin:1) + found: ["foo", "bar"] are not null + diff --git a/pkg/validations/filetests/one_of=/invalid-when-value-not-in-enum.tpltest b/pkg/validations/filetests/one_of=/invalid-when-value-not-in-enum.tpltest index 9adea891..b52744be 100644 --- a/pkg/validations/filetests/one_of=/invalid-when-value-not-in-enum.tpltest +++ b/pkg/validations/filetests/one_of=/invalid-when-value-not-in-enum.tpltest @@ -4,4 +4,8 @@ foo: critical +++ ERR: -- .foo (stdin:2) requires "one of"; fail: value not in ["debug", "info", "warning", "error", "fatal"] (by stdin:1) + foo + from: stdin:2 + - must be: one of ["debug", "info", "warning", "error", "fatal"] (by: stdin:1) + found: not one of allowed values + diff --git a/pkg/validations/filetests/when_null_skip=/true--and-value-is-null--skips-rules.tpltest b/pkg/validations/filetests/when-null-skips/value-is-null--skips-rules.tpltest similarity index 100% rename from pkg/validations/filetests/when_null_skip=/true--and-value-is-null--skips-rules.tpltest rename to pkg/validations/filetests/when-null-skips/value-is-null--skips-rules.tpltest diff --git a/pkg/validations/filetests/when-null-skips/value-is-null-with-not_null--runs-rules.tpltest b/pkg/validations/filetests/when-null-skips/value-is-null-with-not_null--runs-rules.tpltest new file mode 100644 index 00000000..2d5058b1 --- /dev/null +++ b/pkg/validations/filetests/when-null-skips/value-is-null-with-not_null--runs-rules.tpltest @@ -0,0 +1,11 @@ +#@assert/validate not_null=True +foo: null + ++++ + +ERR: + foo + from: stdin:2 + - must be: not null (by: stdin:1) + found: value is null + diff --git a/pkg/validations/filetests/when-null-skips/value-not-null--runs-rules.tpltest b/pkg/validations/filetests/when-null-skips/value-not-null--runs-rules.tpltest new file mode 100644 index 00000000..248600b9 --- /dev/null +++ b/pkg/validations/filetests/when-null-skips/value-not-null--runs-rules.tpltest @@ -0,0 +1,11 @@ +#@assert/validate ("", lambda v: fail("rules were run")) +foo: "" + ++++ + +ERR: + foo + from: stdin:2 + - must be: (by: stdin:1) + found: rules were run + diff --git a/pkg/validations/filetests/when=/can-use-parent-value.tpltest b/pkg/validations/filetests/when=/can-use-parent-value.tpltest index 6be32a70..74c76e66 100644 --- a/pkg/validations/filetests/when=/can-use-parent-value.tpltest +++ b/pkg/validations/filetests/when=/can-use-parent-value.tpltest @@ -8,4 +8,8 @@ counters: +++ ERR: -- .counters.foo (stdin:4) requires "fail"; fail: fails (by stdin:3) + counters.foo + from: stdin:4 + - must be: fail (by: stdin:3) + found: fails + diff --git a/pkg/validations/filetests/when=/can-use-root-value.tpltest b/pkg/validations/filetests/when=/can-use-root-value.tpltest index 604640f3..e9999b0b 100644 --- a/pkg/validations/filetests/when=/can-use-root-value.tpltest +++ b/pkg/validations/filetests/when=/can-use-root-value.tpltest @@ -8,4 +8,8 @@ counters: +++ ERR: -- .counters.bar (stdin:6) requires "fail"; fail: fails (by stdin:5) + counters.bar + from: stdin:6 + - must be: fail (by: stdin:5) + found: fails + diff --git a/pkg/validations/filetests/when=/rejects-callable-arg-failing.tpltest b/pkg/validations/filetests/when=/rejects-callable-arg-failing.tpltest index ec3956be..682966fd 100644 --- a/pkg/validations/filetests/when=/rejects-callable-arg-failing.tpltest +++ b/pkg/validations/filetests/when=/rejects-callable-arg-failing.tpltest @@ -6,4 +6,4 @@ foo: bar +++ ERR: -Validating .foo: Failure evaluating when=: fail: error from within lambda \ No newline at end of file +Validating foo: Failure evaluating when=: fail: error from within lambda \ No newline at end of file diff --git a/pkg/validations/filetests/when=/rejects-callable-arg-returns-non-bool.tpltest b/pkg/validations/filetests/when=/rejects-callable-arg-returns-non-bool.tpltest index 98e82772..ea746507 100644 --- a/pkg/validations/filetests/when=/rejects-callable-arg-returns-non-bool.tpltest +++ b/pkg/validations/filetests/when=/rejects-callable-arg-returns-non-bool.tpltest @@ -4,4 +4,4 @@ foo: bar +++ ERR: -Validating .foo: want when= to be bool, got string \ No newline at end of file +Validating foo: want when= to be bool, got string \ No newline at end of file diff --git a/pkg/validations/filetests/when=/true--runs-rules.tpltest b/pkg/validations/filetests/when=/true--runs-rules.tpltest index 1b51c74b..9351587d 100644 --- a/pkg/validations/filetests/when=/true--runs-rules.tpltest +++ b/pkg/validations/filetests/when=/true--runs-rules.tpltest @@ -4,4 +4,8 @@ foo: "" +++ ERR: -- .foo (stdin:2) requires "fail"; fail: fails (by stdin:1) + foo + from: stdin:2 + - must be: fail (by: stdin:1) + found: fails + diff --git a/pkg/validations/filetests/when_null_skip=/true--and-not_null--runs-rules.tpltest b/pkg/validations/filetests/when_null_skip=/true--and-not_null--runs-rules.tpltest deleted file mode 100644 index 68cb907d..00000000 --- a/pkg/validations/filetests/when_null_skip=/true--and-not_null--runs-rules.tpltest +++ /dev/null @@ -1,7 +0,0 @@ -#@assert/validate not_null=True -foo: null - -+++ - -ERR: -- .foo (stdin:2) requires "not null"; fail: value is null (by stdin:1) diff --git a/pkg/validations/filetests/when_null_skip=/true--and-value-not-null--runs-rules.tpltest b/pkg/validations/filetests/when_null_skip=/true--and-value-not-null--runs-rules.tpltest deleted file mode 100644 index c3d4e041..00000000 --- a/pkg/validations/filetests/when_null_skip=/true--and-value-not-null--runs-rules.tpltest +++ /dev/null @@ -1,7 +0,0 @@ -#@assert/validate ("", lambda v: fail("rules were run")) -foo: "" - -+++ - -ERR: -- .foo (stdin:2) requires ""; fail: rules were run (by stdin:1) diff --git a/pkg/validations/validate.go b/pkg/validations/validate.go index b71ea3be..83a5559a 100644 --- a/pkg/validations/validate.go +++ b/pkg/validations/validate.go @@ -7,6 +7,7 @@ import ( "fmt" "reflect" "sort" + "strings" "github.com/k14s/starlark-go/starlark" "github.com/vmware-tanzu/carvel-ytt/pkg/filepos" @@ -78,7 +79,7 @@ type validationRun struct { } func newValidationRun(threadName string, root yamlmeta.Node) *validationRun { - return &validationRun{thread: &starlark.Thread{Name: threadName}, chk: Check{[]error{}}, root: root} + return &validationRun{thread: &starlark.Thread{Name: threadName}, root: root} } // VisitWithParent if `node` has validations in its meta. @@ -93,13 +94,12 @@ func (a *validationRun) VisitWithParent(value yamlmeta.Node, parent yamlmeta.Nod return nil } for _, v := range validations { - // possible refactor to check validationRun kwargs prior to validating rules - violations, err := v.Validate(value, parent, a.root, path, a.thread) + invalid, err := v.Validate(value, parent, a.root, path, a.thread) if err != nil { return err } - if violations != nil { - a.chk.Violations = append(a.chk.Violations, violations...) + if len(invalid.Violations) > 0 { + a.chk.Invalidations = append(a.chk.Invalidations, invalid) } } @@ -111,37 +111,55 @@ func (a *validationRun) VisitWithParent(value yamlmeta.Node, parent yamlmeta.Nod // // Returns an error if the assertion returns False (not-None), or assert.fail()s. // Otherwise, returns nil. -func (v NodeValidation) Validate(node yamlmeta.Node, parent yamlmeta.Node, root yamlmeta.Node, path string, thread *starlark.Thread) ([]error, error) { +func (v NodeValidation) Validate(node yamlmeta.Node, parent yamlmeta.Node, root yamlmeta.Node, path string, thread *starlark.Thread) (Invalidation, error) { nodeValue := v.newStarlarkValue(node) parentValue := v.newStarlarkValue(parent) rootValue := v.newStarlarkValue(root) executeRules, err := v.kwargs.shouldValidate(nodeValue, parentValue, thread, rootValue) if err != nil { - return nil, fmt.Errorf("Validating %s: %s", path, err) + return Invalidation{}, fmt.Errorf("Validating %s: %s", path, err) } if !executeRules { - return nil, nil + return Invalidation{}, nil + } + + displayedPath := path + if displayedPath == "" { + displayedPath = fmt.Sprintf("(%s)", yamlmeta.TypeName(node)) + } + invalid := Invalidation{ + Path: displayedPath, + ValueSource: node.GetPosition(), } - var violations []error for _, rul := range byPriority(v.rules) { result, err := starlark.Call(thread, rul.assertion, starlark.Tuple{nodeValue}, []starlark.Tuple{}) if err != nil { - violations = append(violations, fmt.Errorf("%s (%s) requires %q; %s (by %s)", path, node.GetPosition().AsCompactString(), rul.msg, err.Error(), v.position.AsCompactString())) + violation := Violation{ + RuleSource: v.position, + Description: rul.msg, + Results: strings.TrimPrefix(strings.TrimPrefix(err.Error(), "fail: "), "check: "), + } + invalid.Violations = append(invalid.Violations, violation) if rul.isCritical { break } } else { if !(result == starlark.True) { - violations = append(violations, fmt.Errorf("%s (%s) requires %q (by %s)", path, node.GetPosition().AsCompactString(), rul.msg, v.position.AsCompactString())) + violation := Violation{ + RuleSource: v.position, + Description: rul.msg, + Results: "", + } + invalid.Violations = append(invalid.Violations, violation) if rul.isCritical { break } } } } - return violations, nil + return invalid, nil } // shouldValidate uses validationKwargs and the node's value to run checks on the value. If the value satisfies the checks, @@ -203,25 +221,25 @@ func (v validationKwargs) asRules() []rule { if v.minLength != nil { rules = append(rules, rule{ - msg: fmt.Sprintf("length greater or equal to %v", *v.minLength), + msg: fmt.Sprintf("length >= %v", *v.minLength), assertion: yttlibrary.NewAssertMinLen(*v.minLength).CheckFunc(), }) } if v.maxLength != nil { rules = append(rules, rule{ - msg: fmt.Sprintf("length less than or equal to %v", *v.maxLength), + msg: fmt.Sprintf("length <= %v", *v.maxLength), assertion: yttlibrary.NewAssertMaxLen(*v.maxLength).CheckFunc(), }) } if v.min != nil { rules = append(rules, rule{ - msg: fmt.Sprintf("a value greater or equal to %v", v.min), + msg: fmt.Sprintf("a value >= %v", v.min), assertion: yttlibrary.NewAssertMin(v.min).CheckFunc(), }) } if v.max != nil { rules = append(rules, rule{ - msg: fmt.Sprintf("a value less than or equal to %v", v.max), + msg: fmt.Sprintf("a value <= %v", v.max), assertion: yttlibrary.NewAssertMax(v.max).CheckFunc(), }) } @@ -235,30 +253,33 @@ func (v validationKwargs) asRules() []rule { } if v.oneNotNull != nil { var assertion *yttlibrary.Assertion + var childKeys = "" switch oneNotNull := v.oneNotNull.(type) { case starlark.Bool: if oneNotNull { assertion = yttlibrary.NewAssertOneNotNull(nil) + childKeys = "all children" } else { // should have been caught when args were parsed panic("one_not_null= cannot be False") } case starlark.Sequence: assertion = yttlibrary.NewAssertOneNotNull(oneNotNull) + childKeys = oneNotNull.String() default: // should have been caught when args were parsed panic(fmt.Sprintf("Unexpected type \"%s\" for one_not_null=", v.oneNotNull.Type())) } rules = append(rules, rule{ - msg: fmt.Sprintf("exactly one child not null"), + msg: fmt.Sprintf("exactly one of %s to be not null", childKeys), assertion: assertion.CheckFunc(), }) } if v.oneOf != nil { rules = append(rules, rule{ - msg: fmt.Sprintf("one of"), + msg: fmt.Sprintf("one of %s", v.oneOf.String()), assertion: yttlibrary.NewAssertOneOf(v.oneOf).CheckFunc(), }) } @@ -266,45 +287,61 @@ func (v validationKwargs) asRules() []rule { return rules } -// Check holds the resulting violations from executing Validations on a node. +// Invalidation describes a value that was invalidated, and how. +type Invalidation struct { + Path string + ValueSource *filepos.Position + Violations []Violation +} + +// Violation describes how a value failed to satisfy a rule. +type Violation struct { + RuleSource *filepos.Position + Description string + Results string +} + +// Check holds the complete set of Invalidations (if any) resulting from checking all validation rules. type Check struct { - Violations []error + Invalidations []Invalidation } -// Error generates the error message composed of the total set of Check.Violations. -func (c Check) Error() string { - if !c.HasViolations() { +// ResultsAsString generates the error message composed of the total set of Check.Invalidations. +func (c Check) ResultsAsString() string { + if !c.HasInvalidations() { return "" } msg := "" - for _, err := range c.Violations { - msg = msg + "- " + err.Error() + "\n" + for _, inval := range c.Invalidations { + msg += fmt.Sprintf(" %s\n from: %s\n", inval.Path, inval.ValueSource.AsCompactString()) + for _, viol := range inval.Violations { + msg += fmt.Sprintf(" - must be: %s (by: %s)\n", viol.Description, viol.RuleSource.AsCompactString()) + if viol.Results != "" { + msg += fmt.Sprintf(" found: %s\n", viol.Results) + } + } + msg += "\n" } return msg } -// HasViolations indicates whether this Check contains any violations. -func (c *Check) HasViolations() bool { - return len(c.Violations) > 0 +// HasInvalidations indicates whether this Check contains any violations. +func (c Check) HasInvalidations() bool { + return len(c.Invalidations) > 0 } func (v NodeValidation) newStarlarkValue(node yamlmeta.Node) starlark.Value { - var nodeValue starlark.Value - switch typedNode := node.(type) { - case *yamlmeta.DocumentSet: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode).AsStarlarkValue() - case *yamlmeta.Array: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode).AsStarlarkValue() - case *yamlmeta.Map: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode).AsStarlarkValue() - case *yamlmeta.Document: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode.Value).AsStarlarkValue() - case *yamlmeta.MapItem: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode.Value).AsStarlarkValue() - case *yamlmeta.ArrayItem: - nodeValue = yamltemplate.NewGoValueWithYAML(typedNode.Value).AsStarlarkValue() + if node == nil || reflect.ValueOf(node).IsNil() { + return starlark.None } - return nodeValue + switch node.(type) { + case *yamlmeta.DocumentSet, *yamlmeta.Array, *yamlmeta.Map: + return yamltemplate.NewGoValueWithYAML(node).AsStarlarkValue() + case *yamlmeta.Document, *yamlmeta.MapItem, *yamlmeta.ArrayItem: + return yamltemplate.NewGoValueWithYAML(node.GetValues()[0]).AsStarlarkValue() + default: + panic(fmt.Sprintf("Unexpected node type %T (at or near %s)", node, node.GetPosition().AsCompactString())) + } } diff --git a/pkg/validations/validations_test.go b/pkg/validations/validations_test.go index d06e62df..a57fc455 100644 --- a/pkg/validations/validations_test.go +++ b/pkg/validations/validations_test.go @@ -70,8 +70,8 @@ func EvalAndValidateTemplate(ft filetests.FileTests) filetests.EvaluateTemplate return nil, filetests.NewTestErr(err, fmt.Errorf("Unexpected error (did you include the \"ERR:\" marker in the output?):%v", err)) } // TODO: proper error handling! - if chk.HasViolations() { - err := fmt.Errorf("\n%s", chk.Error()) + if chk.HasInvalidations() { + err := fmt.Errorf("\n%s", chk.ResultsAsString()) return nil, filetests.NewTestErr(err, fmt.Errorf("Unexpected violations (did you include the \"ERR:\" marker in the output?):%v", err)) } diff --git a/pkg/workspace/library_execution.go b/pkg/workspace/library_execution.go index 2e2ac01e..1d540a6e 100644 --- a/pkg/workspace/library_execution.go +++ b/pkg/workspace/library_execution.go @@ -4,6 +4,7 @@ package workspace import ( + "errors" "fmt" "strings" @@ -102,7 +103,7 @@ func (ll *LibraryExecution) Values(valuesOverlays []*datavalues.Envelope, schema if !ll.skipDataValuesValidation { err = ll.validateValues(values) if err != nil { - return nil, nil, fmt.Errorf("Validating final data values: %s", err) + return nil, nil, fmt.Errorf("Validating final data values:\n%s", err) } } return values, libValues, err @@ -124,14 +125,13 @@ func (ll *LibraryExecution) validateValues(values *datavalues.Envelope) error { return err } - assertCheck, err := validations.Run(values.Doc, "run-data-values-validations") + chk, err := validations.Run(values.Doc, "run-data-values-validations") if err != nil { return err } - if assertCheck.HasViolations() { - combinedViolations := assertCheck.Error() - return fmt.Errorf("One or more data values were invalid:\n%s", combinedViolations) + if chk.HasInvalidations() { + return errors.New(chk.ResultsAsString()) } return nil diff --git a/pkg/yamlmeta/walk.go b/pkg/yamlmeta/walk.go index 52923d1b..698645e3 100644 --- a/pkg/yamlmeta/walk.go +++ b/pkg/yamlmeta/walk.go @@ -49,7 +49,7 @@ func WalkWithParent(node Node, parent Node, path string, v VisitorWithParent) er for idx, child := range node.GetValues() { if childNode, ok := child.(Node); ok { - err = WalkWithParent(childNode, node, path+keyOrIdxPart(childNode, idx), v) + err = WalkWithParent(childNode, node, pathToNode(path, childNode, idx), v) if err != nil { return err } @@ -58,13 +58,16 @@ func WalkWithParent(node Node, parent Node, path string, v VisitorWithParent) er return nil } -func keyOrIdxPart(node Node, idx int) string { +func pathToNode(path string, node Node, idx int) string { switch typedNode := node.(type) { case *MapItem: - return fmt.Sprintf(".%s", typedNode.Key) + if path == "" { + return fmt.Sprintf("%s", typedNode.Key) + } + return fmt.Sprintf("%s.%s", path, typedNode.Key) case *ArrayItem: - return fmt.Sprintf("[%d]", idx) + return fmt.Sprintf("%s[%d]", path, idx) default: - return "" + return path } } diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/max/value-fails-assertion.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/max/value-fails-assertion.tpltest index ce830376..cc198ab7 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/max/value-fails-assertion.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/max/value-fails-assertion.tpltest @@ -5,7 +5,7 @@ max_of_int: #@ assert.max(2).check(3) +++ ERR: -- fail: value is more than 2 +- fail: value > 2 in lambda assert.max:? | __ytt_tplXXX_set_ctx_type("yaml") (generated) stdin:1 | #@ load("@ytt:assert", "assert") diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/max_len/value-fails-assertion.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/max_len/value-fails-assertion.tpltest index 807b9db0..327aa437 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/max_len/value-fails-assertion.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/max_len/value-fails-assertion.tpltest @@ -4,7 +4,7 @@ max_len_of_string: #@ assert.max_len(2).check("abc") +++ ERR: -- fail: length of 3 is more than 2 +- fail: length = 3 in lambda assert.max_len:? | __ytt_tplXXX_set_ctx_type("yaml") (generated) stdin:1 | #@ load("@ytt:assert", "assert") diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/min/value-fails-assertion.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/min/value-fails-assertion.tpltest index 726dcb4d..a204fe47 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/min/value-fails-assertion.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/min/value-fails-assertion.tpltest @@ -5,7 +5,7 @@ min_of_int: #@ assert.min(3).check(2) +++ ERR: -- fail: value is less than 3 +- fail: value < 3 in lambda assert.min:? | __ytt_tplXXX_set_ctx_type("yaml") (generated) stdin:1 | #@ load("@ytt:assert", "assert") diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/min_len/value-fails-assertion.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/min_len/value-fails-assertion.tpltest index f4fbc3de..124ec4c9 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/min_len/value-fails-assertion.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/min_len/value-fails-assertion.tpltest @@ -4,7 +4,7 @@ min_len_of_string: #@ assert.min_len(2).check("a") +++ ERR: -- fail: length of 1 is less than 2 +- fail: length = 1 in lambda assert.min_len:? | __ytt_tplXXX_set_ctx_type("yaml") (generated) stdin:1 | #@ load("@ytt:assert", "assert") diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/err-in-check.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/err-in-check.tpltest index 97c43c9e..249d2546 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/err-in-check.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/err-in-check.tpltest @@ -14,6 +14,6 @@ multiple_values_are_not_null: #@ assert.one_not_null().check(multiple_values_are +++ ERR: -- check: multiple values are not null ["foo" "qux"] +- check: ["foo", "qux"] are not null in stdin:12 | multiple_values_are_not_null: #@ assert.one_not_null().check(multiple_values_are_not_null()) \ No newline at end of file diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/main.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/main.tpltest index 82b9e582..c4787ee6 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/main.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/main.tpltest @@ -55,7 +55,7 @@ when_keys_are_not_given: - 'check: all values are null' multiple_values_are_not_null: - null - - 'check: multiple values are not null ["foo" "qux"]' + - 'check: ["foo", "qux"] are not null' value_is_empty: - null - 'check: value is empty' diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/works-with-structs.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/works-with-structs.tpltest index fb89d9d7..650df22a 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/works-with-structs.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/one_not_null/works-with-structs.tpltest @@ -18,6 +18,6 @@ keys_are_specified: when_keys_are_not_given: multiple_values_are_not_null: - null - - 'check: multiple values are not null ["foo" "qux"]' + - 'check: ["foo", "qux"] are not null' keys_are_specified: one_value_not_null_is_ok: true diff --git a/pkg/yamltemplate/filetests/ytt-library/assert/one_of/main.tpltest b/pkg/yamltemplate/filetests/ytt-library/assert/one_of/main.tpltest index 94d79937..79ed3264 100644 --- a/pkg/yamltemplate/filetests/ytt-library/assert/one_of/main.tpltest +++ b/pkg/yamltemplate/filetests/ytt-library/assert/one_of/main.tpltest @@ -21,7 +21,7 @@ pass: fail: not_in_enum: - null - - 'fail: value not in ["aws", "azure", "vsphere"]' + - 'fail: not one of allowed values' enum_not_a_sequence: int: - null diff --git a/pkg/yttlibrary/assert.go b/pkg/yttlibrary/assert.go index d048c4a9..6b9cdadb 100644 --- a/pkg/yttlibrary/assert.go +++ b/pkg/yttlibrary/assert.go @@ -182,7 +182,7 @@ func NewAssertionFromStarlarkFunc(funcName string, checkFunc core.StarlarkFunc) func NewAssertMaxLen(maximum starlark.Int) *Assertion { return NewAssertionFromSource( "assert.max_len", - `lambda sequence: True if len(sequence) <= maximum else fail ("length of {} is more than {}".format(len(sequence), maximum))`, + `lambda sequence: True if len(sequence) <= maximum else fail ("length = {}".format(len(sequence)))`, starlark.StringDict{"maximum": maximum}, ) } @@ -206,7 +206,7 @@ func (m AssertModule) MaxLen(thread *starlark.Thread, f *starlark.Builtin, args func NewAssertMinLen(minimum starlark.Int) *Assertion { return NewAssertionFromSource( "assert.min_len", - `lambda sequence: True if len(sequence) >= minimum else fail ("length of {} is less than {}".format(len(sequence), minimum))`, + `lambda sequence: True if len(sequence) >= minimum else fail ("length = {}".format(len(sequence)))`, starlark.StringDict{"minimum": minimum}, ) } @@ -230,7 +230,7 @@ func (m AssertModule) MinLen(thread *starlark.Thread, f *starlark.Builtin, args func NewAssertMin(min starlark.Value) *Assertion { return NewAssertionFromSource( "assert.min", - `lambda val: True if yaml.decode(yaml.encode(val)) >= yaml.decode(yaml.encode(min)) else fail("value is less than {}".format(yaml.decode(yaml.encode(min))))`, + `lambda val: yaml.decode(yaml.encode(val)) >= yaml.decode(yaml.encode(min)) or fail("value < {}".format(yaml.decode(yaml.encode(min))))`, starlark.StringDict{"min": min, "yaml": YAMLAPI["yaml"]}, ) } @@ -251,7 +251,7 @@ func (m AssertModule) Min(thread *starlark.Thread, f *starlark.Builtin, args sta func NewAssertMax(max starlark.Value) *Assertion { return NewAssertionFromSource( "assert.max", - `lambda val: True if yaml.decode(yaml.encode(val)) <= yaml.decode(yaml.encode(max)) else fail("value is more than {}".format(yaml.decode(yaml.encode(max))))`, + `lambda val: yaml.decode(yaml.encode(val)) <= yaml.decode(yaml.encode(max)) or fail("value > {}".format(yaml.decode(yaml.encode(max))))`, starlark.StringDict{"max": max, "yaml": YAMLAPI["yaml"]}, ) } @@ -270,7 +270,7 @@ func (m AssertModule) Max(thread *starlark.Thread, f *starlark.Builtin, args sta func NewAssertNotNull() *Assertion { return NewAssertionFromSource( "assert.not_null", - `lambda value: fail("value is null") if value == None else True`, + `lambda value: value != None or fail("value is null")`, starlark.StringDict{}, ) } @@ -350,7 +350,7 @@ func (m AssertModule) oneNotNullCheck(keys starlark.Sequence) core.StarlarkFunc } } - var nulls, notNulls []string + var nulls, notNulls []starlark.Value for _, key := range keysToCheck { value, found, err := dict.Get(key) @@ -359,12 +359,12 @@ func (m AssertModule) oneNotNullCheck(keys starlark.Sequence) core.StarlarkFunc } if !found { // allow schema to catch this (see also https://github.com/vmware-tanzu/carvel-ytt/issues/722) - nulls = append(nulls, key.String()) + nulls = append(nulls, key) } if value == starlark.None { - nulls = append(nulls, key.String()) + nulls = append(nulls, key) } else { - notNulls = append(notNulls, key.String()) + notNulls = append(notNulls, key) } } @@ -377,7 +377,7 @@ func (m AssertModule) oneNotNullCheck(keys starlark.Sequence) core.StarlarkFunc case 1: return starlark.True, nil default: - return nil, fmt.Errorf("check: multiple values are not null %s", notNulls) + return nil, fmt.Errorf("check: %s are not null", starlark.NewList(notNulls).String()) } } } @@ -388,7 +388,7 @@ func (m AssertModule) oneNotNullCheck(keys starlark.Sequence) core.StarlarkFunc func NewAssertOneOf(enum starlark.Sequence) *Assertion { return NewAssertionFromSource( "assert.one_of", - `lambda val: True if yaml.decode(yaml.encode(val)) in yaml.decode(yaml.encode(enum)) else fail("value not in {}".format(yaml.decode(yaml.encode(enum))))`, + `lambda val: yaml.decode(yaml.encode(val)) in yaml.decode(yaml.encode(enum)) or fail("not one of allowed values")`, starlark.StringDict{"enum": enum, "yaml": YAMLAPI["yaml"]}, ) }