From 8aa042675dd6a7951644be0acb134430ed0dd58b Mon Sep 17 00:00:00 2001 From: Filip Filmar Date: Fri, 6 Apr 2018 17:13:49 -0700 Subject: [PATCH] Adds a hook to specify unmarshal options. This makes it possible to configure the parser to error out if an unknown tagged field has been found in the byte stream that is being unmarshaled to a go object. NOTE: This change requires go 1.10. It seems that we should all upgrade, see: https://pocketgophers.com/when-should-you-upgrade-go/ For example like this (also see ExampleUnknown in yaml_test.go): ``` func ExampleUnknown() { type WithTaggedField struct { Field string `json:"field"` } y := []byte(`unknown: "hello"`) v := WithTaggedField{} fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields)) // Ouptut: // unmarshaling JSON: while decoding JSON: json: unknown field "unknown" } ``` If you want to manipulate the decoder used in the unmarshal function, you can define a custom option to your liking and apply it in the Unmarshal call. The way I found out about this is kind of comical. Some nonprintable characters made their way into my YAML file. You wouldn't see them in the editor, but they'd be there. Now when the YAML file is parsed, the nonprintable chars together with the key characters were made into a weird UTF-8 key (e.g. like "\u00c2some_key"), which would then become a key unknown to the tags of my struct, and would silently get dropped in the parsing steps. So as result, you would get a perfectly normal parse, except the data that you wanted to be in the struct are not there. This meant that silently dropping unknown keys is not always a good idea, so I went to fix it, while retaining the same interface. JSONOpt type was fairly easy to add in the Unmarshal because it can be silently ignored by the old code. --- .travis.yml | 5 +++-- yaml.go | 26 +++++++++++++++++++++++--- yaml_go110.go | 14 ++++++++++++++ yaml_go110_test.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ yaml_test.go | 4 ++-- 5 files changed, 88 insertions(+), 7 deletions(-) create mode 100644 yaml_go110.go create mode 100644 yaml_go110_test.go diff --git a/.travis.yml b/.travis.yml index 0e9d6ed..930860e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: go go: - - 1.3 - - 1.4 + - "1.3" + - "1.4" + - "1.10" script: - go test - go build diff --git a/yaml.go b/yaml.go index 4fb4054..c48aec3 100644 --- a/yaml.go +++ b/yaml.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "reflect" "strconv" @@ -26,15 +27,19 @@ func Marshal(o interface{}) ([]byte, error) { return y, nil } -// Converts YAML to JSON then uses JSON to unmarshal into an object. -func Unmarshal(y []byte, o interface{}) error { +// JSONOpt is a decoding option for decoding from JSON format. +type JSONOpt func(*json.Decoder) *json.Decoder + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { vo := reflect.ValueOf(o) j, err := yamlToJSON(y, &vo) if err != nil { return fmt.Errorf("error converting YAML to JSON: %v", err) } - err = json.Unmarshal(j, o) + err = jsonUnmarshal(bytes.NewReader(j), o, opts...) if err != nil { return fmt.Errorf("error unmarshaling JSON: %v", err) } @@ -42,6 +47,21 @@ func Unmarshal(y []byte, o interface{}) error { return nil } +// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the +// object, optionally applying decoder options prior to decoding. We are not +// using json.Unmarshal directly as we want the chance to pass in non-default +// options. +func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error { + d := json.NewDecoder(r) + for _, opt := range opts { + d = opt(d) + } + if err := d.Decode(&o); err != nil { + return fmt.Errorf("while decoding JSON: %v", err) + } + return nil +} + // Convert JSON to YAML. func JSONToYAML(j []byte) ([]byte, error) { // Convert the JSON to an object. diff --git a/yaml_go110.go b/yaml_go110.go new file mode 100644 index 0000000..ab3e06a --- /dev/null +++ b/yaml_go110.go @@ -0,0 +1,14 @@ +// This file contains changes that are only compatible with go 1.10 and onwards. + +// +build go1.10 + +package yaml + +import "encoding/json" + +// DisallowUnknownFields configures the JSON decoder to error out if unknown +// fields come along, instead of dropping them by default. +func DisallowUnknownFields(d *json.Decoder) *json.Decoder { + d.DisallowUnknownFields() + return d +} diff --git a/yaml_go110_test.go b/yaml_go110_test.go new file mode 100644 index 0000000..b7767b7 --- /dev/null +++ b/yaml_go110_test.go @@ -0,0 +1,46 @@ +// +build go1.10 + +package yaml + +import ( + "fmt" + "testing" +) + +func TestUnmarshalWithTags(t *testing.T) { + type WithTaggedField struct { + Field string `json:"field"` + } + + t.Run("Known tagged field", func(t *testing.T) { + y := []byte(`field: "hello"`) + v := WithTaggedField{} + if err := Unmarshal(y, &v, DisallowUnknownFields); err != nil { + t.Errorf("unexpected error: %v", err) + } + if v.Field != "hello" { + t.Errorf("v.Field=%v, want 'hello'", v.Field) + } + + }) + t.Run("With unknown tagged field", func(t *testing.T) { + y := []byte(`unknown: "hello"`) + v := WithTaggedField{} + err := Unmarshal(y, &v, DisallowUnknownFields) + if err == nil { + t.Errorf("want error because of unknown field, got : v=%#v", v) + } + }) + +} + +func ExampleUnknown() { + type WithTaggedField struct { + Field string `json:"field"` + } + y := []byte(`unknown: "hello"`) + v := WithTaggedField{} + fmt.Printf("%v\n", Unmarshal(y, &v, DisallowUnknownFields)) + // Ouptut: + // unmarshaling JSON: while decoding JSON: json: unknown field "unknown" +} diff --git a/yaml_test.go b/yaml_test.go index 505af45..48f0c61 100644 --- a/yaml_test.go +++ b/yaml_test.go @@ -106,8 +106,8 @@ b: unmarshal(t, y, &s5, &e5) } -func unmarshal(t *testing.T, y []byte, s, e interface{}) { - err := Unmarshal(y, s) +func unmarshal(t *testing.T, y []byte, s, e interface{}, opts ...JSONOpt) { + err := Unmarshal(y, s, opts...) if err != nil { t.Errorf("error unmarshaling YAML: %v", err) }