Support expanded and flat objects

code/go/go.mod

@@ -10,7 +10,5 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415
 	github.com/xeipuuv/gojsonschema v1.2.0
-	golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
-	golang.org/x/tools v0.0.0-20201005185003-576e169c3de7 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
 )

code/go/go.sum

@@ -20,37 +20,6 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
-golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY=
-golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2 h1:daAzF/Ytp6YSqJDu1hZJthJIhOrsAa7UbIkziU1t0K4=
-golang.org/x/tools v0.0.0-20200909210914-44a2922940c2/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE=
-golang.org/x/tools v0.0.0-20200925191224-5d1fdd8fa346 h1:hzJjkvxUIF3bSt+v8N5tBQNx/605vszZJ+3XsIamzZo=
-golang.org/x/tools v0.0.0-20200925191224-5d1fdd8fa346/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
-golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0zYfkB7UCa+ayT6uVvbyI=
-golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
-golang.org/x/tools v0.0.0-20201005185003-576e169c3de7 h1:YTAUHYgZh/ZOA35/OrjTDmFFKb6ddkBL1Zgtl9r8Di8=
-golang.org/x/tools v0.0.0-20201005185003-576e169c3de7/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

code/go/internal/validator/folder_item_content.go

@@ -0,0 +1,85 @@
+package validator
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"mime"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v3"
+)
+
+func loadItemContent(itemPath, mediaType string) ([]byte, error) {
+	itemData, err := ioutil.ReadFile(itemPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "reading item file failed")
+	}
+
+	if len(itemData) == 0 {
+		return nil, errors.New("file is empty")
+	}
+
+	if mediaType == "" {
+		return itemData, nil // no item's schema defined
+	}
+
+	basicMediaType, params, err := mime.ParseMediaType(mediaType)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid media type (%s)", mediaType)
+	}
+
+	switch basicMediaType {
+	case "application/x-yaml":
+		// TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
+		if v, _ := params["require-document-dashes"]; v == "true" && !bytes.HasPrefix(itemData, []byte("---\n")) {
+			return nil, errors.New("document dashes are required (start the document with '---')")
+		}
+
+		var c interface{}
+		err = yaml.Unmarshal(itemData, &c)
+		if err != nil {
+			return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", itemPath)
+		}
+		c = expandItemKey(c)
+
+		itemData, err = json.Marshal(&c)
+		if err != nil {
+			return nil, errors.Wrapf(err, "converting YAML file to JSON failed (path: %s)", itemPath)
+		}
+	case "application/json": // no need to convert the item content
+	default:
+		return nil, fmt.Errorf("unsupported media type (%s)", mediaType)
+	}
+	return itemData, nil
+}
+
+func expandItemKey(c interface{}) interface{} {
+	if c == nil {
+		return c
+	}
+
+	// c is an array
+	if cArr, isArray := c.([]interface{}); isArray {
+		var arr []interface{}
+		for _, ca := range cArr {
+			arr = append(arr, expandItemKey(ca))
+		}
+		return arr
+	}
+
+	// c is map[string]interface{}
+	if cMap, isMapString := c.(map[string]interface{}); isMapString {
+		expanded := MapStr{}
+		for k, v := range cMap {
+			ex := expandItemKey(v)
+			_, err := expanded.Put(k, ex)
+			if err != nil {
+				panic(errors.Wrapf(err, "unexpected error while setting key value (key: %s)", k))
+			}
+		}
+		return expanded
+	}
+	return c // c is something else, e.g. string, int, etc.
+}

code/go/internal/validator/folder_item_spec.go

@@ -1,20 +1,16 @@
 package validator
 
 import (
-	"bytes"
-	"encoding/json"
 	"fmt"
-	"github.com/elastic/package-spec/code/go/internal/yamlschema"
-	"github.com/xeipuuv/gojsonschema"
-	"io/ioutil"
-	"mime"
 	"net/http"
 	"os"
 	"path/filepath"
 	"regexp"
 
 	"github.com/pkg/errors"
-	"gopkg.in/yaml.v3"
+	"github.com/xeipuuv/gojsonschema"
+
+	"github.com/elastic/package-spec/code/go/internal/yamlschema"
 )
 
 type folderItemSpec struct {
@@ -96,46 +92,3 @@ func (s *folderItemSpec) validate(fs http.FileSystem, folderSpecPath string, ite
 	}
 	return errs
 }
-
-func loadItemContent(itemPath, mediaType string) ([]byte, error) {
-	itemData, err := ioutil.ReadFile(itemPath)
-	if err != nil {
-		return nil, errors.Wrap(err, "reading item file failed")
-	}
-
-	if len(itemData) == 0 {
-		return nil, errors.New("file is empty")
-	}
-
-	if mediaType == "" {
-		return itemData, nil // no item's schema defined
-	}
-
-	basicMediaType, params, err := mime.ParseMediaType(mediaType)
-	if err != nil {
-		return nil, errors.Wrapf(err, "invalid media type (%s)", mediaType)
-	}
-
-	switch basicMediaType {
-	case "application/x-yaml":
-		// TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
-		if v, _ := params["require-document-dashes"]; v == "true" && !bytes.HasPrefix(itemData, []byte("---\n")) {
-			return nil, errors.New("document dashes are required (start the document with '---')")
-		}
-
-		var c interface{}
-		err = yaml.Unmarshal(itemData, &c)
-		if err != nil {
-			return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", itemPath)
-		}
-
-		itemData, err = json.Marshal(&c)
-		if err != nil {
-			return nil, errors.Wrapf(err, "converting YAML file to JSON failed (path: %s)", itemPath)
-		}
-	case "application/json": // no need to convert the item content
-	default:
-		return nil, fmt.Errorf("unsupported media type (%s)", mediaType)
-	}
-	return itemData, nil
-}

code/go/internal/validator/mapstr.go

@@ -0,0 +1,136 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package validator
+
+// WARNING: This code is copied from https://github.com/elastic/beats/blob/master/libbeat/common/mapstr.go
+// This was done to not have to import the full common package and all its dependencies
+// Not needed methods / variables were removed, but no changes made to the logic.
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// errKeyNotFound indicates that the specified key was not found.
+	errKeyNotFound = errors.New("key not found")
+)
+
+// MapStr is a map[string]interface{} wrapper with utility methods for common
+// map operations like converting to JSON.
+type MapStr map[string]interface{}
+
+// GetValue gets a value from the map. If the key does not exist then an error
+// is returned.
+func (m MapStr) GetValue(key string) (interface{}, error) {
+	_, _, v, found, err := mapFind(key, m, false)
+	if err != nil {
+		return nil, err
+	}
+	if !found {
+		return nil, errKeyNotFound
+	}
+	return v, nil
+}
+
+// Put associates the specified value with the specified key. If the map
+// previously contained a mapping for the key, the old value is replaced and
+// returned. The key can be expressed in dot-notation (e.g. x.y) to put a value
+// into a nested map.
+//
+// If you need insert keys containing dots then you must use bracket notation
+// to insert values (e.g. m[key] = value).
+func (m MapStr) Put(key string, value interface{}) (interface{}, error) {
+	// XXX `safemapstr.Put` mimics this implementation, both should be updated to have similar behavior
+	k, d, old, _, err := mapFind(key, m, true)
+	if err != nil {
+		return nil, err
+	}
+
+	d[k] = value
+	return old, nil
+}
+
+// StringToPrint returns the MapStr as pretty JSON.
+func (m MapStr) StringToPrint() string {
+	j, err := json.MarshalIndent(m, "", "  ")
+	if err != nil {
+		return fmt.Sprintf("Not valid json: %v", err)
+	}
+	return string(j)
+}
+
+// tomapStr performs a type assertion on v and returns a MapStr. v can be either
+// a MapStr or a map[string]interface{}. If it's any other type or nil then
+// an error is returned.
+func toMapStr(v interface{}) (MapStr, error) {
+	m, ok := tryToMapStr(v)
+	if !ok {
+		return nil, errors.Errorf("expected map but type is %T", v)
+	}
+	return m, nil
+}
+
+func tryToMapStr(v interface{}) (MapStr, bool) {
+	switch m := v.(type) {
+	case MapStr:
+		return m, true
+	case map[string]interface{}:
+		return MapStr(m), true
+	default:
+		return nil, false
+	}
+}
+
+// mapFind iterates a MapStr based on a the given dotted key, finding the final
+// subMap and subKey to operate on.
+// An error is returned if some intermediate is no map or the key doesn't exist.
+// If createMissing is set to true, intermediate maps are created.
+// The final map and un-dotted key to run further operations on are returned in
+// subKey and subMap. The subMap already contains a value for subKey, the
+// present flag is set to true and the oldValue return will hold
+// the original value.
+func mapFind(
+	key string,
+	data MapStr,
+	createMissing bool,
+) (subKey string, subMap MapStr, oldValue interface{}, present bool, err error) {
+	// XXX `safemapstr.mapFind` mimics this implementation, both should be updated to have similar behavior
+
+	for {
+		// Fast path, key is present as is.
+		if v, exists := data[key]; exists {
+			return key, data, v, true, nil
+		}
+
+		idx := strings.IndexRune(key, '.')
+		if idx < 0 {
+			return key, data, nil, false, nil
+		}
+
+		k := key[:idx]
+		d, exists := data[k]
+		if !exists {
+			if createMissing {
+				d = MapStr{}
+				data[k] = d
+			} else {
+				return "", nil, nil, false, errKeyNotFound
+			}
+		}
+
+		v, err := toMapStr(d)
+		if err != nil {
+			return "", nil, nil, false, err
+		}
+
+		// advance to sub-map
+		key = key[idx+1:]
+		data = v
+	}
+}

code/go/internal/validator/test/packages/expanded/data_stream/foo/_dev/test/pipeline/test-access-event.json

@@ -0,0 +1,12 @@
+{
+    "events": [
+        {
+            "@timestamp": "2016-10-25T12:49:34.000Z",
+            "message": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"\n"
+        },
+        {
+            "@timestamp": "2016-10-25T12:49:34.000Z",
+            "message": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\n"
+        }
+    ]
+}