-
Notifications
You must be signed in to change notification settings - Fork 72
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support expanded and flat objects (#56)
* Load item content every time * Require document dashes * Validate expanded and flat objects in YAML files * Fix: kibana.version
- Loading branch information
Showing
20 changed files
with
1,435 additions
and
91 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package validator | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "encoding/json" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "mime" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "gopkg.in/yaml.v3" | ||
| ) | ||
|
|
||
| func loadItemContent(itemPath, mediaType string) ([]byte, error) { | ||
| itemData, err := ioutil.ReadFile(itemPath) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "reading item file failed") | ||
| } | ||
|
|
||
| if len(itemData) == 0 { | ||
| return nil, errors.New("file is empty") | ||
| } | ||
|
|
||
| if mediaType == "" { | ||
| return itemData, nil // no item's schema defined | ||
| } | ||
|
|
||
| basicMediaType, params, err := mime.ParseMediaType(mediaType) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "invalid media type (%s)", mediaType) | ||
| } | ||
|
|
||
| switch basicMediaType { | ||
| case "application/x-yaml": | ||
| // TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54) | ||
| if v, _ := params["require-document-dashes"]; v == "true" && !bytes.HasPrefix(itemData, []byte("---\n")) { | ||
| return nil, errors.New("document dashes are required (start the document with '---')") | ||
| } | ||
|
|
||
| var c interface{} | ||
| err = yaml.Unmarshal(itemData, &c) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "unmarshalling YAML file failed (path: %s)", itemPath) | ||
| } | ||
| c = expandItemKey(c) | ||
|
|
||
| itemData, err = json.Marshal(&c) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "converting YAML file to JSON failed (path: %s)", itemPath) | ||
| } | ||
| case "application/json": // no need to convert the item content | ||
| default: | ||
| return nil, fmt.Errorf("unsupported media type (%s)", mediaType) | ||
| } | ||
| return itemData, nil | ||
| } | ||
|
|
||
| func expandItemKey(c interface{}) interface{} { | ||
| if c == nil { | ||
| return c | ||
| } | ||
|
|
||
| // c is an array | ||
| if cArr, isArray := c.([]interface{}); isArray { | ||
| var arr []interface{} | ||
| for _, ca := range cArr { | ||
| arr = append(arr, expandItemKey(ca)) | ||
| } | ||
| return arr | ||
| } | ||
|
|
||
| // c is map[string]interface{} | ||
| if cMap, isMapString := c.(map[string]interface{}); isMapString { | ||
| expanded := MapStr{} | ||
| for k, v := range cMap { | ||
| ex := expandItemKey(v) | ||
| _, err := expanded.Put(k, ex) | ||
| if err != nil { | ||
| panic(errors.Wrapf(err, "unexpected error while setting key value (key: %s)", k)) | ||
| } | ||
| } | ||
| return expanded | ||
| } | ||
| return c // c is something else, e.g. string, int, etc. | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package validator | ||
|
|
||
| // WARNING: This code is copied from https://github.com/elastic/beats/blob/master/libbeat/common/mapstr.go | ||
| // This was done to not have to import the full common package and all its dependencies | ||
| // Not needed methods / variables were removed, but no changes made to the logic. | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| var ( | ||
| // errKeyNotFound indicates that the specified key was not found. | ||
| errKeyNotFound = errors.New("key not found") | ||
| ) | ||
|
|
||
| // MapStr is a map[string]interface{} wrapper with utility methods for common | ||
| // map operations like converting to JSON. | ||
| type MapStr map[string]interface{} | ||
|
|
||
| // GetValue gets a value from the map. If the key does not exist then an error | ||
| // is returned. | ||
| func (m MapStr) GetValue(key string) (interface{}, error) { | ||
| _, _, v, found, err := mapFind(key, m, false) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if !found { | ||
| return nil, errKeyNotFound | ||
| } | ||
| return v, nil | ||
| } | ||
|
|
||
| // Put associates the specified value with the specified key. If the map | ||
| // previously contained a mapping for the key, the old value is replaced and | ||
| // returned. The key can be expressed in dot-notation (e.g. x.y) to put a value | ||
| // into a nested map. | ||
| // | ||
| // If you need insert keys containing dots then you must use bracket notation | ||
| // to insert values (e.g. m[key] = value). | ||
| func (m MapStr) Put(key string, value interface{}) (interface{}, error) { | ||
| // XXX `safemapstr.Put` mimics this implementation, both should be updated to have similar behavior | ||
| k, d, old, _, err := mapFind(key, m, true) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| d[k] = value | ||
| return old, nil | ||
| } | ||
|
|
||
| // StringToPrint returns the MapStr as pretty JSON. | ||
| func (m MapStr) StringToPrint() string { | ||
| j, err := json.MarshalIndent(m, "", " ") | ||
| if err != nil { | ||
| return fmt.Sprintf("Not valid json: %v", err) | ||
| } | ||
| return string(j) | ||
| } | ||
|
|
||
| // tomapStr performs a type assertion on v and returns a MapStr. v can be either | ||
| // a MapStr or a map[string]interface{}. If it's any other type or nil then | ||
| // an error is returned. | ||
| func toMapStr(v interface{}) (MapStr, error) { | ||
| m, ok := tryToMapStr(v) | ||
| if !ok { | ||
| return nil, errors.Errorf("expected map but type is %T", v) | ||
| } | ||
| return m, nil | ||
| } | ||
|
|
||
| func tryToMapStr(v interface{}) (MapStr, bool) { | ||
| switch m := v.(type) { | ||
| case MapStr: | ||
| return m, true | ||
| case map[string]interface{}: | ||
| return MapStr(m), true | ||
| default: | ||
| return nil, false | ||
| } | ||
| } | ||
|
|
||
| // mapFind iterates a MapStr based on a the given dotted key, finding the final | ||
| // subMap and subKey to operate on. | ||
| // An error is returned if some intermediate is no map or the key doesn't exist. | ||
| // If createMissing is set to true, intermediate maps are created. | ||
| // The final map and un-dotted key to run further operations on are returned in | ||
| // subKey and subMap. The subMap already contains a value for subKey, the | ||
| // present flag is set to true and the oldValue return will hold | ||
| // the original value. | ||
| func mapFind( | ||
| key string, | ||
| data MapStr, | ||
| createMissing bool, | ||
| ) (subKey string, subMap MapStr, oldValue interface{}, present bool, err error) { | ||
| // XXX `safemapstr.mapFind` mimics this implementation, both should be updated to have similar behavior | ||
|
|
||
| for { | ||
| // Fast path, key is present as is. | ||
| if v, exists := data[key]; exists { | ||
| return key, data, v, true, nil | ||
| } | ||
|
|
||
| idx := strings.IndexRune(key, '.') | ||
| if idx < 0 { | ||
| return key, data, nil, false, nil | ||
| } | ||
|
|
||
| k := key[:idx] | ||
| d, exists := data[k] | ||
| if !exists { | ||
| if createMissing { | ||
| d = MapStr{} | ||
| data[k] = d | ||
| } else { | ||
| return "", nil, nil, false, errKeyNotFound | ||
| } | ||
| } | ||
|
|
||
| v, err := toMapStr(d) | ||
| if err != nil { | ||
| return "", nil, nil, false, err | ||
| } | ||
|
|
||
| // advance to sub-map | ||
| key = key[idx+1:] | ||
| data = v | ||
| } | ||
| } |
12 changes: 12 additions & 0 deletions
12
...alidator/test/packages/expanded/data_stream/foo/_dev/test/pipeline/test-access-event.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "events": [ | ||
| { | ||
| "@timestamp": "2016-10-25T12:49:34.000Z", | ||
| "message": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"\n" | ||
| }, | ||
| { | ||
| "@timestamp": "2016-10-25T12:49:34.000Z", | ||
| "message": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\n" | ||
| } | ||
| ] | ||
| } |
Oops, something went wrong.