From 7f420f6b17ebe6cfa1f59082d53df94e076fa613 Mon Sep 17 00:00:00 2001 From: simitt Date: Sat, 22 Dec 2018 13:22:28 +0100 Subject: [PATCH 1/5] Allow multiple object types per field. --- libbeat/common/field.go | 47 +++++---- libbeat/template/processor.go | 55 +++++----- libbeat/template/processor_test.go | 158 +++++++++++++++++++---------- 3 files changed, 163 insertions(+), 97 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 3ab60355b7b..c3e40099e8d 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -34,25 +34,28 @@ import ( type Fields []Field type Field struct { - Name string `config:"name"` - Type string `config:"type"` - Description string `config:"description"` - Format string `config:"format"` - ScalingFactor int `config:"scaling_factor"` - Fields Fields `config:"fields"` - MultiFields Fields `config:"multi_fields"` - ObjectType string `config:"object_type"` - ObjectTypeMappingType string `config:"object_type_mapping_type"` - Enabled *bool `config:"enabled"` - Analyzer string `config:"analyzer"` - SearchAnalyzer string `config:"search_analyzer"` - Norms bool `config:"norms"` - Dynamic DynamicType `config:"dynamic"` - Index *bool `config:"index"` - DocValues *bool `config:"doc_values"` - CopyTo string `config:"copy_to"` - IgnoreAbove int `config:"ignore_above"` - AliasPath string `config:"path"` + Name string `config:"name"` + Type string `config:"type"` + Description string `config:"description"` + Format string `config:"format"` + Fields Fields `config:"fields"` + MultiFields Fields `config:"multi_fields"` + Enabled *bool `config:"enabled"` + Analyzer string `config:"analyzer"` + SearchAnalyzer string `config:"search_analyzer"` + Norms bool `config:"norms"` + Dynamic DynamicType `config:"dynamic"` + Index *bool `config:"index"` + DocValues *bool `config:"doc_values"` + CopyTo string `config:"copy_to"` + IgnoreAbove int `config:"ignore_above"` + AliasPath string `config:"path"` + + ScalingFactor int `config:"scaling_factor"` + ObjectType string `config:"object_type"` + ObjectTypeMappingType string `config:"object_type_mapping_type"` + + ObjectTypeParams []ObjectTypeCfg `config:"object_type_params"` // Kibana specific Analyzed *bool `config:"analyzed"` @@ -73,6 +76,12 @@ type Field struct { Path string } +type ObjectTypeCfg struct { + ObjectType string `config:"object_type"` + ObjectTypeMappingType string `config:"object_type_mapping_type"` + ScalingFactor int `config:"scaling_factor"` +} + type VersionizedString struct { MinVersion string `config:"min_version"` Value string `config:"value"` diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index e0fbc9e8025..a1b36965693 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -49,7 +49,7 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map case "ip": mapping = p.ip(&field) case "scaled_float": - mapping = p.scaledFloat(&field) + mapping = p.scaledFloat(&field, field.ScalingFactor) case "half_float": mapping = p.halfFloat(&field) case "integer": @@ -119,14 +119,13 @@ func (p *Processor) integer(f *common.Field) common.MapStr { return property } -func (p *Processor) scaledFloat(f *common.Field) common.MapStr { +func (p *Processor) scaledFloat(f *common.Field, scalingFactor int) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" if p.EsVersion.IsMajor(2) { property["type"] = "float" } else { - scalingFactor := f.ScalingFactor // Set default scaling factor if scalingFactor == 0 { scalingFactor = defaultScalingFactor @@ -256,33 +255,41 @@ func (p *Processor) alias(f *common.Field) common.MapStr { } func (p *Processor) object(f *common.Field) common.MapStr { - dynProperties := getDefaultProperties(f) - - matchType := func(onlyType string) string { - if f.ObjectTypeMappingType != "" { - return f.ObjectTypeMappingType + matchType := func(onlyType string, mt string) string { + if mt != "" { + return mt } return onlyType } - switch f.ObjectType { - case "scaled_float": - dynProperties = p.scaledFloat(f) - addDynamicTemplate(f, dynProperties, matchType("*")) - case "text": - dynProperties["type"] = "text" + otParams := f.ObjectTypeParams + if f.ObjectType != "" { + objectTypeParam := common.ObjectTypeCfg{f.ObjectType, f.ObjectTypeMappingType, f.ScalingFactor} + otParams = append(otParams, objectTypeParam) + } + + for _, otp := range otParams { + dynProperties := getDefaultProperties(f) - if p.EsVersion.IsMajor(2) { - dynProperties["type"] = "string" - dynProperties["index"] = "analyzed" + switch otp.ObjectType { + case "scaled_float": + dynProperties = p.scaledFloat(f, otp.ScalingFactor) + addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType)) + case "text": + dynProperties["type"] = "text" + + if p.EsVersion.IsMajor(2) { + dynProperties["type"] = "string" + dynProperties["index"] = "analyzed" + } + addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + case "keyword": + dynProperties["type"] = otp.ObjectType + addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType)) + case "byte", "double", "float", "long", "short", "boolean": + dynProperties["type"] = otp.ObjectType + addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType)) } - addDynamicTemplate(f, dynProperties, matchType("string")) - case "keyword": - dynProperties["type"] = f.ObjectType - addDynamicTemplate(f, dynProperties, matchType("string")) - case "byte", "double", "float", "long", "short": - dynProperties["type"] = f.ObjectType - addDynamicTemplate(f, dynProperties, matchType(f.ObjectType)) } properties := getDefaultProperties(f) diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 1064a9e91a2..6cb2317e9f2 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -18,6 +18,7 @@ package template import ( + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -43,21 +44,21 @@ func TestProcessor(t *testing.T) { expected: common.MapStr{"type": "long"}, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}), + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, 0), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}, 10), expected: common.MapStr{ "type": "scaled_float", - "scaling_factor": 100, + "scaling_factor": 10, }, }, { - output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), + output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}, 0), expected: common.MapStr{"type": "float"}, }, { @@ -264,22 +265,24 @@ func TestProcessor(t *testing.T) { } } -func TestDynamicTemplate(t *testing.T) { +func TestDynamicTemplates(t *testing.T) { p := &Processor{} tests := []struct { field common.Field - expected common.MapStr + expected []common.MapStr }{ { field: common.Field{ Type: "object", ObjectType: "keyword", Name: "context", }, - expected: common.MapStr{ - "context": common.MapStr{ - "mapping": common.MapStr{"type": "keyword"}, - "match_mapping_type": "string", - "path_match": "context.*", + expected: []common.MapStr{ + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "keyword"}, + "match_mapping_type": "string", + "path_match": "context.*", + }, }, }, }, @@ -288,11 +291,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", ObjectTypeMappingType: "futuretype", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "futuretype", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "futuretype", + "path_match": "language.english.*", + }, }, }, }, @@ -301,11 +306,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", ObjectTypeMappingType: "*", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "*", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "*", + "path_match": "language.english.*", + }, }, }, }, @@ -314,11 +321,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "long", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "long"}, - "match_mapping_type": "long", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "long"}, + "match_mapping_type": "long", + "path_match": "language.english.*", + }, }, }, }, @@ -327,11 +336,13 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "text", Path: "language", Name: "english", }, - expected: common.MapStr{ - "language.english": common.MapStr{ - "mapping": common.MapStr{"type": "text"}, - "match_mapping_type": "string", - "path_match": "language.english.*", + expected: []common.MapStr{ + common.MapStr{ + "language.english": common.MapStr{ + "mapping": common.MapStr{"type": "text"}, + "match_mapping_type": "string", + "path_match": "language.english.*", + }, }, }, }, @@ -340,14 +351,16 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", }, - expected: common.MapStr{ - "core.*.pct": common.MapStr{ - "mapping": common.MapStr{ - "type": "scaled_float", - "scaling_factor": defaultScalingFactor, + expected: []common.MapStr{ + common.MapStr{ + "core.*.pct": common.MapStr{ + "mapping": common.MapStr{ + "type": "scaled_float", + "scaling_factor": defaultScalingFactor, + }, + "match_mapping_type": "*", + "path_match": "core.*.pct", }, - "match_mapping_type": "*", - "path_match": "core.*.pct", }, }, }, @@ -356,35 +369,72 @@ func TestDynamicTemplate(t *testing.T) { Type: "object", ObjectType: "scaled_float", Name: "core.*.pct", ScalingFactor: 100, ObjectTypeMappingType: "float", }, - expected: common.MapStr{ - "core.*.pct": common.MapStr{ - "mapping": common.MapStr{ - "type": "scaled_float", - "scaling_factor": 100, + expected: []common.MapStr{ + common.MapStr{ + "core.*.pct": common.MapStr{ + "mapping": common.MapStr{ + "type": "scaled_float", + "scaling_factor": 100, + }, + "match_mapping_type": "float", + "path_match": "core.*.pct", + }, + }, + }, + }, + { + field: common.Field{ + Type: "object", ObjectTypeParams: []common.ObjectTypeCfg{ + {ObjectType: "float", ObjectTypeMappingType: "float"}, + {ObjectType: "boolean"}, + {ObjectType: "scaled_float", ScalingFactor: 10000}, + }, + Name: "context", + }, + expected: []common.MapStr{ + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "float"}, + "match_mapping_type": "float", + "path_match": "context.*", + }, + }, + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "boolean"}, + "match_mapping_type": "boolean", + "path_match": "context.*", + }, + }, + common.MapStr{ + "context": common.MapStr{ + "mapping": common.MapStr{"type": "scaled_float", "scaling_factor": 10000}, + "match_mapping_type": "*", + "path_match": "context.*", }, - "match_mapping_type": "float", - "path_match": "core.*.pct", }, }, }, } - for _, numericType := range []string{"byte", "double", "float", "long", "short"} { + for _, numericType := range []string{"byte", "double", "float", "long", "short", "boolean"} { gen := struct { field common.Field - expected common.MapStr + expected []common.MapStr }{ field: common.Field{ Type: "object", ObjectType: numericType, Name: "somefield", ObjectTypeMappingType: "long", }, - expected: common.MapStr{ - "somefield": common.MapStr{ - "mapping": common.MapStr{ - "type": numericType, + expected: []common.MapStr{ + common.MapStr{ + "somefield": common.MapStr{ + "mapping": common.MapStr{ + "type": numericType, + }, + "match_mapping_type": "long", + "path_match": "somefield.*", }, - "match_mapping_type": "long", - "path_match": "somefield.*", }, }, } @@ -394,7 +444,7 @@ func TestDynamicTemplate(t *testing.T) { for _, test := range tests { dynamicTemplates = nil p.object(&test.field) - assert.Equal(t, test.expected, dynamicTemplates[0]) + assert.Equal(t, test.expected, dynamicTemplates) } } From b9210de0ecb3f820b27ac09772cc845cf2f17344 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 2 Jan 2019 12:57:33 +0100 Subject: [PATCH 2/5] Adapt handling according to PR comments. --- libbeat/common/field.go | 26 +++++++++++++++---- libbeat/common/field_test.go | 40 ++++++++++++++++++++++++++++++ libbeat/template/processor.go | 30 ++++++++++++++++------ libbeat/template/processor_test.go | 28 ++++++++++++++++++--- 4 files changed, 107 insertions(+), 17 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index c3e40099e8d..ecac0aad38a 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/elastic/go-ucfg/yaml" + "github.com/pkg/errors" ) //This reflects allowed attributes for field definitions in the fields.yml. @@ -51,11 +52,10 @@ type Field struct { IgnoreAbove int `config:"ignore_above"` AliasPath string `config:"path"` - ScalingFactor int `config:"scaling_factor"` - ObjectType string `config:"object_type"` - ObjectTypeMappingType string `config:"object_type_mapping_type"` - - ObjectTypeParams []ObjectTypeCfg `config:"object_type_params"` + ObjectType string `config:"object_type"` + ObjectTypeMappingType string `config:"object_type_mapping_type"` + ScalingFactor int `config:"scaling_factor"` + ObjectTypeParams []ObjectTypeCfg `config:"object_type_params"` // Kibana specific Analyzed *bool `config:"analyzed"` @@ -76,6 +76,7 @@ type Field struct { Path string } +// Config for defining type of object attributes type ObjectTypeCfg struct { ObjectType string `config:"object_type"` ObjectTypeMappingType string `config:"object_type_mapping_type"` @@ -103,6 +104,21 @@ func (d *DynamicType) Unpack(s string) error { return nil } +func (f *Field) Unpack(c *Config) error { + type tmpField Field + var tf tmpField + if err := c.Unpack(&tf); err != nil { + return err + } + + if len(tf.ObjectTypeParams) != 0 && (tf.ScalingFactor != 0 || tf.ObjectTypeMappingType != "" || tf.ObjectType != "") { + return errors.New("Mixing top level objectType configuration with array of object type configurations is forbidden.") + } + + *f = Field(tf) + return nil +} + func LoadFieldsYaml(path string) (Fields, error) { keys := []Field{} diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go index ea1fe2729c0..6854e16ff0d 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/common/field_test.go @@ -20,6 +20,8 @@ package common import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/elastic/go-ucfg/yaml" @@ -189,3 +191,41 @@ func TestGetKeys(t *testing.T) { assert.Equal(t, test.keys, test.fields.GetKeys()) } } + +func TestFieldUnpack(t *testing.T) { + tests := []struct { + cfg MapStr + field Field + err bool + }{ + { + cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, + field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10}, + err: false, + }, { + cfg: MapStr{"object_type_params": []MapStr{ + {"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}}, + field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}}, + err: false, + }, { + cfg: MapStr{ + "object_type": "scaled_float", "object_type_mapping_type": "float", + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + }, + } + + for _, test := range tests { + cfg, err := NewConfigFrom(test.cfg) + require.NoError(t, err) + var f Field + err = cfg.Unpack(&f) + if test.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.field, f) + } + } + +} diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index a1b36965693..12d819c205f 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -34,6 +34,8 @@ var ( defaultIgnoreAbove = 1024 ) +const scalingFactorKey = "scalingFactor" + // Process recursively processes the given fields and writes the template in the given output func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error { for _, field := range fields { @@ -49,7 +51,7 @@ func (p *Processor) Process(fields common.Fields, path string, output common.Map case "ip": mapping = p.ip(&field) case "scaled_float": - mapping = p.scaledFloat(&field, field.ScalingFactor) + mapping = p.scaledFloat(&field) case "half_float": mapping = p.halfFloat(&field) case "integer": @@ -119,7 +121,7 @@ func (p *Processor) integer(f *common.Field) common.MapStr { return property } -func (p *Processor) scaledFloat(f *common.Field, scalingFactor int) common.MapStr { +func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr { property := getDefaultProperties(f) property["type"] = "scaled_float" @@ -127,9 +129,19 @@ func (p *Processor) scaledFloat(f *common.Field, scalingFactor int) common.MapSt property["type"] = "float" } else { // Set default scaling factor - if scalingFactor == 0 { + var scalingFactor int + if f.ScalingFactor != 0 { + scalingFactor = f.ScalingFactor + } else { scalingFactor = defaultScalingFactor } + + if len(params) > 0 { + if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 { + scalingFactor = s + } + } + property["scaling_factor"] = scalingFactor } return property @@ -262,10 +274,12 @@ func (p *Processor) object(f *common.Field) common.MapStr { return onlyType } - otParams := f.ObjectTypeParams - if f.ObjectType != "" { - objectTypeParam := common.ObjectTypeCfg{f.ObjectType, f.ObjectTypeMappingType, f.ScalingFactor} - otParams = append(otParams, objectTypeParam) + var otParams []common.ObjectTypeCfg + if len(f.ObjectTypeParams) != 0 { + otParams = f.ObjectTypeParams + } else { + otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{ + ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}} } for _, otp := range otParams { @@ -273,7 +287,7 @@ func (p *Processor) object(f *common.Field) common.MapStr { switch otp.ObjectType { case "scaled_float": - dynProperties = p.scaledFloat(f, otp.ScalingFactor) + dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor}) addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType)) case "text": dynProperties["type"] = "text" diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 6cb2317e9f2..c56c45dfead 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -18,7 +18,6 @@ package template import ( - "fmt" "testing" "github.com/stretchr/testify/assert" @@ -44,21 +43,42 @@ func TestProcessor(t *testing.T) { expected: common.MapStr{"type": "long"}, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float"}, 0), + output: p.scaledFloat(&common.Field{Type: "scaled_float"}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 1000, }, }, { - output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}, 10), + output: p.scaledFloat(&common.Field{Type: "scaled_float", ScalingFactor: 100}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 100, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 0}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{"someKey": 10}), + expected: common.MapStr{ + "type": "scaled_float", + "scaling_factor": 1000, + }, + }, + { + output: p.scaledFloat(&common.Field{Type: "scaled_float"}, common.MapStr{scalingFactorKey: 10}), expected: common.MapStr{ "type": "scaled_float", "scaling_factor": 10, }, }, { - output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}, 0), + output: pEsVersion2.scaledFloat(&common.Field{Type: "scaled_float"}), expected: common.MapStr{"type": "float"}, }, { From 389b07cae3bd79767af575be387cdeede592d456 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 2 Jan 2019 13:00:55 +0100 Subject: [PATCH 3/5] fix fmt --- libbeat/common/field.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index ecac0aad38a..0e94fe82ab6 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -76,7 +76,7 @@ type Field struct { Path string } -// Config for defining type of object attributes +// ObjectTypeCfg defines type and configuration of object attributes type ObjectTypeCfg struct { ObjectType string `config:"object_type"` ObjectTypeMappingType string `config:"object_type_mapping_type"` @@ -104,6 +104,8 @@ func (d *DynamicType) Unpack(s string) error { return nil } +// Unpack checks if objectTypeParams are mixed with top level objectType configuration +// and creates Field out of config func (f *Field) Unpack(c *Config) error { type tmpField Field var tf tmpField @@ -112,7 +114,7 @@ func (f *Field) Unpack(c *Config) error { } if len(tf.ObjectTypeParams) != 0 && (tf.ScalingFactor != 0 || tf.ObjectTypeMappingType != "" || tf.ObjectType != "") { - return errors.New("Mixing top level objectType configuration with array of object type configurations is forbidden.") + return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden") } *f = Field(tf) From 426be2defdc18b60534cbaaaadd075667ee3cdf1 Mon Sep 17 00:00:00 2001 From: simitt Date: Wed, 2 Jan 2019 14:09:38 +0100 Subject: [PATCH 4/5] Add changelog, make fmt --- CHANGELOG-developer.next.asciidoc | 2 ++ libbeat/common/field.go | 3 ++- libbeat/template/processor.go | 27 +++++++++++++-------------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 4cacdc3a60b..b425914e40f 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -23,3 +23,5 @@ The list below covers the major changes between 7.0.0-alpha2 and master only. ==== Bugfixes ==== Added + +- Allow multiple object type configurations per field. {pull}9772[9772] diff --git a/libbeat/common/field.go b/libbeat/common/field.go index 0e94fe82ab6..b058639b043 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -21,8 +21,9 @@ import ( "fmt" "strings" - "github.com/elastic/go-ucfg/yaml" "github.com/pkg/errors" + + "github.com/elastic/go-ucfg/yaml" ) //This reflects allowed attributes for field definitions in the fields.yml. diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 12d819c205f..aa933aa89d2 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -127,23 +127,22 @@ func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common if p.EsVersion.IsMajor(2) { property["type"] = "float" - } else { - // Set default scaling factor - var scalingFactor int - if f.ScalingFactor != 0 { - scalingFactor = f.ScalingFactor - } else { - scalingFactor = defaultScalingFactor - } + return property + } - if len(params) > 0 { - if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 { - scalingFactor = s - } - } + // Set scaling factor + scalingFactor := defaultScalingFactor + if f.ScalingFactor != 0 && len(f.ObjectTypeParams) == 0 { + scalingFactor = f.ScalingFactor + } - property["scaling_factor"] = scalingFactor + if len(params) > 0 { + if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 { + scalingFactor = s + } } + + property["scaling_factor"] = scalingFactor return property } From b460d80fc8ca6d32690b1b7d095a085ccef7610a Mon Sep 17 00:00:00 2001 From: simitt Date: Thu, 3 Jan 2019 09:43:42 +0100 Subject: [PATCH 5/5] Refactor with PR comments --- libbeat/common/field.go | 16 ++++--------- libbeat/common/field_test.go | 44 +++++++++++++++++++++++++----------- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/libbeat/common/field.go b/libbeat/common/field.go index b058639b043..95aff0aeac4 100644 --- a/libbeat/common/field.go +++ b/libbeat/common/field.go @@ -105,20 +105,14 @@ func (d *DynamicType) Unpack(s string) error { return nil } -// Unpack checks if objectTypeParams are mixed with top level objectType configuration -// and creates Field out of config -func (f *Field) Unpack(c *Config) error { - type tmpField Field - var tf tmpField - if err := c.Unpack(&tf); err != nil { - return err +// Validate ensures objectTypeParams are not mixed with top level objectType configuration +func (f *Field) Validate() error { + if len(f.ObjectTypeParams) == 0 { + return nil } - - if len(tf.ObjectTypeParams) != 0 && (tf.ScalingFactor != 0 || tf.ObjectTypeMappingType != "" || tf.ObjectType != "") { + if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" { return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden") } - - *f = Field(tf) return nil } diff --git a/libbeat/common/field_test.go b/libbeat/common/field_test.go index 6854e16ff0d..8f41ace61ef 100644 --- a/libbeat/common/field_test.go +++ b/libbeat/common/field_test.go @@ -192,40 +192,58 @@ func TestGetKeys(t *testing.T) { } } -func TestFieldUnpack(t *testing.T) { +func TestFieldValidate(t *testing.T) { tests := []struct { cfg MapStr field Field err bool + name string }{ { cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10}, field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10}, err: false, + name: "top level object type config", }, { cfg: MapStr{"object_type_params": []MapStr{ {"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}}, field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}}, err: false, + name: "multiple object type configs", }, { cfg: MapStr{ - "object_type": "scaled_float", "object_type_mapping_type": "float", + "object_type": "scaled_float", "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, - err: true, + err: true, + name: "invalid config mixing object_type and object_type_params", + }, { + cfg: MapStr{ + "object_type_mapping_type": "float", + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + name: "invalid config mixing object_type_mapping_type and object_type_params", + }, { + cfg: MapStr{ + "scaling_factor": 100, + "object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}}, + err: true, + name: "invalid config mixing scaling_factor and object_type_params", }, } for _, test := range tests { - cfg, err := NewConfigFrom(test.cfg) - require.NoError(t, err) - var f Field - err = cfg.Unpack(&f) - if test.err { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, test.field, f) - } + t.Run(test.name, func(t *testing.T) { + cfg, err := NewConfigFrom(test.cfg) + require.NoError(t, err) + var f Field + err = cfg.Unpack(&f) + if test.err { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, test.field, f) + } + }) } }