Skip to content

Commit

Permalink
Allow multiple object types per field. (elastic#9772)
Browse files Browse the repository at this point in the history
  • Loading branch information
@simitt
simitt committed Jan 11, 2019
1 parent d97d603 commit 9d1c830
Show file tree
Hide file tree
Showing 5 changed files with 269 additions and 97 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG-developer.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,5 @@ The list below covers the major changes between 6.6.0 and 6.x only.
==== Bugfixes

==== Added

- Allow multiple object type configurations per field. {pull}9772[9772]
60 changes: 41 additions & 19 deletions libbeat/common/field.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"fmt"
"strings"

"github.com/pkg/errors"

"github.com/elastic/go-ucfg/yaml"
)

Expand All @@ -34,25 +36,27 @@ import (
type Fields []Field

type Field struct {
Name string `config:"name"`
Type string `config:"type"`
Description string `config:"description"`
Format string `config:"format"`
ScalingFactor int `config:"scaling_factor"`
Fields Fields `config:"fields"`
MultiFields Fields `config:"multi_fields"`
ObjectType string `config:"object_type"`
ObjectTypeMappingType string `config:"object_type_mapping_type"`
Enabled *bool `config:"enabled"`
Analyzer string `config:"analyzer"`
SearchAnalyzer string `config:"search_analyzer"`
Norms bool `config:"norms"`
Dynamic DynamicType `config:"dynamic"`
Index *bool `config:"index"`
DocValues *bool `config:"doc_values"`
CopyTo string `config:"copy_to"`
IgnoreAbove int `config:"ignore_above"`
AliasPath string `config:"path"`
Name string `config:"name"`
Type string `config:"type"`
Description string `config:"description"`
Format string `config:"format"`
Fields Fields `config:"fields"`
MultiFields Fields `config:"multi_fields"`
Enabled *bool `config:"enabled"`
Analyzer string `config:"analyzer"`
SearchAnalyzer string `config:"search_analyzer"`
Norms bool `config:"norms"`
Dynamic DynamicType `config:"dynamic"`
Index *bool `config:"index"`
DocValues *bool `config:"doc_values"`
CopyTo string `config:"copy_to"`
IgnoreAbove int `config:"ignore_above"`
AliasPath string `config:"path"`

ObjectType string `config:"object_type"`
ObjectTypeMappingType string `config:"object_type_mapping_type"`
ScalingFactor int `config:"scaling_factor"`
ObjectTypeParams []ObjectTypeCfg `config:"object_type_params"`

// Kibana specific
Analyzed *bool `config:"analyzed"`
Expand All @@ -72,6 +76,13 @@ type Field struct {
Path string
}

// ObjectTypeCfg defines type and configuration of object attributes
type ObjectTypeCfg struct {
ObjectType string `config:"object_type"`
ObjectTypeMappingType string `config:"object_type_mapping_type"`
ScalingFactor int `config:"scaling_factor"`
}

type VersionizedString struct {
MinVersion string `config:"min_version"`
Value string `config:"value"`
Expand All @@ -93,6 +104,17 @@ func (d *DynamicType) Unpack(s string) error {
return nil
}

// Validate ensures objectTypeParams are not mixed with top level objectType configuration
func (f *Field) Validate() error {
if len(f.ObjectTypeParams) == 0 {
return nil
}
if f.ScalingFactor != 0 || f.ObjectTypeMappingType != "" || f.ObjectType != "" {
return errors.New("mixing top level objectType configuration with array of object type configurations is forbidden")
}
return nil
}

func LoadFieldsYaml(path string) (Fields, error) {
keys := []Field{}

Expand Down
58 changes: 58 additions & 0 deletions libbeat/common/field_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package common
import (
"testing"

"github.com/stretchr/testify/require"

"github.com/stretchr/testify/assert"

"github.com/elastic/go-ucfg/yaml"
Expand Down Expand Up @@ -189,3 +191,59 @@ func TestGetKeys(t *testing.T) {
assert.Equal(t, test.keys, test.fields.GetKeys())
}
}

func TestFieldValidate(t *testing.T) {
tests := []struct {
cfg MapStr
field Field
err bool
name string
}{
{
cfg: MapStr{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 10},
field: Field{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 10},
err: false,
name: "top level object type config",
}, {
cfg: MapStr{"object_type_params": []MapStr{
{"object_type": "scaled_float", "object_type_mapping_type": "float", "scaling_factor": 100}}},
field: Field{ObjectTypeParams: []ObjectTypeCfg{{ObjectType: "scaled_float", ObjectTypeMappingType: "float", ScalingFactor: 100}}},
err: false,
name: "multiple object type configs",
}, {
cfg: MapStr{
"object_type": "scaled_float",
"object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
err: true,
name: "invalid config mixing object_type and object_type_params",
}, {
cfg: MapStr{
"object_type_mapping_type": "float",
"object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
err: true,
name: "invalid config mixing object_type_mapping_type and object_type_params",
}, {
cfg: MapStr{
"scaling_factor": 100,
"object_type_params": []MapStr{{"object_type": "scaled_float", "object_type_mapping_type": "float"}}},
err: true,
name: "invalid config mixing scaling_factor and object_type_params",
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg, err := NewConfigFrom(test.cfg)
require.NoError(t, err)
var f Field
err = cfg.Unpack(&f)
if test.err {
assert.Error(t, err)
} else {
assert.NoError(t, err)
assert.Equal(t, test.field, f)
}
})
}

}
76 changes: 48 additions & 28 deletions libbeat/template/processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ var (
defaultIgnoreAbove = 1024
)

const scalingFactorKey = "scalingFactor"

// Process recursively processes the given fields and writes the template in the given output
func (p *Processor) Process(fields common.Fields, path string, output common.MapStr) error {
for _, field := range fields {
Expand Down Expand Up @@ -119,20 +121,28 @@ func (p *Processor) integer(f *common.Field) common.MapStr {
return property
}

func (p *Processor) scaledFloat(f *common.Field) common.MapStr {
func (p *Processor) scaledFloat(f *common.Field, params ...common.MapStr) common.MapStr {
property := getDefaultProperties(f)
property["type"] = "scaled_float"

if p.EsVersion.IsMajor(2) {
property["type"] = "float"
} else {
scalingFactor := f.ScalingFactor
// Set default scaling factor
if scalingFactor == 0 {
scalingFactor = defaultScalingFactor
return property
}

// Set scaling factor
scalingFactor := defaultScalingFactor
if f.ScalingFactor != 0 && len(f.ObjectTypeParams) == 0 {
scalingFactor = f.ScalingFactor
}

if len(params) > 0 {
if s, ok := params[0][scalingFactorKey].(int); ok && s != 0 {
scalingFactor = s
}
property["scaling_factor"] = scalingFactor
}

property["scaling_factor"] = scalingFactor
return property
}

Expand Down Expand Up @@ -261,33 +271,43 @@ func (p *Processor) alias(f *common.Field) common.MapStr {
}

func (p *Processor) object(f *common.Field) common.MapStr {
dynProperties := getDefaultProperties(f)

matchType := func(onlyType string) string {
if f.ObjectTypeMappingType != "" {
return f.ObjectTypeMappingType
matchType := func(onlyType string, mt string) string {
if mt != "" {
return mt
}
return onlyType
}

switch f.ObjectType {
case "scaled_float":
dynProperties = p.scaledFloat(f)
addDynamicTemplate(f, dynProperties, matchType("*"))
case "text":
dynProperties["type"] = "text"
var otParams []common.ObjectTypeCfg
if len(f.ObjectTypeParams) != 0 {
otParams = f.ObjectTypeParams
} else {
otParams = []common.ObjectTypeCfg{common.ObjectTypeCfg{
ObjectType: f.ObjectType, ObjectTypeMappingType: f.ObjectTypeMappingType, ScalingFactor: f.ScalingFactor}}
}

if p.EsVersion.IsMajor(2) {
dynProperties["type"] = "string"
dynProperties["index"] = "analyzed"
for _, otp := range otParams {
dynProperties := getDefaultProperties(f)

switch otp.ObjectType {
case "scaled_float":
dynProperties = p.scaledFloat(f, common.MapStr{scalingFactorKey: otp.ScalingFactor})
addDynamicTemplate(f, dynProperties, matchType("*", otp.ObjectTypeMappingType))
case "text":
dynProperties["type"] = "text"

if p.EsVersion.IsMajor(2) {
dynProperties["type"] = "string"
dynProperties["index"] = "analyzed"
}
addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
case "keyword":
dynProperties["type"] = otp.ObjectType
addDynamicTemplate(f, dynProperties, matchType("string", otp.ObjectTypeMappingType))
case "byte", "double", "float", "long", "short", "boolean":
dynProperties["type"] = otp.ObjectType
addDynamicTemplate(f, dynProperties, matchType(otp.ObjectType, otp.ObjectTypeMappingType))
}
addDynamicTemplate(f, dynProperties, matchType("string"))
case "keyword":
dynProperties["type"] = f.ObjectType
addDynamicTemplate(f, dynProperties, matchType("string"))
case "byte", "double", "float", "long", "short":
dynProperties["type"] = f.ObjectType
addDynamicTemplate(f, dynProperties, matchType(f.ObjectType))
}

properties := getDefaultProperties(f)
Expand Down
Loading

0 comments on commit 9d1c830

Please sign in to comment.