diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index ad738a47c3a..3b9479b7379 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -52,6 +52,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154] - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093] - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193] +- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668] ==== Bugfixes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9ed505c73f5..eebe2314dfb 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix the ability for subcommands to be ran properly from the beats containers. {pull}30452[30452] - Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534] +- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668] *Auditbeat* diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 8be4b89e0b8..8dc9795e968 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -317,11 +317,11 @@ func (p *Processor) wildcard(f *mapping.Field) common.MapStr { property["type"] = "wildcard" - switch f.IgnoreAbove { - case 0: // Use libbeat default - property["ignore_above"] = defaultIgnoreAbove - case -1: // Use ES default - default: // Use user value + /* For wildcard fields, unlike keywords, don't force a default ignore_above limit. + The default in ES will be used unless an explicit limit is set. + This is to take advantage of wildcard type benefits when indexing large strings. + */ + if f.IgnoreAbove > 0 { property["ignore_above"] = f.IgnoreAbove } diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index dd177c5aea8..95bd6b0faeb 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -812,45 +812,76 @@ func TestProcessWildcardOSS(t *testing.T) { } func TestProcessWildcardElastic(t *testing.T) { - // Test common fields are combined even if they come from different objects - fields := mapping.Fields{ - mapping.Field{ - Name: "test", - Type: "group", - Fields: mapping.Fields{ + for _, test := range []struct { + title string + fields mapping.Fields + expected common.MapStr + }{ + { + title: "default", + fields: mapping.Fields{ mapping.Field{ - Name: "one", - Type: "wildcard", + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + }, + }, + }, + }, + expected: common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "type": "wildcard", + }, + }, }, }, }, - } - - output := common.MapStr{} - version, err := common.NewVersion("8.0.0") - if err != nil { - t.Fatal(err) - } - - p := Processor{EsVersion: *version, ElasticLicensed: true} - err = p.Process(fields, nil, output) - if err != nil { - t.Fatal(err) - } - - // Make sure fields without a name are skipped during template generation - expectedOutput := common.MapStr{ - "test": common.MapStr{ - "properties": common.MapStr{ - "one": common.MapStr{ - "ignore_above": 1024, - "type": "wildcard", + { + title: "explicit ignore_above", + fields: mapping.Fields{ + mapping.Field{ + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + IgnoreAbove: 4096, + }, + }, + }, + }, + expected: common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "ignore_above": 4096, + "type": "wildcard", + }, + }, }, }, }, + } { + t.Run(test.title, func(t *testing.T) { + output := common.MapStr{} + version, err := common.NewVersion("8.0.0") + if err != nil { + t.Fatal(err) + } + p := Processor{EsVersion: *version, ElasticLicensed: true} + err = p.Process(test.fields, nil, output) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, test.expected, output) + }) } - - assert.Equal(t, expectedOutput, output) } func TestProcessWildcardPreSupport(t *testing.T) {