diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index 770019d0c8c..476f686b1d0 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -59,6 +59,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only. - Removed Beat generators. {pull}28816[28816] - libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573] - Removed deprecated disk spool from Beats. Use disk queue instead. {pull}28869[28869] +- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668] ==== Bugfixes diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5a868ced7cb..d211690372d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -45,6 +45,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - Update docker/distribution dependency library to fix a security issues concerning OCI Manifest Type Confusion Issue. {pull}30462[30462] - Fixes Beats crashing when glibc >= 2.35 is used {issue}30576[30576] - Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534] +- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668] *Auditbeat* diff --git a/libbeat/template/processor.go b/libbeat/template/processor.go index 9e4214ac04e..4186aa8ac6e 100644 --- a/libbeat/template/processor.go +++ b/libbeat/template/processor.go @@ -317,11 +317,11 @@ func (p *Processor) wildcard(f *mapping.Field, analyzers common.MapStr) common.M property["type"] = "wildcard" - switch f.IgnoreAbove { - case 0: // Use libbeat default - property["ignore_above"] = defaultIgnoreAbove - case -1: // Use ES default - default: // Use user value + /* For wildcard fields, unlike keywords, don't force a default ignore_above limit. + The default in ES will be used unless an explicit limit is set. + This is to take advantage of wildcard type benefits when indexing large strings. + */ + if f.IgnoreAbove > 0 { property["ignore_above"] = f.IgnoreAbove } diff --git a/libbeat/template/processor_test.go b/libbeat/template/processor_test.go index 4b35da89815..9f40529c7ac 100644 --- a/libbeat/template/processor_test.go +++ b/libbeat/template/processor_test.go @@ -805,46 +805,77 @@ func TestProcessWildcardOSS(t *testing.T) { } func TestProcessWildcardElastic(t *testing.T) { - // Test common fields are combined even if they come from different objects - fields := mapping.Fields{ - mapping.Field{ - Name: "test", - Type: "group", - Fields: mapping.Fields{ + for _, test := range []struct { + title string + fields mapping.Fields + expected common.MapStr + }{ + { + title: "default", + fields: mapping.Fields{ mapping.Field{ - Name: "one", - Type: "wildcard", + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + }, + }, + }, + }, + expected: common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "type": "wildcard", + }, + }, }, }, }, - } - - output := common.MapStr{} - analyzers := common.MapStr{} - version, err := common.NewVersion("8.0.0") - if err != nil { - t.Fatal(err) - } - - p := Processor{EsVersion: *version, ElasticLicensed: true} - err = p.Process(fields, nil, output, analyzers) - if err != nil { - t.Fatal(err) - } - - // Make sure fields without a name are skipped during template generation - expectedOutput := common.MapStr{ - "test": common.MapStr{ - "properties": common.MapStr{ - "one": common.MapStr{ - "ignore_above": 1024, - "type": "wildcard", + { + title: "explicit ignore_above", + fields: mapping.Fields{ + mapping.Field{ + Name: "test", + Type: "group", + Fields: mapping.Fields{ + mapping.Field{ + Name: "one", + Type: "wildcard", + IgnoreAbove: 4096, + }, + }, + }, + }, + expected: common.MapStr{ + "test": common.MapStr{ + "properties": common.MapStr{ + "one": common.MapStr{ + "ignore_above": 4096, + "type": "wildcard", + }, + }, }, }, }, + } { + t.Run(test.title, func(t *testing.T) { + output := common.MapStr{} + analyzers := common.MapStr{} + version, err := common.NewVersion("8.0.0") + if err != nil { + t.Fatal(err) + } + p := Processor{EsVersion: *version, ElasticLicensed: true} + err = p.Process(test.fields, nil, output, analyzers) + if err != nil { + t.Fatal(err) + } + assert.Equal(t, test.expected, output) + }) } - - assert.Equal(t, expectedOutput, output) } func TestProcessWildcardPreSupport(t *testing.T) {