[7.17](backport #30668) libbeat: Don't force an ignore_above limit on wildcard fields

CHANGELOG-developer.next.asciidoc

@@ -52,6 +52,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -34,6 +34,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix the ability for subcommands to be ran properly from the beats containers. {pull}30452[30452]
 - Log errors when parsing and applying config blocks and if the input is disabled. {pull}30534[30534]
+- Wildcard fields no longer have a default ignore_above setting of 1024. {issue}30096[30096] {pull}30668[30668]
 
 *Auditbeat*
 

libbeat/template/processor.go

@@ -317,11 +317,11 @@ func (p *Processor) wildcard(f *mapping.Field) common.MapStr {
 
 	property["type"] = "wildcard"
 
-	switch f.IgnoreAbove {
-	case 0: // Use libbeat default
-		property["ignore_above"] = defaultIgnoreAbove
-	case -1: // Use ES default
-	default: // Use user value
+	/* For wildcard fields, unlike keywords, don't force a default ignore_above limit.
+	   The default in ES will be used unless an explicit limit is set.
+	   This is to take advantage of wildcard type benefits when indexing large strings.
+	*/
+	if f.IgnoreAbove > 0 {
 		property["ignore_above"] = f.IgnoreAbove
 	}
 

libbeat/template/processor_test.go

@@ -812,45 +812,76 @@ func TestProcessWildcardOSS(t *testing.T) {
 }
 
 func TestProcessWildcardElastic(t *testing.T) {
-	// Test common fields are combined even if they come from different objects
-	fields := mapping.Fields{
-		mapping.Field{
-			Name: "test",
-			Type: "group",
-			Fields: mapping.Fields{
+	for _, test := range []struct {
+		title    string
+		fields   mapping.Fields
+		expected common.MapStr
+	}{
+		{
+			title: "default",
+			fields: mapping.Fields{
 				mapping.Field{
-					Name: "one",
-					Type: "wildcard",
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name: "one",
+							Type: "wildcard",
+						},
+					},
+				},
+			},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"type": "wildcard",
+						},
+					},
 				},
 			},
 		},
-	}
-
-	output := common.MapStr{}
-	version, err := common.NewVersion("8.0.0")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	p := Processor{EsVersion: *version, ElasticLicensed: true}
-	err = p.Process(fields, nil, output)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Make sure fields without a name are skipped during template generation
-	expectedOutput := common.MapStr{
-		"test": common.MapStr{
-			"properties": common.MapStr{
-				"one": common.MapStr{
-					"ignore_above": 1024,
-					"type":         "wildcard",
+		{
+			title: "explicit ignore_above",
+			fields: mapping.Fields{
+				mapping.Field{
+					Name: "test",
+					Type: "group",
+					Fields: mapping.Fields{
+						mapping.Field{
+							Name:        "one",
+							Type:        "wildcard",
+							IgnoreAbove: 4096,
+						},
+					},
+				},
+			},
+			expected: common.MapStr{
+				"test": common.MapStr{
+					"properties": common.MapStr{
+						"one": common.MapStr{
+							"ignore_above": 4096,
+							"type":         "wildcard",
+						},
+					},
 				},
 			},
 		},
+	} {
+		t.Run(test.title, func(t *testing.T) {
+			output := common.MapStr{}
+			version, err := common.NewVersion("8.0.0")
+			if err != nil {
+				t.Fatal(err)
+			}
+			p := Processor{EsVersion: *version, ElasticLicensed: true}
+			err = p.Process(test.fields, nil, output)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Equal(t, test.expected, output)
+		})
 	}
-
-	assert.Equal(t, expectedOutput, output)
 }
 
 func TestProcessWildcardPreSupport(t *testing.T) {