Allow to override the ignore_above option for field of the type (#7238

)

By default all fields of the type keyword were limitted to 1024
characters, this PR keep the default but allow developers to redefine that limit using the
`ignore_above` directive in the field.yml.

Ref: elastic/ecs#6

CHANGELOG.asciidoc

@@ -67,6 +67,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Fix delays on autodiscovery events handling caused by blocking runner stops. {pull}7170[7170]
 - Error out on invalid Autodiscover template conditions settings. {pull}7200[7200]
 - Do not emit Kubernetes autodiscover events for Pods without IP address. {pull}7235[7235]
+- Allow to override the `ignore_above` option when defining new field with the type keyword. {pull}7238[7238]
 
 *Auditbeat*
 

libbeat/common/field.go

@@ -34,6 +34,7 @@ type Field struct {
 	Index                 *bool       `config:"index"`
 	DocValues             *bool       `config:"doc_values"`
 	CopyTo                string      `config:"copy_to"`
+	IgnoreAbove           int         `config:"ignore_above"`
 
 	// Kibana specific
 	Analyzed     *bool  `config:"analyzed"`

libbeat/template/processor.go

@@ -14,6 +14,7 @@ type Processor struct {
 
 var (
 	defaultScalingFactor = 1000
+	defaultIgnoreAbove   = 1024
 )
 
 // Process recursively processes the given fields and writes the template in the given output
@@ -150,11 +151,14 @@ func (p *Processor) keyword(f *common.Field) common.MapStr {
 	defaultFields = append(defaultFields, fullName)
 
 	property["type"] = "keyword"
-	property["ignore_above"] = 1024
+	if f.IgnoreAbove == 0 {
+		property["ignore_above"] = defaultIgnoreAbove
+	} else {
+		property["ignore_above"] = f.IgnoreAbove
+	}
 
 	if p.EsVersion.IsMajor(2) {
 		property["type"] = "string"
-		property["ignore_above"] = 1024
 		property["index"] = "not_analyzed"
 	}
 

libbeat/template/processor_test.go

@@ -130,6 +130,20 @@ func TestProcessor(t *testing.T) {
 				},
 			},
 		},
+		{
+			output: p.keyword(&common.Field{Type: "keyword", IgnoreAbove: 256}),
+			expected: common.MapStr{
+				"type":         "keyword",
+				"ignore_above": 256,
+			},
+		},
+		{
+			output: p.keyword(&common.Field{Type: "keyword"}),
+			expected: common.MapStr{
+				"type":         "keyword",
+				"ignore_above": 1024,
+			},
+		},
 		{
 			output: p.text(&common.Field{Type: "text", MultiFields: common.Fields{
 				common.Field{Name: "raw", Type: "keyword"},