diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a765a4bfa20..f732e24f388 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Gracefully shut down on SIGHUP {pull}10704[10704] - New processor: `copy_fields`. {pull}11303[11303] - Add `error.message` to events when `fail_on_error` is set in `rename` and `copy_fields` processors. {pull}11303[11303] +- New processor: `truncate_fields`. {pull}11297[11297] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index a67509d1efe..b7c15b13391 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -339,6 +339,32 @@ auditbeat.modules: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index ff8a5b0b050..7877244192b 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1058,6 +1058,32 @@ filebeat.inputs: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/filebeat/tests/system/test_processors.py b/filebeat/tests/system/test_processors.py index b381ed31452..5450d80a355 100644 --- a/filebeat/tests/system/test_processors.py +++ b/filebeat/tests/system/test_processors.py @@ -1,4 +1,6 @@ +# -*- coding: utf-8 -*- from filebeat import BaseTest +import io import os """ @@ -192,3 +194,74 @@ def test_dissect_bad_tokenizer(self): )[0] assert "extracted.key" not in output assert output["message"] == "Hello world" + + def test_truncate_bytes(self): + """ + Check if truncate_fields with max_bytes can truncate long lines and leave short lines as is + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/test.log", + processors=[{ + "truncate_fields": { + "max_bytes": 10, + "fields": ["message"], + }, + }] + ) + + self._init_and_read_test_input([ + u"This is my super long line\n", + u"This is an even longer long line\n", + u"A végrehajtás során hiba történt\n", # Error occured during execution (Hungarian) + u"This is OK\n", + ]) + + self._assert_expected_lines([ + u"This is my", + u"This is an", + u"A végreha", + u"This is OK", + ]) + + def test_truncate_characters(self): + """ + Check if truncate_fields with max_charaters can truncate long lines and leave short lines as is + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/test.log", + processors=[{ + "truncate_fields": { + "max_characters": 10, + "fields": ["message"], + }, + }] + ) + + self._init_and_read_test_input([ + u"This is my super long line\n", + u"A végrehajtás során hiba történt\n", # Error occured during execution (Hungarian) + u"This is OK\n", + ]) + + self._assert_expected_lines([ + u"This is my", + u"A végrehaj", + u"This is OK", + ]) + + def _init_and_read_test_input(self, input_lines): + with io.open(self.working_dir + "/test.log", "w", encoding="utf-8") as f: + for line in input_lines: + f.write((line)) + + filebeat = self.start_beat() + self.wait_until(lambda: self.output_has(lines=len(input_lines))) + filebeat.check_kill_and_wait() + + def _assert_expected_lines(self, expected_lines): + output = self.read_output() + + assert len(output) == len(expected_lines) + + for i in range(len(expected_lines)): + assert output[i]["message"] == expected_lines[i] diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e5317c15121..351455bdc58 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -483,6 +483,32 @@ heartbeat.scheduler: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index f48abb827a9..00a03fae29a 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -279,6 +279,32 @@ setup.template.settings: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 7224aa9e5f5..d1781694c6a 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -227,6 +227,32 @@ # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/libbeat/processors/actions/checks.go b/libbeat/processors/actions/checks.go index 2800ecef1bf..d554409cd9d 100644 --- a/libbeat/processors/actions/checks.go +++ b/libbeat/processors/actions/checks.go @@ -79,3 +79,25 @@ func allowedFields(fields ...string) func(*common.Config) error { return nil } } + +func mutuallyExclusiveRequiredFields(fields ...string) func(*common.Config) error { + return func(cfg *common.Config) error { + var foundField string + for _, field := range cfg.GetFields() { + for _, f := range fields { + if field == f { + if len(foundField) == 0 { + foundField = field + } else { + return fmt.Errorf("field %s and %s are mutually exclusive", foundField, field) + } + } + } + } + + if len(foundField) == 0 { + return fmt.Errorf("missing option, select one from %v", fields) + } + return nil + } +} diff --git a/libbeat/processors/actions/checks_test.go b/libbeat/processors/actions/checks_test.go new file mode 100644 index 00000000000..ef8a2368a28 --- /dev/null +++ b/libbeat/processors/actions/checks_test.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "testing" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/processors" +) + +type mockProcessor struct{} + +func newMock(c *common.Config) (processors.Processor, error) { + return &mockProcessor{}, nil +} + +func (m *mockProcessor) Run(event *beat.Event) (*beat.Event, error) { + return event, nil +} + +func (m *mockProcessor) String() string { + return "mockProcessor" +} + +func TestRequiredFields(t *testing.T) { + tests := map[string]struct { + Config map[string]interface{} + Required []string + Valid bool + }{ + "one required field present in the configuration": { + Config: map[string]interface{}{ + "required_field": nil, + "not_required": nil, + }, + Required: []string{ + "required_field", + }, + Valid: true, + }, + "two required field present in the configuration": { + Config: map[string]interface{}{ + "required_field": nil, + "another_required_field": nil, + "not_required": nil, + }, + Required: []string{ + "required_field", + "another_required_field", + }, + Valid: true, + }, + "one required field present and one missing in the configuration": { + Config: map[string]interface{}{ + "required_field": nil, + "not_required": nil, + }, + Required: []string{ + "required_field", + "one_more_required_field", + }, + Valid: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + runTest(t, requireFields, test.Config, test.Required, test.Valid) + }) + } +} + +func TestAllowedFields(t *testing.T) { + tests := map[string]struct { + Config map[string]interface{} + Allowed []string + Valid bool + }{ + "one allowed field present in the configuration": { + Config: map[string]interface{}{ + "allowed_field": nil, + }, + Allowed: []string{ + "allowed_field", + }, + Valid: true, + }, + "two allowed field present in the configuration": { + Config: map[string]interface{}{ + "allowed_field": nil, + "another_allowed_field": nil, + }, + Allowed: []string{ + "allowed_field", + "another_allowed_field", + }, + Valid: true, + }, + "one allowed field present and one not allowed is present in the configuration": { + Config: map[string]interface{}{ + "allowed_field": nil, + "not_allowed": nil, + }, + Allowed: []string{ + "allowed_field", + }, + Valid: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + runTest(t, allowedFields, test.Config, test.Allowed, test.Valid) + }) + } +} + +func TestMutuallyExclusiveRequiredFields(t *testing.T) { + tests := map[string]struct { + Config map[string]interface{} + MutuallyExclusive []string + Valid bool + }{ + "one mutually exclusive field is present in the configuration": { + Config: map[string]interface{}{ + "first_option": nil, + }, + MutuallyExclusive: []string{ + "first_option", + "second_option", + }, + Valid: true, + }, + "two mutually exclusive field is present in the configuration": { + Config: map[string]interface{}{ + "first_option": nil, + "second_option": nil, + }, + MutuallyExclusive: []string{ + "first_option", + "second_option", + }, + Valid: false, + }, + "no mutually exclusive field is present in the configuration": { + Config: map[string]interface{}{ + "third_option": nil, + }, + MutuallyExclusive: []string{ + "first_option", + "second_option", + }, + Valid: false, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + runTest(t, mutuallyExclusiveRequiredFields, test.Config, test.MutuallyExclusive, test.Valid) + }) + } +} + +func runTest( + t *testing.T, + check func(fields ...string) func(*common.Config) error, + config map[string]interface{}, + fields []string, + valid bool, +) { + cfg, err := common.NewConfigFrom(config) + if err != nil { + t.Fatalf("Unexpected error while creating configuration: %+v\n", err) + } + factory := configChecked(newMock, check(fields...)) + _, err = factory(cfg) + + if err != nil && valid { + t.Errorf("Unexpected error when validating configuration of processor: %+v\n", err) + } + + if err == nil && !valid { + t.Errorf("Expected error but nothing was reported\n") + } +} diff --git a/libbeat/processors/actions/truncate_fields.go b/libbeat/processors/actions/truncate_fields.go new file mode 100644 index 00000000000..b6b5db6b69c --- /dev/null +++ b/libbeat/processors/actions/truncate_fields.go @@ -0,0 +1,198 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "bytes" + "fmt" + "strings" + "unicode/utf8" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/processors" +) + +type truncateFieldsConfig struct { + Fields []string `config:"fields"` + MaxBytes int `config:"max_bytes" validate:"min=0"` + MaxChars int `config:"max_characters" validate:"min=0"` + IgnoreMissing bool `config:"ignore_missing"` + FailOnError bool `config:"fail_on_error"` +} + +type truncateFields struct { + config truncateFieldsConfig + truncate truncater +} + +type truncater func(*truncateFields, []byte) ([]byte, bool, error) + +func init() { + processors.RegisterPlugin("truncate_fields", + configChecked(newTruncateFields, + requireFields("fields"), + mutuallyExclusiveRequiredFields("max_bytes", "max_characters"), + ), + ) +} + +func newTruncateFields(c *common.Config) (processors.Processor, error) { + var config truncateFieldsConfig + err := c.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("fail to unpack the truncate_fields configuration: %s", err) + } + + var truncateFunc truncater + if config.MaxBytes > 0 { + truncateFunc = (*truncateFields).truncateBytes + } else { + truncateFunc = (*truncateFields).truncateCharacters + } + + return &truncateFields{ + config: config, + truncate: truncateFunc, + }, nil +} + +func (f *truncateFields) Run(event *beat.Event) (*beat.Event, error) { + var backup common.MapStr + if f.config.FailOnError { + backup = event.Fields.Clone() + } + + for _, field := range f.config.Fields { + event, err := f.truncateSingleField(field, event) + if err != nil && f.config.FailOnError { + logp.Debug("truncate_fields", "Failed to truncate fields: %s", err) + event.Fields = backup + return event, err + } + } + + return event, nil +} + +func (f *truncateFields) truncateSingleField(field string, event *beat.Event) (*beat.Event, error) { + v, err := event.GetValue(field) + if err != nil { + if f.config.IgnoreMissing && errors.Cause(err) == common.ErrKeyNotFound { + return event, nil + } + return event, errors.Wrapf(err, "could not fetch value for key: %s", field) + } + + switch value := v.(type) { + case []byte: + return f.addTruncatedByte(field, value, event) + case string: + return f.addTruncatedString(field, value, event) + default: + return event, fmt.Errorf("value cannot be truncated: %+v", value) + } + +} + +func (f *truncateFields) addTruncatedString(field, value string, event *beat.Event) (*beat.Event, error) { + truncated, isTruncated, err := f.truncate(f, []byte(value)) + if err != nil { + return event, err + } + _, err = event.PutValue(field, string(truncated)) + if err != nil { + return event, fmt.Errorf("could not add truncated string value for key: %s, Error: %+v", field, err) + } + + if isTruncated { + common.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + } + + return event, nil +} + +func (f *truncateFields) addTruncatedByte(field string, value []byte, event *beat.Event) (*beat.Event, error) { + truncated, isTruncated, err := f.truncate(f, value) + if err != nil { + return event, err + } + _, err = event.PutValue(field, truncated) + if err != nil { + return event, fmt.Errorf("could not add truncated byte slice value for key: %s, Error: %+v", field, err) + } + + if isTruncated { + common.AddTagsWithKey(event.Fields, "log.flags", []string{"truncated"}) + } + + return event, nil +} + +func (f *truncateFields) truncateBytes(value []byte) ([]byte, bool, error) { + size := len(value) + if size <= f.config.MaxBytes { + return value, false, nil + } + + size = f.config.MaxBytes + truncated := make([]byte, size) + n := copy(truncated, value[:size]) + if n != size { + return nil, false, fmt.Errorf("unexpected number of bytes were copied") + } + return truncated, true, nil +} + +func (f *truncateFields) truncateCharacters(value []byte) ([]byte, bool, error) { + count := utf8.RuneCount(value) + if count <= f.config.MaxChars { + return value, false, nil + } + + count = f.config.MaxChars + r := bytes.NewReader(value) + w := bytes.NewBuffer(nil) + + for i := 0; i < count; i++ { + r, _, err := r.ReadRune() + if err != nil { + return nil, false, err + } + + _, err = w.WriteRune(r) + if err != nil { + return nil, false, err + } + } + + return w.Bytes(), true, nil +} + +func (f *truncateFields) String() string { + var limit string + if f.config.MaxBytes > 0 { + limit = fmt.Sprintf("max_bytes=%d", f.config.MaxBytes) + } else { + limit = fmt.Sprintf("max_characters=%d", f.config.MaxChars) + } + return "truncate_fields=" + strings.Join(f.config.Fields, ", ") + limit +} diff --git a/libbeat/processors/actions/truncate_fields_test.go b/libbeat/processors/actions/truncate_fields_test.go new file mode 100644 index 00000000000..45d8fb64272 --- /dev/null +++ b/libbeat/processors/actions/truncate_fields_test.go @@ -0,0 +1,177 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package actions + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/beat" + "github.com/elastic/beats/libbeat/common" +) + +func TestTruncateFields(t *testing.T) { + var tests = map[string]struct { + MaxBytes int + MaxChars int + Input common.MapStr + Output common.MapStr + ShouldError bool + TruncateFunc truncater + }{ + "truncate bytes of too long string line": { + MaxBytes: 3, + Input: common.MapStr{ + "message": "too long line", + }, + Output: common.MapStr{ + "message": "too", + "log": common.MapStr{ + "flags": []string{"truncated"}, + }, + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "truncate bytes of too long byte line": { + MaxBytes: 3, + Input: common.MapStr{ + "message": []byte("too long line"), + }, + Output: common.MapStr{ + "message": []byte("too"), + "log": common.MapStr{ + "flags": []string{"truncated"}, + }, + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "do not truncate short string line": { + MaxBytes: 15, + Input: common.MapStr{ + "message": "shorter line", + }, + Output: common.MapStr{ + "message": "shorter line", + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "do not truncate short byte line": { + MaxBytes: 15, + Input: common.MapStr{ + "message": []byte("shorter line"), + }, + Output: common.MapStr{ + "message": []byte("shorter line"), + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "try to truncate integer and get error": { + MaxBytes: 5, + Input: common.MapStr{ + "message": 42, + }, + Output: common.MapStr{ + "message": 42, + }, + ShouldError: true, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "do not truncate characters of short byte line": { + MaxChars: 6, + Input: common.MapStr{ + "message": []byte("ez jó"), // this is good (hungarian) + }, + Output: common.MapStr{ + "message": []byte("ez jó"), // this is good (hungarian) + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateCharacters, + }, + "do not truncate bytes of short byte line with multibyte runes": { + MaxBytes: 6, + Input: common.MapStr{ + "message": []byte("ez jó"), // this is good (hungarian) + }, + Output: common.MapStr{ + "message": []byte("ez jó"), // this is good (hungarian) + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + "truncate characters of too long byte line": { + MaxChars: 10, + Input: common.MapStr{ + "message": []byte("ez egy túl hosszú sor"), // this is a too long line (hungarian) + }, + Output: common.MapStr{ + "message": []byte("ez egy túl"), // this is a too (hungarian) + "log": common.MapStr{ + "flags": []string{"truncated"}, + }, + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateCharacters, + }, + "truncate bytes of too long byte line with multibyte runes": { + MaxBytes: 10, + Input: common.MapStr{ + "message": []byte("ez egy túl hosszú sor"), // this is a too long line (hungarian) + }, + Output: common.MapStr{ + "message": []byte("ez egy tú"), // this is a "to" (hungarian) + "log": common.MapStr{ + "flags": []string{"truncated"}, + }, + }, + ShouldError: false, + TruncateFunc: (*truncateFields).truncateBytes, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + p := truncateFields{ + config: truncateFieldsConfig{ + Fields: []string{"message"}, + MaxBytes: test.MaxBytes, + MaxChars: test.MaxChars, + FailOnError: true, + }, + truncate: test.TruncateFunc, + } + + event := &beat.Event{ + Fields: test.Input, + } + + newEvent, err := p.Run(event) + if test.ShouldError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + + assert.Equal(t, test.Output, newEvent.Fields) + }) + } +} diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index bc30e786090..c978b78ec18 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -953,6 +953,32 @@ metricbeat.modules: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 63263147f22..410d56139a1 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -707,6 +707,32 @@ packetbeat.ignore_outgoing: false # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index fb8853f30cb..37c41b9ab3b 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -256,6 +256,32 @@ winlogbeat.event_logs: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml index 60c5fa02fa9..ebe3473d616 100644 --- a/x-pack/auditbeat/auditbeat.reference.yml +++ b/x-pack/auditbeat/auditbeat.reference.yml @@ -374,6 +374,32 @@ auditbeat.modules: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 6cf32cc03e8..aaa8d07fed0 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1164,6 +1164,32 @@ filebeat.inputs: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 67679ba65df..b52a8f074f9 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -372,6 +372,32 @@ functionbeat.provider.aws.functions: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ================================== diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 8d72144103c..2c0b0f08d04 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -982,6 +982,32 @@ metricbeat.modules: # to: message_copied # fail_on_error: true # ignore_missing: false +# +# The following example truncates the value of message to 1024 bytes +# +#processors: +#- truncate_fields: +# fields: +# - message +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true +# +# The following example preserves the raw message under event.original +# +#processors: +#- copy_fields: +# fields: +# - from: message +# to: event.original +# fail_on_error: false +# ignore_missing: true +#- truncate_fields: +# fields: +# - event.original +# max_bytes: 1024 +# fail_on_error: false +# ignore_missing: true #============================= Elastic Cloud ==================================