diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 41cb29fb403..ce641793d7b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -34,6 +34,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Preserve case of http.request.method. ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359] - Disable the option of running --machine-learning on its own. {pull}20241[20241] - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993] +- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547] +- Removes old module aliases for `googlecloud` (moved to gcp) and `apache2` (moved to apache). {pull}27919[27919] +- Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915] +- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526] +- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762] +- Remove deprecated fields in Kafka module. {pull}27938[27938] +- Remove deprecated fields in coredns module. {pull}28196[28196] +- Remove old `httpjson` config implementation. {pull}28054[28054] +- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423] +- Fail to start Filebat if none between `queue_url`, `bucket_arn` or `non_aws_bucket_name` is set for a configured aws-s3 input {issue}13911[13911] {pull}28666[28666] +- All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620] +- Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620] +- Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817] +- Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014] +- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998] *Heartbeat* diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index a47e70d98c2..8da4a2e75fd 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -94,7 +94,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -348,7 +349,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 2a410d93c28..6e9874e578a 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -501,7 +501,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -755,7 +756,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/libbeat/reader/readfile/line_terminator.go b/libbeat/reader/readfile/line_terminator.go index 68ab24736c2..bcd82e36741 100644 --- a/libbeat/reader/readfile/line_terminator.go +++ b/libbeat/reader/readfile/line_terminator.go @@ -44,6 +44,8 @@ const ( LineSeparator // ParagraphSeparator is the unicode char PS ParagraphSeparator + // NullTerminator + NullTerminator ) var ( @@ -57,6 +59,7 @@ var ( "next_line": NextLine, "line_separator": LineSeparator, "paragraph_separator": ParagraphSeparator, + "null_terminator": NullTerminator, } lineTerminatorCharacters = map[LineTerminator][]byte{ @@ -69,6 +72,7 @@ var ( NextLine: []byte{'\u0085'}, LineSeparator: []byte("\u2028"), ParagraphSeparator: []byte("\u2029"), + NullTerminator: []byte{'\u0000'}, } ) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index e524916c25d..1d7da9bd184 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2590,7 +2590,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -2844,7 +2845,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it