From 046d76578418e1e7cc08dfdbd45d00f87990fd2b Mon Sep 17 00:00:00 2001 From: Tiago Queiroz <tiago.queiroz@elastic.co> Date: Thu, 18 Nov 2021 10:31:41 +0100 Subject: [PATCH] Add null (`\u0000`) as a valid line terminator Add null (`\u0000`) as a valid line terminator called `null_terminator`. Closes: #27061 --- CHANGELOG.next.asciidoc | 1 + filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl | 6 ++++-- filebeat/filebeat.reference.yml | 6 ++++-- libbeat/reader/readfile/line_terminator.go | 4 ++++ x-pack/filebeat/filebeat.reference.yml | 6 ++++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 6bb189ee301..006249120a0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -63,6 +63,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620] - Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817] - Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014] +- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998] *Heartbeat* diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index a47e70d98c2..8da4a2e75fd 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -94,7 +94,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -348,7 +349,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index dc7250e402c..2acac4dd00a 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -501,7 +501,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -755,7 +756,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/libbeat/reader/readfile/line_terminator.go b/libbeat/reader/readfile/line_terminator.go index 68ab24736c2..bcd82e36741 100644 --- a/libbeat/reader/readfile/line_terminator.go +++ b/libbeat/reader/readfile/line_terminator.go @@ -44,6 +44,8 @@ const ( LineSeparator // ParagraphSeparator is the unicode char PS ParagraphSeparator + // NullTerminator + NullTerminator ) var ( @@ -57,6 +59,7 @@ var ( "next_line": NextLine, "line_separator": LineSeparator, "paragraph_separator": ParagraphSeparator, + "null_terminator": NullTerminator, } lineTerminatorCharacters = map[LineTerminator][]byte{ @@ -69,6 +72,7 @@ var ( NextLine: []byte{'\u0085'}, LineSeparator: []byte("\u2028"), ParagraphSeparator: []byte("\u2029"), + NullTerminator: []byte{'\u0000'}, } ) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 428230017f7..92cc3cfd2c9 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2489,7 +2489,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -2743,7 +2744,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it