From c2fb9a3d28a570787822c394e7dc2e7f4ed157af Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 24 Nov 2021 10:49:46 +0100 Subject: [PATCH] Add null (`\u0000`) as a valid line terminator (#28998) (#29067) Add null (`\u0000`) as a valid line terminator called `null_terminator`. Closes: #27061 (cherry picked from commit 668da78eb298f6fc7637b724e73977ec290902ea) Co-authored-by: Tiago Queiroz --- CHANGELOG.next.asciidoc | 1 + filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl | 6 ++++-- filebeat/filebeat.reference.yml | 6 ++++-- libbeat/reader/readfile/line_terminator.go | 4 ++++ x-pack/filebeat/filebeat.reference.yml | 6 ++++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index a30dd4fedf74..d7c6f70e5f00 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620] - Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817] - Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014] +- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998] *Heartbeat* - Change behavior in case of duplicate monitor IDs in configs to be last monitor wins. {pull}29041[29041] diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index a47e70d98c20..8da4a2e75fd1 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -94,7 +94,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -348,7 +349,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index f586e8965aed..45d679cc1944 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -501,7 +501,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -755,7 +756,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it diff --git a/libbeat/reader/readfile/line_terminator.go b/libbeat/reader/readfile/line_terminator.go index 68ab24736c24..bcd82e36741a 100644 --- a/libbeat/reader/readfile/line_terminator.go +++ b/libbeat/reader/readfile/line_terminator.go @@ -44,6 +44,8 @@ const ( LineSeparator // ParagraphSeparator is the unicode char PS ParagraphSeparator + // NullTerminator + NullTerminator ) var ( @@ -57,6 +59,7 @@ var ( "next_line": NextLine, "line_separator": LineSeparator, "paragraph_separator": ParagraphSeparator, + "null_terminator": NullTerminator, } lineTerminatorCharacters = map[LineTerminator][]byte{ @@ -69,6 +72,7 @@ var ( NextLine: []byte{'\u0085'}, LineSeparator: []byte("\u2028"), ParagraphSeparator: []byte("\u2029"), + NullTerminator: []byte{'\u0000'}, } ) diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index aeedf518df54..e96b8e7f97dc 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2489,7 +2489,8 @@ filebeat.inputs: #max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto ### Recursive glob configuration @@ -2743,7 +2744,8 @@ filebeat.inputs: #message_max_bytes: 10485760 # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, - # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator, + # null_terminator #line_terminator: auto # The ingest pipeline ID associated with this input. If this is set, it