Add null (\u0000) as a valid line terminator (#28998) (#29067)

Add null (`\u0000`) as a valid line terminator called `null_terminator`. Closes: #27061 (cherry picked from commit 668da78) Co-authored-by: Tiago Queiroz <[email protected]>
elastic · Nov 24, 2021 · c2fb9a3 · c2fb9a3
1 parent d08050f
commit c2fb9a3
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 6 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -65,6 +65,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620]
 - Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817]
 - Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014]
+- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998]
 
 *Heartbeat*
 - Change behavior in case of duplicate monitor IDs in configs to be last monitor wins. {pull}29041[29041]

diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl
@@ -94,7 +94,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -348,7 +349,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -501,7 +501,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -755,7 +756,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it

diff --git a/libbeat/reader/readfile/line_terminator.go b/libbeat/reader/readfile/line_terminator.go
@@ -44,6 +44,8 @@ const (
 	LineSeparator
 	// ParagraphSeparator is the unicode char PS
 	ParagraphSeparator
+	// NullTerminator
+	NullTerminator
 )
 
 var (
@@ -57,6 +59,7 @@ var (
 		"next_line":                 NextLine,
 		"line_separator":            LineSeparator,
 		"paragraph_separator":       ParagraphSeparator,
+		"null_terminator":           NullTerminator,
 	}
 
 	lineTerminatorCharacters = map[LineTerminator][]byte{
@@ -69,6 +72,7 @@ var (
 		NextLine:               []byte{'\u0085'},
 		LineSeparator:          []byte("\u2028"),
 		ParagraphSeparator:     []byte("\u2029"),
+		NullTerminator:         []byte{'\u0000'},
 	}
 )
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -2489,7 +2489,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -2743,7 +2744,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it