Add null (\u0000) as a valid line terminator (#28998)

Add null (`\u0000`) as a valid line terminator called `null_terminator`.

Closes: #27061
(cherry picked from commit 668da78)

CHANGELOG.next.asciidoc

@@ -34,6 +34,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Preserve case of http.request.method.  ECS prior to 1.6 specified normalizing to lowercase, which lost information. Affects filesets: apache/access, elasticsearch/audit, iis/access, iis/error, nginx/access, nginx/ingress_controller, aws/elb, suricata/eve, zeek/http. {issue}18154[18154] {pull}18359[18359]
 - Disable the option of running --machine-learning on its own. {pull}20241[20241]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
+- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
+- Removes old module aliases for `googlecloud` (moved to gcp) and `apache2` (moved to apache). {pull}27919[27919]
+- Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915]
+- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
+- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]
+- Remove deprecated fields in Kafka module. {pull}27938[27938]
+- Remove deprecated fields in coredns module. {pull}28196[28196]
+- Remove old `httpjson` config implementation. {pull}28054[28054]
+- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
+- Fail to start Filebat if none between `queue_url`, `bucket_arn` or `non_aws_bucket_name` is set for a configured aws-s3 input {issue}13911[13911] {pull}28666[28666]
+- All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620]
+- Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620]
+- Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817]
+- Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014]
+- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998]
 
 *Heartbeat*
 

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -94,7 +94,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -348,7 +349,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it

filebeat/filebeat.reference.yml

@@ -501,7 +501,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -755,7 +756,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it

libbeat/reader/readfile/line_terminator.go

@@ -44,6 +44,8 @@ const (
 	LineSeparator
 	// ParagraphSeparator is the unicode char PS
 	ParagraphSeparator
+	// NullTerminator
+	NullTerminator
 )
 
 var (
@@ -57,6 +59,7 @@ var (
 		"next_line":                 NextLine,
 		"line_separator":            LineSeparator,
 		"paragraph_separator":       ParagraphSeparator,
+		"null_terminator":           NullTerminator,
 	}
 
 	lineTerminatorCharacters = map[LineTerminator][]byte{
@@ -69,6 +72,7 @@ var (
 		NextLine:               []byte{'\u0085'},
 		LineSeparator:          []byte("\u2028"),
 		ParagraphSeparator:     []byte("\u2029"),
+		NullTerminator:         []byte{'\u0000'},
 	}
 )
 

x-pack/filebeat/filebeat.reference.yml

@@ -2590,7 +2590,8 @@ filebeat.inputs:
   #max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   ### Recursive glob configuration
@@ -2844,7 +2845,8 @@ filebeat.inputs:
   #message_max_bytes: 10485760
 
   # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed,
-  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator.
+  # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator,
+  # null_terminator
   #line_terminator: auto
 
   # The ingest pipeline ID associated with this input. If this is set, it