Ignore trailing spaces in CEF messages (elastic#17253)

This patch updates the ragel state machine to skip trailing spaces
at the end of CEF messages.

Some CEF exporters, Check Point for example, have been observed to add a
trailing space to CEF messages:

> "CEF:0:| [...] src=127.0.0.1 "

Currently, this space character is interpreted as part of the last
field's value, which can cause decoding errors if the value is an
integer or an IP address.

For maximizing compatibility, we also want to ignore other kinds of
space characters (new line, carriage return, tab). For example we
can get a trailing newline when processing CEF messages from UDP 
input instead of syslog, which removes newlines.

Spaces in non-final extensions are preserved, as the CEF standard
permits (but discourages) it's use in non-final extensions.

CHANGELOG.next.asciidoc

@@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix default index pattern in IBM MQ filebeat dashboard. {pull}17146[17146]
 - Fix `elasticsearch.gc` fileset to not collect _all_ logs when Elasticsearch is running in Docker. {issue}13164[13164] {issue}16583[16583] {pull}17164[17164]
 - Fixed a mapping exception when ingesting CEF logs that used the spriv or dpriv extensions. {issue}17216[17216] {pull}17220[17220]
+- CEF: Fixed decoding errors caused by trailing spaces in messages. {pull}17253[17253]
 
 *Heartbeat*
 

x-pack/filebeat/module/cef/log/test/cef.log

@@ -1,3 +1,4 @@
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Web request|low|eventId=3457 requestMethod=POST slat=38.915 slong=-77.511 proto=TCP sourceServiceName=httpd requestContext=https://www.google.com src=6.7.8.9 spt=33876 dst=192.168.10.1 dpt=443 request=https://www.example.com/cart
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|eventId=123 src=6.7.8.9 spt=33876 dst=1.2.3.4 dpt=443 duser=alice suser=bob destinationTranslatedAddress=10.10.10.10 fileHash=bc8bbe52f041fd17318f08a0f73762ce oldFileHash=a9796280592f86b74b27e370662d41eb
 CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|spriv=user dpriv=root
+CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4     

x-pack/filebeat/module/cef/log/test/cef.log-expected.json

@@ -150,5 +150,39 @@
         "tags": [
             "cef"
         ]
+    },
+    {
+        "cef.device.event_class_id": "18",
+        "cef.device.product": "Vaporware",
+        "cef.device.vendor": "Elastic",
+        "cef.device.version": "1.0.0-alpha",
+        "cef.extensions.destinationAddress": "192.168.1.2",
+        "cef.extensions.message": "This event is padded with whitespace",
+        "cef.extensions.sourceAddress": "192.168.3.4",
+        "cef.name": "Authentication",
+        "cef.severity": "low",
+        "cef.version": "0",
+        "destination.ip": "192.168.1.2",
+        "event.code": "18",
+        "event.dataset": "cef.log",
+        "event.module": "cef",
+        "event.original": "CEF:0|Elastic|Vaporware|1.0.0-alpha|18|Authentication|low|message=This event is padded with whitespace dst=192.168.1.2 src=192.168.3.4     ",
+        "event.severity": 0,
+        "fileset.name": "log",
+        "input.type": "log",
+        "log.offset": 611,
+        "message": "This event is padded with whitespace",
+        "observer.product": "Vaporware",
+        "observer.vendor": "Elastic",
+        "observer.version": "1.0.0-alpha",
+        "related.ip": [
+            "192.168.1.2",
+            "192.168.3.4"
+        ],
+        "service.type": "cef",
+        "source.ip": "192.168.3.4",
+        "tags": [
+            "cef"
+        ]
     }
 ]

x-pack/filebeat/processors/decode_cef/cef/cef.go

@@ -14,7 +14,10 @@ import (
 // Parser is generated from a ragel state machine using the following command:
 //go:generate ragel -Z -G1 cef.rl -o parser.go
 //go:generate goimports -l -w parser.go
-
+//
+// Run go vet and remove any unreachable code in the generated parser.go.
+// The go generator outputs duplicated goto statements sometimes.
+//
 // An SVG rendering of the state machine can be viewed by opening cef.svg in
 // Chrome / Firefox.
 //go:generate ragel -V -p cef.rl -o cef.dot

x-pack/filebeat/processors/decode_cef/cef/cef.rl

@@ -124,13 +124,13 @@ func (e *Event) unpack(data string) error {
         extension_key_start_chars = alnum | '_';
         extension_key_chars = extension_key_start_chars | '.' | ',' | '[' | ']';
         extension_key_pattern = extension_key_start_chars extension_key_chars*;
-        extension_value_chars = backslash | escape_equal | (any -- equal -- escape);
+        extension_value_chars_nospace = backslash | escape_equal | (any -- equal -- escape -- space);
 
         # Extension fields.
         extension_key = extension_key_pattern >mark %extension_key;
-        extension_value = (extension_value_chars @extension_value_mark)* >extension_value_start $err(extension_err);
-        extension = extension_key equal extension_value %/extension_eof;
-        extensions = " "* extension (" " extension)*;
+        extension_value = (space* extension_value_chars_nospace @extension_value_mark)* >extension_value_start $err(extension_err);
+        extension = extension_key equal extension_value;
+        extensions = " "* extension (space* " " extension)* space* %/extension_eof;
 
         # gobble_extension attempts recovery from a malformed value by trying to
         # advance to the next extension key and re-entering the main state machine.

x-pack/filebeat/processors/decode_cef/cef/cef_test.go

@@ -44,6 +44,14 @@ const (
 	malformedExtensionEscape = `CEF:0|FooBar|Web Gateway|1.2.3.45.67|200|Success|2|rt=Sep 07 2018 14:50:39 cat=Access Log dst=1.1.1.1 dhost=foo.example.com suser=redacted src=2.2.2.2 requestMethod=POST request='https://foo.example.com/bar/bingo/1' requestClientApplication='Foo-Bar/2018.1.7; =Email:[email protected]; Guid:test=' cs1= cs1Label=Foo Bar`
 
 	multipleMalformedExtensionValues = `CEF:0|vendor|product|version|event_id|name|Very-High| msg=Hello World error=Failed because id==old_id user=root angle=106.7<=180`
+
+	paddedMessage = `CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in non-final extensions is  preserved    src=10.0.0.192 `
+
+	crlfMessage = "CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Trailing space in final extensions is not preserved\t \r\n"
+
+	tabMessage = "CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Tabs\tand\rcontrol\ncharacters are preserved\t src=127.0.0.1"
+
+	tabNoSepMessage = "CEF:0|security|threatmanager|1.0|100|message has tabs|10|spt=1232 msg=Tab is not a separator\tsrc=127.0.0.1"
 )
 
 var testMessages = []string{
@@ -60,6 +68,9 @@ var testMessages = []string{
 	escapesInExtension,
 	malformedExtensionEscape,
 	multipleMalformedExtensionValues,
+	paddedMessage,
+	crlfMessage,
+	tabMessage,
 }
 
 func TestGenerateFuzzCorpus(t *testing.T) {
@@ -322,6 +333,47 @@ func TestEventUnpack(t *testing.T) {
 		err := e.Unpack("CEF:0|||||||a=")
 		assert.NoError(t, err)
 	})
+
+	t.Run("padded", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(paddedMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"src": IPField("10.0.0.192"),
+			"spt": IntegerField(1232),
+			"msg": StringField("Trailing space in non-final extensions is  preserved   "),
+		}, e.Extensions)
+	})
+
+	t.Run("padded with extra whitespace chars", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(crlfMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+			"msg": StringField("Trailing space in final extensions is not preserved"),
+		}, e.Extensions)
+	})
+
+	t.Run("internal whitespace chars", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(tabMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+			"src": IPField("127.0.0.1"),
+			"msg": StringField("Tabs\tand\rcontrol\ncharacters are preserved\t"),
+		}, e.Extensions)
+	})
+
+	t.Run("No tab as separator", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(tabNoSepMessage)
+		assert.Error(t, err)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+		}, e.Extensions)
+	})
 }
 
 func TestEventUnpackWithFullExtensionNames(t *testing.T) {