Improve system/syslog date parsing fix in elastic#12543 (elastic#12568)

The original fix hardcoded some date formates that are a subset of ISO8601. This adds full ISO8601 date support to the pipeline. Related elastic#12543
adriansr · Jun 17, 2019 · 83a4b25 · 83a4b25
1 parent f31fb67
commit 83a4b25
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -200,6 +200,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
 - Add specific date processor to convert timezones so same pipeline can be used when convert_timezone is enabled or disabled. {pull}12253[12253]
 - Add MSSQL module {pull}12079[12079]
+- Add full ISO8601 date parsing support for system/syslog module. {pull}12568[12568]
 
 *Heartbeat*
 

diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json
@@ -34,10 +34,7 @@
                 "formats": [
                     "MMM  d HH:mm:ss",
                     "MMM dd HH:mm:ss",
-                    "yyyy-MM-dd'T'HH:mm:ss.SSSZZ",
-                    "yyyy-MM-dd'T'HH:mm:ss.SSSSSSZZ",
-                    "yyyy-MM-dd'T'HH:mm:ss.SSSXXX",
-                    "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX"
+                    "ISO8601"
                 ],
                 "ignore_failure": true
             }

diff --git a/filebeat/module/system/syslog/test/tz-offset.log b/filebeat/module/system/syslog/test/tz-offset.log
@@ -1,2 +1,3 @@
 1986-04-26T01:23:45.101+0400 rmbkmonitor04 shutdown[2649]: shutting down for system halt
 1986-04-26T01:23:45.388424+04:00 rmbkmonitor04 thermald: constraint_0_power_limit_uw exceeded.
+2019-06-14T10:40:20.912134 localhost sudo: pam_unix(sudo-i:session): session opened for user root by userauth3(uid=0)
diff --git a/filebeat/module/system/syslog/test/tz-offset.log-expected.json b/filebeat/module/system/syslog/test/tz-offset.log-expected.json
@@ -27,5 +27,19 @@
         "message": "constraint_0_power_limit_uw exceeded.",
         "process.name": "thermald",
         "service.type": "system"
+    },
+    {
+        "@timestamp": "2019-06-14T10:40:20.912Z",
+        "event.dataset": "system.syslog",
+        "event.module": "system",
+        "event.timezone": "+00:00",
+        "fileset.name": "syslog",
+        "host.hostname": "localhost",
+        "input.type": "log",
+        "log.file.path": "tz-offset.log",
+        "log.offset": 184,
+        "message": "pam_unix(sudo-i:session): session opened for user root by userauth3(uid=0)",
+        "process.name": "sudo",
+        "service.type": "system"
     }
 ]