Fix timezone parsing of logstash module ingest pipelines (#13890)

Use the same date format that we are using now for Elasticsearch timestamps without timezone, that is the same format used for timestamps in Logstash plain text logs. Similar to #13308, related to #13877.
elastic · Oct 3, 2019 · 774efc2 · 774efc2
1 parent 49cb774
commit 774efc2
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 15 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -153,6 +153,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed early expiration of templates (Netflow v9 and IPFIX). {pull}13821[13821]
 - Fixed bad handling of sequence numbers when multiple observation domains were exported by a single device (Netflow V9 and IPFIX). {pull}13821[13821]
 - Fix conditions and error checking of date processors in ingest pipelines that use `event.timezone` to parse dates. {pull}13883[13883]
+- Fix timezone parsing of logstash module ingest pipelines. {pull}13890[13890]
 
 *Heartbeat*
 

diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -30,19 +30,19 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "logstash.log.timestamp",
                 "target_field": "@timestamp",
-                "formats": [
-                  "ISO8601"
-              ],
-              "ignore_failure": true
+                "formats": ["yyyy-MM-dd'T'HH:mm:ss,SSS"],
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {
             "date": {
                 "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
+                "field": "logstash.log.timestamp",
+                "target_field": "@timestamp",
+                "formats": ["yyyy-MM-dd'T'HH:mm:ss,SSS"],
                 "timezone": "{{ event.timezone }}",
                 "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }

diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2017-10-23T12:20:12.046-02:00",
+        "@timestamp": "2017-10-23T14:20:12.046-02:00",
         "event.dataset": "logstash.log",
         "event.module": "logstash",
         "event.timezone": "-02:00",
@@ -13,7 +13,7 @@
         "service.type": "logstash"
     },
     {
-        "@timestamp": "2017-11-20T01:55:00.318-02:00",
+        "@timestamp": "2017-11-20T03:55:00.318-02:00",
         "event.dataset": "logstash.log",
         "event.module": "logstash",
         "event.timezone": "-02:00",

diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json b/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json
@@ -45,19 +45,19 @@
         },
         {
             "date": {
+                "if": "ctx.event.timezone == null",
                 "field": "logstash.slowlog.timestamp",
                 "target_field": "@timestamp",
-                "formats": [
-                  "ISO8601"
-              ],
-              "ignore_failure": true
+                "formats": ["yyyy-MM-dd'T'HH:mm:ss,SSS"],
+                "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }
         },
         {
             "date": {
                 "if": "ctx.event.timezone != null",
-                "field": "@timestamp",
-                "formats": ["ISO8601"],
+                "field": "logstash.slowlog.timestamp",
+                "target_field": "@timestamp",
+                "formats": ["yyyy-MM-dd'T'HH:mm:ss,SSS"],
                 "timezone": "{{ event.timezone }}",
                 "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}]
             }

diff --git a/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json b/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json
@@ -1,6 +1,6 @@
 [
     {
-        "@timestamp": "2017-10-30T07:57:58.243-02:00",
+        "@timestamp": "2017-10-30T09:57:58.243-02:00",
         "event.dataset": "logstash.slowlog",
         "event.duration": 3027675106,
         "event.module": "logstash",