Allow users to convert timezone in logstash module filesets (#9797)

* Allow users to convert timezone in logstash/log fileset * Forgot to remove unnecessary field * Allow users to convert timezone in logstash/slowlog fileset * Adding CHANGELOG entry * Updating expected results of tests
elastic · Dec 28, 2018 · d6c0394 · d6c0394
1 parent d716d89
commit d6c0394
Show file tree

Hide file tree

Showing 11 changed files with 95 additions and 38 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -202,6 +202,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the
 - Rename many `icinga.*` fields to map to ECS. {pull}9294[9294]
 - Rename many `postgresql.log.*` fields to map to ECS. {pull}9303[9303]
 - Rename many `kafka.log.*` fields to map to ECS. {pull}9297[9297]
+- Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797]
 
 *Metricbeat*
 

diff --git a/filebeat/module/logstash/_meta/config.yml b/filebeat/module/logstash/_meta/config.yml
@@ -7,9 +7,15 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Slow logs
   slowlog:
    enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml
@@ -8,3 +8,8 @@ multiline:
   pattern: ^\[[0-9]{4}-[0-9]{2}-[0-9]{2}
   negate: true
   match: after
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/logstash/log/ingest/pipeline-plain.json b/filebeat/module/logstash/log/ingest/pipeline-plain.json
@@ -34,9 +34,19 @@
             }
         },
         {
-            "rename": {
+            "date": {
                 "field": "logstash.log.timestamp",
-                "target_field": "@timestamp"
+                "target_field": "@timestamp",
+                "formats": [
+                  "ISO8601"
+              ],
+              {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+              "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "logstash.log.timestamp"
             }
         }
     ]

diff --git a/filebeat/module/logstash/log/manifest.yml b/filebeat/module/logstash/log/manifest.yml
@@ -8,6 +8,13 @@ var:
       - /var/log/logstash/logstash-{{.format}}*.log
     os.windows:
       - c:/programdata/logstash/logs/logstash-{{.format}}*.log
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline-{{.format}}.json
 input: config/log.yml
diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json
@@ -1,27 +1,27 @@
 [
     {
-        "@timestamp": "2017-10-23T14:20:12,046", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "log", 
-        "event.module": "logstash", 
-        "input.type": "log", 
-        "log.offset": 0, 
-        "logstash.log.level": "INFO", 
-        "logstash.log.message": "Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}", 
+        "@timestamp": "2017-10-23T14:20:12.046Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "log",
+        "event.module": "logstash",
+        "input.type": "log",
+        "log.offset": 0,
+        "logstash.log.level": "INFO",
+        "logstash.log.message": "Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}",
         "logstash.log.module": "logstash.modules.scaffold"
-    }, 
+    },
     {
-        "@timestamp": "2017-11-20T03:55:00,318", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "log", 
-        "event.module": "logstash", 
-        "input.type": "log", 
+        "@timestamp": "2017-11-20T03:55:00.318Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "log",
+        "event.module": "logstash",
+        "input.type": "log",
         "log.flags": [
             "multiline"
-        ], 
-        "log.offset": 175, 
-        "logstash.log.level": "INFO", 
-        "logstash.log.message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n", 
+        ],
+        "log.offset": 175,
+        "logstash.log.level": "INFO",
+        "logstash.log.message": "(0.058950s) Select Name as [person.name]\n, Address as [person.address]\nfrom people\n",
         "logstash.log.module": "logstash.inputs.jdbc     "
     }
-]
+]
diff --git a/filebeat/module/logstash/slowlog/config/slowlog.yml b/filebeat/module/logstash/slowlog/config/slowlog.yml
@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
diff --git a/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json b/filebeat/module/logstash/slowlog/ingest/pipeline-plain.json
@@ -49,9 +49,19 @@
             }
         },
         {
-            "rename": {
+            "date": {
                 "field": "logstash.slowlog.timestamp",
-                "target_field": "@timestamp"
+                "target_field": "@timestamp",
+                "formats": [
+                  "ISO8601"
+              ],
+              {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+              "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "logstash.slowlog.timestamp"
             }
         },
         {

diff --git a/filebeat/module/logstash/slowlog/manifest.yml b/filebeat/module/logstash/slowlog/manifest.yml
@@ -8,6 +8,13 @@ var:
       - /var/log/logstash/logstash-slowlog-{{.format}}*.log
     os.windows:
       - c:/programdata/logstash/logs/logstash-slowlog-{{.format}}*.log
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline-{{.format}}.json
 input: config/slowlog.yml
diff --git a/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json b/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json
@@ -1,19 +1,19 @@
 [
     {
-        "@timestamp": "2017-10-30T09:57:58,243", 
-        "ecs.version": "1.0.0-beta2", 
-        "event.dataset": "slowlog", 
-        "event.module": "logstash", 
-        "input.type": "log", 
-        "log.offset": 0, 
-        "logstash.slowlog.event": "\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"", 
-        "logstash.slowlog.level": "WARN", 
-        "logstash.slowlog.message": "event processing time {:plugin_params=>{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"}", 
-        "logstash.slowlog.module": "slowlog.logstash.filters.sleep", 
-        "logstash.slowlog.plugin_name": "sleep", 
-        "logstash.slowlog.plugin_params": "{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}", 
-        "logstash.slowlog.plugin_type": "filters", 
-        "logstash.slowlog.took_in_millis": 3027, 
+        "@timestamp": "2017-10-30T09:57:58.243Z",
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "slowlog",
+        "event.module": "logstash",
+        "input.type": "log",
+        "log.offset": 0,
+        "logstash.slowlog.event": "\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"",
+        "logstash.slowlog.level": "WARN",
+        "logstash.slowlog.message": "event processing time {:plugin_params=>{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"}",
+        "logstash.slowlog.module": "slowlog.logstash.filters.sleep",
+        "logstash.slowlog.plugin_name": "sleep",
+        "logstash.slowlog.plugin_params": "{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}",
+        "logstash.slowlog.plugin_type": "filters",
+        "logstash.slowlog.took_in_millis": 3027,
         "logstash.slowlog.took_in_nanos": 3027675106
     }
-]
+]
diff --git a/filebeat/modules.d/logstash.yml.disabled b/filebeat/modules.d/logstash.yml.disabled
@@ -10,9 +10,15 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   # Slow logs
   slowlog:
    enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false