Cherrypick to 6.x: Support different timestamp format in postgresql m…

…odule (elastic#9650) (elastic#9794) * Support different timestamp format in postgresql module (elastic#9650) * Support different timestamp format in postgresql module * Update changelog (cherry picked from commit 9d60da5) * Rerun test with 6.x ES
urso · Dec 27, 2018 · baa22e9 · baa22e9
1 parent 818bfac
commit baa22e9
Show file tree

Hide file tree

Showing 4 changed files with 93 additions and 1 deletion.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -223,6 +223,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits]
 - Fix RFC3339 timezone and nanoseconds parsing with the syslog input. {pull}8346[8346]
 - Mark the TCP and UDP input as GA. {pull}8125[8125]
 - Support multiline logs in logstash/log fileset of Filebeat. {pull}8562[8562]
+- Support different timestamp format in postgresql module. {issue}9494[9494] {pull}9650[9650]
 
 *Heartbeat*
 

diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json
@@ -20,7 +20,7 @@
         "field": "postgresql.log.timestamp",
         "target_field": "@timestamp",
         "formats": [
-          "yyyy-MM-dd HH:mm:ss.SSS"
+          "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss"
         ],
         "ignore_failure": true
       }

diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log
@@ -0,0 +1,5 @@
+2017-07-31 13:36:43 EST [835-1] LOG:  autovacuum launcher started
+2017-07-31 13:36:44 EST [832-1] LOG:  checkpoints are occurring too frequently (25 seconds apart)
+2017-07-31 13:46:02 EST [832-2] HINT:  Consider increasing the configuration parameter "max_wal_size".
+2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL:  the database system is starting up
+2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL:  the database system is starting up
diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-new-timestamp.log-expected.json
@@ -0,0 +1,86 @@
+[
+    {
+        "@timestamp": "2017-07-31T13:36:43.000Z", 
+        "event.dataset": "postgresql.log", 
+        "fileset.module": "postgresql", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "message": "2017-07-31 13:36:43 EST [835-1] LOG:  autovacuum launcher started", 
+        "offset": 0, 
+        "postgresql.log.core_id": "1", 
+        "postgresql.log.level": "LOG", 
+        "postgresql.log.message": "autovacuum launcher started", 
+        "postgresql.log.thread_id": "835", 
+        "postgresql.log.timestamp": "2017-07-31 13:36:43", 
+        "postgresql.log.timezone": "EST", 
+        "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-07-31T13:36:44.000Z", 
+        "event.dataset": "postgresql.log", 
+        "fileset.module": "postgresql", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "message": "2017-07-31 13:36:44 EST [832-1] LOG:  checkpoints are occurring too frequently (25 seconds apart)", 
+        "offset": 66, 
+        "postgresql.log.core_id": "1", 
+        "postgresql.log.level": "LOG", 
+        "postgresql.log.message": "checkpoints are occurring too frequently (25 seconds apart)", 
+        "postgresql.log.thread_id": "832", 
+        "postgresql.log.timestamp": "2017-07-31 13:36:44", 
+        "postgresql.log.timezone": "EST", 
+        "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-07-31T13:46:02.000Z", 
+        "event.dataset": "postgresql.log", 
+        "fileset.module": "postgresql", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "message": "2017-07-31 13:46:02 EST [832-2] HINT:  Consider increasing the configuration parameter \"max_wal_size\".", 
+        "offset": 164, 
+        "postgresql.log.core_id": "2", 
+        "postgresql.log.level": "HINT", 
+        "postgresql.log.message": "Consider increasing the configuration parameter \"max_wal_size\".", 
+        "postgresql.log.thread_id": "832", 
+        "postgresql.log.timestamp": "2017-07-31 13:46:02", 
+        "postgresql.log.timezone": "EST", 
+        "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-07-31T13:46:23.000Z", 
+        "event.dataset": "postgresql.log", 
+        "fileset.module": "postgresql", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "message": "2017-07-31 13:46:23 EST [768-1] postgres@postgres FATAL:  the database system is starting up", 
+        "offset": 267, 
+        "postgresql.log.core_id": "1", 
+        "postgresql.log.database": "postgres", 
+        "postgresql.log.level": "FATAL", 
+        "postgresql.log.message": "the database system is starting up", 
+        "postgresql.log.thread_id": "768", 
+        "postgresql.log.timestamp": "2017-07-31 13:46:23", 
+        "postgresql.log.timezone": "EST", 
+        "postgresql.log.user": "postgres", 
+        "prospector.type": "log"
+    }, 
+    {
+        "@timestamp": "2017-07-31T13:46:55.000Z", 
+        "event.dataset": "postgresql.log", 
+        "fileset.module": "postgresql", 
+        "fileset.name": "log", 
+        "input.type": "log", 
+        "message": "2017-07-31 13:46:55 EST [771-1] postgres@postgres FATAL:  the database system is starting up", 
+        "offset": 360, 
+        "postgresql.log.core_id": "1", 
+        "postgresql.log.database": "postgres", 
+        "postgresql.log.level": "FATAL", 
+        "postgresql.log.message": "the database system is starting up", 
+        "postgresql.log.thread_id": "771", 
+        "postgresql.log.timestamp": "2017-07-31 13:46:55", 
+        "postgresql.log.timezone": "EST", 
+        "postgresql.log.user": "postgres", 
+        "prospector.type": "log"
+    }
+]