[Filebeat] Upgrade Postgres module to work with version 11 (elastic#1…

…3053)
breml · Aug 9, 2019 · dfe4f27 · dfe4f27
1 parent b1035b0
commit dfe4f27
Show file tree

Hide file tree

Showing 6 changed files with 466 additions and 2 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -12670,6 +12670,15 @@ example: SELECT * FROM users;
 
 --
 
+*`postgresql.log.error.code`*::
++
+--
+Error code returned by Postgres (if any)
+
+type: long
+
+--
+
 *`postgresql.log.timezone`*::
 +
 --

diff --git a/filebeat/module/postgresql/fields.go b/filebeat/module/postgresql/fields.go
diff --git a/filebeat/module/postgresql/log/_meta/fields.yml b/filebeat/module/postgresql/log/_meta/fields.yml
@@ -20,6 +20,10 @@
       description:
         Query statement.
 
+    - name: error.code
+      type: long
+      description: Error code returned by Postgres (if any)
+
     - name: timezone
       type: alias
       path: event.timezone

diff --git a/filebeat/module/postgresql/log/ingest/pipeline.json b/filebeat/module/postgresql/log/ingest/pipeline.json
@@ -6,7 +6,7 @@
         "field": "message",
         "ignore_missing": true,
         "patterns": [
-            "^%{DATETIME:postgresql.log.timestamp} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}:  (duration: %{NUMBER:temp.duration:float} ms  statement: %{GREEDYDATA:postgresql.log.query}|%{GREEDYDATA:message})"
+            "^%{DATETIME:postgresql.log.timestamp} \\[%{NUMBER:process.pid:long}(-%{BASE16FLOAT:postgresql.log.core_id:long})?\\] ((\\[%{USERNAME:user.name}\\]@\\[%{POSTGRESQL_DB_NAME:postgresql.log.database}\\]|%{USERNAME:user.name}@%{POSTGRESQL_DB_NAME:postgresql.log.database}) )?%{WORD:log.level}:  (?:%{NUMBER:postgresql.log.error.code:long}|%{SPACE})(duration: %{NUMBER:temp.duration:float} ms  statement: %{GREEDYDATA:postgresql.log.query}|: %{GREEDYDATA:message}|%{GREEDYDATA:message})"
         ],
         "pattern_definitions": {
           "DATETIME": "[-0-9]+ %{TIME} %{WORD:event.timezone}",

diff --git a/filebeat/module/postgresql/log/test/postgresql-11.4.log b/filebeat/module/postgresql/log/test/postgresql-11.4.log
@@ -0,0 +1,29 @@
+2019-07-23 12:06:24.406 UTC [25] LOG:  00000: database system was interrupted; last known up at 2019-07-23 12:03:20 UTC
+2019-07-23 12:06:24.406 UTC [25] LOCATION:  StartupXLOG, xlog.c:6388
+2019-07-23 12:06:24.478 UTC [25] LOG:  00000: database system was not properly shut down; automatic recovery in progress
+2019-07-23 12:06:24.478 UTC [25] LOCATION:  StartupXLOG, xlog.c:6879
+2019-07-23 12:06:24.485 UTC [25] LOG:  00000: redo starts at 0/1651398
+2019-07-23 12:06:24.485 UTC [25] LOCATION:  StartupXLOG, xlog.c:7150
+2019-07-23 12:06:24.485 UTC [25] LOG:  00000: invalid record length at 0/16513D0: wanted 24, got 0
+2019-07-23 12:06:24.485 UTC [25] LOCATION:  ReadRecord, xlog.c:4233
+2019-07-23 12:06:24.485 UTC [25] LOG:  00000: redo done at 0/1651398
+2019-07-23 12:06:24.485 UTC [25] LOCATION:  StartupXLOG, xlog.c:7422
+2019-07-23 12:06:24.507 UTC [1] LOG:  00000: database system is ready to accept connections
+2019-07-23 12:06:24.507 UTC [1] LOCATION:  reaper, postmaster.c:2889
+2019-07-23 12:06:30.536 UTC [44] LOG:  00000: connection received: host=[local]
+2019-07-23 12:06:30.536 UTC [44] LOCATION:  BackendInitialize, postmaster.c:4206
+2019-07-23 12:06:30.537 UTC [44] LOG:  00000: connection authorized: user=postgres database=postgres
+2019-07-23 12:06:30.537 UTC [44] LOCATION:  PerformAuthentication, postinit.c:279
+2019-07-23 12:06:33.732 UTC [44] LOG:  00000: statement: show config_filel;
+2019-07-23 12:06:33.732 UTC [44] LOCATION:  exec_simple_query, postgres.c:975
+2019-07-23 12:06:33.732 UTC [44] ERROR:  42704: unrecognized configuration parameter "config_filel"
+2019-07-23 12:06:33.732 UTC [44] LOCATION:  GetConfigOptionByName, guc.c:8342
+2019-07-23 12:06:33.732 UTC [44] STATEMENT:  show config_filel;
+2019-07-23 12:06:34.877 UTC [44] LOG:  00000: statement: show config_file;
+2019-07-23 12:06:34.877 UTC [44] LOCATION:  exec_simple_query, postgres.c:975
+2019-07-23 12:06:34.878 UTC [44] LOG:  00000: duration: 0.524 ms
+2019-07-23 12:06:34.878 UTC [44] LOCATION:  exec_simple_query, postgres.c:1219
+2019-07-23 12:09:57.563 UTC [44] LOG:  00000: statement: SELECT * FROM pg_catalog.pg_tables;
+2019-07-23 12:09:57.563 UTC [44] LOCATION:  exec_simple_query, postgres.c:975
+2019-07-23 12:09:57.565 UTC [44] LOG:  00000: duration: 2.139 ms
+2019-07-23 12:09:57.565 UTC [44] LOCATION:  exec_simple_query, postgres.c:1219