[Filebeat] Improve ECS categorization field mappings for mysql module (…

…elastic#17491) (elastic#17522) * Improve ECS categorization field mappings for mysql module - error & slowlog filesets - event.category - event.code - event.kind - event.provider - event.type Closes elastic#16172 (cherry picked from commit 787dd62)
blakerouse · Apr 6, 2020 · e3b5e20 · e3b5e20
1 parent ddcc60c
commit e3b5e20
Show file tree

Hide file tree

Showing 23 changed files with 2,165 additions and 66 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -307,6 +307,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings for mssql module. {issue}16171[16171] {pull}17376[17376]
 - Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275]
 - Added access_key_id, secret_access_key and session_token into aws module config. {pull}17456[17456]
+- Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491]
 
 *Heartbeat*
 

diff --git a/filebeat/module/mysql/error/ingest/pipeline.json b/filebeat/module/mysql/error/ingest/pipeline.json
diff --git a/filebeat/module/mysql/error/ingest/pipeline.yml b/filebeat/module/mysql/error/ingest/pipeline.yml
@@ -0,0 +1,70 @@
+description: Pipeline for parsing MySQL error logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{MYSQLDATETIME}%{SPACE}(%{NUMBER:mysql.thread_id:long}%{SPACE})?(\[%{DATA:log.level}\]%{SPACE})?%{GREEDYMULTILINE:message}'
+    - '%{GREEDYDATA:message}'
+    ignore_missing: true
+    pattern_definitions:
+      LOCALDATETIME: (?:%{YEAR}-%{MONTHNUM}-%{MONTHDAY}|%{NUMBER})%{SPACE}%{TIME}
+      MYSQLDATETIME: (?:%{LOCALDATETIME:_tmp.local_timestamp}|%{TIMESTAMP_ISO8601:_tmp.timestamp})
+      GREEDYMULTILINE: |-
+        (.|
+        )+
+- grok:
+    field: message
+    patterns:
+    - '(\[%{DATA:event.code}\])%{SPACE}(\[%{DATA:event.provider}\])%{SPACE}%{GREEDYMULTILINE}'
+    - '%{GREEDYDATA}'
+    ignore_missing: true
+    ignore_failure: true
+    pattern_definitions:
+      GREEDYMULTILINE: |-
+        (.|
+        )+
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- date:
+    if: ctx._tmp?.local_timestamp != null && ctx.event?.timezone == null
+    field: _tmp.local_timestamp
+    formats:
+    - yyMMdd H:m:s
+    - yyMMdd  H:m:s
+    - yyyy-MM-dd H:m:s
+    - yyyy-MM-dd  H:m:s
+- date:
+    if: ctx._tmp?.local_timestamp != null && ctx.event?.timezone != null
+    field: _tmp.local_timestamp
+    timezone: '{{ event.timezone }}'
+    formats:
+    - yyMMdd H:m:s
+    - yyMMdd  H:m:s
+    - yyyy-MM-dd H:m:s
+    - yyyy-MM-dd  H:m:s
+- date:
+    if: ctx._tmp?.timestamp != null
+    field: _tmp.timestamp
+    formats:
+    - ISO8601
+- remove:
+    field: _tmp
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: database
+- append:
+    field: event.type
+    value: info
+- append:
+    field: event.type
+    value: error
+    if: "ctx?.log?.level != null && ctx.log.level.toLowerCase() == 'error'"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
diff --git a/filebeat/module/mysql/error/manifest.yml b/filebeat/module/mysql/error/manifest.yml
@@ -10,5 +10,5 @@ var:
     os.windows:
       - "c:/programdata/MySQL/MySQL Server*/error.log*"
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/error.yml
diff --git a/filebeat/module/mysql/error/test/error.log-expected.json b/filebeat/module/mysql/error/test/error.log-expected.json
@@ -1,9 +1,16 @@
 [
     {
         "@timestamp": "2016-12-09T13:08:33.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.offset": 0,
@@ -12,9 +19,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.335Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Warning",
@@ -25,9 +39,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.335Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Warning",
@@ -38,9 +59,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.336Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Note",
@@ -51,9 +79,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.345Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Warning",
@@ -64,9 +99,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.351Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Note",
@@ -77,9 +119,16 @@
     },
     {
         "@timestamp": "2016-12-09T12:08:33.784Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.flags": [
@@ -93,9 +142,16 @@
     },
     {
         "@timestamp": "2016-12-09T22:21:02.443Z",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Note",
@@ -106,9 +162,16 @@
     },
     {
         "@timestamp": "2016-12-09T14:18:50.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Warning",
@@ -118,9 +181,16 @@
     },
     {
         "@timestamp": "2016-12-09T14:18:50.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.level": "Note",
@@ -130,9 +200,16 @@
     },
     {
         "@timestamp": "2016-12-09T14:18:50.000-02:00",
+        "event.category": [
+            "database"
+        ],
         "event.dataset": "mysql.error",
+        "event.kind": "event",
         "event.module": "mysql",
         "event.timezone": "-02:00",
+        "event.type": [
+            "info"
+        ],
         "fileset.name": "error",
         "input.type": "log",
         "log.offset": 1422,