Convert Filebeat system.syslog to ECS

CHANGELOG.asciidoc

@@ -123,6 +123,7 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits]
 - Rename `offset` to `log.offset`. {pull}8923[8923]
 - Rename `source_ecs` to `source` in the Filebeat Suricata module. {pull}8983[8983]
 - Remove warnings for deprecated options: "spool_size", "publish_async", "idle_timeout". {pull}9002[9002]
+- Rename many `system.syslog.*` fields to map to ECS. {pull}9135[9135]
 
 *Metricbeat*
 

dev-tools/ecs-migration.yml

@@ -85,6 +85,26 @@
   alias: true
   copy_to: false
 
+- from: system.syslog.hostname
+  to: host.hostname
+  alias: true
+  copy_to: false
+
+- from: system.syslog.program
+  to: process.name
+  alias: true
+  copy_to: false
+
+- from: system.syslog.pid
+  to: process.pid
+  alias: true
+  copy_to: false
+
+- from: system.syslog.message
+  to: message
+  alias: true
+  copy_to: false
+
 # From Auditbeat's auditd module.
 - from: source.hostname
   to: source.domain

filebeat/docs/fields.asciidoc

@@ -6039,6 +6039,13 @@ Module for parsing system log files.
 
 
 
+*`host.hostname`*::
++
+--
+type: keyword
+
+--
+
 [float]
 == system fields
 
@@ -6363,45 +6370,38 @@ Contains fields from the syslog system logs.
 
 
 
-*`system.syslog.timestamp`*::
+*`@timestamp`*::
 +
 --
-The timestamp as read from the syslog message.
-
+type: alias
 
 --
 
-*`system.syslog.hostname`*::
+*`host.hostname`*::
 +
 --
-The hostname as read from the syslog message.
-
+type: alias
 
 --
 
-*`system.syslog.program`*::
+*`process.name`*::
 +
 --
-The process name as read from the syslog message.
-
+type: alias
 
 --
 
-*`system.syslog.pid`*::
+*`process.pid`*::
 +
 --
-The PID of the process that sent the syslog message.
-
+type: alias
 
 --
 
-*`system.syslog.message`*::
+*`message`*::
 +
 --
-type: text
-
-The message in the log line.
-
+type: alias
 
 --
 

filebeat/module/system/_meta/fields.yml

@@ -4,6 +4,15 @@
     Module for parsing system log files.
   short_config: true
   fields:
+    - name: host.hostname
+      type: keyword
+      decription: >
+        Hostname of the host.
+
+        It can contain what `hostname` returns on Unix systems, the fully
+        qualified domain name, or a name specified by the user. The sender
+        decides which value to use.
+
     - name: system
       type: group
       description: >

filebeat/module/system/syslog/_meta/fields.yml

@@ -4,18 +4,17 @@
     Contains fields from the syslog system logs.
   fields:
     - name: timestamp
-      description: >
-        The timestamp as read from the syslog message.
+      type: alias
+      path: '@timestamp'
     - name: hostname
-      description: >
-        The hostname as read from the syslog message.
+      type: alias
+      path: host.hostname
     - name: program
-      description: >
-        The process name as read from the syslog message.
+      type: alias
+      path: process.name
     - name: pid
-      description: >
-        The PID of the process that sent the syslog message.
+      type: alias
+      path: process.pid
     - name: message
-      type: text
-      description: >
-        The message in the log line.
+      type: alias
+      path: message

filebeat/module/system/syslog/ingest/pipeline.json

@@ -1,41 +1,53 @@
 {
-	"description": "Pipeline for parsing Syslog messages.",
-	"processors": [
-		{
-			"grok": {
-				"field": "message",
-				"patterns": [
-					"%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}",
-					"%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}"
-				],
-        "pattern_definitions" : {
-          "GREEDYMULTILINE" : "(.|\n)*"
+    "description": "Pipeline for parsing Syslog messages.",
+    "processors": [
+        {
+            "grok": {
+                "field": "message",
+                "patterns": [
+                    "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYMULTILINE:system.syslog.message}",
+                    "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}"
+                ],
+                "pattern_definitions" : {
+                    "GREEDYMULTILINE" : "(.|\n)*"
+                },
+                "ignore_missing": true
+            }
         },
-        "ignore_missing": true
-			}
-    },
-		{
-      "remove": {
-        "field": "message"
-      }
-    },
-    {
-      "date": {
-        "field": "system.syslog.timestamp",
-        "target_field": "@timestamp",
-        "formats": [
-					"MMM  d HH:mm:ss",
-					"MMM dd HH:mm:ss"
-        ],
-        {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >}
-        "ignore_failure": true
-      }
-    }
-	],
-  "on_failure" : [{
-    "set" : {
-      "field" : "error.message",
-      "value" : "{{ _ingest.on_failure_message }}"
-    }
-  }]
+        {
+            "remove": {
+                "field": "message"
+            }
+        },
+        {
+            "rename": {
+                "field": "system.syslog.message",
+                "target_field": "message",
+                "ignore_missing": true
+            }
+        },
+        {
+            "date": {
+                "field": "system.syslog.timestamp",
+                "target_field": "@timestamp",
+                "formats": [
+                    "MMM  d HH:mm:ss",
+                    "MMM dd HH:mm:ss"
+                ],
+                {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >}
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "system.syslog.timestamp"
+            }
+        }
+    ],
+    "on_failure" : [{
+        "set" : {
+            "field" : "error.message",
+            "value" : "{{ _ingest.on_failure_message }}"
+        }
+    }]
 }

filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json

@@ -3,36 +3,33 @@
         "@timestamp": "2018-12-13T11:35:28.000Z", 
         "event.dataset": "syslog", 
         "event.module": "system", 
+        "host.hostname": "a-mac-with-esc-key", 
         "input.type": "log", 
         "log.flags": [
             "multiline"
         ], 
         "log.offset": 0, 
-        "system.syslog.hostname": "a-mac-with-esc-key", 
-        "system.syslog.message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine <KSUpdateEngine:0x100341a00\n\t\tticketStore=<KSPersistentTicketStore:0x100204520 store=<KSKeyedPersistentStore:0x100213290\n\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore\"\n\t\t\tlockFile=<KSLockFile:0x1002160e0\n\t\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore.lock\"\n\t\t\t\tlocked=NO\n\t\t\t>\n\t\t>>\n\t\tprocessor=<KSActionProcessor:0x1003bb5f0\n\t\t\tdelegate=<KSUpdateEngine:0x100341a00>\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", 
-        "system.syslog.pid": "21412", 
-        "system.syslog.program": "GoogleSoftwareUpdateAgent", 
-        "system.syslog.timestamp": "Dec 13 11:35:28"
+        "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine <KSUpdateEngine:0x100341a00\n\t\tticketStore=<KSPersistentTicketStore:0x100204520 store=<KSKeyedPersistentStore:0x100213290\n\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore\"\n\t\t\tlockFile=<KSLockFile:0x1002160e0\n\t\t\t\tpath=\"/Users/tsg/Library/Google/GoogleSoftwareUpdate/TicketStore/Keystone.ticketstore.lock\"\n\t\t\t\tlocked=NO\n\t\t\t>\n\t\t>>\n\t\tprocessor=<KSActionProcessor:0x1003bb5f0\n\t\t\tdelegate=<KSUpdateEngine:0x100341a00>\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", 
+        "process.name": "GoogleSoftwareUpdateAgent", 
+        "process.pid": 21412
     }, 
     {
         "@timestamp": "2018-12-13T11:35:28.000Z", 
         "event.dataset": "syslog", 
         "event.module": "system", 
+        "host.hostname": "a-mac-with-esc-key", 
         "input.type": "log", 
         "log.offset": 907, 
-        "system.syslog.hostname": "a-mac-with-esc-key", 
-        "system.syslog.message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", 
-        "system.syslog.pid": "21412", 
-        "system.syslog.program": "GoogleSoftwareUpdateAgent", 
-        "system.syslog.timestamp": "Dec 13 11:35:28"
+        "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", 
+        "process.name": "GoogleSoftwareUpdateAgent", 
+        "process.pid": 21412
     }, 
     {
         "@timestamp": "2018-04-04T03:39:57.000Z", 
         "event.dataset": "syslog", 
         "event.module": "system", 
         "input.type": "log", 
         "log.offset": 1176, 
-        "system.syslog.message": "--- last message repeated 1 time ---", 
-        "system.syslog.timestamp": "Apr 4 03:39:57"
+        "message": "--- last message repeated 1 time ---"
     }
 ]