Migration of apache2 error fileset to ECS (#8963)

* Migration of apache2 error fileset to ECS

Fields migrated:

* apache.error.level -> log.level
* apache.error.message -> message
* apache.error.pid -> process.pid
* apache.error.tid -> process.tid
* apache.error.client -> source.ip

Other changes

* Changelog updated
* Migration file updated
* Field alias added for the old fields
* Test outputs updated
* Fix source.ip to be IP only. Before had a HOSTORIP pattern.

Note: The ECS fields used were on purpose only put on the module level and are put into filebeat or libbeat when they are used in more then 1 place.

CHANGELOG.asciidoc

@@ -24,6 +24,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 *Filebeat*
 
 - Allow beats to blacklist certain part of the configuration while using Central Management. {pull}9099[9099]
+- Modify apache/error dataset to follow ECS. {pull}8963[8963]
 
 *Heartbeat*
 

auditbeat/docs/fields.asciidoc

@@ -5090,6 +5090,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

dev-tools/ecs-migration.yml

@@ -188,7 +188,9 @@
   to: source.geo.*
   alias: true
 
-## Apache
+# Filebeat modules
+
+## Apache module
 
 - from: apache2.access.user_name
   to: user.name
@@ -231,6 +233,27 @@
   to: user_agent.*
   alias: true
 
+### Error fileset
+- from: apache.error.message
+  to: message
+  alias: true
+
+- from: apache.error.level
+  to: log.level
+  alias: true
+
+- from: apache2.error.client
+  to: source.address
+  alias: true
+
+- from: apache2.error.pid
+  to: process.pid
+  alias: true
+
+- from: apache2.error.tid
+  to: process.thread.id
+  alias: true
+
 # IIS module
 
 - from: iis.access.server_ip
@@ -440,3 +463,4 @@
   to: event.dataset
   alias: false
   comment: No alias mapping as field did not always exist
+

filebeat/docs/fields.asciidoc

@@ -301,7 +301,9 @@ Fields from the Apache error logs.
 *`apache2.error.level`*::
 +
 --
-type: keyword
+type: alias
+
+alias to: log.level
 
 The severity level of the message.
 
@@ -321,7 +323,9 @@ The IP address of the client that generated the error.
 *`apache2.error.message`*::
 +
 --
-type: text
+type: alias
+
+alias to: message
 
 The logged message.
 
@@ -331,7 +335,9 @@ The logged message.
 *`apache2.error.pid`*::
 +
 --
-type: long
+type: alias
+
+alias to: process.pid
 
 The process ID.
 
@@ -341,7 +347,9 @@ The process ID.
 *`apache2.error.tid`*::
 +
 --
-type: long
+type: alias
+
+alias to: process.thread.id
 
 The thread ID.
 
@@ -3106,6 +3114,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

filebeat/module/apache2/error/_meta/fields.yml

@@ -4,23 +4,27 @@
     Fields from the Apache error logs.
   fields:
     - name: level
-      type: keyword
+      type: alias
+      path: log.level
       description: >
         The severity level of the message.
     - name: client
       type: keyword
       description: >
         The IP address of the client that generated the error.
     - name: message
-      type: text
+      type: alias
+      path: message
       description: >
         The logged message.
     - name: pid
-      type: long
+      type: alias
+      path: process.pid
       description: >
         The process ID.
     - name: tid
-      type: long
+      type: alias
+      path: process.thread.id
       description: >
         The thread ID.
     - name: module

filebeat/module/apache2/error/ingest/pipeline.json

@@ -5,27 +5,15 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "\\[%{APACHE_TIME:apache2.error.timestamp}\\] \\[%{LOGLEVEL:apache2.error.level}\\]( \\[client %{IPORHOST:apache2.error.client}\\])? %{GREEDYDATA:apache2.error.message}",
-          "\\[%{APACHE_TIME:apache2.error.timestamp}\\] \\[%{DATA:apache2.error.module}:%{LOGLEVEL:apache2.error.level}\\] \\[pid %{NUMBER:apache2.error.pid}(:tid %{NUMBER:apache2.error.tid})?\\]( \\[client %{IPORHOST:apache2.error.client}\\])? %{GREEDYDATA:apache2.error.message1}"
+          "\\[%{APACHE_TIME:apache2.error.timestamp}\\] \\[%{LOGLEVEL:log.level}\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}",
+          "\\[%{APACHE_TIME:apache2.error.timestamp}\\] \\[%{DATA:apache2.error.module}:%{LOGLEVEL:log.level}\\] \\[pid %{NUMBER:process.pid:long}(:tid %{NUMBER:process.thread.id:long})?\\]( \\[client %{IPORHOST:source.address}\\])? %{GREEDYDATA:message}"
         ],
         "pattern_definitions": {
           "APACHE_TIME": "%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}"
         },
         "ignore_missing": true
       }
     },
-    {
-      "remove":{
-        "field": "message"
-      }
-    },
-    {
-      "rename": {
-        "field": "apache2.error.message1",
-        "target_field": "apache2.error.message",
-        "ignore_failure": true
-      }
-    },
     {
       "date": {
         "field": "apache2.error.timestamp",

filebeat/module/apache2/error/test/test.log-expected.json

@@ -1,36 +1,36 @@
 [
     {
         "@timestamp": "2016-12-26T16:22:08.000Z", 
-        "apache2.error.client": "192.168.33.1", 
-        "apache2.error.level": "error", 
-        "apache2.error.message": "File does not exist: /var/www/favicon.ico", 
         "event.dataset": "error", 
         "event.module": "apache2", 
         "input.type": "log", 
-        "log.offset": 0
+        "log.level": "error", 
+        "log.offset": 0, 
+        "message": "File does not exist: /var/www/favicon.ico", 
+        "source.address": "192.168.33.1"
     }, 
     {
         "@timestamp": "2016-12-26T16:15:55.103Z", 
-        "apache2.error.level": "notice", 
-        "apache2.error.message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", 
         "apache2.error.module": "core", 
-        "apache2.error.pid": "11379", 
         "event.dataset": "error", 
         "event.module": "apache2", 
         "input.type": "log", 
-        "log.offset": 99
+        "log.level": "notice", 
+        "log.offset": 99, 
+        "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", 
+        "process.pid": 11379
     }, 
     {
         "@timestamp": "2011-09-09T10:42:29.902Z", 
-        "apache2.error.client": "72.15.99.187", 
-        "apache2.error.level": "error", 
-        "apache2.error.message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", 
         "apache2.error.module": "core", 
-        "apache2.error.pid": "35708", 
-        "apache2.error.tid": "4328636416", 
         "event.dataset": "error", 
         "event.module": "apache2", 
         "input.type": "log", 
-        "log.offset": 229
+        "log.level": "error", 
+        "log.offset": 229, 
+        "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", 
+        "process.pid": 35708, 
+        "process.thread.id": 4328636416, 
+        "source.address": "72.15.99.187"
     }
 ]

heartbeat/docs/fields.asciidoc

@@ -2681,6 +2681,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

journalbeat/docs/fields.asciidoc

@@ -2974,6 +2974,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

libbeat/_meta/fields.ecs.yml

@@ -1740,7 +1740,17 @@
         packet/event. Source fields are usually populated in conjunction with destination fields.
       type: group
       fields:
-
+        - name: address
+          level: extended
+          type: keyword
+          description: >
+            Some event source addresses are defined ambiguously. The event will
+            sometimes list an IP, a domain or a unix socket.  You should always
+            store the raw address in the `.address` field.
+
+            Then it should be duplicated to `.ip` or `.domain`, depending on which
+            one it is.
+
         - name: ip
           level: core
           type: ip

metricbeat/docs/fields.asciidoc

@@ -5716,6 +5716,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

packetbeat/docs/fields.asciidoc

@@ -4577,6 +4577,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

winlogbeat/docs/fields.asciidoc

@@ -2594,6 +2594,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --

x-pack/functionbeat/docs/fields.asciidoc

@@ -2575,6 +2575,17 @@ Source fields describe details about the source of a packet/event. Source fields
 
 
 
+*`source.address`*::
++
+--
+type: keyword
+
+Some event source addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket.  You should always store the raw address in the `.address` field.
+Then it should be duplicated to `.ip` or `.domain`, depending on which one it is.
+
+
+--
+
 *`source.ip`*::
 +
 --