Convert Filebeat iis.error to ECS and add IPv6 zone support for IIS a…

…ccess logs (#9955) - Introduce IPv6 zone workaround iis.access log as well, resolving #9836. - Update the IPv6 zone fix (#9869) for iis.error to use the ECS `.address` field instead of a transient field. - Convert many fields under `iis.error.*` to ECS. Previous field names are field aliases towards the new corresponding ECS field: - iis.error.remote_ip => source.address - iis.error.remote_port => source.port - iis.error.server_ip => destination.address - iis.error.server_port => destination.port - iis.error.http_version => http.version - iis.error.method => http.request.method - iis.error.url => url.original - iis.error.response_code => http.response.status_code - iis.error.geoip.* => source.geo.* - read_timestamp => event.created (not aliased, still used elsewhere) - Update field aliases introduced in #9084 to point to `.address` instead of `.ip`, since this value can be ambiguous. The IP field is populated with the cleaned up IP without the zone. This is also true for the `.ip` fields under populated by the error logs. - iis.access.remote_ip => source.address - iis.access.server_ip => destination.address - Coerce to long: source.port, destination.port and http.response.status_code in the iis.error fileset
elastic · Jan 11, 2019 · cf7b152 · cf7b152
1 parent 383ccce
commit cf7b152
Show file tree

Hide file tree

Showing 14 changed files with 323 additions and 197 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Modify apache/error dataset to follow ECS. {pull}8963[8963]
 - Rename many `traefik.access.*` fields to map to ECS. {pull}9005[9005]
 - Fix parsing of GC entries in elasticsearch server log. {issue}9513[9513] {pull}9810[9810]
+- Rename many `iis.error.*` fields to map to ECS. {pull}9955[9955]
 - Rename a few `logstash.*` fields to map to ECS, remove logstash.slowlog.message. {pull}9935[9935]
 - Rename a few `mysql.*` fields to map to ECS. {pull}10008[10008]
 - Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007]
@@ -67,7 +68,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Filebeat*
 
 - Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761]
-- Support IPv6 addresses with zone id in IIS ingest pipeline. {issue}9836[9836] {pull}9869[9869]
+- Support IPv6 addresses with zone id in IIS ingest pipeline.
+  {issue}9836[9836] error log: {pull}9869[9869], access log: {pull}9955[9955].
 - Support haproxy log lines without captured headers. {issue}9463[9463] {pull}9958[9958]
 
 *Heartbeat*

diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -281,11 +281,11 @@
 ## IIS module
 
 - from: iis.access.server_ip
-  to: destination.ip
+  to: destination.address
   alias: true
 
 - from: iis.access.remote_ip
-  to: source.ip
+  to: source.address
   alias: true
 
 - from: iis.access.url
@@ -348,6 +348,62 @@
   to: http.request.referrer
   alias: true
 
+- from: iis.error.remote_ip
+  to: source.address
+  alias: true
+
+- from: iis.error.remote_port
+  to: source.port
+  alias: true
+
+- from: iis.error.server_ip
+  to: destination.address
+  alias: true
+
+- from: iis.error.server_port
+  to: destination.port
+  alias: true
+
+- from: iis.error.http_version
+  to: http.version
+  alias: true
+
+- from: iis.error.method
+  to: http.request.method
+  alias: true
+
+- from: iis.error.url
+  to: url.original
+  alias: true
+
+- from: iis.error.response_code
+  to: http.response.status_code
+  alias: true
+
+- from: iis.error.geoip.continent_name
+  to: source.geo.continent_name
+  alias: true
+
+- from: iis.error.geoip.country_iso_code
+  to: source.geo.country_iso_code
+  alias: true
+
+- from: iis.error.geoip.location
+  to: source.geo.location
+  alias: true
+
+- from: iis.error.geoip.region_name
+  to: source.geo.region_name
+  alias: true
+
+- from: iis.error.geoip.city_name
+  to: source.geo.city_name
+  alias: true
+
+- from: iis.error.geoip.region_iso_code
+  to: source.geo.region_iso_code
+  alias: true
+
 ## HAProxy module
 
 - from: haproxy.client.port

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -5163,7 +5163,7 @@ The number of bytes of the server request body.
 --
 type: alias
 
-alias to: destination.ip
+alias to: destination.address
 
 --
 
@@ -5217,7 +5217,7 @@ alias to: user.name
 --
 type: alias
 
-alias to: source.ip
+alias to: source.address
 
 --
 
@@ -5410,172 +5410,150 @@ Contains fields for IIS error logs.
 
 
 
-*`iis.error.remote_ip`*::
+*`iis.error.reason_phrase`*::
 +
 --
 type: keyword
 
-The client IP address.
+The HTTP reason phrase.
 
 
 --
 
-*`iis.error.remote_port`*::
+*`iis.error.queue_name`*::
 +
 --
-type: long
+type: keyword
 
-The client port number.
+The IIS application pool name.
 
 
 --
 
-*`iis.error.server_ip`*::
+*`iis.error.remote_ip`*::
 +
 --
-type: keyword
-
-The server IP address.
+type: alias
 
+alias to: source.address
 
 --
 
-*`iis.error.server_port`*::
+*`iis.error.remote_port`*::
 +
 --
-type: long
-
-The server port number.
+type: alias
 
+alias to: source.port
 
 --
 
-*`iis.error.http_version`*::
+*`iis.error.server_ip`*::
 +
 --
-type: keyword
-
-The HTTP version.
+type: alias
 
+alias to: destination.address
 
 --
 
-*`iis.error.method`*::
+*`iis.error.server_port`*::
 +
 --
-type: keyword
-
-example: GET
-
-The request HTTP method.
+type: alias
 
+alias to: destination.port
 
 --
 
-*`iis.error.url`*::
+*`iis.error.http_version`*::
 +
 --
-type: keyword
-
-The request HTTP URL.
+type: alias
 
+alias to: http.version
 
 --
 
-*`iis.error.response_code`*::
+*`iis.error.method`*::
 +
 --
-type: long
-
-The HTTP response code.
+type: alias
 
+alias to: http.request.method
 
 --
 
-*`iis.error.reason_phrase`*::
+*`iis.error.url`*::
 +
 --
-type: keyword
-
-The HTTP reason phrase.
+type: alias
 
+alias to: url.original
 
 --
 
-*`iis.error.queue_name`*::
+*`iis.error.response_code`*::
 +
 --
-type: keyword
-
-The IIS application pool name.
+type: alias
 
+alias to: http.response.status_code
 
 --
 
-[float]
-== geoip fields
-
-Contains GeoIP information gathered based on the remote_ip field. Only present if the GeoIP Elasticsearch plugin is available and used.
-
-
 
 *`iis.error.geoip.continent_name`*::
 +
 --
-type: keyword
-
-The name of the continent.
+type: alias
 
+alias to: source.geo.continent_name
 
 --
 
 *`iis.error.geoip.country_iso_code`*::
 +
 --
-type: keyword
-
-Country ISO code.
+type: alias
 
+alias to: source.geo.country_iso_code
 
 --
 
 *`iis.error.geoip.location`*::
 +
 --
-type: geo_point
-
-The longitude and latitude.
+type: alias
 
+alias to: source.geo.location
 
 --
 
 *`iis.error.geoip.region_name`*::
 +
 --
-type: keyword
-
-The region name.
+type: alias
 
+alias to: source.geo.region_name
 
 --
 
 *`iis.error.geoip.city_name`*::
 +
 --
-type: keyword
-
-The city name.
+type: alias
 
+alias to: source.geo.city_name
 
 --
 
 *`iis.error.geoip.region_iso_code`*::
 +
 --
-type: keyword
-
-Region ISO code.
+type: alias
 
+alias to: source.geo.region_iso_code
 
 --
 

diff --git a/filebeat/module/iis/access/_meta/fields.yml b/filebeat/module/iis/access/_meta/fields.yml
@@ -40,7 +40,7 @@
 
     - name: server_ip
       type: alias
-      path: destination.ip
+      path: destination.address
       migration: true
     - name: method
       type: alias
@@ -64,7 +64,7 @@
       migration: true
     - name: remote_ip
       type: alias
-      path: source.ip
+      path: source.address
       migration: true
     - name: referrer
       type: alias