[filebeat] add 8.x kibana logs ingest pipeline (#31286)

* add routing pipeline to 7 or ecs

* simplify ecs pipeline

* flatten headers

* kibana 8.x logs integration test

* shorter condition

filebeat/docs/fields.asciidoc

@@ -86813,66 +86813,17 @@ type: object
 
 --
 
-*`kibana.log.kibana.log.meta.req.headers.referer`*::
+*`kibana.log.meta.req.headers`*::
 +
 --
-type: alias
-
-alias to: http.request.referrer
-
---
-
-*`kibana.log.kibana.log.meta.req.referer`*::
-+
---
-type: alias
-
-alias to: http.request.referrer
-
---
-
-*`kibana.log.kibana.log.meta.req.headers.user-agent`*::
-+
---
-type: alias
-
-alias to: user_agent.original
-
---
-
-*`kibana.log.kibana.log.meta.req.remoteAddress`*::
-+
---
-type: alias
-
-alias to: source.address
-
---
-
-*`kibana.log.kibana.log.meta.req.url`*::
-+
---
-type: alias
-
-alias to: url.original
-
---
-
-*`kibana.log.kibana.log.meta.statusCode`*::
-+
---
-type: alias
-
-alias to: http.response.status_code
+type: flattened
 
 --
 
-*`kibana.log.kibana.log.meta.method`*::
+*`kibana.log.meta.res.headers`*::
 +
 --
-type: alias
-
-alias to: http.request.method
+type: flattened
 
 --
 

filebeat/module/kibana/log/_meta/fields.yml

@@ -15,31 +15,7 @@
       type: object
       object_type: keyword
 
-    - name: kibana.log.meta.req.headers.referer
-      type: alias
-      path: http.request.referrer
-      migration: true
-    - name: kibana.log.meta.req.referer
-      type: alias
-      path: http.request.referrer
-      migration: true
-    - name: kibana.log.meta.req.headers.user-agent
-      type: alias
-      path: user_agent.original
-      migration: true
-    - name: kibana.log.meta.req.remoteAddress
-      type: alias
-      path: source.address
-      migration: true
-    - name: kibana.log.meta.req.url
-      type: alias
-      path: url.original
-      migration: true
-    - name: kibana.log.meta.statusCode
-      type: alias
-      path: http.response.status_code
-      migration: true
-    - name: kibana.log.meta.method
-      type: alias
-      path: http.request.method
-      migration: true
+    - name: meta.req.headers
+      type: flattened
+    - name: meta.res.headers
+      type: flattened

filebeat/module/kibana/log/ingest/pipeline-7.yml

@@ -0,0 +1,104 @@
+description: Pipeline for parsing Kibana logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- rename:
+    field: json
+    target_field: kibana.log.meta
+- date:
+    field: kibana.log.meta.@timestamp
+    formats:
+    - ISO8601
+    target_field: '@timestamp'
+- remove:
+    field: kibana.log.meta.@timestamp
+- rename:
+    field: kibana.log.meta.message
+    target_field: message
+- rename:
+    field: kibana.log.meta.state
+    target_field: kibana.log.state
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.pid
+    target_field: process.pid
+- rename:
+    field: kibana.log.meta.tags
+    target_field: kibana.log.tags
+- rename:
+    field: kibana.log.meta.res.statusCode
+    target_field: http.response.status_code
+    ignore_missing: true
+- script:
+    lang: painless
+    source: ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * 1000000L)
+    if: ctx?.kibana?.log?.meta?.res?.responseTime != null
+- remove:
+    field: kibana.log.meta.res.responseTime
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.res.contentLength
+    target_field: http.response.body.bytes
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.method
+    target_field: http.request.method
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.referer
+    target_field: http.request.referrer
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.user-agent
+    target_field: user_agent.original
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.remoteAddress
+    target_field: source.address
+    ignore_missing: true
+- set:
+    field: source.ip
+    value: '{{source.address}}'
+    ignore_empty_value: true
+- rename:
+    field: kibana.log.meta.req.url
+    target_field: url.original
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.req.referer
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.statusCode
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.method
+    ignore_missing: true
+- append:
+    field: service.name
+    value: kibana
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      if (ctx?.kibana?.log?.state != null) {
+        if (ctx.kibana.log.state == "red") {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"

filebeat/module/kibana/log/ingest/pipeline-ecs.yml

@@ -0,0 +1,29 @@
+description: Pipeline for parsing Kibana ecs logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- script:
+    lang: painless
+    inline: 'ctx.json.keySet().each (key -> ctx[key] = ctx.json.get(key))'
+- remove:
+    field: json
+- rename:
+    field: http.request.headers
+    target_field: kibana.log.meta.req.headers
+    ignore_missing: true
+- rename:
+    field: http.response.headers
+    target_field: kibana.log.meta.res.headers
+    ignore_missing: true
+- set:
+    field: event.outcome
+    value: success
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
+- set:
+    field: event.outcome
+    value: failure
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'

filebeat/module/kibana/log/ingest/pipeline.yml

@@ -4,105 +4,9 @@ on_failure:
     field: error.message
     value: '{{ _ingest.on_failure_message }}'
 processors:
-- set:
-    field: event.ingested
-    value: '{{_ingest.timestamp}}'
-- set:
-    copy_from: '@timestamp'
-    field: event.created
-- rename:
-    field: json
-    target_field: kibana.log.meta
-- date:
-    field: kibana.log.meta.@timestamp
-    formats:
-    - ISO8601
-    target_field: '@timestamp'
-- remove:
-    field: kibana.log.meta.@timestamp
-- rename:
-    field: kibana.log.meta.message
-    target_field: message
-- rename:
-    field: kibana.log.meta.state
-    target_field: kibana.log.state
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.pid
-    target_field: process.pid
-- rename:
-    field: kibana.log.meta.tags
-    target_field: kibana.log.tags
-- rename:
-    field: kibana.log.meta.res.statusCode
-    target_field: http.response.status_code
-    ignore_missing: true
-- script:
-    lang: painless
-    source: ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * 1000000L)
-    if: ctx?.kibana?.log?.meta?.res?.responseTime != null
-- remove:
-    field: kibana.log.meta.res.responseTime
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.res.contentLength
-    target_field: http.response.body.bytes
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.req.method
-    target_field: http.request.method
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.req.headers.referer
-    target_field: http.request.referrer
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.req.headers.user-agent
-    target_field: user_agent.original
-    ignore_missing: true
-- rename:
-    field: kibana.log.meta.req.remoteAddress
-    target_field: source.address
-    ignore_missing: true
-- set:
-    field: source.ip
-    value: '{{source.address}}'
-    ignore_empty_value: true
-- rename:
-    field: kibana.log.meta.req.url
-    target_field: url.original
-    ignore_missing: true
-- remove:
-    field: kibana.log.meta.req.referer
-    ignore_missing: true
-- remove:
-    field: kibana.log.meta.statusCode
-    ignore_missing: true
-- remove:
-    field: kibana.log.meta.method
-    ignore_missing: true
-- append:
-    field: service.name
-    value: kibana
-- set:
-    field: event.kind
-    value: event
-- script:
-    lang: painless
-    source: >-
-      if (ctx?.kibana?.log?.state != null) {
-        if (ctx.kibana.log.state == "red") {
-          ctx.event.type = "error";
-        } else {
-          ctx.event.type = "info";
-        }
-      }
-
-- set:
-    field: event.outcome
-    value: success
-    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
-- set:
-    field: event.outcome
-    value: failure
-    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
+- pipeline:
+    if: 'ctx?.json?.ecs?.version == null'
+    name: '{< IngestPipeline "pipeline-7" >}'
+- pipeline:
+    if: 'ctx?.json?.ecs?.version != null'
+    name: '{< IngestPipeline "pipeline-ecs" >}'

filebeat/module/kibana/log/manifest.yml

@@ -5,5 +5,8 @@ var:
     default:
       - /var/log/kibana/kibana.stdout
 
-ingest_pipeline: ingest/pipeline.yml
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/pipeline-7.yml
+  - ingest/pipeline-ecs.yml
 input: config/log.yml