[filebeat] add 8.x kibana logs ingest pipeline (#31286)

* add routing pipeline to 7 or ecs

* simplify ecs pipeline

* flatten headers

* kibana 8.x logs integration test

* shorter condition

(cherry picked from commit 47777ec)

# Conflicts:
#	filebeat/docs/fields.asciidoc
#	filebeat/module/kibana/log/ingest/pipeline.yml

filebeat/docs/fields.asciidoc

@@ -86862,6 +86862,7 @@ type: keyword
 
 --
 
+<<<<<<< HEAD
 *`rsa.time.gmttime`*::
 +
 --
@@ -86910,6 +86911,19 @@ type: keyword
 This key is used to capture incomplete timestamp that explicitly refers to an expiration.
 
 type: keyword
+=======
+*`kibana.log.meta.req.headers`*::
++
+--
+type: flattened
+
+--
+
+*`kibana.log.meta.res.headers`*::
++
+--
+type: flattened
+>>>>>>> 47777ec1dc ([filebeat] add 8.x kibana logs ingest pipeline (#31286))
 
 --
 

filebeat/module/kibana/log/_meta/fields.yml

@@ -15,31 +15,7 @@
       type: object
       object_type: keyword
 
-    - name: kibana.log.meta.req.headers.referer
-      type: alias
-      path: http.request.referrer
-      migration: true
-    - name: kibana.log.meta.req.referer
-      type: alias
-      path: http.request.referrer
-      migration: true
-    - name: kibana.log.meta.req.headers.user-agent
-      type: alias
-      path: user_agent.original
-      migration: true
-    - name: kibana.log.meta.req.remoteAddress
-      type: alias
-      path: source.address
-      migration: true
-    - name: kibana.log.meta.req.url
-      type: alias
-      path: url.original
-      migration: true
-    - name: kibana.log.meta.statusCode
-      type: alias
-      path: http.response.status_code
-      migration: true
-    - name: kibana.log.meta.method
-      type: alias
-      path: http.request.method
-      migration: true
+    - name: meta.req.headers
+      type: flattened
+    - name: meta.res.headers
+      type: flattened

filebeat/module/kibana/log/ingest/pipeline-7.yml

@@ -0,0 +1,104 @@
+description: Pipeline for parsing Kibana logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- rename:
+    field: json
+    target_field: kibana.log.meta
+- date:
+    field: kibana.log.meta.@timestamp
+    formats:
+    - ISO8601
+    target_field: '@timestamp'
+- remove:
+    field: kibana.log.meta.@timestamp
+- rename:
+    field: kibana.log.meta.message
+    target_field: message
+- rename:
+    field: kibana.log.meta.state
+    target_field: kibana.log.state
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.pid
+    target_field: process.pid
+- rename:
+    field: kibana.log.meta.tags
+    target_field: kibana.log.tags
+- rename:
+    field: kibana.log.meta.res.statusCode
+    target_field: http.response.status_code
+    ignore_missing: true
+- script:
+    lang: painless
+    source: ctx.event.duration = Math.round(ctx.kibana.log.meta.res.responseTime * 1000000L)
+    if: ctx?.kibana?.log?.meta?.res?.responseTime != null
+- remove:
+    field: kibana.log.meta.res.responseTime
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.res.contentLength
+    target_field: http.response.body.bytes
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.method
+    target_field: http.request.method
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.referer
+    target_field: http.request.referrer
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.headers.user-agent
+    target_field: user_agent.original
+    ignore_missing: true
+- rename:
+    field: kibana.log.meta.req.remoteAddress
+    target_field: source.address
+    ignore_missing: true
+- set:
+    field: source.ip
+    value: '{{source.address}}'
+    ignore_empty_value: true
+- rename:
+    field: kibana.log.meta.req.url
+    target_field: url.original
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.req.referer
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.statusCode
+    ignore_missing: true
+- remove:
+    field: kibana.log.meta.method
+    ignore_missing: true
+- append:
+    field: service.name
+    value: kibana
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    source: >-
+      if (ctx?.kibana?.log?.state != null) {
+        if (ctx.kibana.log.state == "red") {
+          ctx.event.type = "error";
+        } else {
+          ctx.event.type = "info";
+        }
+      }
+
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400"
+- set:
+    field: event.outcome
+    value: failure
+    if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"

filebeat/module/kibana/log/ingest/pipeline-ecs.yml

@@ -0,0 +1,29 @@
+description: Pipeline for parsing Kibana ecs logs
+processors:
+- set:
+    field: event.ingested
+    value: '{{_ingest.timestamp}}'
+- set:
+    copy_from: '@timestamp'
+    field: event.created
+- script:
+    lang: painless
+    inline: 'ctx.json.keySet().each (key -> ctx[key] = ctx.json.get(key))'
+- remove:
+    field: json
+- rename:
+    field: http.request.headers
+    target_field: kibana.log.meta.req.headers
+    ignore_missing: true
+- rename:
+    field: http.response.headers
+    target_field: kibana.log.meta.res.headers
+    ignore_missing: true
+- set:
+    field: event.outcome
+    value: success
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code < 400'
+- set:
+    field: event.outcome
+    value: failure
+    if: 'ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400'

filebeat/module/kibana/log/ingest/pipeline.yml

@@ -4,6 +4,7 @@ on_failure:
     field: error.message
     value: '{{ _ingest.on_failure_message }}'
 processors:
+<<<<<<< HEAD
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
@@ -106,3 +107,11 @@ processors:
     field: event.outcome
     value: failure
     if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400"
+=======
+- pipeline:
+    if: 'ctx?.json?.ecs?.version == null'
+    name: '{< IngestPipeline "pipeline-7" >}'
+- pipeline:
+    if: 'ctx?.json?.ecs?.version != null'
+    name: '{< IngestPipeline "pipeline-ecs" >}'
+>>>>>>> 47777ec1dc ([filebeat] add 8.x kibana logs ingest pipeline (#31286))

filebeat/module/kibana/log/manifest.yml

@@ -5,5 +5,8 @@ var:
     default:
       - /var/log/kibana/kibana.stdout
 
-ingest_pipeline: ingest/pipeline.yml
+ingest_pipeline:
+  - ingest/pipeline.yml
+  - ingest/pipeline-7.yml
+  - ingest/pipeline-ecs.yml
 input: config/log.yml

filebeat/module/kibana/log/test/log.830.log

@@ -0,0 +1,4 @@
+{"http":{"request":{"id":"unknownId","method":"POST","headers":{"user-agent":"elastic-transport-js/8.0.2 (darwin 21.4.0-x64; Node.js v16.14.2)","x-elastic-product-origin":"kibana","x-elastic-client-meta":"es=8.2.0p,js=16.14.2,t=8.0.2,hc=16.14.2","x-opaque-id":"unknownId","accept":"application/vnd.elasticsearch+json; compatible-with=8,text/plain"}},"response":{"body":{"bytes":201},"status_code":200,"headers":{"x-opaque-id":"unknownId","x-elastic-product":"Elasticsearch","content-type":"application/json","content-length":"201"}}},"url":{"path":"/.kibana_task_manager_8.3.0_001/_pit","query":"keep_alive=10m"},"ecs":{"version":"8.0.0"},"@timestamp":"2022-04-20T18:46:21.484+02:00","message":"200 - 201.0B\nPOST /.kibana_task_manager_8.3.0_001/_pit?keep_alive=10m","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":78667},"span":{"id":"bc7bc9d23d6710b8"},"trace":{"id":"0f59c7a01606546871443db5ea8ee81a"}}
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-04-20T18:46:21.484+02:00","message":"[.kibana_task_manager] OUTDATED_DOCUMENTS_SEARCH_OPEN_PIT -> OUTDATED_DOCUMENTS_SEARCH_READ. took: 5ms.","log":{"level":"INFO","logger":"savedobjects-service"},"process":{"pid":78667},"span":{"id":"35084baec283deb4"},"trace":{"id":"0f59c7a01606546871443db5ea8ee81a"}}
+{"http":{"request":{"id":"unknownId","method":"POST","headers":{"user-agent":"elastic-transport-js/8.0.2 (darwin 21.4.0-x64; Node.js v16.14.2)","x-elastic-product-origin":"kibana","x-elastic-client-meta":"es=8.2.0p,js=16.14.2,t=8.0.2,hc=16.14.2","x-opaque-id":"unknownId","content-type":"application/vnd.elasticsearch+json; compatible-with=8","accept":"application/vnd.elasticsearch+json; compatible-with=8","content-length":"6076"}},"response":{"body":{"bytes":344},"status_code":200,"headers":{"x-opaque-id":"unknownId","x-elastic-product":"Elasticsearch","content-type":"application/vnd.elasticsearch+json;compatible-with=8","content-length":"344"}}},"url":{"path":"/_search","query":""},"ecs":{"version":"8.0.0"},"@timestamp":"2022-04-20T18:46:21.486+02:00","message":"200 - 344.0B\nPOST /_search\n{\"sort\":{\"_shard_doc\":{\"order\":\"asc\"}},\"pit\":{\"id\":\"k4_qAwERLmtpYmFuYV84LjMuMF8wMDEWMFh6RkhHN2NUdDZ2cS16WjRsUUs1UQAWVjFzSkhLV21RNzJKY1NJYlRKQkh2QQAAAAAAAACGkhZNMWx0T1Nhd1M2MnNWbjJ3VTVYTDVRAAEWMFh6RkhHN2NUdDZ2cS16WjRsUUs1UQAA\",\"keep_alive\":\"10m\"},\"size\":1000,\"track_total_hits\":true,\"query\":{\"bool\":{\"should\":[{\"bool\":{\"must\":{\"term\":{\"type\":\"core-usage-stats\"}},\"must_not\":{\"term\":{\"migrationVersion.core-usage-stats\":\"7.14.1\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"legacy-url-alias\"}},\"must_not\":{\"term\":{\"migrationVersion.legacy-url-alias\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"config\"}},\"must_not\":{\"term\":{\"migrationVersion.config\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"task\"}},\"must_not\":{\"term\":{\"migrationVersion.task\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"index-pattern\"}},\"must_not\":{\"term\":{\"migrationVersion.index-pattern\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"space\"}},\"must_not\":{\"term\":{\"migrationVersion.space\":\"6.6.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"spaces-usage-stats\"}},\"must_not\":{\"term\":{\"migrationVersion.spaces-usage-stats\":\"7.14.1\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"exception-list-agnostic\"}},\"must_not\":{\"term\":{\"migrationVersion.exception-list-agnostic\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"exception-list\"}},\"must_not\":{\"term\":{\"migrationVersion.exception-list\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"action\"}},\"must_not\":{\"term\":{\"migrationVersion.action\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"action_task_params\"}},\"must_not\":{\"term\":{\"migrationVersion.action_task_params\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"query\"}},\"must_not\":{\"term\":{\"migrationVersion.query\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search-telemetry\"}},\"must_not\":{\"term\":{\"migrationVersion.search-telemetry\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search-session\"}},\"must_not\":{\"term\":{\"migrationVersion.search-session\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"alert\"}},\"must_not\":{\"term\":{\"migrationVersion.alert\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest_manager_settings\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest_manager_settings\":\"7.13.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-agent-policies\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-agent-policies\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-outputs\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-outputs\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ingest-package-policies\"}},\"must_not\":{\"term\":{\"migrationVersion.ingest-package-policies\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"epm-packages\"}},\"must_not\":{\"term\":{\"migrationVersion.epm-packages\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"graph-workspace\"}},\"must_not\":{\"term\":{\"migrationVersion.graph-workspace\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"tag\"}},\"must_not\":{\"term\":{\"migrationVersion.tag\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"visualization\"}},\"must_not\":{\"term\":{\"migrationVersion.visualization\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-element\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-element\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-workpad\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-workpad\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"canvas-workpad-template\"}},\"must_not\":{\"term\":{\"migrationVersion.canvas-workpad-template\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"dashboard\"}},\"must_not\":{\"term\":{\"migrationVersion.dashboard\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"search\"}},\"must_not\":{\"term\":{\"migrationVersion.search\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"lens\"}},\"must_not\":{\"term\":{\"migrationVersion.lens\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"map\"}},\"must_not\":{\"term\":{\"migrationVersion.map\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-job\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-job\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-trained-model\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-trained-model\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"ml-module\"}},\"must_not\":{\"term\":{\"migrationVersion.ml-module\":\"7.10.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-comments\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-comments\":\"8.3.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-configure\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-configure\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-connector-mappings\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-connector-mappings\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases\"}},\"must_not\":{\"term\":{\"migrationVersion.cases\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"cases-user-actions\"}},\"must_not\":{\"term\":{\"migrationVersion.cases-user-actions\":\"8.1.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline-note\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline-note\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline-pinned-event\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline-pinned-event\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-detection-engine-rule-actions\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-detection-engine-rule-actions\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-detection-engine-rule-execution-info\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-detection-engine-rule-execution-info\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"siem-ui-timeline\"}},\"must_not\":{\"term\":{\"migrationVersion.siem-ui-timeline\":\"8.0.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"endpoint:user-artifact-manifest\"}},\"must_not\":{\"term\":{\"migrationVersion.endpoint:user-artifact-manifest\":\"7.12.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"uptime-dynamic-settings\"}},\"must_not\":{\"term\":{\"migrationVersion.uptime-dynamic-settings\":\"8.2.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"infrastructure-ui-source\"}},\"must_not\":{\"term\":{\"migrationVersion.infrastructure-ui-source\":\"7.16.2\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"upgrade-assistant-telemetry\"}},\"must_not\":{\"term\":{\"migrationVersion.upgrade-assistant-telemetry\":\"7.16.0\"}}}},{\"bool\":{\"must\":{\"term\":{\"type\":\"apm-indices\"}},\"must_not\":{\"term\":{\"migrationVersion.apm-indices\":\"8.2.0\"}}}}]}}}","log":{"level":"DEBUG","logger":"elasticsearch.query.data"},"process":{"pid":78667},"span":{"id":"109794cfe74be5ee"},"trace":{"id":"0f59c7a01606546871443db5ea8ee81a"}}
+{"ecs":{"version":"8.0.0"},"@timestamp":"2022-04-20T18:46:21.500+02:00","message":"[.kibana_task_manager] UPDATE_TARGET_MAPPINGS -> UPDATE_TARGET_MAPPINGS_WAIT_FOR_TASK. took: 8ms.","log":{"level":"INFO","logger":"savedobjects-service"},"process":{"pid":78667},"span":{"id":"35084baec283deb4"},"trace":{"id":"0f59c7a01606546871443db5ea8ee81a"}}