Skip to content

Commit

Permalink
[Filebeat] Update handling of elasticsearch server logs (#30018) (#30094
Browse files Browse the repository at this point in the history
)

Co-authored-by: Noémi Ványi <[email protected]>
Co-authored-by: klacabane <[email protected]>
(cherry picked from commit 27d44ce)

Co-authored-by: Mat Schaffer <[email protected]>
  • Loading branch information
mergify[bot] and matschaffer authored Jan 28, 2022
1 parent 5eb2a73 commit 3c131c7
Showing 32 changed files with 881 additions and 746 deletions.
47 changes: 47 additions & 0 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
@@ -49735,6 +49735,39 @@ example: 0

--

*`elasticsearch.elastic_product_origin`*::
+
--
Used by Elastic stack to identify which component of the stack sent the request

type: keyword

example: kibana

--

*`elasticsearch.http.request.x_opaque_id`*::
+
--
Used by Elasticsearch to throttle and deduplicate deprecation warnings

type: keyword

example: v7app

--

*`elasticsearch.event.category`*::
+
--
Category of the deprecation event

type: keyword

example: compatible_api

--


*`elasticsearch.audit.layer`*::
+
@@ -49922,6 +49955,20 @@ type: boolean

--

*`elasticsearch.audit.authentication.type`*::
+
--
type: keyword

--

*`elasticsearch.audit.opaque_id`*::
+
--
type: text

--

[float]
=== deprecation

12 changes: 12 additions & 0 deletions filebeat/module/elasticsearch/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -40,3 +40,15 @@
description: "Id of the shard"
example: "0"
type: keyword
- name: elastic_product_origin
type: keyword
description: "Used by Elastic stack to identify which component of the stack sent the request"
example: "kibana"
- name: http.request.x_opaque_id
description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings"
example: "v7app"
type: keyword
- name: event.category
description: "Category of the deprecation event"
example: "compatible_api"
type: keyword
4 changes: 4 additions & 0 deletions filebeat/module/elasticsearch/audit/_meta/fields.yml
Original file line number Diff line number Diff line change
@@ -70,3 +70,7 @@
type: text
- name: invalidate.apikeys.owned_by_authenticated_user
type: boolean
- name: authentication.type
type: keyword
- name: opaque_id
type: text
10 changes: 10 additions & 0 deletions filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
Original file line number Diff line number Diff line change
@@ -176,6 +176,16 @@ processors:
field: elasticsearch.audit.level
target_field: log.level
ignore_missing: true
- dot_expander:
field: trace.id
path: elasticsearch.audit
- rename:
field: elasticsearch.audit.trace.id
target_field: trace.id
ignore_missing: true
- remove:
field: elasticsearch.audit.trace.id
ignore_missing: true
- date:
field: elasticsearch.audit.@timestamp
target_field: "@timestamp"
6 changes: 3 additions & 3 deletions filebeat/module/elasticsearch/audit/ingest/pipeline.yml
Original file line number Diff line number Diff line change
@@ -3,9 +3,9 @@ processors:
- set:
field: event.ingested
value: '{{_ingest.timestamp}}'
- rename:
field: '@timestamp'
target_field: event.created
- set:
copy_from: "@timestamp"
field: event.created
- grok:
field: message
patterns:
3 changes: 3 additions & 0 deletions filebeat/module/elasticsearch/audit/test/test-audit-800.log
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{"type":"audit", "timestamp":"2022-01-27T14:16:25,271+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"yEUG-8deS2y8ZxGgeyeUnw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_1"], "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
{"type":"audit", "timestamp":"2022-01-27T14:16:28,601+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"qo04VI2qRzKrE1dlrsjYgw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_2"]}
{"type":"audit", "timestamp":"2022-01-27T14:16:30,950+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"rest", "event.action":"anonymous_access_denied", "origin.type":"rest", "origin.address":"[::1]:64583", "url.path":"/test_3", "request.method":"PUT", "request.id":"0ybRdKGYRAekov1eKI6nIw", "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
[
{
"@timestamp": "2022-01-27T13:16:25.271Z",
"elasticsearch.audit.action": "indices:admin/create",
"elasticsearch.audit.authentication.type": "REALM",
"elasticsearch.audit.indices": [
"test_1"
],
"elasticsearch.audit.layer": "transport",
"elasticsearch.audit.opaque_id": "myApp1",
"elasticsearch.audit.origin.type": "rest",
"elasticsearch.audit.request.id": "yEUG-8deS2y8ZxGgeyeUnw",
"elasticsearch.audit.request.name": "CreateIndexRequest",
"elasticsearch.audit.user.realm": "reserved",
"elasticsearch.audit.user.roles": [
"superuser"
],
"elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
"event.action": "access_granted",
"event.category": "database",
"event.dataset": "elasticsearch.audit",
"event.kind": "event",
"event.module": "elasticsearch",
"event.outcome": "success",
"fileset.name": "audit",
"host.id": "O8SFUsk8QpGG16JVJcNgUw",
"http.request.id": "yEUG-8deS2y8ZxGgeyeUnw",
"input.type": "log",
"log.offset": 0,
"message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:25,271+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"yEUG-8deS2y8ZxGgeyeUnw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_1\"], \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
"related.user": [
"elastic"
],
"service.type": "elasticsearch",
"source.address": "[::1]:64583",
"source.ip": "::1",
"source.port": 64583,
"trace.id": "0af7651916cd43dd8448eb211c80319c",
"user.name": "elastic"
},
{
"@timestamp": "2022-01-27T13:16:28.601Z",
"elasticsearch.audit.action": "indices:admin/create",
"elasticsearch.audit.authentication.type": "REALM",
"elasticsearch.audit.indices": [
"test_2"
],
"elasticsearch.audit.layer": "transport",
"elasticsearch.audit.origin.type": "rest",
"elasticsearch.audit.request.id": "qo04VI2qRzKrE1dlrsjYgw",
"elasticsearch.audit.request.name": "CreateIndexRequest",
"elasticsearch.audit.user.realm": "reserved",
"elasticsearch.audit.user.roles": [
"superuser"
],
"elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
"event.action": "access_granted",
"event.category": "database",
"event.dataset": "elasticsearch.audit",
"event.kind": "event",
"event.module": "elasticsearch",
"event.outcome": "success",
"fileset.name": "audit",
"host.id": "O8SFUsk8QpGG16JVJcNgUw",
"http.request.id": "qo04VI2qRzKrE1dlrsjYgw",
"input.type": "log",
"log.offset": 517,
"message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:28,601+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"qo04VI2qRzKrE1dlrsjYgw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_2\"]}",
"related.user": [
"elastic"
],
"service.type": "elasticsearch",
"source.address": "[::1]:64583",
"source.ip": "::1",
"source.port": 64583,
"user.name": "elastic"
},
{
"@timestamp": "2022-01-27T13:16:30.950Z",
"elasticsearch.audit.layer": "rest",
"elasticsearch.audit.opaque_id": "myApp1",
"elasticsearch.audit.origin.type": "rest",
"elasticsearch.audit.request.id": "0ybRdKGYRAekov1eKI6nIw",
"elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
"event.action": "anonymous_access_denied",
"event.category": "database",
"event.dataset": "elasticsearch.audit",
"event.kind": "event",
"event.module": "elasticsearch",
"event.outcome": "failure",
"fileset.name": "audit",
"host.id": "O8SFUsk8QpGG16JVJcNgUw",
"http.request.id": "0ybRdKGYRAekov1eKI6nIw",
"http.request.method": "PUT",
"input.type": "log",
"log.offset": 965,
"message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:30,950+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"url.path\":\"/test_3\", \"request.method\":\"PUT\", \"request.id\":\"0ybRdKGYRAekov1eKI6nIw\", \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
"service.type": "elasticsearch",
"source.address": "[::1]:64583",
"source.ip": "::1",
"source.port": 64583,
"trace.id": "0af7651916cd43dd8448eb211c80319c",
"url.original": "/test_3"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
processors:
- json:
field: message
target_field: elasticsearch.deprecation
- drop:
if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)'
- remove:
field: elasticsearch.deprecation.type
- dot_expander:
field: service.name
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.service.name
target_field: service.name
ignore_missing: true
- rename:
field: elasticsearch.deprecation.level
target_field: log.level
ignore_missing: true
- dot_expander:
field: log.level
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.log.level
target_field: log.level
ignore_missing: true
- dot_expander:
field: log.logger
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.log.logger
target_field: log.logger
ignore_missing: true
- dot_expander:
field: process.thread.name
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.process.thread.name
target_field: process.thread.name
ignore_missing: true
- rename:
field: elasticsearch.deprecation.component
target_field: elasticsearch.component
ignore_missing: true
- dot_expander:
field: cluster.name
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.cluster.name
target_field: elasticsearch.cluster.name
- dot_expander:
field: node.name
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.node.name
target_field: elasticsearch.node.name
- dot_expander:
field: cluster.uuid
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.cluster.uuid
target_field: elasticsearch.cluster.uuid
ignore_missing: true
- dot_expander:
field: node.id
path: elasticsearch.deprecation
- rename:
field: elasticsearch.deprecation.node.id
target_field: elasticsearch.node.id
ignore_missing: true
- remove:
field: message
- rename:
field: elasticsearch.deprecation.message
target_field: message
- date:
field: 'elasticsearch.deprecation.@timestamp'
formats:
- ISO8601
ignore_failure: true
if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null'
- date:
field: 'elasticsearch.deprecation.timestamp'
formats:
- ISO8601
ignore_failure: true
if: 'ctx.elasticsearch?.deprecation?.timestamp != null'
- remove:
field:
- elasticsearch.deprecation.timestamp
- elasticsearch.deprecation.@timestamp
ignore_missing: true
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
on_failure:
- set:
field: error.message
value: '{{ _ingest.on_failure_message }}'
processors:
- json:
field: message
add_to_root: true
- dot_expander:
field: '*'
override: true
- set:
field: event.dataset
value: elasticsearch.deprecation
Loading

0 comments on commit 3c131c7

Please sign in to comment.