[Filebeat] Update handling of elasticsearch server logs (#30018)

Co-authored-by: Noémi Ványi <[email protected]> Co-authored-by: klacabane <[email protected]>
elastic · Jan 28, 2022 · 27d44ce · 27d44ce
1 parent 1306e5f
commit 27d44ce
Show file tree

Hide file tree

Showing 32 changed files with 881 additions and 746 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -49735,6 +49735,39 @@ example: 0
 
 --
 
+*`elasticsearch.elastic_product_origin`*::
++
+--
+Used by Elastic stack to identify which component of the stack sent the request
+
+type: keyword
+
+example: kibana
+
+--
+
+*`elasticsearch.http.request.x_opaque_id`*::
++
+--
+Used by Elasticsearch to throttle and deduplicate deprecation warnings
+
+type: keyword
+
+example: v7app
+
+--
+
+*`elasticsearch.event.category`*::
++
+--
+Category of the deprecation event
+
+type: keyword
+
+example: compatible_api
+
+--
+
 
 *`elasticsearch.audit.layer`*::
 +
@@ -49922,6 +49955,20 @@ type: boolean
 
 --
 
+*`elasticsearch.audit.authentication.type`*::
++
+--
+type: keyword
+
+--
+
+*`elasticsearch.audit.opaque_id`*::
++
+--
+type: text
+
+--
+
 [float]
 === deprecation
 

diff --git a/filebeat/module/elasticsearch/_meta/fields.yml b/filebeat/module/elasticsearch/_meta/fields.yml
@@ -40,3 +40,15 @@
           description: "Id of the shard"
           example: "0"
           type: keyword
+        - name: elastic_product_origin
+          type: keyword
+          description: "Used by Elastic stack to identify which component of the stack sent the request"
+          example: "kibana"
+        - name: http.request.x_opaque_id
+          description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings"
+          example: "v7app"
+          type: keyword
+        - name: event.category
+          description: "Category of the deprecation event"
+          example: "compatible_api"
+          type: keyword
diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml
@@ -70,3 +70,7 @@
       type: text
     - name: invalidate.apikeys.owned_by_authenticated_user
       type: boolean
+    - name: authentication.type
+      type: keyword
+    - name: opaque_id
+      type: text
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
@@ -176,6 +176,16 @@ processors:
       field: elasticsearch.audit.level
       target_field: log.level
       ignore_missing: true
+  - dot_expander:
+      field: trace.id
+      path: elasticsearch.audit
+  - rename:
+      field: elasticsearch.audit.trace.id
+      target_field: trace.id
+      ignore_missing: true
+  - remove:
+      field: elasticsearch.audit.trace.id
+      ignore_missing: true
   - date:
       field: elasticsearch.audit.@timestamp
       target_field: "@timestamp"

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml
@@ -3,9 +3,9 @@ processors:
 - set:
     field: event.ingested
     value: '{{_ingest.timestamp}}'
-- rename:
-    field: '@timestamp'
-    target_field: event.created
+- set:
+    copy_from: "@timestamp"
+    field: event.created
 - grok:
     field: message
     patterns:

diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log b/filebeat/module/elasticsearch/audit/test/test-audit-800.log
@@ -0,0 +1,3 @@
+{"type":"audit", "timestamp":"2022-01-27T14:16:25,271+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"yEUG-8deS2y8ZxGgeyeUnw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_1"], "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
+{"type":"audit", "timestamp":"2022-01-27T14:16:28,601+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"qo04VI2qRzKrE1dlrsjYgw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_2"]}
+{"type":"audit", "timestamp":"2022-01-27T14:16:30,950+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"rest", "event.action":"anonymous_access_denied", "origin.type":"rest", "origin.address":"[::1]:64583", "url.path":"/test_3", "request.method":"PUT", "request.id":"0ybRdKGYRAekov1eKI6nIw", "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"}
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json
@@ -0,0 +1,105 @@
+[
+    {
+        "@timestamp": "2022-01-27T13:16:25.271Z",
+        "elasticsearch.audit.action": "indices:admin/create",
+        "elasticsearch.audit.authentication.type": "REALM",
+        "elasticsearch.audit.indices": [
+            "test_1"
+        ],
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.opaque_id": "myApp1",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "yEUG-8deS2y8ZxGgeyeUnw",
+        "elasticsearch.audit.request.name": "CreateIndexRequest",
+        "elasticsearch.audit.user.realm": "reserved",
+        "elasticsearch.audit.user.roles": [
+            "superuser"
+        ],
+        "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "event.action": "access_granted",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.audit",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.outcome": "success",
+        "fileset.name": "audit",
+        "host.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "http.request.id": "yEUG-8deS2y8ZxGgeyeUnw",
+        "input.type": "log",
+        "log.offset": 0,
+        "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:25,271+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"yEUG-8deS2y8ZxGgeyeUnw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_1\"], \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
+        "related.user": [
+            "elastic"
+        ],
+        "service.type": "elasticsearch",
+        "source.address": "[::1]:64583",
+        "source.ip": "::1",
+        "source.port": 64583,
+        "trace.id": "0af7651916cd43dd8448eb211c80319c",
+        "user.name": "elastic"
+    },
+    {
+        "@timestamp": "2022-01-27T13:16:28.601Z",
+        "elasticsearch.audit.action": "indices:admin/create",
+        "elasticsearch.audit.authentication.type": "REALM",
+        "elasticsearch.audit.indices": [
+            "test_2"
+        ],
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "qo04VI2qRzKrE1dlrsjYgw",
+        "elasticsearch.audit.request.name": "CreateIndexRequest",
+        "elasticsearch.audit.user.realm": "reserved",
+        "elasticsearch.audit.user.roles": [
+            "superuser"
+        ],
+        "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "event.action": "access_granted",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.audit",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.outcome": "success",
+        "fileset.name": "audit",
+        "host.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "http.request.id": "qo04VI2qRzKrE1dlrsjYgw",
+        "input.type": "log",
+        "log.offset": 517,
+        "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:28,601+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"qo04VI2qRzKrE1dlrsjYgw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_2\"]}",
+        "related.user": [
+            "elastic"
+        ],
+        "service.type": "elasticsearch",
+        "source.address": "[::1]:64583",
+        "source.ip": "::1",
+        "source.port": 64583,
+        "user.name": "elastic"
+    },
+    {
+        "@timestamp": "2022-01-27T13:16:30.950Z",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.opaque_id": "myApp1",
+        "elasticsearch.audit.origin.type": "rest",
+        "elasticsearch.audit.request.id": "0ybRdKGYRAekov1eKI6nIw",
+        "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "event.action": "anonymous_access_denied",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.audit",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.outcome": "failure",
+        "fileset.name": "audit",
+        "host.id": "O8SFUsk8QpGG16JVJcNgUw",
+        "http.request.id": "0ybRdKGYRAekov1eKI6nIw",
+        "http.request.method": "PUT",
+        "input.type": "log",
+        "log.offset": 965,
+        "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:30,950+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"url.path\":\"/test_3\", \"request.method\":\"PUT\", \"request.id\":\"0ybRdKGYRAekov1eKI6nIw\", \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}",
+        "service.type": "elasticsearch",
+        "source.address": "[::1]:64583",
+        "source.ip": "::1",
+        "source.port": 64583,
+        "trace.id": "0af7651916cd43dd8448eb211c80319c",
+        "url.original": "/test_3"
+    }
+]
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml
@@ -0,0 +1,97 @@
+description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
+processors:
+- json:
+    field: message
+    target_field: elasticsearch.deprecation
+- drop:
+    if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)'
+- remove:
+    field: elasticsearch.deprecation.type
+- dot_expander:
+    field: service.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.service.name
+    target_field: service.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.deprecation.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.level
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.level
+    target_field: log.level
+    ignore_missing: true
+- dot_expander:
+    field: log.logger
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.log.logger
+    target_field: log.logger
+    ignore_missing: true
+- dot_expander:
+    field: process.thread.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.process.thread.name
+    target_field: process.thread.name
+    ignore_missing: true
+- rename:
+    field: elasticsearch.deprecation.component
+    target_field: elasticsearch.component
+    ignore_missing: true
+- dot_expander:
+    field: cluster.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.cluster.name
+    target_field: elasticsearch.cluster.name
+- dot_expander:
+    field: node.name
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.node.name
+    target_field: elasticsearch.node.name
+- dot_expander:
+    field: cluster.uuid
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.cluster.uuid
+    target_field: elasticsearch.cluster.uuid
+    ignore_missing: true
+- dot_expander:
+    field: node.id
+    path: elasticsearch.deprecation
+- rename:
+    field: elasticsearch.deprecation.node.id
+    target_field: elasticsearch.node.id
+    ignore_missing: true
+- remove:
+    field: message
+- rename:
+    field: elasticsearch.deprecation.message
+    target_field: message
+- date:
+    field: 'elasticsearch.deprecation.@timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null'
+- date:
+    field: 'elasticsearch.deprecation.timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation?.timestamp != null'
+- remove:
+    field:
+      - elasticsearch.deprecation.timestamp
+      - elasticsearch.deprecation.@timestamp
+    ignore_missing: true
diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml
@@ -0,0 +1,15 @@
+description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format.
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'
+processors:
+- json:
+    field: message
+    add_to_root: true
+- dot_expander:
+    field: '*'
+    override: true
+- set:
+    field: event.dataset
+    value: elasticsearch.deprecation