diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 2c921af2f39..9030022bc67 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -49735,6 +49735,39 @@ example: 0 -- +*`elasticsearch.elastic_product_origin`*:: ++ +-- +Used by Elastic stack to identify which component of the stack sent the request + +type: keyword + +example: kibana + +-- + +*`elasticsearch.http.request.x_opaque_id`*:: ++ +-- +Used by Elasticsearch to throttle and deduplicate deprecation warnings + +type: keyword + +example: v7app + +-- + +*`elasticsearch.event.category`*:: ++ +-- +Category of the deprecation event + +type: keyword + +example: compatible_api + +-- + *`elasticsearch.audit.layer`*:: + @@ -49922,6 +49955,20 @@ type: boolean -- +*`elasticsearch.audit.authentication.type`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.audit.opaque_id`*:: ++ +-- +type: text + +-- + [float] === deprecation diff --git a/filebeat/module/elasticsearch/_meta/fields.yml b/filebeat/module/elasticsearch/_meta/fields.yml index 721f33a4879..8ae4789d929 100644 --- a/filebeat/module/elasticsearch/_meta/fields.yml +++ b/filebeat/module/elasticsearch/_meta/fields.yml @@ -40,3 +40,15 @@ description: "Id of the shard" example: "0" type: keyword + - name: elastic_product_origin + type: keyword + description: "Used by Elastic stack to identify which component of the stack sent the request" + example: "kibana" + - name: http.request.x_opaque_id + description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings" + example: "v7app" + type: keyword + - name: event.category + description: "Category of the deprecation event" + example: "compatible_api" + type: keyword diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml index 38774e4f8b9..ce0ffdf1fda 100644 --- a/filebeat/module/elasticsearch/audit/_meta/fields.yml +++ b/filebeat/module/elasticsearch/audit/_meta/fields.yml @@ -70,3 +70,7 @@ type: text - name: invalidate.apikeys.owned_by_authenticated_user type: boolean + - name: authentication.type + type: keyword + - name: opaque_id + type: text diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml index 359af0ab196..14e6a03538e 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -176,6 +176,16 @@ processors: field: elasticsearch.audit.level target_field: log.level ignore_missing: true + - dot_expander: + field: trace.id + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.trace.id + target_field: trace.id + ignore_missing: true + - remove: + field: elasticsearch.audit.trace.id + ignore_missing: true - date: field: elasticsearch.audit.@timestamp target_field: "@timestamp" diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 1ae5da8dbb7..e241acafacb 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log b/filebeat/module/elasticsearch/audit/test/test-audit-800.log new file mode 100644 index 00000000000..75c7ebb6055 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-800.log @@ -0,0 +1,3 @@ +{"type":"audit", "timestamp":"2022-01-27T14:16:25,271+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"yEUG-8deS2y8ZxGgeyeUnw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_1"], "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"} +{"type":"audit", "timestamp":"2022-01-27T14:16:28,601+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"qo04VI2qRzKrE1dlrsjYgw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_2"]} +{"type":"audit", "timestamp":"2022-01-27T14:16:30,950+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"rest", "event.action":"anonymous_access_denied", "origin.type":"rest", "origin.address":"[::1]:64583", "url.path":"/test_3", "request.method":"PUT", "request.id":"0ybRdKGYRAekov1eKI6nIw", "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"} diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json new file mode 100644 index 00000000000..6477bb708e5 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json @@ -0,0 +1,105 @@ +[ + { + "@timestamp": "2022-01-27T13:16:25.271Z", + "elasticsearch.audit.action": "indices:admin/create", + "elasticsearch.audit.authentication.type": "REALM", + "elasticsearch.audit.indices": [ + "test_1" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.opaque_id": "myApp1", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "yEUG-8deS2y8ZxGgeyeUnw", + "elasticsearch.audit.request.name": "CreateIndexRequest", + "elasticsearch.audit.user.realm": "reserved", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "yEUG-8deS2y8ZxGgeyeUnw", + "input.type": "log", + "log.offset": 0, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:25,271+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"yEUG-8deS2y8ZxGgeyeUnw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_1\"], \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}", + "related.user": [ + "elastic" + ], + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "trace.id": "0af7651916cd43dd8448eb211c80319c", + "user.name": "elastic" + }, + { + "@timestamp": "2022-01-27T13:16:28.601Z", + "elasticsearch.audit.action": "indices:admin/create", + "elasticsearch.audit.authentication.type": "REALM", + "elasticsearch.audit.indices": [ + "test_2" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "qo04VI2qRzKrE1dlrsjYgw", + "elasticsearch.audit.request.name": "CreateIndexRequest", + "elasticsearch.audit.user.realm": "reserved", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "qo04VI2qRzKrE1dlrsjYgw", + "input.type": "log", + "log.offset": 517, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:28,601+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"qo04VI2qRzKrE1dlrsjYgw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_2\"]}", + "related.user": [ + "elastic" + ], + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "user.name": "elastic" + }, + { + "@timestamp": "2022-01-27T13:16:30.950Z", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.opaque_id": "myApp1", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "0ybRdKGYRAekov1eKI6nIw", + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "anonymous_access_denied", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "failure", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "0ybRdKGYRAekov1eKI6nIw", + "http.request.method": "PUT", + "input.type": "log", + "log.offset": 965, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:30,950+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"url.path\":\"/test_3\", \"request.method\":\"PUT\", \"request.id\":\"0ybRdKGYRAekov1eKI6nIw\", \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}", + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "trace.id": "0af7651916cd43dd8448eb211c80319c", + "url.original": "/test_3" + } +] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..08f044e68d5 --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml @@ -0,0 +1,97 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.deprecation +- drop: + if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)' +- remove: + field: elasticsearch.deprecation.type +- dot_expander: + field: service.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- remove: + field: message +- rename: + field: elasticsearch.deprecation.message + target_field: message +- date: + field: 'elasticsearch.deprecation.@timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' +- date: + field: 'elasticsearch.deprecation.timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation?.timestamp != null' +- remove: + field: + - elasticsearch.deprecation.timestamp + - elasticsearch.deprecation.@timestamp + ignore_missing: true diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..89c7b4083f6 --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml @@ -0,0 +1,15 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + add_to_root: true +- dot_expander: + field: '*' + override: true +- set: + field: event.dataset + value: elasticsearch.deprecation diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml index 08f044e68d5..d4647fbff10 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml @@ -4,94 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- json: - field: message - target_field: elasticsearch.deprecation -- drop: - if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)' -- remove: - field: elasticsearch.deprecation.type -- dot_expander: - field: service.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.process.thread.name - target_field: process.thread.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: node.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: cluster.uuid - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: node.id - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- remove: - field: message -- rename: - field: elasticsearch.deprecation.message - target_field: message -- date: - field: 'elasticsearch.deprecation.@timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' -- date: - field: 'elasticsearch.deprecation.timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation?.timestamp != null' -- remove: - field: - - elasticsearch.deprecation.timestamp - - elasticsearch.deprecation.@timestamp - ignore_missing: true + - pipeline: + if: '!ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-7" >}' + - pipeline: + if: 'ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-8" >}' diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index e1f4838df9b..7c64e431021 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/deprecation/manifest.yml b/filebeat/module/elasticsearch/deprecation/manifest.yml index 8dfbaec866b..93b1ef80b09 100644 --- a/filebeat/module/elasticsearch/deprecation/manifest.yml +++ b/filebeat/module/elasticsearch/deprecation/manifest.yml @@ -16,4 +16,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/log.yml diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log index 888a5d92080..40157a6d5e2 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log @@ -1,15 +1,2 @@ -{"@timestamp":"2020-04-15T12:35:20.315Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.316Z", "log.level": "WARN", "message":"Field parameter [tree] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.366Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.367Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.479Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.480Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.481Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.487Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T13:46:33.582Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#3]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.219Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#4]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.339Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#5]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.455Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#6]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:47:36.309Z", "log.level": "WARN", "message":"index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:55:56.365Z", "log.level": "WARN", "message":"index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:56:14.697Z", "log.level": "WARN", "message":"[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][search][T#7]","log.logger":"org.elasticsearch.deprecation.index.query.QueryShardContext","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} +{"@timestamp":"2022-01-27T11:48:45.809Z", "log.level":"CRITICAL", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.elastic_product_origin":"","elasticsearch.event.category":"compatible_api","elasticsearch.http.request.x_opaque_id":"v7app","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#8]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:52:39.882Z", "log.level":"CRITICAL", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"compatible_api","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#9]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json index 89f625d1f17..20dd03fab88 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json @@ -1,331 +1,58 @@ [ { - "@timestamp": "2020-04-15T12:35:20.315Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:48:45.809Z", + "data_stream.dataset": "deprecation.elasticsearch", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.elastic_product_origin": "", + "elasticsearch.event.category": "compatible_api", + "elasticsearch.http.request.x_opaque_id": "v7app", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "event.category": "database", + "event.code": "create_index_with_types", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", + "log.level": "CRITICAL", + "log.logger": "org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction", "log.offset": 0, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", + "message": "[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version.", + "process.thread.name": "elasticsearch[runTask-0][transport_worker][T#8]", "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.316Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 501, - "message": "Field parameter [tree] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.366Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 997, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.367Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 1498, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.479Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 1998, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.480Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 2507, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.481Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 3015, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.487Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 3524, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:33.582Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 4032, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#3]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.219Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 4523, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#4]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.339Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 5014, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#5]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.455Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 5505, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#6]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:47:36.309Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService", - "log.offset": 5996, - "message": "index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", - "process.thread.name": "elasticsearch[n1][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:55:56.365Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService", - "log.offset": 6560, - "message": "index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", - "process.thread.name": "elasticsearch[n1][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2020-04-16T13:56:14.697Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", + "@timestamp": "2022-01-27T11:52:39.882Z", + "data_stream.dataset": "deprecation.elasticsearch", + "data_stream.namespace": "default", + "data_stream.type": "logs", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.event.category": "compatible_api", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "event.category": "database", + "event.code": "create_index_with_types", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.query.QueryShardContext", - "log.offset": 7128, - "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead.", - "process.thread.name": "elasticsearch[n1][search][T#7]", + "log.level": "CRITICAL", + "log.logger": "org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction", + "log.offset": 989, + "message": "[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version.", + "process.thread.name": "elasticsearch[runTask-0][transport_worker][T#9]", "service.name": "ES_ECS", "service.type": "elasticsearch" } diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 889d94d042a..525d0c50eac 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUml9z2zYSwN/zKXb0cu2MzZP/1Fdr5m6mVRzbmSZxLdu5VvFwVuCKQgQCNABKVjv57jcAKVmiSOrPtbmeXxKKAPa3i8ViF8QhjGnWARJoLGeGULPRKwDLraAOtC6Wf2+9AtAkCA11IMZXABEZpnlquZId+NcrAFgdCd6pKBP0CmDISUSm45scgsSE1oW6PztL3eBaZWnxS4WM1eGWh2QqSZUkaRdvSgOsavTSHoZaJTAdkSawIwKhYqCJe6E0j7lES1FraVB6xiT1JlIBBSxIgndk8TVa7GpCS9cyouce6QlntNwv129Ms6nS0Tq+yIwlHWQZj2o1uL+/fg1q6DGLDtVkl8lED67E+zvee/iF3wy/Hz/H5/HuNO6pluY9JrQVTaTYmPRhRZtmCqkiChrM8WIM17Ja9use/8juZnQ3+mjv//3Tj+dv2z++m+7IsLUZ6jkmH9+/Nb+ebC+YOzdqluw9zTevljnkggaE9tCSsYdcppndVX6T9b10XrM28MNl/Ho6uL8ddh+++8cPPfY06MY72N2MUEeN4qO50X3Taor29gIxi7hda70cjqAi+iyPIHBGeuVNmfnOhRfXah5zOBuBHXGzFnA6oMnYA7AapUmVdu+Ap+GQi9ISWlXY9Sq/rdZ7mdxLD127jfiukTN8DmxHaEExlmntmFEqOUtUZkJkjIwJI5KcogPAzI5IWs7QDRUOkQv/c6lV/hhrlNY9MyUlMd+j6rd5N4tJSpqiUNNT5q2mMxni0kDFc96h3nir8nc3Yz59wUY7flzsNQXx2sTDN+tvcp9BuL3o3cEPN9fzzt8ue8mi3xQNaGLEJxSBkl7aSzM2QilJfHsAQjEUoYtb8E2++zEUPo4BNyajaJnz23rbvYyzu900oUg2et6qD+WdPFzphdN8goJH3mgYI5fra6IAb7ndiYaYCeuW1h7smSEdbKeAa/o3U6nHAfDh8otaL215N7V8QmHENTGr9GxfaCXINELfuhZg1SJQEaSaS8ZTFDAgoWRsaj2iD60xH6DEEKOEy9YBtNxeZIpHeNyT2i/l8r64+wBVU7Z5hKokc7ue6GPWRh+RS8lE3iWPsc6p6ZlYVu8aHWgVGVYnUZJbpf+eIJd7eIcWQYoakw3e4eLQ/e01+LZkSdc7Q+t3Z3s3/D8/IxtLzkbHX1qV0rmMONvgmNd5m2LDoAgGs8JaTe44VOrwuH10HrSPgvapc8iVX07Wfjnbx0uLULmauqyrcC/5U0aQp7BFn3rzffztp3A8OHvoTT6Mfnhq2+nN5OrDz/tE2hyuYvnUb/XzLWUHR+wKQt1jWglxW63b1qzhQEWzys4oOJb9JEU76sDI2jSY6+r6B0xJu75sEx5rzDW2OqOGTT3EKNJkyuI2gRiVaUYBT/cQnGm+ozS3cIt8QewhcBHbdxVr1ivFbWUmZAzG1aHc0rOtCRHz/T3AlI9pZgI1lRSFg1m4somGDq1y7IFSglCuVQERpZrynXljLVB5PAENRxQx239MgMsuuBTFkC0EBFvWJekITbWFy9I3ELi/N14QmJQYH3LmsoPLbi4iKDWuYlrmqnAZaIwMWwG6v+WS/LILTAmR1wzVoEvTn+UeGxpitWhDobAcSbYE65ZIFgLdVqZ0xGXsLOq43+IEYcK1zVBAgmzEZQO4YTobhGaWDJQILQ4EhZYn9GfpATeYGQInArgEQ0zJyABza8rpkKWQs4BnMRvBreYy/grgW3B7lI3cU8JxqGlowlQrl4V4/j+R/M4xm9RV3S8SPQZoGpIm6TKiF6Xq0V2+JgSJUJNhKL8W9ZK9E9RjRy/4hEANPhOzxhUZggDTVMxLI27AWJWmFNUrwwQaE2ZSKIy+lia5NO8vMnPpp4fY0voszTxnLWNVUN6S8SZ3DOje3Oc+XvgL6aHSiQN+CYUViPUhG0pVVI2RYaOht1TE/ZWUUJk1PMqPTcakJYkqBZYCy8z8Dyi5LENCI6UrQ78G5p2yKIAEps5fS9BW+cJWkM3Jl/ZLf+ZkLGrfasglN6OgMsv4PElCncmaJVivyAYFfCHiUD3J24d3BU2WLq22A0ADmA/vvDxVXFqQWTIgXU1rR5owMqF1dgldlKkLHnuTX6IeYLxizUIqeKk+thXTUBU0Fo7sQqDfXebMf7SJHYJVauymOIcqOBu5LMbV5VB16rbJWl0QKo7zrTeuETkiLEfGvRPZK8IUUAhVbDYoo/m88N92zmVdn3A8qA3qXFqK12qRLTBhsXid8l6Oc/wxF2ows00ZituZ/jSkexdGPFE9zKKCFlEYU/nsa++J+yAiiElSkTgrxrIUJZv99WfQT54aOoMsa/AXmM5am26e3ZnKZPxHzu8vbsD/8xmelXX4C8xxg12r6RZ2Iz1ZEbp6VNjzr/21ifInlG2/ob5sdcjGViNbzY6X5LU60HONwLdy4MyV0WoIpLXSqxuS/4TdgSGKlfOPyuOYslb5frR6alnn0k2HL94TmhZAK5+Xy279aWr12WnV0qpeAotALNerjlWWsqQmijmHUGsKLlKEqfoaAhf6TUiPCKPQ0FOjyXv0lLl6uUgRay1/cnp6fn5+XGn+WoqXfC+cn+4EGz6lrFbJl90D90/CheBFBlZLeHTWbm+ZBy6sNHALGncD9NHN56rOyMUHwaXMdoqmGJiiHei/34p+ER6EmgoV10ei/H1+N8HkFcPaNbY1iFb/uH30/WH77PD4/O6o3WmfdY5OD85PTh771+/ffIDHfn4VJh8iKCCCp4z07BH6k/Dh7ejzwyP0E7KaM3/h5iw4CdqHbtygfRYcnz32248+xe6fBt8l5vHAP4S5kfqn/tkVIiNuTf/o/PTkO/fTLCXTfzxwYdHm//EI/qJG/+f7i9tfwruri/fhm4u77tViDH8dxvSPXHv/+aH/+6eWp/3U6vz+qZWgZaMQhcgfB0oZ+6nVOQraX758eTz4b+K3y+BL29PqDP3kG6xdWVqejUpjD8muzl59rbGIPUqNG0j8kuN2UfcU37R8/euNVcd30m4nZkcUN5FNLO59nbzdRHlXaRDVc+/zGa2V6N8e7Sj3xTObpOdXL12rOuFlt94Rwzt86CewiUOoafMs77BkdiOkZ6sxzDkbCC9cs0Id4HKodILr37f39ZKXYNPklXnVyW2do5we7yE0j04bxTrjc4ryu311AMe7AWiVWV7atMsXXnyLOiOb9tHVr8c//zg+/zw9jW2Mb6zczfClSwEr0q+jP2Zum5fgXcPaixTbZ7nVS+vl/quGECmWJYsbgy5b8HGeogZ5/wkAAP//ZMWUGw==" + return "eJzUWltz2zb2f8+nOKOXfztj8y9f6taa2Z1plcRxprk0sp1tFQ/nCDyiEIEADYCS1U6++w5AShYpkrpsm+3qxSZxOb9zPwfEMUxp0QMSaCxnhlCzyTMAy62gHnRerL/vPAPQJAgN9SDGZwARGaZ5armSPfjnMwAo7wRvVJQJegYw5iQi0/NTjkFiQptE3c8uUre5VllavKmhUd5ufUumklRJknY1UtmgzNHTfBhrlcB8QprATgiEioFmbkBpHnOJlqLO2qb0iEnqRaQCCliQBG/I4nO02NeElq5lRI8D0jPOaH1dzt+UFnOlo034IjOWdJBlPGrk4Pb2+jmosYdZLKhHdpXM9OiVeHvDB3e/8vfjH6aP8WW8Pxr31IjmLSa0E5pIsSnp45o57SikiihoEceTMNzMetrPB/wju1nQzeSjvf3Xzz9dvu7+9Ga+J4adxdCMY/bx7Wvz29nuhLkzo3bK3tL89HqaYy5oRGiPLRl7zGWa2X3pt0nfU+cNvoHvruLn89Hth3H/7rvvfxywh1E/3kPuZoI6aiUfLYXup9aj6O5OsAhJYapVlDEb5s6/dfmGixqKYLSAItiAscimYBXwiKTl4wXMJ7wUfpZc+InGvXGPmh4yMraerSkfocTOBg8Ta9OgWBk8hirFh4zCtoBSRluERqvATrSyVhCgjCCiKEsFZ2gJIko1MXTrYY5achmbBov/HtN0D/m7kBs4GrHSi0bE/WLCUmzrePwW9WCcuNHykaAQU747KswibjdmrycpqMlJ6zsIXJAujVRZunFJx81aZiJnH3bCzUYa6oEmY4/AapQmVdqNAU/DMReVwFrmXlcsqc2cKxoJ3byt8N0kp5EcsJ2gBcVYprXDjFLJRaIyEyJjZEwYkeQUHQFmduKcIldfOEYu/OvKrPwx1iite2ZKSmJ+Rd275TKLSUqaorDwhyPQmQxxbaPiOV/QLLwy/f3FmKsv2CrHj6sKpEC8oXj4ZnMktxmEDy8GN/Dj++vl4m/XrWS1bo4GNDHiM4pASU/taRqboJQkvj0CoRiK0GUz+CaviRgKn92AG5NRtI7z22bZPe2zv9w0oUi2Wl7ZhvJFHlxlwHE+Q8EjLzSMkctNnyiAd1zNQmPMhHWudQD2zJAOdmPATf0/U8vHEfDx+kCjlXa8mVo+ozDimphVenEoaCXItIL+4Ga4HLEMVASp5pLxFAWMSKhKRihZxHCZvEKMEi47R9BxFYopHuH+QNTelavV0v4b1Kls+w51rcduK9HHrK02ItdKzHxJHmOdUdMjsazZNFzuy+vuXqIkt0r/f4JcHmAdWgQpaky2WIeLQ7cfrsHPJUu62Rg6fzjZu+3/8RnZVHI2Of3SqaXOZcTZFsO8zucUCSMvbHJptZnjWKnj0+7JZdA9CbrnziBLb8423lwcYqXLooxHrSzcSv6QEeSNTU0JWBbfx99/Dqeji7vB7N3kx4eunb+fvXr3yyGRNgdX4z7NqX6ZUvYwxL4g1AOmlRAf6nnbGWs4UtGidjEKjlU7SdFOKtWxWx8wJe2m2yY81phzbHVGLUk9xCjSZKrktgExKtOMAp4eQDjTfE9qznGLekEcQHAV2/clazbPD3almZAxGNeHckuPtiFELPN7gCmf0sIEai4pCkeLsJREQwetdu+RUoJQ1ofqUoKuK+d2KAVrGrIGzpZL1vqbrW1I7XkZtJyZxezwPQGu+uCqI0O2IBDs2BKlEzT1wqtS34LA/V56QmBSYnzMmStMrvo5iaAyuQ7TOq4aa4VWve4E0P3Wz4iu+sCUEHm7Ug90Tf1Z7iyhIdYIbSwUVoPYjsD6FSQrgi6LKh1xGefHAQSvcYYw49pmKCBBNuGyBbhhOhuFZpGMlAgtusbb8oT+Kj7gPWaGwJEALsEQUzIywJw7Ox6yFHIs4LGYrcCt5jL+CsB3wO2hbMU9J5yGmsYmTLVyBZDH/xciv3GYTeoa/ieKHgZoGpMm6YqxJ6aaobtSUQgSoSbDUH4t1GvyTlBPHXrBZwRq9JmYNa6/EQSY5kdfzie4AWNVmlLUzAwTaEyYSaEw+lqc5NS8vcjMVb4exI7SZ2nmcTZirAvKO2J8nxsG9N/f5jZe2AvpsdKJA/wUCmsgNodsqDRwDUKGrYLekRH3qzChMmt4lJ/YTElLEnUMrAWWhfkvoOSyChJaUboO+GvAvFEWBZDA1NlrBbRVvqcWZHPka/nSH3cZi9rPGnPJzSSorTI+z5JQZ7LBBZsZ2cKA74EcVI/k9d2bAk2WrnnbEaABzLd3Vp4qLi3ILBmRrkdrJ5owMqF1cgldlGkKHgcjv0I9wrgkzYIqeKo+thVqqAsaK0N2IdBnlyXmP1vEDoJVyn83yUEVOFtxWYzrO7H60m2btPogVBznqTduIDkhrEbGgwvZV4QpoBCqSDYoo6Ve+O9717JuTTgdNQZ1Li3FG23QDjBh5byOeU/HGf6UCzVa2LYKxWWmvwyS/5LlETWDWbVhIgpjqh67Hay4dyKCmCQVhbNiLEtRssXfX4NeeWrsBLLOwd9AnY0y3a7dhcpk/Gfq91e34f+4hhdVHv4GOm6Raz26ldxIz0pEy6eUAz/s7/FUv97s+vn2KdUhm1qNrFwdr9Hr9GCQf+J3sxxw5tpoNQbSWulyQvJ3KnowRlE6/6g9jqlyleej8oFpk0m3Hb54S2hzgE6ul6t+80Fu/bFtnWvVu8AqEMvNrqOMpUqpDcUSh1AbDK5KhLn6GgRX/M1ITwij0NBDq8gH9JC5frkoERslf3Z+fnl5eVor/kYUT/VeuDzdCbZ8xSl3yVf9I/cn4ULwogJrRHhy0e3uWAeupDRyDo37AfTRzdeqTsirSzWrynaOptiYoj3Q/7AT+lV4EGouVNwcifLx/FqEyTuGjXuVGyA6w9PuyQ/H3Yvj08ubk26ve9E7OT+6PDu7H16/ffkO7of53ax8i6AAETxkpBf3MJyFd68nn+/uYZiQ1Zz5G2AXwVnQPXb7Bt2L4PTifti99yX28Dz4LjH3R/4hzIU0PPfPrhGZcGuGJ5fnZ9+5V4uUzPD+yIVFm//jIfg7IsNfbl98+DW8efXibfjyxU3/1WoPfz/LDE/cfP/lY/jHp45H+6nT++NTJ0HLJiEKkT+OlDL2U6d3EnS/fPlyf/SfxG9XwVfSU1lDP/sJG3fo1rVRK+wx2bL2mnuNVexRatqCxLsct6u+p/ic5vtfL6wmfGfdbmL2hOIU2YbFjTfR24+UN5UWUgM3nmu0kaIfPdmT7pNltlEvLrwt0kb1V816Txje4EOvwDYcQs3btbyHy+yHkB6txjDH2YLwhZtWsANcjpVOcPPT+qFW8hRs2qwy7zq5bTKU89MDiObRaStZJ3xOUX7ZtAnA6X4AtMosryTt6l0bP6NJyKZ78uq3019+ml5+np/HNsaXVu4n+MqnyBL16+jP0W27C960+F6k2CHu1kxtkNuvGkOkWJasLiu6asHHeYpa6P07AAD//wFdWko=" } diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index d0980763ecc..6d3c9006a20 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -36,9 +36,9 @@ processors: PROCTIME: '\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs\]' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - date: field: timestamp target_field: '@timestamp' diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..d92dd640772 --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml @@ -0,0 +1,101 @@ +description: Pipeline for parsing the Elasticsearch 7.x server log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- drop: + if: ctx.elasticsearch.server.type != 'server' +- remove: + field: elasticsearch.server.type +- dot_expander: + field: service.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.server.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.server.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true +- grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} +- remove: + field: elasticsearch.server.message +- set: + field: '@timestamp' + value: '{{ elasticsearch.server.timestamp }}' + ignore_empty_value: true +- remove: + field: elasticsearch.server.timestamp +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..7619050a4cb --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml @@ -0,0 +1,109 @@ +description: Pipeline for parsing the Elasticsearch 8.0 server log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- dot_expander: + field: event.dataset + path: elasticsearch.server +- drop: + if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' +- set: + value: '{{ elasticsearch.server.event.dataset }}' + field: event.dataset + ignore_empty_value: true +- remove: + field: elasticsearch.server.event.dataset +- dot_expander: + field: ecs.version + path: elasticsearch.server +- set: + value: '{{ elasticsearch.server.ecs.version }}' + field: ecs.version + ignore_empty_value: true +- remove: + field: elasticsearch.server.ecs.version +- dot_expander: + field: service.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true +- dot_expander: + field: elasticsearch.cluster.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: elasticsearch.node.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: elasticsearch.cluster.uuid + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: elasticsearch.node.id + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true +- grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} +- remove: + field: elasticsearch.server.message +- set: + field: '@timestamp' + value: '{{ elasticsearch.server.@timestamp }}' + ignore_empty_value: true +- remove: + field: elasticsearch.server.@timestamp +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml index c3b655643ed..e5b23aabfd9 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -7,100 +7,9 @@ processors: - json: field: message target_field: elasticsearch.server -- drop: - if: ctx.elasticsearch.server.type != 'server' -- remove: - field: elasticsearch.server.type -- dot_expander: - field: service.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.server.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: node.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: cluster.uuid - path: elasticsearch.server -- rename: - field: elasticsearch.server.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: node.id - path: elasticsearch.server -- rename: - field: elasticsearch.server.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- rename: - field: elasticsearch.server.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.server -- rename: - field: elasticsearch.server.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.server -- rename: - field: elasticsearch.server.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.process.thread.name - target_field: process.thread.name - ignore_missing: true -- grok: - field: elasticsearch.server.message - pattern_definitions: - GREEDYMULTILINE: |- - (.| - )* - INDEXNAME: '[a-zA-Z0-9_.-]*' - GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent - \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] - collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] - GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} - patterns: - - '%{GC_ALL}' - - '%{GC_YOUNG}' - - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} -- remove: - field: elasticsearch.server.message -- rename: - field: elasticsearch.server.@timestamp - target_field: '@timestamp' - ignore_missing: true -- rename: - field: elasticsearch.server.timestamp - target_field: '@timestamp' - ignore_missing: true -- date: - field: '@timestamp' - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true +- pipeline: + if: ctx.elasticsearch.server.containsKey('type') + name: '{< IngestPipeline "pipeline-json-7" >}' +- pipeline: + if: ctx.elasticsearch.server.containsKey('ecs.version') + name: '{< IngestPipeline "pipeline-json-8" >}' diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 4d4e634cc4b..32abc88dae4 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/server/manifest.yml b/filebeat/module/elasticsearch/server/manifest.yml index 406972cba56..d9d5d4e398b 100644 --- a/filebeat/module/elasticsearch/server/manifest.yml +++ b/filebeat/module/elasticsearch/server/manifest.yml @@ -16,4 +16,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/log.yml diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log index b7119ffc069..78c9f51fd7d 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log @@ -1,3 +1,3 @@ -{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"} -{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} -{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} +{"@timestamp":"2022-01-25T15:12:08.472Z", "log.level": "INFO", "message":"adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} +{"@timestamp":"2022-01-25T15:12:08.588Z", "log.level": "INFO", "message":"adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} +{"@timestamp":"2022-01-25T15:12:08.686Z", "log.level": "INFO", "message":"adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json index 817cadf6002..228661fc9c3 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json @@ -1,70 +1,67 @@ [ { - "@timestamp": "2020-04-14T14:05:58.019Z", - "elasticsearch.cluster.name": "elasticsearch", - "elasticsearch.cluster.uuid": "ECEBF2VPQuCF9tbBKaLqXQ", - "elasticsearch.node.id": "suOYiQwuRvialOY-c0wHLA", - "elasticsearch.node.name": "CBR-MBP.local", + "@timestamp": "2022-01-25T15:12:08.472Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "suOYiQwuRvialOY-c0wHLA", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", "log.offset": 0, - "message": "adding template [.management-beats] for index patterns [.management-beats]", - "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]", + "message": "adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" }, { - "@timestamp": "2020-04-14T20:57:49.663Z", - "elasticsearch.cluster.name": "docker-cluster", - "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", - "elasticsearch.index.name": "test-filebeat-modules", - "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", - "elasticsearch.node.name": "7debcb878699", + "@timestamp": "2022-01-25T15:12:08.588Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "kZnYdakGTqihZQT_1rM92g", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetadataCreateIndexService", - "log.offset": 489, - "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", - "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", + "log.offset": 608, + "message": "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" }, { - "@timestamp": "2020-04-14T20:57:49.772Z", - "elasticsearch.cluster.name": "docker-cluster", - "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", - "elasticsearch.index.id": "IW1jJcOBTFeIDihqjoT8yQ", - "elasticsearch.index.name": "test-filebeat-modules", - "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", - "elasticsearch.node.name": "7debcb878699", + "@timestamp": "2022-01-25T15:12:08.686Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "kZnYdakGTqihZQT_1rM92g", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetadataMappingService", - "log.offset": 1031, - "message": "update_mapping [_doc]", - "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", + "log.offset": 1220, + "message": "adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" } diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..00ce95ccaa1 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml @@ -0,0 +1,120 @@ +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.slowlog +- drop: + if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type + != 'index_search_slowlog' +- remove: + field: elasticsearch.slowlog.type +- dot_expander: + field: service.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.slowlog.doc_type + target_field: elasticsearch.slowlog.types + ignore_missing: true +- convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true +- rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true +- grok: + field: elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- remove: + field: elasticsearch.slowlog.message +- set: + value: "{{ elasticsearch.slowlog.@timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- set: + value: "{{ elasticsearch.slowlog.timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.@timestamp + ignore_missing: true +- remove: + field: elasticsearch.slowlog.timestamp + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..3e0479d59ea --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -0,0 +1,36 @@ +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + add_to_root: true +- dot_expander: + field: '*' + override: true +- convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true +- rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true +- grok: + field: elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- set: + field: message + value: '{{ elasticsearch.slowlog.message }}' + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.message diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 174a429946a..614c9f7aa43 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -4,111 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- json: - field: message - target_field: elasticsearch.slowlog -- drop: - if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type - != 'index_search_slowlog' -- remove: - field: elasticsearch.slowlog.type -- dot_expander: - field: service.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.slowlog.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.process.thread.name - target_field: process.thread.name - ignore_missing: true -- rename: - field: elasticsearch.slowlog.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: node.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: cluster.uuid - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: node.id - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- rename: - field: elasticsearch.slowlog.doc_type - target_field: elasticsearch.slowlog.types - ignore_missing: true -- convert: - field: elasticsearch.slowlog.took_millis - type: float - ignore_missing: true -- rename: - field: elasticsearch.slowlog.took_millis - target_field: elasticsearch.slowlog.duration - ignore_missing: true -- grok: - field: elasticsearch.slowlog.message - pattern_definitions: - GREEDYMULTILINE: |- - (.| - )* - INDEXNAME: '[a-zA-Z0-9_.-]*' - patterns: - - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? - - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] -- remove: - field: elasticsearch.slowlog.message -- rename: - field: elasticsearch.slowlog.@timestamp - target_field: '@timestamp' - ignore_missing: true -- rename: - field: elasticsearch.slowlog.timestamp - target_field: '@timestamp' - ignore_missing: true -- date: - field: '@timestamp' - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true + - pipeline: + if: '!ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-7" >}' + - pipeline: + if: 'ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-8" >}' diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index ea501d9b3e0..440220f1dd7 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/slowlog/manifest.yml b/filebeat/module/elasticsearch/slowlog/manifest.yml index caddd94158b..08b49643108 100644 --- a/filebeat/module/elasticsearch/slowlog/manifest.yml +++ b/filebeat/module/elasticsearch/slowlog/manifest.yml @@ -22,4 +22,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/slowlog.yml diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log index 3704f88d189..b580682cda0 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log @@ -1,2 +1,2 @@ -{"@timestamp":"2020-04-16T11:20:02.069Z", "log.level":"TRACE", "id":"5xy3gnEBmUEb0NJ1lijF", "message":"[test_index/M4fNwSWlTfek9m1SNL49Kg]", "source":"{\\\"f", "took":"15.1ms", "took_millis":"15" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#2]","log.logger":"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T11:20:02.777Z", "log.level":"TRACE", "id":"6By3gnEBmUEb0NJ1mSij", "message":"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]", "source":"{\\\"field\\\":123}", "took":"10.4ms", "took_millis":"10" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#4]","log.logger":"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2022-01-27T11:36:49.421Z", "log.level":"TRACE", "elasticsearch.slowlog.id":"_YRSm34B7FprLQsj6fZg","elasticsearch.slowlog.message":"[test_1/8pT6xiN_Tt-dcJWRR3LX6A]","elasticsearch.slowlog.source":"{\\\"a\\\":\\\"b\\\"}","elasticsearch.slowlog.took":"31.9ms","elasticsearch.slowlog.took_millis":"31" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#3]","log.logger":"index.indexing.slowlog.index","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:39:29.508Z", "log.level":"TRACE", "elasticsearch.slowlog.id":"_oRVm34B7FprLQsjW_Zh","elasticsearch.slowlog.message":"[test_1/8pT6xiN_Tt-dcJWRR3LX6A]","elasticsearch.slowlog.source":"{\\\"a\\\":","elasticsearch.slowlog.took":"1.7ms","elasticsearch.slowlog.took_millis":"1" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#5]","log.logger":"index.indexing.slowlog.index","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json index cce5652340a..c3571decd25 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json @@ -1,58 +1,60 @@ [ { - "@timestamp": "2020-04-16T11:20:02.069Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.id": "M4fNwSWlTfek9m1SNL49Kg", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", - "elasticsearch.slowlog.id": "5xy3gnEBmUEb0NJ1lijF", - "elasticsearch.slowlog.source": "{\\\"f", - "elasticsearch.slowlog.took": "15.1ms", + "@timestamp": "2022-01-27T11:36:49.421Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.id": "8pT6xiN_Tt-dcJWRR3LX6A", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": "_YRSm34B7FprLQsj6fZg", + "elasticsearch.slowlog.source": "{\\\"a\\\":\\\"b\\\"}", + "elasticsearch.slowlog.took": "31.9ms", "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 15000000, + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 31000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", "log.level": "TRACE", - "log.logger": "index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg", + "log.logger": "index.indexing.slowlog.index", "log.offset": 0, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.069Z\", \"log.level\":\"TRACE\", \"id\":\"5xy3gnEBmUEb0NJ1lijF\", \"message\":\"[test_index/M4fNwSWlTfek9m1SNL49Kg]\", \"source\":\"{\\\\\\\"f\", \"took\":\"15.1ms\", \"took_millis\":\"15\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#2]\",\"log.logger\":\"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][write][T#2]", + "message": "[test_1/8pT6xiN_Tt-dcJWRR3LX6A]", + "process.thread.name": "elasticsearch[runTask-0][write][T#3]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2020-04-16T11:20:02.777Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.id": "Jsz7IUYMQ9ubo2ahiMgCbQ", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", - "elasticsearch.slowlog.id": "6By3gnEBmUEb0NJ1mSij", - "elasticsearch.slowlog.source": "{\\\"field\\\":123}", - "elasticsearch.slowlog.took": "10.4ms", + "@timestamp": "2022-01-27T11:39:29.508Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.id": "8pT6xiN_Tt-dcJWRR3LX6A", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": "_oRVm34B7FprLQsjW_Zh", + "elasticsearch.slowlog.source": "{\\\"a\\\":", + "elasticsearch.slowlog.took": "1.7ms", "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 10000000, + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 1000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", "log.level": "TRACE", - "log.logger": "index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ", - "log.offset": 514, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.777Z\", \"log.level\":\"TRACE\", \"id\":\"6By3gnEBmUEb0NJ1mSij\", \"message\":\"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]\", \"source\":\"{\\\\\\\"field\\\\\\\":123}\", \"took\":\"10.4ms\", \"took_millis\":\"10\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#4]\",\"log.logger\":\"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][write][T#4]", + "log.logger": "index.indexing.slowlog.index", + "log.offset": 750, + "message": "[test_1/8pT6xiN_Tt-dcJWRR3LX6A]", + "process.thread.name": "elasticsearch[runTask-0][write][T#5]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" } ] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log index b817ea08fe7..d113ad63f1f 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log @@ -1,2 +1,3 @@ -{"@timestamp":"2020-04-16T11:20:02.828Z", "log.level":"TRACE", "id":"null", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"10ms", "took_millis":"10", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#1]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T11:20:02.839Z", "log.level":"TRACE", "id":"my-identifier", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"76.4micros", "took_millis":"0", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#3]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2022-01-27T11:36:57.424Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":"myApp1","elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"8ms","elasticsearch.slowlog.took_millis":8,"elasticsearch.slowlog.total_hits":"1 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#1]","log.logger":"index.search.slowlog.query","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:42:17.693Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"164.7micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"2 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#3]","log.logger":"index.search.slowlog.query","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:42:31.395Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"115.3micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"2 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#5]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json index 39cd0679087..af79450e828 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json @@ -1,65 +1,99 @@ [ { - "@timestamp": "2020-04-16T11:20:02.828Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:36:57.424Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "elasticsearch.shard.id": "0", - "elasticsearch.slowlog.id": "null", + "elasticsearch.slowlog.id": "myApp1", "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", - "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.source": "{}", "elasticsearch.slowlog.stats": "[]", - "elasticsearch.slowlog.took": "10ms", - "elasticsearch.slowlog.total_hits": "0 hits", - "elasticsearch.slowlog.total_shards": "1", + "elasticsearch.slowlog.took": "8ms", + "elasticsearch.slowlog.total_hits": "1 hits", + "elasticsearch.slowlog.total_shards": 1, "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 10000000, + "event.dataset": "elasticsearch.index_search_slowlog", + "event.duration": 8000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "TRACE", - "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", "log.offset": 0, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.828Z\", \"log.level\":\"TRACE\", \"id\":\"null\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"10ms\", \"took_millis\":\"10\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#1]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][search][T#1]", + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#1]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" + }, + { + "@timestamp": "2022-01-27T11:42:17.693Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{}", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.took": "164.7micros", + "elasticsearch.slowlog.total_hits": "2 hits", + "elasticsearch.slowlog.total_shards": 1, + "event.category": "database", + "event.dataset": "elasticsearch.index_search_slowlog", + "event.duration": 0, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "tVLnAGLgQum5ca6z50aqbw", + "input.type": "log", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", + "log.offset": 861, + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#3]", + "service.name": "ES_ECS", + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2020-04-16T11:20:02.839Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:42:31.395Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "elasticsearch.shard.id": "0", - "elasticsearch.slowlog.id": "my-identifier", + "elasticsearch.slowlog.id": null, "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", - "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.source": "{}", "elasticsearch.slowlog.stats": "[]", - "elasticsearch.slowlog.took": "76.4micros", - "elasticsearch.slowlog.total_hits": "0 hits", - "elasticsearch.slowlog.total_shards": "1", + "elasticsearch.slowlog.took": "115.3micros", + "elasticsearch.slowlog.total_hits": "2 hits", + "elasticsearch.slowlog.total_shards": 1, "event.category": "database", - "event.dataset": "elasticsearch.slowlog", + "event.dataset": "elasticsearch.index_search_slowlog", "event.duration": 0, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "TRACE", - "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", - "log.offset": 613, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.839Z\", \"log.level\":\"TRACE\", \"id\":\"my-identifier\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"76.4micros\", \"took_millis\":\"0\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#3]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][search][T#3]", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", + "log.offset": 1726, + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#5]", "service.name": "ES_ECS", "service.type": "elasticsearch" }