From 9bb84b7cc451b7bb7a69db36fbef62f0eec60b72 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Wed, 26 Jan 2022 17:15:49 +0900 Subject: [PATCH 01/27] New log samples from 8.0 branch https://github.com/elastic/elasticsearch/commit/c29e0d41d39 --- .../elasticsearch/server/test/elasticsearch-json.800.log | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log index b7119ffc069..78c9f51fd7d 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log @@ -1,3 +1,3 @@ -{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"} -{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} -{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"} +{"@timestamp":"2022-01-25T15:12:08.472Z", "log.level": "INFO", "message":"adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} +{"@timestamp":"2022-01-25T15:12:08.588Z", "log.level": "INFO", "message":"adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} +{"@timestamp":"2022-01-25T15:12:08.686Z", "log.level": "INFO", "message":"adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"} From ccb7ad5516f9329938a28fba58c09c845bb4b7c3 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Wed, 26 Jan 2022 17:58:48 +0900 Subject: [PATCH 02/27] Likely pipeline updates --- .../server/ingest/pipeline-json.yml | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml index c3b655643ed..2b9eaf8a8b7 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -7,10 +7,22 @@ processors: - json: field: message target_field: elasticsearch.server +- dot_expander: + field: event.dataset + path: elasticsearch.server - drop: - if: ctx.elasticsearch.server.type != 'server' -- remove: - field: elasticsearch.server.type + if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' +- rename: + field: elasticsearch.server.event.dataset + target_field: event.dataset + ignore_missing: true +- dot_expander: + field: ecs.version + path: elasticsearch.server +- rename: + field: elasticsearch.server.ecs.version + target_field: ecs.version + ignore_missing: true - dot_expander: field: service.name path: elasticsearch.server @@ -18,40 +30,32 @@ processors: field: elasticsearch.server.service.name target_field: service.name ignore_missing: true -- rename: - field: elasticsearch.server.component - target_field: elasticsearch.component - ignore_missing: true - dot_expander: - field: cluster.name + field: elasticsearch.cluster.name path: elasticsearch.server - rename: - field: elasticsearch.server.cluster.name + field: elasticsearch.server.elasticsearch.cluster.name target_field: elasticsearch.cluster.name - dot_expander: - field: node.name + field: elasticsearch.node.name path: elasticsearch.server - rename: - field: elasticsearch.server.node.name + field: elasticsearch.server.elasticsearch.node.name target_field: elasticsearch.node.name - dot_expander: - field: cluster.uuid + field: elasticsearch.cluster.uuid path: elasticsearch.server - rename: - field: elasticsearch.server.cluster.uuid + field: elasticsearch.server.elasticsearch.cluster.uuid target_field: elasticsearch.cluster.uuid ignore_missing: true - dot_expander: - field: node.id + field: elasticsearch.node.id path: elasticsearch.server - rename: - field: elasticsearch.server.node.id + field: elasticsearch.server.elasticsearch.node.id target_field: elasticsearch.node.id ignore_missing: true -- rename: - field: elasticsearch.server.level - target_field: log.level - ignore_missing: true - dot_expander: field: log.level path: elasticsearch.server From 4dc1328136b21e2bd3262483fbef3c7c2fef9038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 26 Jan 2022 14:40:35 +0100 Subject: [PATCH 03/27] Do not rename `@timestamp` instead copy to make sure there is always a timestamp field --- filebeat/module/elasticsearch/audit/ingest/pipeline.yml | 6 +++--- .../module/elasticsearch/deprecation/ingest/pipeline.yml | 6 +++--- filebeat/module/elasticsearch/gc/ingest/pipeline.yml | 6 +++--- filebeat/module/elasticsearch/server/ingest/pipeline.yml | 6 +++--- filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml index 1ae5da8dbb7..e241acafacb 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml index e1f4838df9b..7c64e431021 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml index d0980763ecc..6d3c9006a20 100644 --- a/filebeat/module/elasticsearch/gc/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/gc/ingest/pipeline.yml @@ -36,9 +36,9 @@ processors: PROCTIME: '\[Times: user=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.user_sec} sys=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.sys_sec}, real=%{BASE10NUM:elasticsearch.gc.phase.cpu_time.real_sec} secs\]' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - date: field: timestamp target_field: '@timestamp' diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline.yml b/filebeat/module/elasticsearch/server/ingest/pipeline.yml index 4d4e634cc4b..32abc88dae4 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml index ea501d9b3e0..440220f1dd7 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline.yml @@ -3,9 +3,9 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- rename: - field: '@timestamp' - target_field: event.created +- set: + copy_from: "@timestamp" + field: event.created - grok: field: message patterns: From 344352cff85361cb7052baa0348091582f8df6a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Wed, 26 Jan 2022 16:47:14 +0100 Subject: [PATCH 04/27] override `@timestamp` if needed --- .../slowlog/ingest/pipeline-json.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 174a429946a..482e1a1952c 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -98,13 +98,21 @@ processors: - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] - remove: field: elasticsearch.slowlog.message -- rename: +- set: + copy_from: elasticsearch.slowlog.@timestamp + target_field: "@timestamp" + override: true + if: "ctx.elasticsearch.slowlog.@timestamp != null" +- set: + copy_from: elasticsearch.slowlog.timestamp + target_field: "@timestamp" + override: true + if: "ctx.elasticsearch.slowlog.timestamp != null" +- remove: field: elasticsearch.slowlog.@timestamp - target_field: '@timestamp' ignore_missing: true -- rename: +- remove: field: elasticsearch.slowlog.timestamp - target_field: '@timestamp' ignore_missing: true - date: field: '@timestamp' From 4671444ae03379eaee79189865fb4bc77af2ba59 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 27 Jan 2022 12:16:11 +0900 Subject: [PATCH 05/27] Isolate 7 & 8 pipelines for es server logs The format is quite different, so seemed better to keep them separate. There's no specific "version" to keep off of so using presence of "type" (7-style) or "ecs.version" (8-style) as a switch. --- .../server/ingest/pipeline-json-7.yml | 101 ++++++++++++++++ .../server/ingest/pipeline-json-8.yml | 109 ++++++++++++++++++ .../server/ingest/pipeline-json.yml | 107 +---------------- .../module/elasticsearch/server/manifest.yml | 2 + 4 files changed, 218 insertions(+), 101 deletions(-) create mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml create mode 100644 filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..d92dd640772 --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json-7.yml @@ -0,0 +1,101 @@ +description: Pipeline for parsing the Elasticsearch 7.x server log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- drop: + if: ctx.elasticsearch.server.type != 'server' +- remove: + field: elasticsearch.server.type +- dot_expander: + field: service.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.server.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.server +- rename: + field: elasticsearch.server.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.server +- rename: + field: elasticsearch.server.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.server.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true +- grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} +- remove: + field: elasticsearch.server.message +- set: + field: '@timestamp' + value: '{{ elasticsearch.server.timestamp }}' + ignore_empty_value: true +- remove: + field: elasticsearch.server.timestamp +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..7619050a4cb --- /dev/null +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json-8.yml @@ -0,0 +1,109 @@ +description: Pipeline for parsing the Elasticsearch 8.0 server log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- dot_expander: + field: event.dataset + path: elasticsearch.server +- drop: + if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' +- set: + value: '{{ elasticsearch.server.event.dataset }}' + field: event.dataset + ignore_empty_value: true +- remove: + field: elasticsearch.server.event.dataset +- dot_expander: + field: ecs.version + path: elasticsearch.server +- set: + value: '{{ elasticsearch.server.ecs.version }}' + field: ecs.version + ignore_empty_value: true +- remove: + field: elasticsearch.server.ecs.version +- dot_expander: + field: service.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.service.name + target_field: service.name + ignore_missing: true +- dot_expander: + field: elasticsearch.cluster.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: elasticsearch.node.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: elasticsearch.cluster.uuid + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: elasticsearch.node.id + path: elasticsearch.server +- rename: + field: elasticsearch.server.elasticsearch.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.server +- rename: + field: elasticsearch.server.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.server +- rename: + field: elasticsearch.server.process.thread.name + target_field: process.thread.name + ignore_missing: true +- grok: + field: elasticsearch.server.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent + \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] + collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] + GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} + patterns: + - '%{GC_ALL}' + - '%{GC_YOUNG}' + - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} +- remove: + field: elasticsearch.server.message +- set: + field: '@timestamp' + value: '{{ elasticsearch.server.@timestamp }}' + ignore_empty_value: true +- remove: + field: elasticsearch.server.@timestamp +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml index 2b9eaf8a8b7..e5b23aabfd9 100644 --- a/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/server/ingest/pipeline-json.yml @@ -7,104 +7,9 @@ processors: - json: field: message target_field: elasticsearch.server -- dot_expander: - field: event.dataset - path: elasticsearch.server -- drop: - if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server' -- rename: - field: elasticsearch.server.event.dataset - target_field: event.dataset - ignore_missing: true -- dot_expander: - field: ecs.version - path: elasticsearch.server -- rename: - field: elasticsearch.server.ecs.version - target_field: ecs.version - ignore_missing: true -- dot_expander: - field: service.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.service.name - target_field: service.name - ignore_missing: true -- dot_expander: - field: elasticsearch.cluster.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.elasticsearch.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: elasticsearch.node.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.elasticsearch.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: elasticsearch.cluster.uuid - path: elasticsearch.server -- rename: - field: elasticsearch.server.elasticsearch.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: elasticsearch.node.id - path: elasticsearch.server -- rename: - field: elasticsearch.server.elasticsearch.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.server -- rename: - field: elasticsearch.server.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.server -- rename: - field: elasticsearch.server.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.server -- rename: - field: elasticsearch.server.process.thread.name - target_field: process.thread.name - ignore_missing: true -- grok: - field: elasticsearch.server.message - pattern_definitions: - GREEDYMULTILINE: |- - (.| - )* - INDEXNAME: '[a-zA-Z0-9_.-]*' - GC_ALL: \[gc\]\[%{NUMBER:elasticsearch.server.gc.overhead_seq}\] overhead, spent - \[%{NUMBER:elasticsearch.server.gc.collection_duration.time:float}%{DATA:elasticsearch.server.gc.collection_duration.unit}\] - collecting in the last \[%{NUMBER:elasticsearch.server.gc.observation_duration.time:float}%{DATA:elasticsearch.server.gc.observation_duration.unit}\] - GC_YOUNG: \[gc\]\[young\]\[%{NUMBER:elasticsearch.server.gc.young.one}\]\[%{NUMBER:elasticsearch.server.gc.young.two}\]%{SPACE}%{GREEDYMULTILINE:message} - patterns: - - '%{GC_ALL}' - - '%{GC_YOUNG}' - - ((\[%{INDEXNAME:elasticsearch.index.name}\]|\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\]))?%{SPACE}%{GREEDYMULTILINE:message} -- remove: - field: elasticsearch.server.message -- rename: - field: elasticsearch.server.@timestamp - target_field: '@timestamp' - ignore_missing: true -- rename: - field: elasticsearch.server.timestamp - target_field: '@timestamp' - ignore_missing: true -- date: - field: '@timestamp' - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true +- pipeline: + if: ctx.elasticsearch.server.containsKey('type') + name: '{< IngestPipeline "pipeline-json-7" >}' +- pipeline: + if: ctx.elasticsearch.server.containsKey('ecs.version') + name: '{< IngestPipeline "pipeline-json-8" >}' diff --git a/filebeat/module/elasticsearch/server/manifest.yml b/filebeat/module/elasticsearch/server/manifest.yml index 406972cba56..d9d5d4e398b 100644 --- a/filebeat/module/elasticsearch/server/manifest.yml +++ b/filebeat/module/elasticsearch/server/manifest.yml @@ -16,4 +16,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/log.yml From bd320d84f1716b940f04edea443d78c1df2d9a1a Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 27 Jan 2022 12:16:28 +0900 Subject: [PATCH 06/27] Re-generated expected 8.0.0 server log documents --- .../elasticsearch-json.800.log-expected.json | 61 +++++++++---------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json index 817cadf6002..228661fc9c3 100644 --- a/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log-expected.json @@ -1,70 +1,67 @@ [ { - "@timestamp": "2020-04-14T14:05:58.019Z", - "elasticsearch.cluster.name": "elasticsearch", - "elasticsearch.cluster.uuid": "ECEBF2VPQuCF9tbBKaLqXQ", - "elasticsearch.node.id": "suOYiQwuRvialOY-c0wHLA", - "elasticsearch.node.name": "CBR-MBP.local", + "@timestamp": "2022-01-25T15:12:08.472Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "suOYiQwuRvialOY-c0wHLA", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", "log.offset": 0, - "message": "adding template [.management-beats] for index patterns [.management-beats]", - "process.thread.name": "elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]", + "message": "adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" }, { - "@timestamp": "2020-04-14T20:57:49.663Z", - "elasticsearch.cluster.name": "docker-cluster", - "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", - "elasticsearch.index.name": "test-filebeat-modules", - "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", - "elasticsearch.node.name": "7debcb878699", + "@timestamp": "2022-01-25T15:12:08.588Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "kZnYdakGTqihZQT_1rM92g", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetadataCreateIndexService", - "log.offset": 489, - "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", - "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", + "log.offset": 608, + "message": "adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" }, { - "@timestamp": "2020-04-14T20:57:49.772Z", - "elasticsearch.cluster.name": "docker-cluster", - "elasticsearch.cluster.uuid": "QxYAE76DTAWkgk9CwIRedQ", - "elasticsearch.index.id": "IW1jJcOBTFeIDihqjoT8yQ", - "elasticsearch.index.name": "test-filebeat-modules", - "elasticsearch.node.id": "kZnYdakGTqihZQT_1rM92g", - "elasticsearch.node.name": "7debcb878699", + "@timestamp": "2022-01-25T15:12:08.686Z", + "elasticsearch.cluster.name": "main", + "elasticsearch.cluster.uuid": "28iKoFsvTJ6HEyXbdLL-PQ", + "elasticsearch.node.id": "tc3nhgC0SFCKfwwy6jCmkw", + "elasticsearch.node.name": "matschaffer-mbp2019.lan", "event.category": "database", "event.dataset": "elasticsearch.server", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "server", - "host.id": "kZnYdakGTqihZQT_1rM92g", + "host.id": "tc3nhgC0SFCKfwwy6jCmkw", "input.type": "log", "log.level": "INFO", - "log.logger": "org.elasticsearch.cluster.metadata.MetadataMappingService", - "log.offset": 1031, - "message": "update_mapping [_doc]", - "process.thread.name": "elasticsearch[7debcb878699][masterService#updateTask][T#1]", + "log.logger": "org.elasticsearch.cluster.metadata.MetadataIndexTemplateService", + "log.offset": 1220, + "message": "adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", + "process.thread.name": "elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" } From d0095f9aa01bad81068466cf4a15b2a8f822d4cb Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Thu, 27 Jan 2022 15:20:58 +0900 Subject: [PATCH 07/27] Fix slowlog set pipeline entries --- .../elasticsearch/slowlog/ingest/pipeline-json.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 482e1a1952c..00ce95ccaa1 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -99,15 +99,13 @@ processors: - remove: field: elasticsearch.slowlog.message - set: - copy_from: elasticsearch.slowlog.@timestamp - target_field: "@timestamp" - override: true - if: "ctx.elasticsearch.slowlog.@timestamp != null" + value: "{{ elasticsearch.slowlog.@timestamp }}" + field: "@timestamp" + ignore_empty_value: true - set: - copy_from: elasticsearch.slowlog.timestamp - target_field: "@timestamp" - override: true - if: "ctx.elasticsearch.slowlog.timestamp != null" + value: "{{ elasticsearch.slowlog.timestamp }}" + field: "@timestamp" + ignore_empty_value: true - remove: field: elasticsearch.slowlog.@timestamp ignore_missing: true From 100ad269e56caf9366f39036c7f7eea786ba3a90 Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 27 Jan 2022 13:56:41 +0100 Subject: [PATCH 08/27] support 8.x deprecation logs --- .../deprecation/ingest/pipeline-json-7.yml | 97 ++++++++++++++++ .../deprecation/ingest/pipeline-json-8.yml | 105 ++++++++++++++++++ .../deprecation/ingest/pipeline-json.yml | 95 ++-------------- .../elasticsearch/deprecation/manifest.yml | 2 + 4 files changed, 211 insertions(+), 88 deletions(-) create mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml create mode 100644 filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..08f044e68d5 --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-7.yml @@ -0,0 +1,97 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.deprecation +- drop: + if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)' +- remove: + field: elasticsearch.deprecation.type +- dot_expander: + field: service.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- remove: + field: message +- rename: + field: elasticsearch.deprecation.message + target_field: message +- date: + field: 'elasticsearch.deprecation.@timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' +- date: + field: 'elasticsearch.deprecation.timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation?.timestamp != null' +- remove: + field: + - elasticsearch.deprecation.timestamp + - elasticsearch.deprecation.@timestamp + ignore_missing: true diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..8c9e665424e --- /dev/null +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml @@ -0,0 +1,105 @@ +description: Pipeline for parsing the Elasticsearch deprecation log file in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- drop: + if: ctx.event.dataset != 'elasticsearch.deprecation' +- set: + value: '{{ elasticsearch.deprecation.event.dataset }}' + field: event.dataset + ignore_empty_value: true +- dot_expander: + field: ecs.version + path: elasticsearch.deprecation +- set: + value: '{{ elasticsearch.deprecation.ecs.version }}' + field: ecs.version + ignore_empty_value: true +- remove: + field: elasticsearch.deprecation.ecs.version +- dot_expander: + field: service.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.deprecation.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: elasticsearch.cluster.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.elasticsearch.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: elasticsearch.node.name + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.elasticsearch.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: elasticsearch.cluster.uuid + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.elasticsearch.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: elasticsearch.node.id + path: elasticsearch.deprecation +- rename: + field: elasticsearch.deprecation.elasticsearch.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- remove: + field: message +- rename: + field: elasticsearch.deprecation.message + target_field: message +- date: + field: 'elasticsearch.deprecation.@timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' +- date: + field: 'elasticsearch.deprecation.timestamp' + formats: + - ISO8601 + ignore_failure: true + if: 'ctx.elasticsearch?.deprecation?.timestamp != null' +- remove: + field: + - elasticsearch.deprecation.timestamp + - elasticsearch.deprecation.@timestamp + ignore_missing: true diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml index 08f044e68d5..71854d8c1ed 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml @@ -7,91 +7,10 @@ processors: - json: field: message target_field: elasticsearch.deprecation -- drop: - if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)' -- remove: - field: elasticsearch.deprecation.type -- dot_expander: - field: service.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.process.thread.name - target_field: process.thread.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: node.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: cluster.uuid - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: node.id - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- remove: - field: message -- rename: - field: elasticsearch.deprecation.message - target_field: message -- date: - field: 'elasticsearch.deprecation.@timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' -- date: - field: 'elasticsearch.deprecation.timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation?.timestamp != null' -- remove: - field: - - elasticsearch.deprecation.timestamp - - elasticsearch.deprecation.@timestamp - ignore_missing: true +- pipeline: + if: ctx.elasticsearch.deprecation.containsKey('type') + name: '{< IngestPipeline "pipeline-json-7" >}' +- pipeline: + if: ctx.elasticsearch.deprecation.containsKey('ecs.version') + name: '{< IngestPipeline "pipeline-json-8" >}' + diff --git a/filebeat/module/elasticsearch/deprecation/manifest.yml b/filebeat/module/elasticsearch/deprecation/manifest.yml index 8dfbaec866b..93b1ef80b09 100644 --- a/filebeat/module/elasticsearch/deprecation/manifest.yml +++ b/filebeat/module/elasticsearch/deprecation/manifest.yml @@ -16,4 +16,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/log.yml From 455b4f9132672170bc435d4ebd7db5b0e893bb7c Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 27 Jan 2022 16:10:24 +0100 Subject: [PATCH 09/27] support 8.x slowlog logs --- .../slowlog/ingest/pipeline-json-7.yml | 120 ++++++++++++++++++ .../slowlog/ingest/pipeline-json-8.yml | 117 +++++++++++++++++ .../slowlog/ingest/pipeline-json.yml | 118 +---------------- .../module/elasticsearch/slowlog/manifest.yml | 2 + 4 files changed, 246 insertions(+), 111 deletions(-) create mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml create mode 100644 filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml new file mode 100644 index 00000000000..00ce95ccaa1 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml @@ -0,0 +1,120 @@ +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- json: + field: message + target_field: elasticsearch.slowlog +- drop: + if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type + != 'index_search_slowlog' +- remove: + field: elasticsearch.slowlog.type +- dot_expander: + field: service.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: cluster.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: node.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: cluster.uuid + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: node.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.slowlog.doc_type + target_field: elasticsearch.slowlog.types + ignore_missing: true +- convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true +- rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true +- grok: + field: elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- remove: + field: elasticsearch.slowlog.message +- set: + value: "{{ elasticsearch.slowlog.@timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- set: + value: "{{ elasticsearch.slowlog.timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.@timestamp + ignore_missing: true +- remove: + field: elasticsearch.slowlog.timestamp + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml new file mode 100644 index 00000000000..d511a14c806 --- /dev/null +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -0,0 +1,117 @@ +description: Pipeline for parsing the Elasticsearch slow logs in JSON format. +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' +processors: +- drop: + if: ctx.event.dataset != 'elasticsearch.slowlog' +- dot_expander: + field: service.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.service.name + target_field: service.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.level + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.level + target_field: log.level + ignore_missing: true +- dot_expander: + field: log.logger + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.log.logger + target_field: log.logger + ignore_missing: true +- dot_expander: + field: process.thread.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.process.thread.name + target_field: process.thread.name + ignore_missing: true +- rename: + field: elasticsearch.slowlog.component + target_field: elasticsearch.component + ignore_missing: true +- dot_expander: + field: elasticsearch.cluster.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.cluster.name + target_field: elasticsearch.cluster.name +- dot_expander: + field: elasticsearch.node.name + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.node.name + target_field: elasticsearch.node.name +- dot_expander: + field: elasticsearch.cluster.uuid + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.cluster.uuid + target_field: elasticsearch.cluster.uuid + ignore_missing: true +- dot_expander: + field: elasticsearch.node.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.node.id + target_field: elasticsearch.node.id + ignore_missing: true +- rename: + field: elasticsearch.slowlog.doc_type + target_field: elasticsearch.slowlog.types + ignore_missing: true +- convert: + field: elasticsearch.slowlog.took_millis + type: float + ignore_missing: true +- rename: + field: elasticsearch.slowlog.took_millis + target_field: elasticsearch.slowlog.duration + ignore_missing: true +- grok: + field: elasticsearch.slowlog.elasticsearch.slowlog.message + pattern_definitions: + GREEDYMULTILINE: |- + (.| + )* + INDEXNAME: '[a-zA-Z0-9_.-]*' + patterns: + - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? + - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.message + target_field: message + ignore_missing: true +- set: + value: "{{ elasticsearch.slowlog.@timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- set: + value: "{{ elasticsearch.slowlog.timestamp }}" + field: "@timestamp" + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.@timestamp + ignore_missing: true +- remove: + field: elasticsearch.slowlog.timestamp + ignore_missing: true +- date: + field: '@timestamp' + target_field: '@timestamp' + formats: + - ISO8601 + ignore_failure: true + diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 00ce95ccaa1..03d48f08da4 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -7,114 +7,10 @@ processors: - json: field: message target_field: elasticsearch.slowlog -- drop: - if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type - != 'index_search_slowlog' -- remove: - field: elasticsearch.slowlog.type -- dot_expander: - field: service.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.slowlog.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.process.thread.name - target_field: process.thread.name - ignore_missing: true -- rename: - field: elasticsearch.slowlog.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: cluster.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: node.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: cluster.uuid - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: node.id - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- rename: - field: elasticsearch.slowlog.doc_type - target_field: elasticsearch.slowlog.types - ignore_missing: true -- convert: - field: elasticsearch.slowlog.took_millis - type: float - ignore_missing: true -- rename: - field: elasticsearch.slowlog.took_millis - target_field: elasticsearch.slowlog.duration - ignore_missing: true -- grok: - field: elasticsearch.slowlog.message - pattern_definitions: - GREEDYMULTILINE: |- - (.| - )* - INDEXNAME: '[a-zA-Z0-9_.-]*' - patterns: - - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? - - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] -- remove: - field: elasticsearch.slowlog.message -- set: - value: "{{ elasticsearch.slowlog.@timestamp }}" - field: "@timestamp" - ignore_empty_value: true -- set: - value: "{{ elasticsearch.slowlog.timestamp }}" - field: "@timestamp" - ignore_empty_value: true -- remove: - field: elasticsearch.slowlog.@timestamp - ignore_missing: true -- remove: - field: elasticsearch.slowlog.timestamp - ignore_missing: true -- date: - field: '@timestamp' - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true +- pipeline: + if: ctx.elasticsearch.slowlog.containsKey('type') + name: '{< IngestPipeline "pipeline-json-7" >}' +- pipeline: + if: ctx.elasticsearch.slowlog.containsKey('ecs.version') + name: '{< IngestPipeline "pipeline-json-8" >}' + diff --git a/filebeat/module/elasticsearch/slowlog/manifest.yml b/filebeat/module/elasticsearch/slowlog/manifest.yml index caddd94158b..08b49643108 100644 --- a/filebeat/module/elasticsearch/slowlog/manifest.yml +++ b/filebeat/module/elasticsearch/slowlog/manifest.yml @@ -22,4 +22,6 @@ ingest_pipeline: - ingest/pipeline.yml - ingest/pipeline-plaintext.yml - ingest/pipeline-json.yml + - ingest/pipeline-json-7.yml + - ingest/pipeline-json-8.yml input: config/slowlog.yml From feddd420f1e8ff28e075678e630cabc3f24a6f1c Mon Sep 17 00:00:00 2001 From: klacabane Date: Thu, 27 Jan 2022 18:21:35 +0100 Subject: [PATCH 10/27] adapt deprecation tests to new format --- filebeat/docs/fields.asciidoc | 43 +++ .../deprecation/_meta/fields.yml | 12 + .../test/es_deprecation-json.800.log | 17 +- .../es_deprecation-json.800.log-expected.json | 336 ++---------------- filebeat/module/elasticsearch/fields.go | 2 +- 5 files changed, 89 insertions(+), 321 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 2c921af2f39..afb250f0c78 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -49927,6 +49927,49 @@ type: boolean + +*`elasticsearch.deprecation.data_stream.dataset`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.deprecation.data_stream.namespace`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.deprecation.data_stream.type`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.deprecation.event.code`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.deprecation.event.dataset`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.deprecation.elasticsearch.event.category`*:: ++ +-- +type: keyword + +-- + [float] === gc diff --git a/filebeat/module/elasticsearch/deprecation/_meta/fields.yml b/filebeat/module/elasticsearch/deprecation/_meta/fields.yml index b4f8083631e..bfa5887f6d0 100644 --- a/filebeat/module/elasticsearch/deprecation/_meta/fields.yml +++ b/filebeat/module/elasticsearch/deprecation/_meta/fields.yml @@ -2,3 +2,15 @@ type: group description: > fields: + - name: data_stream.dataset + type: keyword + - name: data_stream.namespace + type: keyword + - name: data_stream.type + type: keyword + - name: event.code + type: keyword + - name: event.dataset + type: keyword + - name: elasticsearch.event.category + type: keyword diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log index 888a5d92080..f1f39cda24b 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log @@ -1,15 +1,2 @@ -{"@timestamp":"2020-04-15T12:35:20.315Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.316Z", "log.level": "WARN", "message":"Field parameter [tree] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.366Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.367Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.479Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.480Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.481Z", "log.level": "WARN", "message":"Field parameter [precision] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-15T12:35:20.487Z", "log.level": "WARN", "message":"Field parameter [strategy] is deprecated and will be removed in a future version." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper","type":"deprecation","cluster.uuid":"a0P-i2H5R9-tJqwtF7BL0A","node.id":"FFMF7MVISuCWZMtxGmcGhg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T13:46:33.582Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#3]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.219Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#4]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.339Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#5]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:46:34.455Z", "log.level": "WARN", "message":"[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][http_server_worker][T#6]","log.logger":"org.elasticsearch.deprecation.rest.RestController","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:47:36.309Z", "log.level": "WARN", "message":"index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:55:56.365Z", "log.level": "WARN", "message":"index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} -{"@timestamp":"2020-04-16T13:56:14.697Z", "log.level": "WARN", "message":"[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead." , "service.name":"ES_ECS","process.thread.name":"elasticsearch[n1][search][T#7]","log.logger":"org.elasticsearch.deprecation.index.query.QueryShardContext","type":"deprecation","cluster.uuid":"ZGYecRsDQPK_-ktRec3ZGQ","node.id":"Ni-9zbrZRm24wm7_zNtMTw","node.name":"n1","cluster.name":"es800"} +{"@timestamp":"2022-01-27T11:25:19.412Z", "log.level": "WARN", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"indices","event.code":"index_name_starts_with_dot","message":"index name [.kibana-event-log-8.0.0] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","elasticsearch.cluster.uuid":"mIMVAJO4TSmq1mu7hCPZ7A","elasticsearch.node.id":"hPDoent5RQ2tj7zTJtOagg","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:26:34.762Z", "log.level": "WARN", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"settings","event.code":"xpack.monitoring.collection.enabled","message":"[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.common.settings.Settings","elasticsearch.cluster.uuid":"mIMVAJO4TSmq1mu7hCPZ7A","elasticsearch.node.id":"hPDoent5RQ2tj7zTJtOagg","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json index 89f625d1f17..5e7c6a2b4c2 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json @@ -1,331 +1,57 @@ [ { - "@timestamp": "2020-04-15T12:35:20.315Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:25:19.412Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "mIMVAJO4TSmq1mu7hCPZ7A", + "elasticsearch.deprecation.data_stream.dataset": "deprecation.elasticsearch", + "elasticsearch.deprecation.data_stream.namespace": "default", + "elasticsearch.deprecation.data_stream.type": "logs", + "elasticsearch.deprecation.elasticsearch.event.category": "indices", + "elasticsearch.deprecation.event.code": "index_name_starts_with_dot", + "elasticsearch.deprecation.event.dataset": "deprecation.elasticsearch", + "elasticsearch.node.id": "hPDoent5RQ2tj7zTJtOagg", + "elasticsearch.node.name": "runTask-0", "event.category": "database", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 0, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.316Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 501, - "message": "Field parameter [tree] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.366Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 997, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.367Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 1498, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.479Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 1998, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.480Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 2507, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.481Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 3015, - "message": "Field parameter [precision] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-15T12:35:20.487Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "a0P-i2H5R9-tJqwtF7BL0A", - "elasticsearch.node.id": "FFMF7MVISuCWZMtxGmcGhg", - "elasticsearch.node.name": "integTest-0", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "FFMF7MVISuCWZMtxGmcGhg", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.mapper.LegacyGeoShapeFieldMapper", - "log.offset": 3524, - "message": "Field parameter [strategy] is deprecated and will be removed in a future version.", - "process.thread.name": "elasticsearch[integTest-0][clusterApplierService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:33.582Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 4032, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#3]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.219Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 4523, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#4]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.339Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 5014, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#5]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:46:34.455Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.rest.RestController", - "log.offset": 5505, - "message": "[PUT /_xpack/security/user/{username}/_password] is deprecated! Use [PUT /_security/user/{username}/_password] instead.", - "process.thread.name": "elasticsearch[n1][http_server_worker][T#6]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:47:36.309Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", + "host.id": "hPDoent5RQ2tj7zTJtOagg", "input.type": "log", "log.level": "WARN", "log.logger": "org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService", - "log.offset": 5996, - "message": "index name [.apm-custom-link] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", - "process.thread.name": "elasticsearch[n1][masterService#updateTask][T#1]", - "service.name": "ES_ECS", - "service.type": "elasticsearch" - }, - { - "@timestamp": "2020-04-16T13:55:56.365Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", - "event.category": "database", - "event.dataset": "elasticsearch.deprecation", - "event.kind": "event", - "event.module": "elasticsearch", - "event.type": "info", - "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", - "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService", - "log.offset": 6560, - "message": "index name [.monitoring-alerts-7] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", - "process.thread.name": "elasticsearch[n1][masterService#updateTask][T#1]", + "log.offset": 0, + "message": "index name [.kibana-event-log-8.0.0] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", + "process.thread.name": "elasticsearch[runTask-0][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" }, { - "@timestamp": "2020-04-16T13:56:14.697Z", - "elasticsearch.cluster.name": "es800", - "elasticsearch.cluster.uuid": "ZGYecRsDQPK_-ktRec3ZGQ", - "elasticsearch.node.id": "Ni-9zbrZRm24wm7_zNtMTw", - "elasticsearch.node.name": "n1", + "@timestamp": "2022-01-27T11:26:34.762Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "mIMVAJO4TSmq1mu7hCPZ7A", + "elasticsearch.deprecation.data_stream.dataset": "deprecation.elasticsearch", + "elasticsearch.deprecation.data_stream.namespace": "default", + "elasticsearch.deprecation.data_stream.type": "logs", + "elasticsearch.deprecation.elasticsearch.event.category": "settings", + "elasticsearch.deprecation.event.code": "xpack.monitoring.collection.enabled", + "elasticsearch.deprecation.event.dataset": "deprecation.elasticsearch", + "elasticsearch.node.id": "hPDoent5RQ2tj7zTJtOagg", + "elasticsearch.node.name": "runTask-0", "event.category": "database", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "Ni-9zbrZRm24wm7_zNtMTw", + "host.id": "hPDoent5RQ2tj7zTJtOagg", "input.type": "log", "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.index.query.QueryShardContext", - "log.offset": 7128, - "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead.", - "process.thread.name": "elasticsearch[n1][search][T#7]", + "log.logger": "org.elasticsearch.deprecation.common.settings.Settings", + "log.offset": 882, + "message": "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", + "process.thread.name": "elasticsearch[runTask-0][masterService#updateTask][T#1]", "service.name": "ES_ECS", "service.type": "elasticsearch" } diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 889d94d042a..46e06a7199b 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUml9z2zYSwN/zKXb0cu2MzZP/1Fdr5m6mVRzbmSZxLdu5VvFwVuCKQgQCNABKVjv57jcAKVmiSOrPtbmeXxKKAPa3i8ViF8QhjGnWARJoLGeGULPRKwDLraAOtC6Wf2+9AtAkCA11IMZXABEZpnlquZId+NcrAFgdCd6pKBP0CmDISUSm45scgsSE1oW6PztL3eBaZWnxS4WM1eGWh2QqSZUkaRdvSgOsavTSHoZaJTAdkSawIwKhYqCJe6E0j7lES1FraVB6xiT1JlIBBSxIgndk8TVa7GpCS9cyouce6QlntNwv129Ms6nS0Tq+yIwlHWQZj2o1uL+/fg1q6DGLDtVkl8lED67E+zvee/iF3wy/Hz/H5/HuNO6pluY9JrQVTaTYmPRhRZtmCqkiChrM8WIM17Ja9use/8juZnQ3+mjv//3Tj+dv2z++m+7IsLUZ6jkmH9+/Nb+ebC+YOzdqluw9zTevljnkggaE9tCSsYdcppndVX6T9b10XrM28MNl/Ho6uL8ddh+++8cPPfY06MY72N2MUEeN4qO50X3Taor29gIxi7hda70cjqAi+iyPIHBGeuVNmfnOhRfXah5zOBuBHXGzFnA6oMnYA7AapUmVdu+Ap+GQi9ISWlXY9Sq/rdZ7mdxLD127jfiukTN8DmxHaEExlmntmFEqOUtUZkJkjIwJI5KcogPAzI5IWs7QDRUOkQv/c6lV/hhrlNY9MyUlMd+j6rd5N4tJSpqiUNNT5q2mMxni0kDFc96h3nir8nc3Yz59wUY7flzsNQXx2sTDN+tvcp9BuL3o3cEPN9fzzt8ue8mi3xQNaGLEJxSBkl7aSzM2QilJfHsAQjEUoYtb8E2++zEUPo4BNyajaJnz23rbvYyzu900oUg2et6qD+WdPFzphdN8goJH3mgYI5fra6IAb7ndiYaYCeuW1h7smSEdbKeAa/o3U6nHAfDh8otaL215N7V8QmHENTGr9GxfaCXINELfuhZg1SJQEaSaS8ZTFDAgoWRsaj2iD60xH6DEEKOEy9YBtNxeZIpHeNyT2i/l8r64+wBVU7Z5hKokc7ue6GPWRh+RS8lE3iWPsc6p6ZlYVu8aHWgVGVYnUZJbpf+eIJd7eIcWQYoakw3e4eLQ/e01+LZkSdc7Q+t3Z3s3/D8/IxtLzkbHX1qV0rmMONvgmNd5m2LDoAgGs8JaTe44VOrwuH10HrSPgvapc8iVX07Wfjnbx0uLULmauqyrcC/5U0aQp7BFn3rzffztp3A8OHvoTT6Mfnhq2+nN5OrDz/tE2hyuYvnUb/XzLWUHR+wKQt1jWglxW63b1qzhQEWzys4oOJb9JEU76sDI2jSY6+r6B0xJu75sEx5rzDW2OqOGTT3EKNJkyuI2gRiVaUYBT/cQnGm+ozS3cIt8QewhcBHbdxVr1ivFbWUmZAzG1aHc0rOtCRHz/T3AlI9pZgI1lRSFg1m4somGDq1y7IFSglCuVQERpZrynXljLVB5PAENRxQx239MgMsuuBTFkC0EBFvWJekITbWFy9I3ELi/N14QmJQYH3LmsoPLbi4iKDWuYlrmqnAZaIwMWwG6v+WS/LILTAmR1wzVoEvTn+UeGxpitWhDobAcSbYE65ZIFgLdVqZ0xGXsLOq43+IEYcK1zVBAgmzEZQO4YTobhGaWDJQILQ4EhZYn9GfpATeYGQInArgEQ0zJyABza8rpkKWQs4BnMRvBreYy/grgW3B7lI3cU8JxqGlowlQrl4V4/j+R/M4xm9RV3S8SPQZoGpIm6TKiF6Xq0V2+JgSJUJNhKL8W9ZK9E9RjRy/4hEANPhOzxhUZggDTVMxLI27AWJWmFNUrwwQaE2ZSKIy+lia5NO8vMnPpp4fY0voszTxnLWNVUN6S8SZ3DOje3Oc+XvgL6aHSiQN+CYUViPUhG0pVVI2RYaOht1TE/ZWUUJk1PMqPTcakJYkqBZYCy8z8Dyi5LENCI6UrQ78G5p2yKIAEps5fS9BW+cJWkM3Jl/ZLf+ZkLGrfasglN6OgMsv4PElCncmaJVivyAYFfCHiUD3J24d3BU2WLq22A0ADmA/vvDxVXFqQWTIgXU1rR5owMqF1dgldlKkLHnuTX6IeYLxizUIqeKk+thXTUBU0Fo7sQqDfXebMf7SJHYJVauymOIcqOBu5LMbV5VB16rbJWl0QKo7zrTeuETkiLEfGvRPZK8IUUAhVbDYoo/m88N92zmVdn3A8qA3qXFqK12qRLTBhsXid8l6Oc/wxF2ows00ZituZ/jSkexdGPFE9zKKCFlEYU/nsa++J+yAiiElSkTgrxrIUJZv99WfQT54aOoMsa/AXmM5am26e3ZnKZPxHzu8vbsD/8xmelXX4C8xxg12r6RZ2Iz1ZEbp6VNjzr/21ifInlG2/ob5sdcjGViNbzY6X5LU60HONwLdy4MyV0WoIpLXSqxuS/4TdgSGKlfOPyuOYslb5frR6alnn0k2HL94TmhZAK5+Xy279aWr12WnV0qpeAotALNerjlWWsqQmijmHUGsKLlKEqfoaAhf6TUiPCKPQ0FOjyXv0lLl6uUgRay1/cnp6fn5+XGn+WoqXfC+cn+4EGz6lrFbJl90D90/CheBFBlZLeHTWbm+ZBy6sNHALGncD9NHN56rOyMUHwaXMdoqmGJiiHei/34p+ER6EmgoV10ei/H1+N8HkFcPaNbY1iFb/uH30/WH77PD4/O6o3WmfdY5OD85PTh771+/ffIDHfn4VJh8iKCCCp4z07BH6k/Dh7ejzwyP0E7KaM3/h5iw4CdqHbtygfRYcnz32248+xe6fBt8l5vHAP4S5kfqn/tkVIiNuTf/o/PTkO/fTLCXTfzxwYdHm//EI/qJG/+f7i9tfwruri/fhm4u77tViDH8dxvSPXHv/+aH/+6eWp/3U6vz+qZWgZaMQhcgfB0oZ+6nVOQraX758eTz4b+K3y+BL29PqDP3kG6xdWVqejUpjD8muzl59rbGIPUqNG0j8kuN2UfcU37R8/euNVcd30m4nZkcUN5FNLO59nbzdRHlXaRDVc+/zGa2V6N8e7Sj3xTObpOdXL12rOuFlt94Rwzt86CewiUOoafMs77BkdiOkZ6sxzDkbCC9cs0Id4HKodILr37f39ZKXYNPklXnVyW2do5we7yE0j04bxTrjc4ryu311AMe7AWiVWV7atMsXXnyLOiOb9tHVr8c//zg+/zw9jW2Mb6zczfClSwEr0q+jP2Zum5fgXcPaixTbZ7nVS+vl/quGECmWJYsbgy5b8HGeogZ5/wkAAP//ZMWUGw==" + return "eJzUmt1v2zgSwN/7VxB+uV0g0Tkfm9sYuAO2bpqk6EcaJ+l13UAYU2OJNUUqJGXHW/R/P5CSHVmWZMvX9np+SSR+zG+Gw+GQ4j6Z4LxHkIM2jGoERaNnhBhmOPZI56z4vvOMEIUcQWOPhPCMkAA1VSwxTIoe+dczQshqT+SNDFKOzwgZM+SB7rkq+0RAjOtC7c/ME9u5kmmSv6mQsdpdsUsq40QKFGZZUupgVaOn+mSsZExmESokJkLCZUhwagukYiETYDDoFDrFR4gTZyLpoUe92HuDBl6Agb5CMHgpAnwcoJoyisV2mX4TnM+kCtbxeaoNKi9NWVCrwe3t5Qsixw4zb1BNdh5P1eiCv71hg7uP7Gr8++QxPA3b09inWpq3EONWNIGkE1T7FXWaKYQM0Gswx5MxbM1q2S8G7AO9meNN9MHc/vv189NX3edvZi0ZtjZDPcf0w9tX+s+j7QUz60bNkp2nuerVMseM4wjB7BvUZp+JJDVt5TdZ30lnNXMD3p2HL2aj2+tx/+63f/wxoA+jftjC7joCFTSKDxZGd1WrKbrbC4Q0YGatdjEckYroU+yBwxzVSkmZ+caGF1trEXMYjYiJmF4LOD2iUJs9YhQInUhlywhL/DHjpSm0qrBtVS6t1rtI7qT7tt5GfFvJGj4DNhEYIilNlbLMIKSYxzLVPlCKWvsBCobBHoHURCgMo2C78sfAuHtdqpU9hgqEsc9UCoHUtah6t2hmIE5QYeArfEid1VQqfCh0lD9nDeqNtyq/vRmz4fM22vHDcq3JidcGnvyyXpL5DJDrs8EN+ePqctH416KXLNvNQBOFFNkUAyKFk/ZUjUYgBPJf9wiXFLhv4xb5JVv9KHAXxwjTOsWgyPlrve2e+mlvN4XA442et+pDWSMHVyqwmk+Bs8AZDUJgYn1O5OAduzrhGFJu7NTagT3VqLztFLBV/6Yr9dgjbFwsqPXSjnNTw6boB0whNVLNd4WWHHUj9LWtQYxcBiokiWKCsgQ4GSGXItS1HjEknQkbgQAfgpiJzh7p2LVI54/kfkdqN5XL62L7DqqGbHMPVUnmdi3BxayNPiIKyUTWJIux1qnxEWla7xo90skzrF4sBTNS/T0GJnbwDsW9BBTEG7zDxqHb60vi6qJBVe8MnS/W9rb7f34GOhGMRodfO5XSmQgY3eCYl1mdfMHAgIzmubWa3HEs5f5h9+DU6x543WPrkCtvjtbenOzipXmoXE1d1lW4FewhRZKlsHmbevN9+Ou1Pxmd3A2m76I/HrpmdjW9ePd+l0ibwVVMn/qlfrGktHDEPkdQA6ok59fVum3N6o9kMK9sDJxB2U8SMFGPRMYk3kJX296jUpj1aRuzUEGmsVEpNizqPgSBQl0WtwlEy1RR9Fiyg+BUsZbS7MTN8wW+g8BlbG8rVq/vFLeVGaPWEFaHcoOPpiZELNZ3DxI2wbn25Exg4I/m/soi6lu0yr5HUnIEsbYLCDBRmK3MG/cClccTZMMmIQADvjYKIfbs/xrbryXFPuwbnQBtvxoWe6lIWbfcNXhUBrs23VX/ldMjL8cAg6FU1ZGibs8X0t0HmZDzPrE5o0aTj7i3pQ8kEehqm5WlbyCwv5dOENEJUjZm1KZr5/1MhFeqXMVU5KqYw/U23BrQ/opnJOd9QiXn2SauGrTgpGkWQnyNtBZtzCWUvWhLsH6JZCnQ5hZSBUyE1qKW+xVMgUyZMilwEgONmGgA11SlI1/P45HkvoERR9+wGL+XHuQKUo3EiiBMEI1UikATaoOc1SFNSMZCHIveCG4UE+EPAN+C26Fs5J4hTHyFY+0nStq00PF/R/Iby6wTFIY8SXQYROEYFQqboj4pVY9uE2jOkfsKNQXxo6gL9o5BTSw9Z1MkcvQZqdF218eRQJLwxV6VaaKNTBIM6pWhHLT2U8ElBD9Kk0ya8xeR2v2Ag9jS+jRJHWctY1VQ3pLxKnMM0r+6zXw89xdUY6liC/wUCisQ60M2KW1ra4xMNhp6S0Xsr6SETI1mQXaONUElkFcpUAgsc/0/oGSiDEkaKRUC/xGYN9IAt5lMYv21BG2kO2ngaDLywnrpDgG1AeVqjZlgOvIqs4zP09hXqaiZgvWKbFDA7QwtqiN5dfcmp0mTwmzbI6AJZN1bL08kE4aINB6hqqY1kUIItG+sXXwbZeqCx87k56BGEK5YM5dKnFQX2/JhqAoaS0e2IdCtLgvmb21ii2CknNghzqByzkYuA2H1/rQ6ddtkrT7hMgyzpTesERkhlCPjzonsBUJCgHOZLzYggsW4sL9a57K2jT8Z1QZ1JgyGa5vDLTDJcvJa5Z0c6/gTxuVobpoyFLsyfTekWxtGHFE9zPJIgwd+iOXDyJ0H7h0PSIgC88RZUpomIOj85x9BN3hybA1S1OAnGM5am24e3blMRfgtx/ej7fD/fITnZR1+gjFusGs13dJuqKYrQlfPbgeu2N1jKX/T2vaj9tNSB3RiVPl4qSCv0yMDW4m4Whac2m20HBNUSqrVBcndKeiRMfCV84/K45iyVtl6tHqMXOfSTYcvzhOaJkAnG5fzfv3xdvVhdtXUqp4Cy0As1ncdqyxlSU0UCw4u1xRcpggz+SMELvWboooQAl/jQ6PJB/iQ2v1yniLWWv7o+Pj09PSw0vy1FE/5nr843fE2fNta3SWf9/fsn5hxzvIMrJbw4KTb3TIPXFppZCc0tAN00c3lqtbI+RfaQmY7A513jEEL+t+3ol+GBy5nXIb1kSgrz457dbZjWLtXuAbRGR52D37f757sH57eHHR73ZPewfHe6dHR/fDy7ct35H6Y3U3Kz39zCO8hRTW/J8Opf/cq+nx3T4YxGsWouwF14h153X3br9c98Q5P7ofde5diD4+932J9v+ce/MxIw2P3bDciETN6eHB6fPSbfTVPUA/v92xYNNk/DsHdnBm+vz27/ujfXJy99V+e3fQvln24+0l6eGDru+9Bwy+fOo72U6f35VMnBkMjHzjPHkdSavOp0zvwul+/fr3f+2/it83gS8vT6gi9dhXW7pAVR6PS2GM0q6NXv9dYxh4pJw0kbsoxs9z35B8Z3f7XGauO76jbjXVLFDuQTSy2vE5eO1HOVRpEDWx5NqK1El3pQUu5T57ZJD27C2tr1Qkvu3VLDOfwvhvAJg4uZ82j3GLKtCPER6PAzzgbCM9stVwdwsRYqhjWLxzs6iVPwabJK7NdJzN1jnJ8uIPQLDptFGuNzzDILlvWARy2A1AyNay0aJdvILkadUbW3YOLPw/fP5+cfp4dhyaEl0a0M3zplsaK9Mvg24xt8xS8aZh7gaS7TLd6aYPMf+WYBJKm8fIKp80WXJzHoEHefwIAAP//Ovb/Tw==" } From 98df0c1701eb0829bd2f094cd1f1f69f0afdbfa0 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 15:43:02 +0900 Subject: [PATCH 11/27] New 8.0.0 slowlog samples from @pgomulka --- .../elasticsearch/slowlog/test/es_indexing_slowlog.800.log | 4 ++-- .../elasticsearch/slowlog/test/es_search_slowlog.800.log | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log index 3704f88d189..b580682cda0 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log @@ -1,2 +1,2 @@ -{"@timestamp":"2020-04-16T11:20:02.069Z", "log.level":"TRACE", "id":"5xy3gnEBmUEb0NJ1lijF", "message":"[test_index/M4fNwSWlTfek9m1SNL49Kg]", "source":"{\\\"f", "took":"15.1ms", "took_millis":"15" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#2]","log.logger":"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T11:20:02.777Z", "log.level":"TRACE", "id":"6By3gnEBmUEb0NJ1mSij", "message":"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]", "source":"{\\\"field\\\":123}", "took":"10.4ms", "took_millis":"10" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][write][T#4]","log.logger":"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_indexing_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2022-01-27T11:36:49.421Z", "log.level":"TRACE", "elasticsearch.slowlog.id":"_YRSm34B7FprLQsj6fZg","elasticsearch.slowlog.message":"[test_1/8pT6xiN_Tt-dcJWRR3LX6A]","elasticsearch.slowlog.source":"{\\\"a\\\":\\\"b\\\"}","elasticsearch.slowlog.took":"31.9ms","elasticsearch.slowlog.took_millis":"31" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#3]","log.logger":"index.indexing.slowlog.index","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:39:29.508Z", "log.level":"TRACE", "elasticsearch.slowlog.id":"_oRVm34B7FprLQsjW_Zh","elasticsearch.slowlog.message":"[test_1/8pT6xiN_Tt-dcJWRR3LX6A]","elasticsearch.slowlog.source":"{\\\"a\\\":","elasticsearch.slowlog.took":"1.7ms","elasticsearch.slowlog.took_millis":"1" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_indexing_slowlog","process.thread.name":"elasticsearch[runTask-0][write][T#5]","log.logger":"index.indexing.slowlog.index","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log index b817ea08fe7..d113ad63f1f 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log @@ -1,2 +1,3 @@ -{"@timestamp":"2020-04-16T11:20:02.828Z", "log.level":"TRACE", "id":"null", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"10ms", "took_millis":"10", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#1]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} -{"@timestamp":"2020-04-16T11:20:02.839Z", "log.level":"TRACE", "id":"my-identifier", "message":"[test_index][0]", "search_type":"QUERY_THEN_FETCH", "source":"{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", "stats":"[]", "took":"76.4micros", "took_millis":"0", "total_hits":"0 hits", "total_shards":"1" , "service.name":"ES_ECS","process.thread.name":"elasticsearch[integTest-0][search][T#3]","log.logger":"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ","type":"index_search_slowlog","cluster.uuid":"HHmOPeWKQlSeaF88DSfFVw","node.id":"wxTr7N_gRWWg3mUdY4spbg","node.name":"integTest-0","cluster.name":"integTest"} +{"@timestamp":"2022-01-27T11:36:57.424Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":"myApp1","elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"8ms","elasticsearch.slowlog.took_millis":8,"elasticsearch.slowlog.total_hits":"1 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#1]","log.logger":"index.search.slowlog.query","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:42:17.693Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"164.7micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"2 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#3]","log.logger":"index.search.slowlog.query","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:42:31.395Z", "log.level":"DEBUG", "elasticsearch.slowlog.id":null,"elasticsearch.slowlog.message":"[test_1][0]","elasticsearch.slowlog.search_type":"QUERY_THEN_FETCH","elasticsearch.slowlog.source":"{}","elasticsearch.slowlog.stats":"[]","elasticsearch.slowlog.took":"115.3micros","elasticsearch.slowlog.took_millis":0,"elasticsearch.slowlog.total_hits":"2 hits","elasticsearch.slowlog.total_shards":1 , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.index_search_slowlog","process.thread.name":"elasticsearch[runTask-0][search][T#5]","log.logger":"index.search.slowlog.query","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} From fc2b942621b96a75b8c66c4befa6f31738370a5f Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 16:56:40 +0900 Subject: [PATCH 12/27] No need to re-parse the message json --- .../module/elasticsearch/slowlog/ingest/pipeline-json-7.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml index 00ce95ccaa1..5ec6fa96866 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml @@ -4,9 +4,6 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- json: - field: message - target_field: elasticsearch.slowlog - drop: if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type != 'index_search_slowlog' From 4e273dba5acf5b9d4d5221369f5604cb73562717 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 16:57:05 +0900 Subject: [PATCH 13/27] Drop unrecognized json We have this in a few test cases due to incorrect filebeat multiline configuration. --- .../module/elasticsearch/slowlog/ingest/pipeline-json.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index 03d48f08da4..d90c36bd4ce 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -7,10 +7,12 @@ processors: - json: field: message target_field: elasticsearch.slowlog + # Drop unrecognized document structures present in test suite until https://github.com/elastic/beats/issues/30080 resolved +- drop: + if: '!ctx.elasticsearch.slowlog.containsKey("type") && !ctx.elasticsearch.slowlog.containsKey("ecs.version")' - pipeline: if: ctx.elasticsearch.slowlog.containsKey('type') name: '{< IngestPipeline "pipeline-json-7" >}' - pipeline: if: ctx.elasticsearch.slowlog.containsKey('ecs.version') name: '{< IngestPipeline "pipeline-json-8" >}' - From 7b8b5cbf97bdb614ea5c4175e530f798f23219d9 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 16:57:57 +0900 Subject: [PATCH 14/27] Map up most of the slowlog 8 fields --- .../slowlog/ingest/pipeline-json-8.yml | 43 +++++++++++-------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index d511a14c806..a8ab28d4819 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -6,6 +6,15 @@ on_failure: processors: - drop: if: ctx.event.dataset != 'elasticsearch.slowlog' +- dot_expander: + field: event.dataset + path: elasticsearch.slowlog +- set: + value: '{{ elasticsearch.slowlog.event.dataset }}' + field: event.dataset + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.event.dataset - dot_expander: field: service.name path: elasticsearch.slowlog @@ -13,10 +22,6 @@ processors: field: elasticsearch.slowlog.service.name target_field: service.name ignore_missing: true -- rename: - field: elasticsearch.slowlog.level - target_field: log.level - ignore_missing: true - dot_expander: field: log.level path: elasticsearch.slowlog @@ -38,10 +43,6 @@ processors: field: elasticsearch.slowlog.process.thread.name target_field: process.thread.name ignore_missing: true -- rename: - field: elasticsearch.slowlog.component - target_field: elasticsearch.component - ignore_missing: true - dot_expander: field: elasticsearch.cluster.name path: elasticsearch.slowlog @@ -68,18 +69,20 @@ processors: field: elasticsearch.slowlog.elasticsearch.node.id target_field: elasticsearch.node.id ignore_missing: true -- rename: - field: elasticsearch.slowlog.doc_type - target_field: elasticsearch.slowlog.types - ignore_missing: true -- convert: +- dot_expander: field: elasticsearch.slowlog.took_millis + path: elasticsearch.slowlog +- convert: + field: elasticsearch.slowlog.elasticsearch.slowlog.took_millis type: float ignore_missing: true - rename: field: elasticsearch.slowlog.took_millis target_field: elasticsearch.slowlog.duration ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.message + path: elasticsearch.slowlog - grok: field: elasticsearch.slowlog.elasticsearch.slowlog.message pattern_definitions: @@ -90,10 +93,17 @@ processors: patterns: - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] -- rename: +- remove: field: elasticsearch.slowlog.elasticsearch.slowlog.message - target_field: message - ignore_missing: true +- dot_expander: + field: ecs.version + path: elasticsearch.slowlog +- set: + value: '{{ elasticsearch.slowlog.ecs.version }}' + field: ecs.version + ignore_empty_value: true +- remove: + field: elasticsearch.slowlog.ecs.version - set: value: "{{ elasticsearch.slowlog.@timestamp }}" field: "@timestamp" @@ -114,4 +124,3 @@ processors: formats: - ISO8601 ignore_failure: true - From 57f77ef49684d15ad4af797f6c787540eb2361ed Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 17:07:28 +0900 Subject: [PATCH 15/27] Actually pull took_millis --- .../module/elasticsearch/slowlog/ingest/pipeline-json-8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index a8ab28d4819..aa8301744e6 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -77,7 +77,7 @@ processors: type: float ignore_missing: true - rename: - field: elasticsearch.slowlog.took_millis + field: elasticsearch.slowlog.elasticsearch.slowlog.took_millis target_field: elasticsearch.slowlog.duration ignore_missing: true - dot_expander: From 91c25397c3cdc4e11add1ddcdf965d1e569dec8c Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 17:08:20 +0900 Subject: [PATCH 16/27] Preserve slow log message field --- .../module/elasticsearch/slowlog/ingest/pipeline-json-8.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index aa8301744e6..7c27e7abedf 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -93,6 +93,10 @@ processors: patterns: - (\[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\])?(%{SPACE})(\[%{INDEXNAME:elasticsearch.index.name}\/%{DATA:elasticsearch.index.id}\])?(%{SPACE})%{SPACE}(took\[%{DATA:elasticsearch.slowlog.took}\],)?%{SPACE}(took_millis\[%{NUMBER:elasticsearch.slowlog.duration:long}\],)?%{SPACE}(type\[%{DATA:elasticsearch.slowlog.type}\],)?%{SPACE}(id\[%{DATA:elasticsearch.slowlog.id}\],)?%{SPACE}(routing\[%{DATA:elasticsearch.slowlog.routing}\],)?%{SPACE}(total_hits\[%{NUMBER:elasticsearch.slowlog.total_hits:int}\],)?%{SPACE}(types\[%{DATA:elasticsearch.slowlog.types}\],)?%{SPACE}(stats\[%{DATA:elasticsearch.slowlog.stats}\],)?%{SPACE}(search_type\[%{DATA:elasticsearch.slowlog.search_type}\],)?%{SPACE}(total_shards\[%{NUMBER:elasticsearch.slowlog.total_shards:int}\],)?%{SPACE}(source\[%{GREEDYMULTILINE:elasticsearch.slowlog.source_query}\])?,?%{SPACE}(extra_source\[%{DATA:elasticsearch.slowlog.extra_source}\])?,? - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] +- set: + field: message + value: '{{ elasticsearch.slowlog.elasticsearch.slowlog.message }}' + ignore_empty_value: true - remove: field: elasticsearch.slowlog.elasticsearch.slowlog.message - dot_expander: From 7fcd2b740d54b1cd622e7fdca5802aaa104e5e7d Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 17:16:20 +0900 Subject: [PATCH 17/27] Raise new slowlog fields to doc root --- .../slowlog/ingest/pipeline-json-8.yml | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index 7c27e7abedf..73ee96d81de 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -2,7 +2,7 @@ description: Pipeline for parsing the Elasticsearch slow logs in JSON format. on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{ _ingest.on_failure_message }} ({{ _ingest.on_failure_pipeline }}/{{ _ingest.on_failure_processor_type }})' processors: - drop: if: ctx.event.dataset != 'elasticsearch.slowlog' @@ -108,6 +108,62 @@ processors: ignore_empty_value: true - remove: field: elasticsearch.slowlog.ecs.version +- dot_expander: + field: elasticsearch.slowlog.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.id + target_field: elasticsearch.slowlog.id + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.search_type + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.search_type + target_field: elasticsearch.slowlog.search_type + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.source + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.source + target_field: elasticsearch.slowlog.source + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.stats + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.stats + target_field: elasticsearch.slowlog.stats + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.took + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.took + target_field: elasticsearch.slowlog.took + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.total_hits + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.total_hits + target_field: elasticsearch.slowlog.total_hits + ignore_missing: true +- dot_expander: + field: elasticsearch.slowlog.total_shards + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.elasticsearch.slowlog.total_shards + target_field: elasticsearch.slowlog.total_shards + ignore_missing: true +- dot_expander: + field: trace.id + path: elasticsearch.slowlog +- rename: + field: elasticsearch.slowlog.trace.id + target_field: trace.id + ignore_missing: true - set: value: "{{ elasticsearch.slowlog.@timestamp }}" field: "@timestamp" From 33e0a6fcd3193dc0d74f4da3ab44d7303ead46fd Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 17:17:50 +0900 Subject: [PATCH 18/27] Update es slowlog expectation files --- .../es_indexing_slowlog.800.log-expected.json | 72 ++++++------ .../es_search_slowlog.800.log-expected.json | 108 ++++++++++++------ 2 files changed, 108 insertions(+), 72 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json index cce5652340a..c3571decd25 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_indexing_slowlog.800.log-expected.json @@ -1,58 +1,60 @@ [ { - "@timestamp": "2020-04-16T11:20:02.069Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.id": "M4fNwSWlTfek9m1SNL49Kg", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", - "elasticsearch.slowlog.id": "5xy3gnEBmUEb0NJ1lijF", - "elasticsearch.slowlog.source": "{\\\"f", - "elasticsearch.slowlog.took": "15.1ms", + "@timestamp": "2022-01-27T11:36:49.421Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.id": "8pT6xiN_Tt-dcJWRR3LX6A", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": "_YRSm34B7FprLQsj6fZg", + "elasticsearch.slowlog.source": "{\\\"a\\\":\\\"b\\\"}", + "elasticsearch.slowlog.took": "31.9ms", "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 15000000, + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 31000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", "log.level": "TRACE", - "log.logger": "index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg", + "log.logger": "index.indexing.slowlog.index", "log.offset": 0, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.069Z\", \"log.level\":\"TRACE\", \"id\":\"5xy3gnEBmUEb0NJ1lijF\", \"message\":\"[test_index/M4fNwSWlTfek9m1SNL49Kg]\", \"source\":\"{\\\\\\\"f\", \"took\":\"15.1ms\", \"took_millis\":\"15\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#2]\",\"log.logger\":\"index.indexing.slowlog.index.M4fNwSWlTfek9m1SNL49Kg\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][write][T#2]", + "message": "[test_1/8pT6xiN_Tt-dcJWRR3LX6A]", + "process.thread.name": "elasticsearch[runTask-0][write][T#3]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2020-04-16T11:20:02.777Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.id": "Jsz7IUYMQ9ubo2ahiMgCbQ", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", - "elasticsearch.slowlog.id": "6By3gnEBmUEb0NJ1mSij", - "elasticsearch.slowlog.source": "{\\\"field\\\":123}", - "elasticsearch.slowlog.took": "10.4ms", + "@timestamp": "2022-01-27T11:39:29.508Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.id": "8pT6xiN_Tt-dcJWRR3LX6A", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.slowlog.id": "_oRVm34B7FprLQsjW_Zh", + "elasticsearch.slowlog.source": "{\\\"a\\\":", + "elasticsearch.slowlog.took": "1.7ms", "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 10000000, + "event.dataset": "elasticsearch.index_indexing_slowlog", + "event.duration": 1000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", "log.level": "TRACE", - "log.logger": "index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ", - "log.offset": 514, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.777Z\", \"log.level\":\"TRACE\", \"id\":\"6By3gnEBmUEb0NJ1mSij\", \"message\":\"[test_index/Jsz7IUYMQ9ubo2ahiMgCbQ]\", \"source\":\"{\\\\\\\"field\\\\\\\":123}\", \"took\":\"10.4ms\", \"took_millis\":\"10\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][write][T#4]\",\"log.logger\":\"index.indexing.slowlog.index.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_indexing_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][write][T#4]", + "log.logger": "index.indexing.slowlog.index", + "log.offset": 750, + "message": "[test_1/8pT6xiN_Tt-dcJWRR3LX6A]", + "process.thread.name": "elasticsearch[runTask-0][write][T#5]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" } ] \ No newline at end of file diff --git a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json index 39cd0679087..af79450e828 100644 --- a/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/es_search_slowlog.800.log-expected.json @@ -1,65 +1,99 @@ [ { - "@timestamp": "2020-04-16T11:20:02.828Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:36:57.424Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "elasticsearch.shard.id": "0", - "elasticsearch.slowlog.id": "null", + "elasticsearch.slowlog.id": "myApp1", "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", - "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.source": "{}", "elasticsearch.slowlog.stats": "[]", - "elasticsearch.slowlog.took": "10ms", - "elasticsearch.slowlog.total_hits": "0 hits", - "elasticsearch.slowlog.total_shards": "1", + "elasticsearch.slowlog.took": "8ms", + "elasticsearch.slowlog.total_hits": "1 hits", + "elasticsearch.slowlog.total_shards": 1, "event.category": "database", - "event.dataset": "elasticsearch.slowlog", - "event.duration": 10000000, + "event.dataset": "elasticsearch.index_search_slowlog", + "event.duration": 8000000, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "TRACE", - "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", "log.offset": 0, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.828Z\", \"log.level\":\"TRACE\", \"id\":\"null\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"10ms\", \"took_millis\":\"10\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#1]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][search][T#1]", + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#1]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" + }, + { + "@timestamp": "2022-01-27T11:42:17.693Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", + "elasticsearch.shard.id": "0", + "elasticsearch.slowlog.id": null, + "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", + "elasticsearch.slowlog.source": "{}", + "elasticsearch.slowlog.stats": "[]", + "elasticsearch.slowlog.took": "164.7micros", + "elasticsearch.slowlog.total_hits": "2 hits", + "elasticsearch.slowlog.total_shards": 1, + "event.category": "database", + "event.dataset": "elasticsearch.index_search_slowlog", + "event.duration": 0, + "event.kind": "event", + "event.module": "elasticsearch", + "event.type": "info", + "fileset.name": "slowlog", + "host.id": "tVLnAGLgQum5ca6z50aqbw", + "input.type": "log", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", + "log.offset": 861, + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#3]", + "service.name": "ES_ECS", + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2020-04-16T11:20:02.839Z", - "elasticsearch.cluster.name": "integTest", - "elasticsearch.cluster.uuid": "HHmOPeWKQlSeaF88DSfFVw", - "elasticsearch.index.name": "test_index", - "elasticsearch.node.id": "wxTr7N_gRWWg3mUdY4spbg", - "elasticsearch.node.name": "integTest-0", + "@timestamp": "2022-01-27T11:42:31.395Z", + "elasticsearch.cluster.name": "runTask", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.index.name": "test_1", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", + "elasticsearch.node.name": "runTask-0", "elasticsearch.shard.id": "0", - "elasticsearch.slowlog.id": "my-identifier", + "elasticsearch.slowlog.id": null, "elasticsearch.slowlog.search_type": "QUERY_THEN_FETCH", - "elasticsearch.slowlog.source": "{\\\"query\\\":{\\\"match_all\\\":{\\\"boost\\\":1.0}}}", + "elasticsearch.slowlog.source": "{}", "elasticsearch.slowlog.stats": "[]", - "elasticsearch.slowlog.took": "76.4micros", - "elasticsearch.slowlog.total_hits": "0 hits", - "elasticsearch.slowlog.total_shards": "1", + "elasticsearch.slowlog.took": "115.3micros", + "elasticsearch.slowlog.total_hits": "2 hits", + "elasticsearch.slowlog.total_shards": 1, "event.category": "database", - "event.dataset": "elasticsearch.slowlog", + "event.dataset": "elasticsearch.index_search_slowlog", "event.duration": 0, "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "slowlog", - "host.id": "wxTr7N_gRWWg3mUdY4spbg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "TRACE", - "log.logger": "index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ", - "log.offset": 613, - "message": "{\"@timestamp\":\"2020-04-16T11:20:02.839Z\", \"log.level\":\"TRACE\", \"id\":\"my-identifier\", \"message\":\"[test_index][0]\", \"search_type\":\"QUERY_THEN_FETCH\", \"source\":\"{\\\\\\\"query\\\\\\\":{\\\\\\\"match_all\\\\\\\":{\\\\\\\"boost\\\\\\\":1.0}}}\", \"stats\":\"[]\", \"took\":\"76.4micros\", \"took_millis\":\"0\", \"total_hits\":\"0 hits\", \"total_shards\":\"1\" , \"service.name\":\"ES_ECS\",\"process.thread.name\":\"elasticsearch[integTest-0][search][T#3]\",\"log.logger\":\"index.search.slowlog.query.Jsz7IUYMQ9ubo2ahiMgCbQ\",\"type\":\"index_search_slowlog\",\"cluster.uuid\":\"HHmOPeWKQlSeaF88DSfFVw\",\"node.id\":\"wxTr7N_gRWWg3mUdY4spbg\",\"node.name\":\"integTest-0\",\"cluster.name\":\"integTest\"}", - "process.thread.name": "elasticsearch[integTest-0][search][T#3]", + "log.level": "DEBUG", + "log.logger": "index.search.slowlog.query", + "log.offset": 1726, + "message": "[test_1][0]", + "process.thread.name": "elasticsearch[runTask-0][search][T#5]", "service.name": "ES_ECS", "service.type": "elasticsearch" } From aee4d571e3bbabad9301f77c799c1e523d35b3fe Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 17:18:08 +0900 Subject: [PATCH 19/27] Revert debugging change --- .../module/elasticsearch/slowlog/ingest/pipeline-json-8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index 73ee96d81de..2a2c2cf0c76 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -2,7 +2,7 @@ description: Pipeline for parsing the Elasticsearch slow logs in JSON format. on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }} ({{ _ingest.on_failure_pipeline }}/{{ _ingest.on_failure_processor_type }})' + value: '{{ _ingest.on_failure_message }}' processors: - drop: if: ctx.event.dataset != 'elasticsearch.slowlog' From 9dc11241eefe79d6f22e6ab939b104f0c1e34cbf Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 18:19:56 +0900 Subject: [PATCH 20/27] Rework slowlogs using strategy idea from @ruflin --- .../slowlog/ingest/pipeline-json-7.yml | 3 + .../slowlog/ingest/pipeline-json-8.yml | 172 ++---------------- .../slowlog/ingest/pipeline-json.yml | 18 +- 3 files changed, 20 insertions(+), 173 deletions(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml index 5ec6fa96866..00ce95ccaa1 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-7.yml @@ -4,6 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: +- json: + field: message + target_field: elasticsearch.slowlog - drop: if: ctx.elasticsearch.slowlog.type != 'index_indexing_slowlog' && ctx.elasticsearch.slowlog.type != 'index_search_slowlog' diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index 2a2c2cf0c76..9514061f630 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -2,89 +2,24 @@ description: Pipeline for parsing the Elasticsearch slow logs in JSON format. on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }}' + value: '{{ _ingest.on_failure_message }} ({{ _ingest.on_failure_pipeline }}/{{ _ingest.on_failure_processor_type }})' processors: -- drop: - if: ctx.event.dataset != 'elasticsearch.slowlog' -- dot_expander: - field: event.dataset - path: elasticsearch.slowlog -- set: - value: '{{ elasticsearch.slowlog.event.dataset }}' - field: event.dataset - ignore_empty_value: true -- remove: - field: elasticsearch.slowlog.event.dataset -- dot_expander: - field: service.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.service.name - target_field: service.name - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.process.thread.name - target_field: process.thread.name - ignore_missing: true -- dot_expander: - field: elasticsearch.cluster.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: elasticsearch.node.name - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: elasticsearch.cluster.uuid - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: elasticsearch.node.id - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.node.id - target_field: elasticsearch.node.id - ignore_missing: true +- json: + field: message + add_to_root: true - dot_expander: - field: elasticsearch.slowlog.took_millis - path: elasticsearch.slowlog + field: '*' + override: true - convert: - field: elasticsearch.slowlog.elasticsearch.slowlog.took_millis + field: elasticsearch.slowlog.took_millis type: float ignore_missing: true - rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.took_millis + field: elasticsearch.slowlog.took_millis target_field: elasticsearch.slowlog.duration ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.message - path: elasticsearch.slowlog - grok: - field: elasticsearch.slowlog.elasticsearch.slowlog.message + field: elasticsearch.slowlog.message pattern_definitions: GREEDYMULTILINE: |- (.| @@ -95,92 +30,7 @@ processors: - \[%{INDEXNAME:elasticsearch.index.name}\]\[%{NUMBER:elasticsearch.shard.id}\] - set: field: message - value: '{{ elasticsearch.slowlog.elasticsearch.slowlog.message }}' - ignore_empty_value: true -- remove: - field: elasticsearch.slowlog.elasticsearch.slowlog.message -- dot_expander: - field: ecs.version - path: elasticsearch.slowlog -- set: - value: '{{ elasticsearch.slowlog.ecs.version }}' - field: ecs.version + value: '{{ elasticsearch.slowlog.message }}' ignore_empty_value: true - remove: - field: elasticsearch.slowlog.ecs.version -- dot_expander: - field: elasticsearch.slowlog.id - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.id - target_field: elasticsearch.slowlog.id - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.search_type - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.search_type - target_field: elasticsearch.slowlog.search_type - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.source - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.source - target_field: elasticsearch.slowlog.source - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.stats - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.stats - target_field: elasticsearch.slowlog.stats - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.took - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.took - target_field: elasticsearch.slowlog.took - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.total_hits - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.total_hits - target_field: elasticsearch.slowlog.total_hits - ignore_missing: true -- dot_expander: - field: elasticsearch.slowlog.total_shards - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.elasticsearch.slowlog.total_shards - target_field: elasticsearch.slowlog.total_shards - ignore_missing: true -- dot_expander: - field: trace.id - path: elasticsearch.slowlog -- rename: - field: elasticsearch.slowlog.trace.id - target_field: trace.id - ignore_missing: true -- set: - value: "{{ elasticsearch.slowlog.@timestamp }}" - field: "@timestamp" - ignore_empty_value: true -- set: - value: "{{ elasticsearch.slowlog.timestamp }}" - field: "@timestamp" - ignore_empty_value: true -- remove: - field: elasticsearch.slowlog.@timestamp - ignore_missing: true -- remove: - field: elasticsearch.slowlog.timestamp - ignore_missing: true -- date: - field: '@timestamp' - target_field: '@timestamp' - formats: - - ISO8601 - ignore_failure: true + field: elasticsearch.slowlog.message diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml index d90c36bd4ce..614c9f7aa43 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json.yml @@ -4,15 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- json: - field: message - target_field: elasticsearch.slowlog - # Drop unrecognized document structures present in test suite until https://github.com/elastic/beats/issues/30080 resolved -- drop: - if: '!ctx.elasticsearch.slowlog.containsKey("type") && !ctx.elasticsearch.slowlog.containsKey("ecs.version")' -- pipeline: - if: ctx.elasticsearch.slowlog.containsKey('type') - name: '{< IngestPipeline "pipeline-json-7" >}' -- pipeline: - if: ctx.elasticsearch.slowlog.containsKey('ecs.version') - name: '{< IngestPipeline "pipeline-json-8" >}' + - pipeline: + if: '!ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-7" >}' + - pipeline: + if: 'ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-8" >}' From adc7cf8c63d2dc2628bb9089f0e8a60abeffa4e5 Mon Sep 17 00:00:00 2001 From: Mat Schaffer Date: Fri, 28 Jan 2022 18:20:40 +0900 Subject: [PATCH 21/27] Revert debugging message (again) --- .../module/elasticsearch/slowlog/ingest/pipeline-json-8.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml index 9514061f630..3e0479d59ea 100644 --- a/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/slowlog/ingest/pipeline-json-8.yml @@ -2,7 +2,7 @@ description: Pipeline for parsing the Elasticsearch slow logs in JSON format. on_failure: - set: field: error.message - value: '{{ _ingest.on_failure_message }} ({{ _ingest.on_failure_pipeline }}/{{ _ingest.on_failure_processor_type }})' + value: '{{ _ingest.on_failure_message }}' processors: - json: field: message From 675403e330a0f361853ff170f58570184d085901 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 13:15:08 +0100 Subject: [PATCH 22/27] update elasticsearch fields --- filebeat/docs/fields.asciidoc | 72 ++++++++----------- .../module/elasticsearch/_meta/fields.yml | 10 +++ .../deprecation/_meta/fields.yml | 12 ---- filebeat/module/elasticsearch/fields.go | 2 +- 4 files changed, 40 insertions(+), 56 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index afb250f0c78..411516e2700 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -49735,6 +49735,35 @@ example: 0 -- +*`elasticsearch.elastic_product_origin`*:: ++ +-- +Used by Elastic stack to identify which component of the stack sent the request + +type: keyword + +example: kibana + +-- + +*`elasticsearch.http.request.x_opaque_id`*:: ++ +-- +Used by Elasticsearch to throttle and deduplicate deprecation warnings + +example: v7app + +-- + +*`elasticsearch.event.category`*:: ++ +-- +Category of the deprecation event + +example: compatible_api + +-- + *`elasticsearch.audit.layer`*:: + @@ -49927,49 +49956,6 @@ type: boolean - -*`elasticsearch.deprecation.data_stream.dataset`*:: -+ --- -type: keyword - --- - -*`elasticsearch.deprecation.data_stream.namespace`*:: -+ --- -type: keyword - --- - -*`elasticsearch.deprecation.data_stream.type`*:: -+ --- -type: keyword - --- - -*`elasticsearch.deprecation.event.code`*:: -+ --- -type: keyword - --- - -*`elasticsearch.deprecation.event.dataset`*:: -+ --- -type: keyword - --- - -*`elasticsearch.deprecation.elasticsearch.event.category`*:: -+ --- -type: keyword - --- - [float] === gc diff --git a/filebeat/module/elasticsearch/_meta/fields.yml b/filebeat/module/elasticsearch/_meta/fields.yml index 721f33a4879..3cb88baf881 100644 --- a/filebeat/module/elasticsearch/_meta/fields.yml +++ b/filebeat/module/elasticsearch/_meta/fields.yml @@ -40,3 +40,13 @@ description: "Id of the shard" example: "0" type: keyword + - name: elastic_product_origin + type: keyword + description: "Used by Elastic stack to identify which component of the stack sent the request" + example: "kibana" + - name: http.request.x_opaque_id + description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings" + example: "v7app" + - name: event.category + description: "Category of the deprecation event" + example: "compatible_api" diff --git a/filebeat/module/elasticsearch/deprecation/_meta/fields.yml b/filebeat/module/elasticsearch/deprecation/_meta/fields.yml index bfa5887f6d0..b4f8083631e 100644 --- a/filebeat/module/elasticsearch/deprecation/_meta/fields.yml +++ b/filebeat/module/elasticsearch/deprecation/_meta/fields.yml @@ -2,15 +2,3 @@ type: group description: > fields: - - name: data_stream.dataset - type: keyword - - name: data_stream.namespace - type: keyword - - name: data_stream.type - type: keyword - - name: event.code - type: keyword - - name: event.dataset - type: keyword - - name: elasticsearch.event.category - type: keyword diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 46e06a7199b..597911929e8 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUmt1v2zgSwN/7VxB+uV0g0Tkfm9sYuAO2bpqk6EcaJ+l13UAYU2OJNUUqJGXHW/R/P5CSHVmWZMvX9np+SSR+zG+Gw+GQ4j6Z4LxHkIM2jGoERaNnhBhmOPZI56z4vvOMEIUcQWOPhPCMkAA1VSwxTIoe+dczQshqT+SNDFKOzwgZM+SB7rkq+0RAjOtC7c/ME9u5kmmSv6mQsdpdsUsq40QKFGZZUupgVaOn+mSsZExmESokJkLCZUhwagukYiETYDDoFDrFR4gTZyLpoUe92HuDBl6Agb5CMHgpAnwcoJoyisV2mX4TnM+kCtbxeaoNKi9NWVCrwe3t5Qsixw4zb1BNdh5P1eiCv71hg7uP7Gr8++QxPA3b09inWpq3EONWNIGkE1T7FXWaKYQM0Gswx5MxbM1q2S8G7AO9meNN9MHc/vv189NX3edvZi0ZtjZDPcf0w9tX+s+j7QUz60bNkp2nuerVMseM4wjB7BvUZp+JJDVt5TdZ30lnNXMD3p2HL2aj2+tx/+63f/wxoA+jftjC7joCFTSKDxZGd1WrKbrbC4Q0YGatdjEckYroU+yBwxzVSkmZ+caGF1trEXMYjYiJmF4LOD2iUJs9YhQInUhlywhL/DHjpSm0qrBtVS6t1rtI7qT7tt5GfFvJGj4DNhEYIilNlbLMIKSYxzLVPlCKWvsBCobBHoHURCgMo2C78sfAuHtdqpU9hgqEsc9UCoHUtah6t2hmIE5QYeArfEid1VQqfCh0lD9nDeqNtyq/vRmz4fM22vHDcq3JidcGnvyyXpL5DJDrs8EN+ePqctH416KXLNvNQBOFFNkUAyKFk/ZUjUYgBPJf9wiXFLhv4xb5JVv9KHAXxwjTOsWgyPlrve2e+mlvN4XA442et+pDWSMHVyqwmk+Bs8AZDUJgYn1O5OAduzrhGFJu7NTagT3VqLztFLBV/6Yr9dgjbFwsqPXSjnNTw6boB0whNVLNd4WWHHUj9LWtQYxcBiokiWKCsgQ4GSGXItS1HjEknQkbgQAfgpiJzh7p2LVI54/kfkdqN5XL62L7DqqGbHMPVUnmdi3BxayNPiIKyUTWJIux1qnxEWla7xo90skzrF4sBTNS/T0GJnbwDsW9BBTEG7zDxqHb60vi6qJBVe8MnS/W9rb7f34GOhGMRodfO5XSmQgY3eCYl1mdfMHAgIzmubWa3HEs5f5h9+DU6x543WPrkCtvjtbenOzipXmoXE1d1lW4FewhRZKlsHmbevN9+Ou1Pxmd3A2m76I/HrpmdjW9ePd+l0ibwVVMn/qlfrGktHDEPkdQA6ok59fVum3N6o9kMK9sDJxB2U8SMFGPRMYk3kJX296jUpj1aRuzUEGmsVEpNizqPgSBQl0WtwlEy1RR9Fiyg+BUsZbS7MTN8wW+g8BlbG8rVq/vFLeVGaPWEFaHcoOPpiZELNZ3DxI2wbn25Exg4I/m/soi6lu0yr5HUnIEsbYLCDBRmK3MG/cClccTZMMmIQADvjYKIfbs/xrbryXFPuwbnQBtvxoWe6lIWbfcNXhUBrs23VX/ldMjL8cAg6FU1ZGibs8X0t0HmZDzPrE5o0aTj7i3pQ8kEehqm5WlbyCwv5dOENEJUjZm1KZr5/1MhFeqXMVU5KqYw/U23BrQ/opnJOd9QiXn2SauGrTgpGkWQnyNtBZtzCWUvWhLsH6JZCnQ5hZSBUyE1qKW+xVMgUyZMilwEgONmGgA11SlI1/P45HkvoERR9+wGL+XHuQKUo3EiiBMEI1UikATaoOc1SFNSMZCHIveCG4UE+EPAN+C26Fs5J4hTHyFY+0nStq00PF/R/Iby6wTFIY8SXQYROEYFQqboj4pVY9uE2jOkfsKNQXxo6gL9o5BTSw9Z1MkcvQZqdF218eRQJLwxV6VaaKNTBIM6pWhHLT2U8ElBD9Kk0ya8xeR2v2Ag9jS+jRJHWctY1VQ3pLxKnMM0r+6zXw89xdUY6liC/wUCisQ60M2KW1ra4xMNhp6S0Xsr6SETI1mQXaONUElkFcpUAgsc/0/oGSiDEkaKRUC/xGYN9IAt5lMYv21BG2kO2ngaDLywnrpDgG1AeVqjZlgOvIqs4zP09hXqaiZgvWKbFDA7QwtqiN5dfcmp0mTwmzbI6AJZN1bL08kE4aINB6hqqY1kUIItG+sXXwbZeqCx87k56BGEK5YM5dKnFQX2/JhqAoaS0e2IdCtLgvmb21ii2CknNghzqByzkYuA2H1/rQ6ddtkrT7hMgyzpTesERkhlCPjzonsBUJCgHOZLzYggsW4sL9a57K2jT8Z1QZ1JgyGa5vDLTDJcvJa5Z0c6/gTxuVobpoyFLsyfTekWxtGHFE9zPJIgwd+iOXDyJ0H7h0PSIgC88RZUpomIOj85x9BN3hybA1S1OAnGM5am24e3blMRfgtx/ej7fD/fITnZR1+gjFusGs13dJuqKYrQlfPbgeu2N1jKX/T2vaj9tNSB3RiVPl4qSCv0yMDW4m4Whac2m20HBNUSqrVBcndKeiRMfCV84/K45iyVtl6tHqMXOfSTYcvzhOaJkAnG5fzfv3xdvVhdtXUqp4Cy0As1ncdqyxlSU0UCw4u1xRcpggz+SMELvWboooQAl/jQ6PJB/iQ2v1yniLWWv7o+Pj09PSw0vy1FE/5nr843fE2fNta3SWf9/fsn5hxzvIMrJbw4KTb3TIPXFppZCc0tAN00c3lqtbI+RfaQmY7A513jEEL+t+3ol+GBy5nXIb1kSgrz457dbZjWLtXuAbRGR52D37f757sH57eHHR73ZPewfHe6dHR/fDy7ct35H6Y3U3Kz39zCO8hRTW/J8Opf/cq+nx3T4YxGsWouwF14h153X3br9c98Q5P7ofde5diD4+932J9v+ce/MxIw2P3bDciETN6eHB6fPSbfTVPUA/v92xYNNk/DsHdnBm+vz27/ujfXJy99V+e3fQvln24+0l6eGDru+9Bwy+fOo72U6f35VMnBkMjHzjPHkdSavOp0zvwul+/fr3f+2/it83gS8vT6gi9dhXW7pAVR6PS2GM0q6NXv9dYxh4pJw0kbsoxs9z35B8Z3f7XGauO76jbjXVLFDuQTSy2vE5eO1HOVRpEDWx5NqK1El3pQUu5T57ZJD27C2tr1Qkvu3VLDOfwvhvAJg4uZ82j3GLKtCPER6PAzzgbCM9stVwdwsRYqhjWLxzs6iVPwabJK7NdJzN1jnJ8uIPQLDptFGuNzzDILlvWARy2A1AyNay0aJdvILkadUbW3YOLPw/fP5+cfp4dhyaEl0a0M3zplsaK9Mvg24xt8xS8aZh7gaS7TLd6aYPMf+WYBJKm8fIKp80WXJzHoEHefwIAAP//Ovb/Tw==" + return "eJzUmm1z2zbywN/nU+zozb+dsfmXH+rWmrmbaZXESaZ5aGwn1yoezgpcUYhAgAZAyWon3/0GACVLFEk9XJvr+U1CEcD+drFY7II4hgnNe0ACjeXMEGo2fgJguRXUg86z1d87TwA0CUJDPUjxCUBChmmeW65kD/75BADWR4LXKikEPQEYcRKJ6fkmxyAxo02h7s/Ocze4VkVe/lIjY3241SGZynIlSdrlm8oA6xo9toeRVhnMxqQJ7JhAqBRo6l4ozVMu0VLSWRmUHjDLvYlURBGLsug1WXyKFvua0NJLmdDDNekpZ7TaL+g3oflM6WQTXxTGko6KgieNGtzevnwKauQxyw71ZFfZVA9fiDc3/PrDr/zd6IfJQ3qZ7k/jnhpp3mBGO9Ekik1IH9e0aaeQKqGoxRyPxnAt62U/veYf2c2cbsYf7e2/fv7p8lX3p9ezPRl2NkMzx/Tjm1fmt7PdBXPnRu2Svaf55vUyR1zQkNAeWzL2mMu8sPvKb7O+l84b1ga+vUqfzoa370f9D999/+M1ux/20z3sbsaok1bxycLovmk9RXd3gWVIinOtkoLZOCz+rd03lqihBIZzKIMNGItsAlYBT0haPprDbMzXws9CC9/QuF/co6b7goytV2vChyixs6HD2No8KntGD7HK8b6guC2grNOWodEqsGOtrBUEKBNIKClywRlagoRyTQxdf5ihllympsHjv8c832T0oTVyY6VKzxvJ+mWDhXlW5foh6oU6s6LlQ0Ex5nxTOhYJtxuTurrpQM0eszqCwDnptTdV9Bu3ibhWi53Fzbcdc7OxrfRAk7FHYDVKkyvt3gHP4xEXlUC5rqWueEabe1YsH7t2W/FdI2f5AGzHaEExVmjtmFEqOc9UYWJkjIyJE5KckiPAwo6dk4dpikfIhf+50io8phqldc9MSUnM96j7bdHNYpaTpiQu/fsIdCFjXBmofA4dmo23Ln9/M4bpi7ba8eMyoyiJNyYevtl8E3wG4f2z6xv48d3LRedvV71k2W+GBjQx4lNKQEkv7bEZG6OUJL49AqEYitjtTvBNyHEYCr9bATemoGSV89tm2z2Os7/dNKHItnreug+FTh6u8sJpPkXBE280TJHLzTVRgndcDkIjLIR1S+sA9sKQjnZTwDX9P1OrxxHw0eqLRi/teDe1fEpxwjUxq/T8UGglyLRCv3ctXMxfBCqCXHPJeI4ChiRUJcKvecRgsRnFmGRcdo6g4zIOUz7C3YHUfilXs5/9B6ibsu0j1JUSu/VEH7O2+ohcSRlDlxBjnVPTA7Gi2TXcHhfy6F6mJLdK/3+GXB7gHVpEOWrMtniHi0O371+Cb0uWdLMzdP5wtnfD/+MzsonkbHz6pVMrncuEsy2O+TK0KTeMkKgEa7W540ip49PuyWXUPYm6584h13452/jl4hAvXSRZPGlV4Vby+4IgFCo1Kd26+T7+/nM8GV58uJ6+Hf9437Wzd9MXb385JNIGuJrl07zVL7aUPRyxLwj1NdNKiPf1uu3MGg9VMq/tjIJj1U9ytONKtuv6R0xJu7lsM55qDBpbXVDLph5jkmgyVXHbQIwqNKOI5wcILjTfU5pbuGW+IA4QuIzt+4o1m+cBu8rMyBhM60O5pQfbECIW+3uEOZ/Q3ERqJimJh/N4bRONHVrt2EOlBKHcqAJWiomttUDtIRS0HESl7PAxAa764FIUQ7YUEO1Yl+RjNPUWrkrfQuD+nntBYHJifMSZyw6u+kFEVGlcx7TKVeMy0BoZdgJ0f6sHL1d9YEqIUDPUg65MfxE8NjbEGtFGQmE1kuwI1q+QLAW6rUzphMs01NgEr3CKMOXaFiggQzbmsgXcMF0MYzPPhkrEFl2Va3lGf5Ue8A4LQ+BEAJdgiCmZGGBuTTkdihwCC3gWsxXcai7TrwC+A7dH2co9I5zEmkYmzrVyWYjn/wvJbxyzyV3V/SjRY4CmEWmSLiN6VKoZ3eVrQpCINRmG8mtRr9g7Qz1x9IJPCdTwMzFrXJEhCDAP50luTXADxqo8p6RZGSbQmLiQQmHytTQJ0ry/yMKlnx5iR+uzvPCcjYx1QXlHxnfBMaD/7jb4eOkvpEdKZw74MRTWIDaHbKhUUQ1Ghq2G3lER91dRQhXW8CQcm0xISxJ1CqwElrn5L1ByWYWEVkpXhn4NzBtlUQAJzJ2/VqCt8oWtIBvIV/ZLf+ZkLGrfasQlN+OoNsv4PM1iXciGJdisyBYFfCHiUD3Jqw+vS5oiX1ltR4AGMAzvvDxXXFqQRTYkXU9rx5owMbF1doldlGkKHgeTX6EeYrpmzVIqeKk+tpXTUBc0lo7sQqDfXRbMf7aJHYJVyn+MCFAlZyuXxbS+HKpP3bZZqw9CpWnYetMGkWPCamQ8OJF9QZgDCqHKzQZlspgX/vveuazrE0+GjUGdS0vpRi2yAyYsF69T3stxjj/hQg3nti1DcTvTX4bkPw95omaYZQUtkjil6tnXwRP3ViSQkqQycVaMFTlKNv/7z6CfPDVyBlnV4G8wnY023T67c1XI9M+c31/dgP/jMzyv6vA3mOMWu9bTLe1GeromdP2o8Nq/9pdjqp9Qdv2G+rjVIZtYjWw9O16R1+nBdfhu7lo5cObKaDUC0lrp9Q3JX1TowQjF2vlH7XFMVauwH62fWja5dNvhi/eEtgXQCfNy1W8+Ta0/O61bWvVLYBmI5WbVsc5SldRGseAQakPBZYowU19D4FK/KekxYRIbum81+TXdF65eLlPERsufnZ9fXl6e1pq/keIx34sXpzvRlk8p61XyVf/I/ZNxIXiZgTUSnlx0uzvmgUsrDd2Cxv0AfXTzuaoz8vKmyjKznaEpB6ZkD/ofdqJfhgehZkKlzZEovA93E0yoGDYuK25AdAan3ZMfjrsXx6eXNyfdXveid3J+dHl2djd4+eb5W7gbhAtPYYiohIjuC9LzOxhM4w+vxp8/3MEgI6s589eqLqKzqHvsxo26F9Hpxd2ge+dT7MF59F1m7o78QxyMNDj3z64QGXNrBieX52ffuZ/mOZnB3ZELizb8xyP4ixqDX26fvf81vnnx7E38/NlN/8VyDH/pyQxOXHv/+WHwx6eOp/3U6f3xqZOhZeMYhQiPQ6WM/dTpnUTdL1++3B39J/HbZfCV7Wl9hn72DTYupq3ORq2xR2TXZ6+51ljGHqUmLSR+yXG7rHvKb1q+/vXGauI763YzsyeKm8g2Fve+Sd5+oryrtIi6du/DjDZK9G9P9pT76Jlt0stbZPO8cfqrbr0nhnf42E9gG4dQs/ZZ3mPJ7EdID1ZjHDhbCJ+5ZqU6wOVI6Qw3v28f6iWPwabNK0PVyW2To5yfHiA0RKetYp3xOSXhBmcTwOl+AFoVllc27eqFF9+iycime/Lit9Nffppcfp6dpzbF51buZ/jKpYA16S+TP2du25fgTcvaSxQ7ZLk1S7sO/qtGkChWZMsbgy5b8HGekhZ5/w4AAP//lrQtsw==" } From 0923cd4bd515d5bd74c52f55fc925ee5810b318a Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 13:16:06 +0100 Subject: [PATCH 23/27] rework 8.x deprecation logs pipeline --- .../deprecation/ingest/pipeline-json-8.yml | 104 ++---------------- .../deprecation/ingest/pipeline-json.yml | 16 +-- .../test/es_deprecation-json.800.log | 4 +- .../es_deprecation-json.800.log-expected.json | 61 +++++----- 4 files changed, 46 insertions(+), 139 deletions(-) diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml index 8c9e665424e..89c7b4083f6 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json-8.yml @@ -4,102 +4,12 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- drop: - if: ctx.event.dataset != 'elasticsearch.deprecation' -- set: - value: '{{ elasticsearch.deprecation.event.dataset }}' - field: event.dataset - ignore_empty_value: true +- json: + field: message + add_to_root: true - dot_expander: - field: ecs.version - path: elasticsearch.deprecation + field: '*' + override: true - set: - value: '{{ elasticsearch.deprecation.ecs.version }}' - field: ecs.version - ignore_empty_value: true -- remove: - field: elasticsearch.deprecation.ecs.version -- dot_expander: - field: service.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.service.name - target_field: service.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.level - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.level - target_field: log.level - ignore_missing: true -- dot_expander: - field: log.logger - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.log.logger - target_field: log.logger - ignore_missing: true -- dot_expander: - field: process.thread.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.process.thread.name - target_field: process.thread.name - ignore_missing: true -- rename: - field: elasticsearch.deprecation.component - target_field: elasticsearch.component - ignore_missing: true -- dot_expander: - field: elasticsearch.cluster.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.elasticsearch.cluster.name - target_field: elasticsearch.cluster.name -- dot_expander: - field: elasticsearch.node.name - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.elasticsearch.node.name - target_field: elasticsearch.node.name -- dot_expander: - field: elasticsearch.cluster.uuid - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.elasticsearch.cluster.uuid - target_field: elasticsearch.cluster.uuid - ignore_missing: true -- dot_expander: - field: elasticsearch.node.id - path: elasticsearch.deprecation -- rename: - field: elasticsearch.deprecation.elasticsearch.node.id - target_field: elasticsearch.node.id - ignore_missing: true -- remove: - field: message -- rename: - field: elasticsearch.deprecation.message - target_field: message -- date: - field: 'elasticsearch.deprecation.@timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null' -- date: - field: 'elasticsearch.deprecation.timestamp' - formats: - - ISO8601 - ignore_failure: true - if: 'ctx.elasticsearch?.deprecation?.timestamp != null' -- remove: - field: - - elasticsearch.deprecation.timestamp - - elasticsearch.deprecation.@timestamp - ignore_missing: true + field: event.dataset + value: elasticsearch.deprecation diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml index 71854d8c1ed..d4647fbff10 100644 --- a/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml @@ -4,13 +4,9 @@ on_failure: field: error.message value: '{{ _ingest.on_failure_message }}' processors: -- json: - field: message - target_field: elasticsearch.deprecation -- pipeline: - if: ctx.elasticsearch.deprecation.containsKey('type') - name: '{< IngestPipeline "pipeline-json-7" >}' -- pipeline: - if: ctx.elasticsearch.deprecation.containsKey('ecs.version') - name: '{< IngestPipeline "pipeline-json-8" >}' - + - pipeline: + if: '!ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-7" >}' + - pipeline: + if: 'ctx.message.contains("ecs.version")' + name: '{< IngestPipeline "pipeline-json-8" >}' diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log index f1f39cda24b..40157a6d5e2 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log @@ -1,2 +1,2 @@ -{"@timestamp":"2022-01-27T11:25:19.412Z", "log.level": "WARN", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"indices","event.code":"index_name_starts_with_dot","message":"index name [.kibana-event-log-8.0.0] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices" , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService","elasticsearch.cluster.uuid":"mIMVAJO4TSmq1mu7hCPZ7A","elasticsearch.node.id":"hPDoent5RQ2tj7zTJtOagg","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} -{"@timestamp":"2022-01-27T11:26:34.762Z", "log.level": "WARN", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"settings","event.code":"xpack.monitoring.collection.enabled","message":"[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.deprecation.common.settings.Settings","elasticsearch.cluster.uuid":"mIMVAJO4TSmq1mu7hCPZ7A","elasticsearch.node.id":"hPDoent5RQ2tj7zTJtOagg","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:48:45.809Z", "log.level":"CRITICAL", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.elastic_product_origin":"","elasticsearch.event.category":"compatible_api","elasticsearch.http.request.x_opaque_id":"v7app","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#8]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","trace.id":"0af7651916cd43dd8448eb211c80319c","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} +{"@timestamp":"2022-01-27T11:52:39.882Z", "log.level":"CRITICAL", "data_stream.dataset":"deprecation.elasticsearch","data_stream.namespace":"default","data_stream.type":"logs","elasticsearch.event.category":"compatible_api","event.code":"create_index_with_types","message":"[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version." , "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"deprecation.elasticsearch","process.thread.name":"elasticsearch[runTask-0][transport_worker][T#9]","log.logger":"org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction","elasticsearch.cluster.uuid":"5alW33KLT16Lp1SevDqDSQ","elasticsearch.node.id":"tVLnAGLgQum5ca6z50aqbw","elasticsearch.node.name":"runTask-0","elasticsearch.cluster.name":"runTask"} diff --git a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json index 5e7c6a2b4c2..20dd03fab88 100644 --- a/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json +++ b/filebeat/module/elasticsearch/deprecation/test/es_deprecation-json.800.log-expected.json @@ -1,57 +1,58 @@ [ { - "@timestamp": "2022-01-27T11:25:19.412Z", + "@timestamp": "2022-01-27T11:48:45.809Z", + "data_stream.dataset": "deprecation.elasticsearch", + "data_stream.namespace": "default", + "data_stream.type": "logs", "elasticsearch.cluster.name": "runTask", - "elasticsearch.cluster.uuid": "mIMVAJO4TSmq1mu7hCPZ7A", - "elasticsearch.deprecation.data_stream.dataset": "deprecation.elasticsearch", - "elasticsearch.deprecation.data_stream.namespace": "default", - "elasticsearch.deprecation.data_stream.type": "logs", - "elasticsearch.deprecation.elasticsearch.event.category": "indices", - "elasticsearch.deprecation.event.code": "index_name_starts_with_dot", - "elasticsearch.deprecation.event.dataset": "deprecation.elasticsearch", - "elasticsearch.node.id": "hPDoent5RQ2tj7zTJtOagg", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.elastic_product_origin": "", + "elasticsearch.event.category": "compatible_api", + "elasticsearch.http.request.x_opaque_id": "v7app", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", "elasticsearch.node.name": "runTask-0", "event.category": "database", + "event.code": "create_index_with_types", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "hPDoent5RQ2tj7zTJtOagg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.cluster.metadata.MetadataCreateIndexService", + "log.level": "CRITICAL", + "log.logger": "org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction", "log.offset": 0, - "message": "index name [.kibana-event-log-8.0.0] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", - "process.thread.name": "elasticsearch[runTask-0][masterService#updateTask][T#1]", + "message": "[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version.", + "process.thread.name": "elasticsearch[runTask-0][transport_worker][T#8]", "service.name": "ES_ECS", - "service.type": "elasticsearch" + "service.type": "elasticsearch", + "trace.id": "0af7651916cd43dd8448eb211c80319c" }, { - "@timestamp": "2022-01-27T11:26:34.762Z", + "@timestamp": "2022-01-27T11:52:39.882Z", + "data_stream.dataset": "deprecation.elasticsearch", + "data_stream.namespace": "default", + "data_stream.type": "logs", "elasticsearch.cluster.name": "runTask", - "elasticsearch.cluster.uuid": "mIMVAJO4TSmq1mu7hCPZ7A", - "elasticsearch.deprecation.data_stream.dataset": "deprecation.elasticsearch", - "elasticsearch.deprecation.data_stream.namespace": "default", - "elasticsearch.deprecation.data_stream.type": "logs", - "elasticsearch.deprecation.elasticsearch.event.category": "settings", - "elasticsearch.deprecation.event.code": "xpack.monitoring.collection.enabled", - "elasticsearch.deprecation.event.dataset": "deprecation.elasticsearch", - "elasticsearch.node.id": "hPDoent5RQ2tj7zTJtOagg", + "elasticsearch.cluster.uuid": "5alW33KLT16Lp1SevDqDSQ", + "elasticsearch.event.category": "compatible_api", + "elasticsearch.node.id": "tVLnAGLgQum5ca6z50aqbw", "elasticsearch.node.name": "runTask-0", "event.category": "database", + "event.code": "create_index_with_types", "event.dataset": "elasticsearch.deprecation", "event.kind": "event", "event.module": "elasticsearch", "event.type": "info", "fileset.name": "deprecation", - "host.id": "hPDoent5RQ2tj7zTJtOagg", + "host.id": "tVLnAGLgQum5ca6z50aqbw", "input.type": "log", - "log.level": "WARN", - "log.logger": "org.elasticsearch.deprecation.common.settings.Settings", - "log.offset": 882, - "message": "[xpack.monitoring.collection.enabled] setting was deprecated in Elasticsearch and will be removed in a future release! See the breaking changes documentation for the next major version.", - "process.thread.name": "elasticsearch[runTask-0][masterService#updateTask][T#1]", + "log.level": "CRITICAL", + "log.logger": "org.elasticsearch.deprecation.rest.action.admin.indices.RestCreateIndexAction", + "log.offset": 989, + "message": "[types removal] Using include_type_name in create index requests is deprecated. The parameter will be removed in the next major version.", + "process.thread.name": "elasticsearch[runTask-0][transport_worker][T#9]", "service.name": "ES_ECS", "service.type": "elasticsearch" } From 3b100089fc963ee792042b1f0cdf5ea7bdcb3537 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 15:10:28 +0100 Subject: [PATCH 24/27] add audit 8.x test logs --- .../audit/test/test-audit-800.log | 3 + .../test/test-audit-800.log-expected.json | 105 ++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-800.log create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log b/filebeat/module/elasticsearch/audit/test/test-audit-800.log new file mode 100644 index 00000000000..75c7ebb6055 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-800.log @@ -0,0 +1,3 @@ +{"type":"audit", "timestamp":"2022-01-27T14:16:25,271+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"yEUG-8deS2y8ZxGgeyeUnw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_1"], "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"} +{"type":"audit", "timestamp":"2022-01-27T14:16:28,601+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"transport", "event.action":"access_granted", "authentication.type":"REALM", "user.name":"elastic", "user.realm":"reserved", "user.roles":["superuser"], "origin.type":"rest", "origin.address":"[::1]:64583", "request.id":"qo04VI2qRzKrE1dlrsjYgw", "action":"indices:admin/create", "request.name":"CreateIndexRequest", "indices":["test_2"]} +{"type":"audit", "timestamp":"2022-01-27T14:16:30,950+0100", "node.id":"O8SFUsk8QpGG16JVJcNgUw", "event.type":"rest", "event.action":"anonymous_access_denied", "origin.type":"rest", "origin.address":"[::1]:64583", "url.path":"/test_3", "request.method":"PUT", "request.id":"0ybRdKGYRAekov1eKI6nIw", "opaque_id":"myApp1", "trace.id":"0af7651916cd43dd8448eb211c80319c"} diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json new file mode 100644 index 00000000000..6477bb708e5 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-800.log-expected.json @@ -0,0 +1,105 @@ +[ + { + "@timestamp": "2022-01-27T13:16:25.271Z", + "elasticsearch.audit.action": "indices:admin/create", + "elasticsearch.audit.authentication.type": "REALM", + "elasticsearch.audit.indices": [ + "test_1" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.opaque_id": "myApp1", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "yEUG-8deS2y8ZxGgeyeUnw", + "elasticsearch.audit.request.name": "CreateIndexRequest", + "elasticsearch.audit.user.realm": "reserved", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "yEUG-8deS2y8ZxGgeyeUnw", + "input.type": "log", + "log.offset": 0, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:25,271+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"yEUG-8deS2y8ZxGgeyeUnw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_1\"], \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}", + "related.user": [ + "elastic" + ], + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "trace.id": "0af7651916cd43dd8448eb211c80319c", + "user.name": "elastic" + }, + { + "@timestamp": "2022-01-27T13:16:28.601Z", + "elasticsearch.audit.action": "indices:admin/create", + "elasticsearch.audit.authentication.type": "REALM", + "elasticsearch.audit.indices": [ + "test_2" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "qo04VI2qRzKrE1dlrsjYgw", + "elasticsearch.audit.request.name": "CreateIndexRequest", + "elasticsearch.audit.user.realm": "reserved", + "elasticsearch.audit.user.roles": [ + "superuser" + ], + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "qo04VI2qRzKrE1dlrsjYgw", + "input.type": "log", + "log.offset": 517, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:28,601+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"request.id\":\"qo04VI2qRzKrE1dlrsjYgw\", \"action\":\"indices:admin/create\", \"request.name\":\"CreateIndexRequest\", \"indices\":[\"test_2\"]}", + "related.user": [ + "elastic" + ], + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "user.name": "elastic" + }, + { + "@timestamp": "2022-01-27T13:16:30.950Z", + "elasticsearch.audit.layer": "rest", + "elasticsearch.audit.opaque_id": "myApp1", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "0ybRdKGYRAekov1eKI6nIw", + "elasticsearch.node.id": "O8SFUsk8QpGG16JVJcNgUw", + "event.action": "anonymous_access_denied", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "failure", + "fileset.name": "audit", + "host.id": "O8SFUsk8QpGG16JVJcNgUw", + "http.request.id": "0ybRdKGYRAekov1eKI6nIw", + "http.request.method": "PUT", + "input.type": "log", + "log.offset": 965, + "message": "{\"type\":\"audit\", \"timestamp\":\"2022-01-27T14:16:30,950+0100\", \"node.id\":\"O8SFUsk8QpGG16JVJcNgUw\", \"event.type\":\"rest\", \"event.action\":\"anonymous_access_denied\", \"origin.type\":\"rest\", \"origin.address\":\"[::1]:64583\", \"url.path\":\"/test_3\", \"request.method\":\"PUT\", \"request.id\":\"0ybRdKGYRAekov1eKI6nIw\", \"opaque_id\":\"myApp1\", \"trace.id\":\"0af7651916cd43dd8448eb211c80319c\"}", + "service.type": "elasticsearch", + "source.address": "[::1]:64583", + "source.ip": "::1", + "source.port": 64583, + "trace.id": "0af7651916cd43dd8448eb211c80319c", + "url.original": "/test_3" + } +] \ No newline at end of file From 1e1c940623597b6bb02675c32218c1084852ff0d Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 15:16:32 +0100 Subject: [PATCH 25/27] add missing audit fields --- filebeat/docs/fields.asciidoc | 14 ++++++++++++++ .../module/elasticsearch/audit/_meta/fields.yml | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 411516e2700..1d2468beb5a 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -49951,6 +49951,20 @@ type: boolean -- +*`elasticsearch.audit.authentication.type`*:: ++ +-- +type: keyword + +-- + +*`elasticsearch.audit.opaque_id`*:: ++ +-- +type: text + +-- + [float] === deprecation diff --git a/filebeat/module/elasticsearch/audit/_meta/fields.yml b/filebeat/module/elasticsearch/audit/_meta/fields.yml index 38774e4f8b9..ce0ffdf1fda 100644 --- a/filebeat/module/elasticsearch/audit/_meta/fields.yml +++ b/filebeat/module/elasticsearch/audit/_meta/fields.yml @@ -70,3 +70,7 @@ type: text - name: invalidate.apikeys.owned_by_authenticated_user type: boolean + - name: authentication.type + type: keyword + - name: opaque_id + type: text From 8a2cc61897955e3fb8256316f53b9fbd18385fd7 Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 15:16:49 +0100 Subject: [PATCH 26/27] handle trace.id in audit logs --- .../elasticsearch/audit/ingest/pipeline-json.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml index 359af0ab196..14e6a03538e 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -176,6 +176,16 @@ processors: field: elasticsearch.audit.level target_field: log.level ignore_missing: true + - dot_expander: + field: trace.id + path: elasticsearch.audit + - rename: + field: elasticsearch.audit.trace.id + target_field: trace.id + ignore_missing: true + - remove: + field: elasticsearch.audit.trace.id + ignore_missing: true - date: field: elasticsearch.audit.@timestamp target_field: "@timestamp" From a254d7b671180537638377261a18cadd66e8e63a Mon Sep 17 00:00:00 2001 From: klacabane Date: Fri, 28 Jan 2022 15:17:14 +0100 Subject: [PATCH 27/27] add elasticsearch missing field types --- filebeat/docs/fields.asciidoc | 4 ++++ filebeat/module/elasticsearch/_meta/fields.yml | 2 ++ filebeat/module/elasticsearch/fields.go | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 1d2468beb5a..9030022bc67 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -49751,6 +49751,8 @@ example: kibana -- Used by Elasticsearch to throttle and deduplicate deprecation warnings +type: keyword + example: v7app -- @@ -49760,6 +49762,8 @@ example: v7app -- Category of the deprecation event +type: keyword + example: compatible_api -- diff --git a/filebeat/module/elasticsearch/_meta/fields.yml b/filebeat/module/elasticsearch/_meta/fields.yml index 3cb88baf881..8ae4789d929 100644 --- a/filebeat/module/elasticsearch/_meta/fields.yml +++ b/filebeat/module/elasticsearch/_meta/fields.yml @@ -47,6 +47,8 @@ - name: http.request.x_opaque_id description: "Used by Elasticsearch to throttle and deduplicate deprecation warnings" example: "v7app" + type: keyword - name: event.category description: "Category of the deprecation event" example: "compatible_api" + type: keyword diff --git a/filebeat/module/elasticsearch/fields.go b/filebeat/module/elasticsearch/fields.go index 597911929e8..525d0c50eac 100644 --- a/filebeat/module/elasticsearch/fields.go +++ b/filebeat/module/elasticsearch/fields.go @@ -32,5 +32,5 @@ func init() { // AssetElasticsearch returns asset data. // This is the base64 encoded zlib format compressed contents of module/elasticsearch. func AssetElasticsearch() string { - return "eJzUmm1z2zbywN/nU+zozb+dsfmXH+rWmrmbaZXESaZ5aGwn1yoezgpcUYhAgAZAyWon3/0GACVLFEk9XJvr+U1CEcD+drFY7II4hgnNe0ACjeXMEGo2fgJguRXUg86z1d87TwA0CUJDPUjxCUBChmmeW65kD/75BADWR4LXKikEPQEYcRKJ6fkmxyAxo02h7s/Ocze4VkVe/lIjY3241SGZynIlSdrlm8oA6xo9toeRVhnMxqQJ7JhAqBRo6l4ozVMu0VLSWRmUHjDLvYlURBGLsug1WXyKFvua0NJLmdDDNekpZ7TaL+g3oflM6WQTXxTGko6KgieNGtzevnwKauQxyw71ZFfZVA9fiDc3/PrDr/zd6IfJQ3qZ7k/jnhpp3mBGO9Ekik1IH9e0aaeQKqGoxRyPxnAt62U/veYf2c2cbsYf7e2/fv7p8lX3p9ezPRl2NkMzx/Tjm1fmt7PdBXPnRu2Svaf55vUyR1zQkNAeWzL2mMu8sPvKb7O+l84b1ga+vUqfzoa370f9D999/+M1ux/20z3sbsaok1bxycLovmk9RXd3gWVIinOtkoLZOCz+rd03lqihBIZzKIMNGItsAlYBT0haPprDbMzXws9CC9/QuF/co6b7goytV2vChyixs6HD2No8KntGD7HK8b6guC2grNOWodEqsGOtrBUEKBNIKClywRlagoRyTQxdf5ihllympsHjv8c832T0oTVyY6VKzxvJ+mWDhXlW5foh6oU6s6LlQ0Ex5nxTOhYJtxuTurrpQM0eszqCwDnptTdV9Bu3ibhWi53Fzbcdc7OxrfRAk7FHYDVKkyvt3gHP4xEXlUC5rqWueEabe1YsH7t2W/FdI2f5AGzHaEExVmjtmFEqOc9UYWJkjIyJE5KckiPAwo6dk4dpikfIhf+50io8phqldc9MSUnM96j7bdHNYpaTpiQu/fsIdCFjXBmofA4dmo23Ln9/M4bpi7ba8eMyoyiJNyYevtl8E3wG4f2z6xv48d3LRedvV71k2W+GBjQx4lNKQEkv7bEZG6OUJL49AqEYitjtTvBNyHEYCr9bATemoGSV89tm2z2Os7/dNKHItnreug+FTh6u8sJpPkXBE280TJHLzTVRgndcDkIjLIR1S+sA9sKQjnZTwDX9P1OrxxHw0eqLRi/teDe1fEpxwjUxq/T8UGglyLRCv3ctXMxfBCqCXHPJeI4ChiRUJcKvecRgsRnFmGRcdo6g4zIOUz7C3YHUfilXs5/9B6ibsu0j1JUSu/VEH7O2+ohcSRlDlxBjnVPTA7Gi2TXcHhfy6F6mJLdK/3+GXB7gHVpEOWrMtniHi0O371+Cb0uWdLMzdP5wtnfD/+MzsonkbHz6pVMrncuEsy2O+TK0KTeMkKgEa7W540ip49PuyWXUPYm6584h13452/jl4hAvXSRZPGlV4Vby+4IgFCo1Kd26+T7+/nM8GV58uJ6+Hf9437Wzd9MXb385JNIGuJrl07zVL7aUPRyxLwj1NdNKiPf1uu3MGg9VMq/tjIJj1U9ytONKtuv6R0xJu7lsM55qDBpbXVDLph5jkmgyVXHbQIwqNKOI5wcILjTfU5pbuGW+IA4QuIzt+4o1m+cBu8rMyBhM60O5pQfbECIW+3uEOZ/Q3ERqJimJh/N4bRONHVrt2EOlBKHcqAJWiomttUDtIRS0HESl7PAxAa764FIUQ7YUEO1Yl+RjNPUWrkrfQuD+nntBYHJifMSZyw6u+kFEVGlcx7TKVeMy0BoZdgJ0f6sHL1d9YEqIUDPUg65MfxE8NjbEGtFGQmE1kuwI1q+QLAW6rUzphMs01NgEr3CKMOXaFiggQzbmsgXcMF0MYzPPhkrEFl2Va3lGf5Ue8A4LQ+BEAJdgiCmZGGBuTTkdihwCC3gWsxXcai7TrwC+A7dH2co9I5zEmkYmzrVyWYjn/wvJbxyzyV3V/SjRY4CmEWmSLiN6VKoZ3eVrQpCINRmG8mtRr9g7Qz1x9IJPCdTwMzFrXJEhCDAP50luTXADxqo8p6RZGSbQmLiQQmHytTQJ0ry/yMKlnx5iR+uzvPCcjYx1QXlHxnfBMaD/7jb4eOkvpEdKZw74MRTWIDaHbKhUUQ1Ghq2G3lER91dRQhXW8CQcm0xISxJ1CqwElrn5L1ByWYWEVkpXhn4NzBtlUQAJzJ2/VqCt8oWtIBvIV/ZLf+ZkLGrfasQlN+OoNsv4PM1iXciGJdisyBYFfCHiUD3Jqw+vS5oiX1ltR4AGMAzvvDxXXFqQRTYkXU9rx5owMbF1doldlGkKHgeTX6EeYrpmzVIqeKk+tpXTUBc0lo7sQqDfXRbMf7aJHYJVyn+MCFAlZyuXxbS+HKpP3bZZqw9CpWnYetMGkWPCamQ8OJF9QZgDCqHKzQZlspgX/vveuazrE0+GjUGdS0vpRi2yAyYsF69T3stxjj/hQg3nti1DcTvTX4bkPw95omaYZQUtkjil6tnXwRP3ViSQkqQycVaMFTlKNv/7z6CfPDVyBlnV4G8wnY023T67c1XI9M+c31/dgP/jMzyv6vA3mOMWu9bTLe1GeromdP2o8Nq/9pdjqp9Qdv2G+rjVIZtYjWw9O16R1+nBdfhu7lo5cObKaDUC0lrp9Q3JX1TowQjF2vlH7XFMVauwH62fWja5dNvhi/eEtgXQCfNy1W8+Ta0/O61bWvVLYBmI5WbVsc5SldRGseAQakPBZYowU19D4FK/KekxYRIbum81+TXdF65eLlPERsufnZ9fXl6e1pq/keIx34sXpzvRlk8p61XyVf/I/ZNxIXiZgTUSnlx0uzvmgUsrDd2Cxv0AfXTzuaoz8vKmyjKznaEpB6ZkD/ofdqJfhgehZkKlzZEovA93E0yoGDYuK25AdAan3ZMfjrsXx6eXNyfdXveid3J+dHl2djd4+eb5W7gbhAtPYYiohIjuC9LzOxhM4w+vxp8/3MEgI6s589eqLqKzqHvsxo26F9Hpxd2ge+dT7MF59F1m7o78QxyMNDj3z64QGXNrBieX52ffuZ/mOZnB3ZELizb8xyP4ixqDX26fvf81vnnx7E38/NlN/8VyDH/pyQxOXHv/+WHwx6eOp/3U6f3xqZOhZeMYhQiPQ6WM/dTpnUTdL1++3B39J/HbZfCV7Wl9hn72DTYupq3ORq2xR2TXZ6+51ljGHqUmLSR+yXG7rHvKb1q+/vXGauI763YzsyeKm8g2Fve+Sd5+oryrtIi6du/DjDZK9G9P9pT76Jlt0stbZPO8cfqrbr0nhnf42E9gG4dQs/ZZ3mPJ7EdID1ZjHDhbCJ+5ZqU6wOVI6Qw3v28f6iWPwabNK0PVyW2To5yfHiA0RKetYp3xOSXhBmcTwOl+AFoVllc27eqFF9+iycime/Lit9Nffppcfp6dpzbF51buZ/jKpYA16S+TP2du25fgTcvaSxQ7ZLk1S7sO/qtGkChWZMsbgy5b8HGekhZ5/w4AAP//lrQtsw==" + return "eJzUWltz2zb2f8+nOKOXfztj8y9f6taa2Z1plcRxprk0sp1tFQ/nCDyiEIEADYCS1U6++w5AShYpkrpsm+3qxSZxOb9zPwfEMUxp0QMSaCxnhlCzyTMAy62gHnRerL/vPAPQJAgN9SDGZwARGaZ5armSPfjnMwAo7wRvVJQJegYw5iQi0/NTjkFiQptE3c8uUre5VllavKmhUd5ufUumklRJknY1UtmgzNHTfBhrlcB8QprATgiEioFmbkBpHnOJlqLO2qb0iEnqRaQCCliQBG/I4nO02NeElq5lRI8D0jPOaH1dzt+UFnOlo034IjOWdJBlPGrk4Pb2+jmosYdZLKhHdpXM9OiVeHvDB3e/8vfjH6aP8WW8Pxr31IjmLSa0E5pIsSnp45o57SikiihoEceTMNzMetrPB/wju1nQzeSjvf3Xzz9dvu7+9Ga+J4adxdCMY/bx7Wvz29nuhLkzo3bK3tL89HqaYy5oRGiPLRl7zGWa2X3pt0nfU+cNvoHvruLn89Hth3H/7rvvfxywh1E/3kPuZoI6aiUfLYXup9aj6O5OsAhJYapVlDEb5s6/dfmGixqKYLSAItiAscimYBXwiKTl4wXMJ7wUfpZc+InGvXGPmh4yMraerSkfocTOBg8Ta9OgWBk8hirFh4zCtoBSRluERqvATrSyVhCgjCCiKEsFZ2gJIko1MXTrYY5achmbBov/HtN0D/m7kBs4GrHSi0bE/WLCUmzrePwW9WCcuNHykaAQU747KswibjdmrycpqMlJ6zsIXJAujVRZunFJx81aZiJnH3bCzUYa6oEmY4/AapQmVdqNAU/DMReVwFrmXlcsqc2cKxoJ3byt8N0kp5EcsJ2gBcVYprXDjFLJRaIyEyJjZEwYkeQUHQFmduKcIldfOEYu/OvKrPwx1iite2ZKSmJ+Rd275TKLSUqaorDwhyPQmQxxbaPiOV/QLLwy/f3FmKsv2CrHj6sKpEC8oXj4ZnMktxmEDy8GN/Dj++vl4m/XrWS1bo4GNDHiM4pASU/taRqboJQkvj0CoRiK0GUz+CaviRgKn92AG5NRtI7z22bZPe2zv9w0oUi2Wl7ZhvJFHlxlwHE+Q8EjLzSMkctNnyiAd1zNQmPMhHWudQD2zJAOdmPATf0/U8vHEfDx+kCjlXa8mVo+ozDimphVenEoaCXItIL+4Ga4HLEMVASp5pLxFAWMSKhKRihZxHCZvEKMEi47R9BxFYopHuH+QNTelavV0v4b1Kls+w51rcduK9HHrK02ItdKzHxJHmOdUdMjsazZNFzuy+vuXqIkt0r/f4JcHmAdWgQpaky2WIeLQ7cfrsHPJUu62Rg6fzjZu+3/8RnZVHI2Of3SqaXOZcTZFsO8zucUCSMvbHJptZnjWKnj0+7JZdA9CbrnziBLb8423lwcYqXLooxHrSzcSv6QEeSNTU0JWBbfx99/Dqeji7vB7N3kx4eunb+fvXr3yyGRNgdX4z7NqX6ZUvYwxL4g1AOmlRAf6nnbGWs4UtGidjEKjlU7SdFOKtWxWx8wJe2m2yY81phzbHVGLUk9xCjSZKrktgExKtOMAp4eQDjTfE9qznGLekEcQHAV2/clazbPD3almZAxGNeHckuPtiFELPN7gCmf0sIEai4pCkeLsJREQwetdu+RUoJQ1ofqUoKuK+d2KAVrGrIGzpZL1vqbrW1I7XkZtJyZxezwPQGu+uCqI0O2IBDs2BKlEzT1wqtS34LA/V56QmBSYnzMmStMrvo5iaAyuQ7TOq4aa4VWve4E0P3Wz4iu+sCUEHm7Ug90Tf1Z7iyhIdYIbSwUVoPYjsD6FSQrgi6LKh1xGefHAQSvcYYw49pmKCBBNuGyBbhhOhuFZpGMlAgtusbb8oT+Kj7gPWaGwJEALsEQUzIywJw7Ox6yFHIs4LGYrcCt5jL+CsB3wO2hbMU9J5yGmsYmTLVyBZDH/xciv3GYTeoa/ieKHgZoGpMm6YqxJ6aaobtSUQgSoSbDUH4t1GvyTlBPHXrBZwRq9JmYNa6/EQSY5kdfzie4AWNVmlLUzAwTaEyYSaEw+lqc5NS8vcjMVb4exI7SZ2nmcTZirAvKO2J8nxsG9N/f5jZe2AvpsdKJA/wUCmsgNodsqDRwDUKGrYLekRH3qzChMmt4lJ/YTElLEnUMrAWWhfkvoOSyChJaUboO+GvAvFEWBZDA1NlrBbRVvqcWZHPka/nSH3cZi9rPGnPJzSSorTI+z5JQZ7LBBZsZ2cKA74EcVI/k9d2bAk2WrnnbEaABzLd3Vp4qLi3ILBmRrkdrJ5owMqF1cgldlGkKHgcjv0I9wrgkzYIqeKo+thVqqAsaK0N2IdBnlyXmP1vEDoJVyn83yUEVOFtxWYzrO7H60m2btPogVBznqTduIDkhrEbGgwvZV4QpoBCqSDYoo6Ve+O9717JuTTgdNQZ1Li3FG23QDjBh5byOeU/HGf6UCzVa2LYKxWWmvwyS/5LlETWDWbVhIgpjqh67Hay4dyKCmCQVhbNiLEtRssXfX4NeeWrsBLLOwd9AnY0y3a7dhcpk/Gfq91e34f+4hhdVHv4GOm6Raz26ldxIz0pEy6eUAz/s7/FUv97s+vn2KdUhm1qNrFwdr9Hr9GCQf+J3sxxw5tpoNQbSWulyQvJ3KnowRlE6/6g9jqlyleej8oFpk0m3Hb54S2hzgE6ul6t+80Fu/bFtnWvVu8AqEMvNrqOMpUqpDcUSh1AbDK5KhLn6GgRX/M1ITwij0NBDq8gH9JC5frkoERslf3Z+fnl5eVor/kYUT/VeuDzdCbZ8xSl3yVf9I/cn4ULwogJrRHhy0e3uWAeupDRyDo37AfTRzdeqTsirSzWrynaOptiYoj3Q/7AT+lV4EGouVNwcifLx/FqEyTuGjXuVGyA6w9PuyQ/H3Yvj08ubk26ve9E7OT+6PDu7H16/ffkO7of53ax8i6AAETxkpBf3MJyFd68nn+/uYZiQ1Zz5G2AXwVnQPXb7Bt2L4PTifti99yX28Dz4LjH3R/4hzIU0PPfPrhGZcGuGJ5fnZ9+5V4uUzPD+yIVFm//jIfg7IsNfbl98+DW8efXibfjyxU3/1WoPfz/LDE/cfP/lY/jHp45H+6nT++NTJ0HLJiEKkT+OlDL2U6d3EnS/fPlyf/SfxG9XwVfSU1lDP/sJG3fo1rVRK+wx2bL2mnuNVexRatqCxLsct6u+p/ic5vtfL6wmfGfdbmL2hOIU2YbFjTfR24+UN5UWUgM3nmu0kaIfPdmT7pNltlEvLrwt0kb1V816Txje4EOvwDYcQs3btbyHy+yHkB6txjDH2YLwhZtWsANcjpVOcPPT+qFW8hRs2qwy7zq5bTKU89MDiObRaStZJ3xOUX7ZtAnA6X4AtMosryTt6l0bP6NJyKZ78uq3019+ml5+np/HNsaXVu4n+MqnyBL16+jP0W27C960+F6k2CHu1kxtkNuvGkOkWJasLiu6asHHeYpa6P07AAD//wFdWko=" }