[Filebeat] Update handling of elasticsearch server logs

filebeat/module/elasticsearch/server/ingest/pipeline-json.yml

@@ -7,51 +7,55 @@ processors:
 - json:
     field: message
     target_field: elasticsearch.server
+- dot_expander:
+    field: event.dataset
+    path: elasticsearch.server
 - drop:
-    if: ctx.elasticsearch.server.type != 'server'
-- remove:
-    field: elasticsearch.server.type
+    if: ctx.elasticsearch.server.event.dataset != 'elasticsearch.server'
+- rename:
+    field: elasticsearch.server.event.dataset
+    target_field: event.dataset
+    ignore_missing: true
+- dot_expander:
+    field: ecs.version
+    path: elasticsearch.server
+- rename:
+    field: elasticsearch.server.ecs.version
+    target_field: ecs.version
+    ignore_missing: true
 - dot_expander:
     field: service.name
     path: elasticsearch.server
 - rename:
     field: elasticsearch.server.service.name
     target_field: service.name
     ignore_missing: true
-- rename:
-    field: elasticsearch.server.component
-    target_field: elasticsearch.component
-    ignore_missing: true
 - dot_expander:
-    field: cluster.name
+    field: elasticsearch.cluster.name
     path: elasticsearch.server
 - rename:
-    field: elasticsearch.server.cluster.name
+    field: elasticsearch.server.elasticsearch.cluster.name
     target_field: elasticsearch.cluster.name
 - dot_expander:
-    field: node.name
+    field: elasticsearch.node.name
     path: elasticsearch.server
 - rename:
-    field: elasticsearch.server.node.name
+    field: elasticsearch.server.elasticsearch.node.name
     target_field: elasticsearch.node.name
 - dot_expander:
-    field: cluster.uuid
+    field: elasticsearch.cluster.uuid
     path: elasticsearch.server
 - rename:
-    field: elasticsearch.server.cluster.uuid
+    field: elasticsearch.server.elasticsearch.cluster.uuid
     target_field: elasticsearch.cluster.uuid
     ignore_missing: true
 - dot_expander:
-    field: node.id
+    field: elasticsearch.node.id
     path: elasticsearch.server
 - rename:
-    field: elasticsearch.server.node.id
+    field: elasticsearch.server.elasticsearch.node.id
     target_field: elasticsearch.node.id
     ignore_missing: true
-- rename:
-    field: elasticsearch.server.level
-    target_field: log.level
-    ignore_missing: true
 - dot_expander:
     field: log.level
     path: elasticsearch.server

filebeat/module/elasticsearch/server/test/elasticsearch-json.800.log

@@ -1,3 +1,3 @@
-{"@timestamp":"2020-04-14T14:05:58.019Z", "log.level": "INFO", "message":"adding template [.management-beats] for index patterns [.management-beats]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[CBR-MBP.local][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetaDataIndexTemplateService","type":"server","cluster.uuid":"ECEBF2VPQuCF9tbBKaLqXQ","node.id":"suOYiQwuRvialOY-c0wHLA","node.name":"CBR-MBP.local","cluster.name":"elasticsearch"}
-{"@timestamp":"2020-04-14T20:57:49.663Z", "log.level": "INFO", "message":"[test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [1]/[1], mappings [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataCreateIndexService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"}
-{"@timestamp":"2020-04-14T20:57:49.772Z", "log.level": "INFO", "message":"[test-filebeat-modules/IW1jJcOBTFeIDihqjoT8yQ] update_mapping [_doc]", "service.name":"ES_ECS","process.thread.name":"elasticsearch[7debcb878699][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataMappingService","type":"server","cluster.uuid":"QxYAE76DTAWkgk9CwIRedQ","node.id":"kZnYdakGTqihZQT_1rM92g","node.name":"7debcb878699","cluster.name":"docker-cluster"}
+{"@timestamp":"2022-01-25T15:12:08.472Z", "log.level": "INFO", "message":"adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"}
+{"@timestamp":"2022-01-25T15:12:08.588Z", "log.level": "INFO", "message":"adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-7-*]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"}
+{"@timestamp":"2022-01-25T15:12:08.686Z", "log.level": "INFO", "message":"adding template [.monitoring-alerts-7] for index patterns [.monitoring-alerts-7]", "ecs.version": "1.2.0","service.name":"ES_ECS","event.dataset":"elasticsearch.server","process.thread.name":"elasticsearch[matschaffer-mbp2019.lan][masterService#updateTask][T#1]","log.logger":"org.elasticsearch.cluster.metadata.MetadataIndexTemplateService","elasticsearch.cluster.uuid":"28iKoFsvTJ6HEyXbdLL-PQ","elasticsearch.node.id":"tc3nhgC0SFCKfwwy6jCmkw","elasticsearch.node.name":"matschaffer-mbp2019.lan","elasticsearch.cluster.name":"main"}