[Filebeat] Update Elasticsearch Deprecation fileset parsing

CHANGELOG.next.asciidoc

@@ -861,6 +861,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
 - Add new template functions and `value_type` parameter to `httpjson` transforms. {pull}26847[26847]
 - Add support to merge registry updates in the filestream input across multiple ACKed batches in case of backpressure in the registry or disk. {pull}25976[25976]
+- Update Elasticsearch module's ingest pipeline for parsing new deprecation logs {issue}26857[26857] {pull}26880[26880]
 
 *Heartbeat*
 

filebeat/module/elasticsearch/deprecation/ingest/pipeline-json.yml

@@ -8,7 +8,7 @@ processors:
     field: message
     target_field: elasticsearch.deprecation
 - drop:
-    if: ctx.elasticsearch.deprecation.type != 'deprecation'
+    if: '!["deprecation", "deprecation.elasticsearch"].contains(ctx.elasticsearch.deprecation.type)'
 - remove:
     field: elasticsearch.deprecation.type
 - dot_expander:
@@ -78,17 +78,20 @@ processors:
 - rename:
     field: elasticsearch.deprecation.message
     target_field: message
-- rename:
-    field: elasticsearch.deprecation.@timestamp
-    target_field: '@timestamp'
-    ignore_missing: true
-- rename:
-    field: elasticsearch.deprecation.timestamp
-    target_field: '@timestamp'
-    ignore_missing: true
 - date:
-    field: '@timestamp'
-    target_field: '@timestamp'
+    field: 'elasticsearch.deprecation.@timestamp'
+    formats:
+    - ISO8601
+    ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation["@timestamp"] != null'
+- date:
+    field: 'elasticsearch.deprecation.timestamp'
     formats:
     - ISO8601
     ignore_failure: true
+    if: 'ctx.elasticsearch?.deprecation?.timestamp != null'
+- remove:
+    field:
+      - elasticsearch.deprecation.timestamp
+      - elasticsearch.deprecation.@timestamp
+    ignore_missing: true

filebeat/module/elasticsearch/deprecation/ingest/pipeline-plaintext.yml

@@ -11,8 +11,9 @@ processors:
       GREEDYMULTILINE: |-
         (.|
         )*
+      DEP_LOGLEVEL: '(%{LOGLEVEL}|DEPRECATION)'
     patterns:
-    - \[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{LOGLEVEL:log.level}%{SPACE}\]\[%{DATA:elasticsearch.component}%{SPACE}\]
+    - \[%{TIMESTAMP_ISO8601:elasticsearch.deprecation.timestamp}\]\[%{DEP_LOGLEVEL:log.level}(%{SPACE})?\]\[%{DATA:elasticsearch.component}(%{SPACE})?\](%{SPACE}\[%{HOSTNAME:elasticsearch.node.name}\])?
       %{GREEDYMULTILINE:message}
 - date:
     if: ctx.event.timezone == null

filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log

@@ -2,3 +2,6 @@
 [2018-04-23T16:40:13,862][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
 [2018-04-23T16:40:14,792][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
 [2018-04-23T16:40:15,127][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] Deprecated field [template] used, replaced by [index_patterns]
+[2021-07-09T10:33:11,962][DEPRECATION][o.e.d.r.RestController   ] [ecenter.local] Legacy index templates are deprecated in favor of composable templates.
+[2021-07-09T10:42:03,065][DEPRECATION][o.e.d.c.m.MetadataCreateIndexService] [ecenter.local] index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices
+[2021-07-09T10:48:46,329][DEPRECATION][o.e.d.c.m.IndexNameExpressionResolver] [ecenter.local] this request accesses system indices: [.apm-agent-configuration, .apm-custom-link, .async-search, .kibana_7.13.3_001, .kibana_task_manager_7.13.3_001], but in a future major version, direct access to system indices will be prevented by default

filebeat/module/elasticsearch/deprecation/test/elasticsearch_deprecation.log-expected.json

@@ -62,5 +62,56 @@
         "log.offset": 411,
         "message": "Deprecated field [template] used, replaced by [index_patterns]",
         "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T10:33:11.962-02:00",
+        "elasticsearch.component": "o.e.d.r.RestController",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 548,
+        "message": "Legacy index templates are deprecated in favor of composable templates.",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T10:42:03.065-02:00",
+        "elasticsearch.component": "o.e.d.c.m.MetadataCreateIndexService",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 702,
+        "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T10:48:46.329-02:00",
+        "elasticsearch.component": "o.e.d.c.m.IndexNameExpressionResolver",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.timezone": "-02:00",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 973,
+        "message": "this request accesses system indices: [.apm-agent-configuration, .apm-custom-link, .async-search, .kibana_7.13.3_001, .kibana_task_manager_7.13.3_001], but in a future major version, direct access to system indices will be prevented by default",
+        "service.type": "elasticsearch"
     }
 ]

filebeat/module/elasticsearch/deprecation/test/test-json.log

@@ -11,3 +11,6 @@
 {"type": "deprecation", "timestamp": "2019-01-30T14:17:17,546-0800", "level": "WARN", "component": "o.e.d.r.a.s.RestSearchAction", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw",  "message": "[types removal] Specifying types in search requests is deprecated."  }
 {"type": "deprecation", "timestamp": "2019-01-30T14:18:33,367-0800", "level": "WARN", "component": "o.e.d.x.w.a.i.IndexAction", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw",  "message": "[types removal] Specifying types in a watcher index action is deprecated."  }
 {"type": "deprecation", "timestamp": "2019-01-30T14:18:46,493-0800", "level": "WARN", "component": "o.e.d.i.q.QueryShardContext", "cluster.name": "es1", "node.name": "es1_1", "cluster.uuid": "S4dWw65ZT1eu3SltmAr84A", "node.id": "gCoNXf3qSQ6a190zBKr7Bw",  "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead."  }
+{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:33:11,962+10:00", "level": "DEPRECATION", "component": "o.e.d.r.RestController", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "Legacy index templates are deprecated in favor of composable templates.", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg"  }
+{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:42:03,065+10:00", "level": "DEPRECATION", "component": "o.e.d.c.m.MetadataCreateIndexService", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg"  }
+{"type": "deprecation.elasticsearch", "timestamp": "2021-07-09T10:48:46,329+10:00", "level": "DEPRECATION", "component": "o.e.d.a.a.i.a.g.TransportGetAliasesAction", "cluster.name": "elasticsearch", "node.name": "ecenter.local", "message": "this request accesses system indices: [.kibana_task_manager_7.13.3_001, .apm-agent-configuration, .kibana_7.13.3_001, .apm-custom-link, .async-search], but in a future major version, direct access to system indices will be prevented by default", "cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw", "node.id": "VLed1qvSSOCitAo-UeM3Jg"  }

filebeat/module/elasticsearch/deprecation/test/test-json.log-expected.json

@@ -258,5 +258,65 @@
         "log.offset": 4459,
         "message": "[types removal] Using the _type field in queries and aggregations is deprecated, prefer to use a field instead.",
         "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T00:33:11.962Z",
+        "elasticsearch.cluster.name": "elasticsearch",
+        "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw",
+        "elasticsearch.component": "o.e.d.r.RestController",
+        "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "host.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 4842,
+        "message": "Legacy index templates are deprecated in favor of composable templates.",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T00:42:03.065Z",
+        "elasticsearch.cluster.name": "elasticsearch",
+        "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw",
+        "elasticsearch.component": "o.e.d.c.m.MetadataCreateIndexService",
+        "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "host.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 5219,
+        "message": "index name [.kibana-event-log-7.13.3-000001] starts with a dot '.', in the next major version, index names starting with a dot are reserved for hidden indices and system indices",
+        "service.type": "elasticsearch"
+    },
+    {
+        "@timestamp": "2021-07-09T00:48:46.329Z",
+        "elasticsearch.cluster.name": "elasticsearch",
+        "elasticsearch.cluster.uuid": "k_63KpgPSDeFRu1NxnlDiw",
+        "elasticsearch.component": "o.e.d.a.a.i.a.g.TransportGetAliasesAction",
+        "elasticsearch.node.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "elasticsearch.node.name": "ecenter.local",
+        "event.category": "database",
+        "event.dataset": "elasticsearch.deprecation",
+        "event.kind": "event",
+        "event.module": "elasticsearch",
+        "event.type": "info",
+        "fileset.name": "deprecation",
+        "host.id": "VLed1qvSSOCitAo-UeM3Jg",
+        "input.type": "log",
+        "log.level": "DEPRECATION",
+        "log.offset": 5716,
+        "message": "this request accesses system indices: [.kibana_task_manager_7.13.3_001, .apm-agent-configuration, .kibana_7.13.3_001, .apm-custom-link, .async-search], but in a future major version, direct access to system indices will be prevented by default",
+        "service.type": "elasticsearch"
     }
 ]