From 20f058b17ba06178fb8c1a14f99b40da2258b48b Mon Sep 17 00:00:00 2001 From: Kent Wang Date: Tue, 27 Aug 2019 21:38:49 +0800 Subject: [PATCH 1/2] Fix filebeat system module timezone parsing (#13308) Fix timezone handling in system module when non-UTC timezones are used. Fix #13306 (cherry picked from commit 44061f4761da3714bc00d3d088a32e3a006e81c0) --- CHANGELOG.next.asciidoc | 16 ++++++++++++++++ filebeat/module/system/auth/ingest/pipeline.json | 9 +++++++-- .../module/system/syslog/ingest/pipeline.json | 11 +++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 39e3f25b7e3..b19fefe3963 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -76,6 +76,22 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve detection of file deletion on Windows. {pull}10747[10747] - Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] - Reduce memory usage if long lines are truncated to fit `max_bytes` limit. The line buffer is copied into a smaller buffer now. This allows the runtime to release unused memory earlier. {pull}11524[11524] +- Fix memory leak in Filebeat pipeline acker. {pull}12063[12063] +- Fix goroutine leak caused on initialization failures of log input. {pull}12125[12125] +- Fix goroutine leak on non-explicit finalization of log input. {pull}12164[12164] +- Skipping unparsable log entries from docker json reader {pull}12268[12268] +- Parse timezone in PostgreSQL logs as part of the timestamp {pull}12338[12338] +- Load correct pipelines when system module is configured in modules.d. {pull}12340[12340] +- Fix timezone offset parsing in system/syslog. {pull}12529[12529] +- When TLS is configured for the TCP input and a `certificate_authorities` is configured we now default to `required` for the `client_authentication`. {pull}12584[12584] +- Apply `max_message_size` to incoming message buffer. {pull}11966[11966] +- Syslog input will now omit the `process` object from events if it is empty. {pull}12700[12700] +- Fix multiline pattern in Postgres which was too permissive {issue}12078[12078] {pull}13069[13069] +- Allow path variables to be used in files loaded from modules.d. {issue}13184[13184] +- Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303] +- Fix timezone parsing of system module ingest pipelines. {pull}13308[13308] +- Change iis url.path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] +- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] *Heartbeat* diff --git a/filebeat/module/system/auth/ingest/pipeline.json b/filebeat/module/system/auth/ingest/pipeline.json index ebdfb4e2504..6c832ea2da5 100644 --- a/filebeat/module/system/auth/ingest/pipeline.json +++ b/filebeat/module/system/auth/ingest/pipeline.json @@ -54,8 +54,13 @@ { "date": { "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], + "field": "system.auth.timestamp", + "target_field": "@timestamp", + "formats": [ + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss", + "ISO8601" + ], "timezone": "{{ event.timezone }}", "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json index 8167ce64b23..1bf54bc08e1 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.json +++ b/filebeat/module/system/syslog/ingest/pipeline.json @@ -34,6 +34,7 @@ "formats": [ "MMM d HH:mm:ss", "MMM dd HH:mm:ss", + "MMM d HH:mm:ss", "ISO8601" ], "ignore_failure": true @@ -42,8 +43,14 @@ { "date": { "if": "ctx.event.timezone != null", - "field": "@timestamp", - "formats": ["ISO8601"], + "field": "system.syslog.timestamp", + "target_field": "@timestamp", + "formats": [ + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss", + "MMM d HH:mm:ss", + "ISO8601" + ], "timezone": "{{ event.timezone }}", "on_failure": [{"append": {"field": "error.message", "value": "{{ _ingest.on_failure_message }}"}}] } From a58ff49cee3e09725b37a3ca6c9cd810d721b462 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 27 Aug 2019 16:32:58 +0200 Subject: [PATCH 2/2] Fix changelog --- CHANGELOG.next.asciidoc | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index b19fefe3963..ccd87a83c9a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -76,22 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve detection of file deletion on Windows. {pull}10747[10747] - Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] - Reduce memory usage if long lines are truncated to fit `max_bytes` limit. The line buffer is copied into a smaller buffer now. This allows the runtime to release unused memory earlier. {pull}11524[11524] -- Fix memory leak in Filebeat pipeline acker. {pull}12063[12063] -- Fix goroutine leak caused on initialization failures of log input. {pull}12125[12125] -- Fix goroutine leak on non-explicit finalization of log input. {pull}12164[12164] -- Skipping unparsable log entries from docker json reader {pull}12268[12268] -- Parse timezone in PostgreSQL logs as part of the timestamp {pull}12338[12338] -- Load correct pipelines when system module is configured in modules.d. {pull}12340[12340] -- Fix timezone offset parsing in system/syslog. {pull}12529[12529] -- When TLS is configured for the TCP input and a `certificate_authorities` is configured we now default to `required` for the `client_authentication`. {pull}12584[12584] -- Apply `max_message_size` to incoming message buffer. {pull}11966[11966] -- Syslog input will now omit the `process` object from events if it is empty. {pull}12700[12700] -- Fix multiline pattern in Postgres which was too permissive {issue}12078[12078] {pull}13069[13069] -- Allow path variables to be used in files loaded from modules.d. {issue}13184[13184] -- Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303] - Fix timezone parsing of system module ingest pipelines. {pull}13308[13308] -- Change iis url.path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225] -- Add timezone information to apache error fileset. {issue}12772[12772] {pull}13304[13304] *Heartbeat*