From 2eb8124469d7fbc4fa2c5a8e9f7dfb43f59616f4 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 1 Apr 2020 10:04:31 -0700 Subject: [PATCH 1/2] Add pattern to handle logs from newer ES versions --- .../audit/ingest/pipeline-json.yml | 1 + .../audit/test/test-audit-761.log | 1 + .../test/test-audit-761.log-expected.json | 46 +++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-761.log create mode 100644 filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml index 434db5cab21..45a096931f8 100644 --- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml +++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml @@ -14,6 +14,7 @@ processors: target_field: elasticsearch.audit.@timestamp formats: - yyyy-MM-dd'T'HH:mm:ss,SSS + - yyyy-MM-dd'T'HH:mm:ss,SSSZ timezone: '{{ event.timezone }}' - remove: if: ctx.elasticsearch.audit['@timestamp'] == null && ctx.event.timezone != null diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log b/filebeat/module/elasticsearch/audit/test/test-audit-761.log new file mode 100644 index 00000000000..c3e9a5d3452 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-761.log @@ -0,0 +1 @@ +{"@timestamp":"2020-04-01T11:21:06,725+0200", "node.id":"vvj136QVQ2Ci2aXmrhyi3Q", "event.type":"transport", "event.action":"access_granted", "user.name":"logstash_manager", "user.realm":"native1", "user.roles":["logstash_admin","cluster_monitor"], "origin.type":"rest", "origin.address":"10.54.25.111:52148", "request.id":"rLBMfPM2Q9q-DQEB_g30ww", "action":"indices:data/read/mget[shard]", "request.name":"MultiGetShardRequest", "indices":[".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash",".logstash"]} diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json new file mode 100644 index 00000000000..c2bb0468065 --- /dev/null +++ b/filebeat/module/elasticsearch/audit/test/test-audit-761.log-expected.json @@ -0,0 +1,46 @@ +[ + { + "@timestamp": "2020-04-01T13:21:06.725Z", + "elasticsearch.audit.action": "indices:data/read/mget[shard]", + "elasticsearch.audit.indices": [ + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash", + ".logstash" + ], + "elasticsearch.audit.layer": "transport", + "elasticsearch.audit.origin.type": "rest", + "elasticsearch.audit.request.id": "rLBMfPM2Q9q-DQEB_g30ww", + "elasticsearch.audit.request.name": "MultiGetShardRequest", + "elasticsearch.audit.user.realm": "native1", + "elasticsearch.audit.user.roles": [ + "logstash_admin", + "cluster_monitor" + ], + "elasticsearch.node.id": "vvj136QVQ2Ci2aXmrhyi3Q", + "event.action": "access_granted", + "event.category": "database", + "event.dataset": "elasticsearch.audit", + "event.kind": "event", + "event.module": "elasticsearch", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "host.id": "vvj136QVQ2Ci2aXmrhyi3Q", + "input.type": "log", + "log.offset": 0, + "message": "{\"@timestamp\":\"2020-04-01T11:21:06,725+0200\", \"node.id\":\"vvj136QVQ2Ci2aXmrhyi3Q\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"user.name\":\"logstash_manager\", \"user.realm\":\"native1\", \"user.roles\":[\"logstash_admin\",\"cluster_monitor\"], \"origin.type\":\"rest\", \"origin.address\":\"10.54.25.111:52148\", \"request.id\":\"rLBMfPM2Q9q-DQEB_g30ww\", \"action\":\"indices:data/read/mget[shard]\", \"request.name\":\"MultiGetShardRequest\", \"indices\":[\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\",\".logstash\"]}", + "related.user": [ + "logstash_manager" + ], + "service.type": "elasticsearch", + "source.address": "10.54.25.111:52148", + "source.ip": "10.54.25.111", + "source.port": 52148, + "user.name": "logstash_manager" + } +] \ No newline at end of file From fd3afa006d9f9a56e4473242b4f67c1fd92fffed Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 1 Apr 2020 10:11:55 -0700 Subject: [PATCH 2/2] Adding CHANGELOG entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bcbf6f40f3a..a8af83e3a9c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -104,6 +104,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - CEF: Fixed decoding errors caused by trailing spaces in messages. {pull}17253[17253] - Fixed a mapping exception when ingesting Logstash plain logs (7.4+) with pipeline ids containing non alphanumeric chars. {issue}17242[17242] {pull}17243[17243] - Fixed MySQL slowlog module causing "regular expression has redundant nested repeat operator" warning in Elasticsearch. {issue}17086[17086] {pull}17156[17156] +- Fix `elasticsearch.audit` data ingest pipeline to be more forgiving with date formats found in Elasticsearch audit logs. {pull}17406[17406] *Heartbeat*