From 32ecd202694d4ed33f3f567e0e88445fc04c2d3b Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 14 Feb 2022 11:45:09 +0100
Subject: [PATCH] Drop not audit logs in elasticsearch/audit fileset ingest
 pipeline (#30164) (#30239)

* adjust elasticsearch/audit fileset

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* fix formatting

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* adjust drop condition; revert test case for docker

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* fix typos

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>

* remove 'remove' of user.name field in pipeline-json for elasticsearch/audit

Signed-off-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
(cherry picked from commit 1e72124675ec58011096c30c4329d22eac82c688)

Co-authored-by: Tetiana Kravchenko <tetiana.kravchenko@elastic.co>
---
 .../audit/ingest/pipeline-json.yml            | 19 ++++++++++++++---
 .../test/test-audit-docker.log-expected.json  | 21 -------------------
 2 files changed, 16 insertions(+), 24 deletions(-)

diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
index 14e6a03538e..01bb4a4c25e 100644
--- a/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
+++ b/filebeat/module/elasticsearch/audit/ingest/pipeline-json.yml
@@ -3,6 +3,13 @@ processors:
   - json:
       field: message
       target_field: elasticsearch.audit
+  - dot_expander:
+      field: event.type
+      path: elasticsearch.audit
+  - drop:
+      if: ctx.elasticsearch.audit.containsKey('type') && ctx.elasticsearch.audit.type != 'audit'
+  - drop:
+      if: '!ctx.elasticsearch.audit.containsKey("type") && !["rest", "transport", "ip_filter", "security_config_change"].contains(ctx.elasticsearch?.audit?.event?.type)'
   - remove:
       field: elasticsearch.audit.type
       ignore_missing: true
@@ -25,13 +32,13 @@ processors:
   - dot_expander:
       field: event.action
       path: elasticsearch.audit
+  - remove:
+      field: event.action
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.event.action
       target_field: event.action
       ignore_missing: true
-  - dot_expander:
-      field: event.type
-      path: elasticsearch.audit
   - rename:
       field: elasticsearch.audit.event.type
       target_field: elasticsearch.audit.layer
@@ -45,6 +52,9 @@ processors:
         - \[%{IPORHOST:source.ip}\]:%{INT:source.port:int}
         - "%{IPORHOST:source.ip}:%{INT:source.port:int}"
       ignore_missing: true
+  - remove:
+      field: source.address
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.origin.address
       target_field: source.address
@@ -76,6 +86,9 @@ processors:
   - dot_expander:
       field: node.name
       path: elasticsearch.audit
+  - remove:
+      field: elasticsearch.node
+      ignore_missing: true
   - rename:
       field: elasticsearch.audit.node
       target_field: elasticsearch.node
diff --git a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
index ff553822c75..c5e72dfe82c 100644
--- a/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
+++ b/filebeat/module/elasticsearch/audit/test/test-audit-docker.log-expected.json
@@ -24,27 +24,6 @@
         "source.port": 40380,
         "url.original": "/"
     },
-    {
-        "@timestamp": "2019-06-11T15:03:32.777Z",
-        "elasticsearch.audit.component": "o.e.x.s.a.AuthenticationService",
-        "elasticsearch.audit.message": "Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]",
-        "elasticsearch.cluster.name": "docker-cluster",
-        "elasticsearch.cluster.uuid": "xEiKc6ipRiyzU8_8czXrJw",
-        "elasticsearch.node.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "elasticsearch.node.name": "dff7befc418f",
-        "event.category": "database",
-        "event.dataset": "elasticsearch.audit",
-        "event.kind": "event",
-        "event.module": "elasticsearch",
-        "event.outcome": "failure",
-        "fileset.name": "audit",
-        "host.id": "Xaq2BFVcQ1OhyMrjL8gNOg",
-        "input.type": "log",
-        "log.level": "INFO",
-        "log.offset": 299,
-        "message": "{\"type\": \"server\", \"timestamp\": \"2019-06-11T15:03:32,777+0000\", \"level\": \"INFO\", \"component\": \"o.e.x.s.a.AuthenticationService\", \"cluster.name\": \"docker-cluster\", \"node.name\": \"dff7befc418f\", \"cluster.uuid\": \"xEiKc6ipRiyzU8_8czXrJw\", \"node.id\": \"Xaq2BFVcQ1OhyMrjL8gNOg\",  \"message\": \"Authentication of [elastic] was terminated by realm [reserved] - failed to authenticate user [elastic]\"  }",
-        "service.type": "elasticsearch"
-    },
     {
         "@timestamp": "2019-06-11T15:03:32.778Z",
         "elasticsearch.audit.layer": "rest",