[Apache] Update grok pattern for accepting user-identity (#9632)

* update grok pattern * update changelog * address review comments * address review comments Co-authored-by: muthu-mps <[email protected]> * address review comments * address review comment --------- Co-authored-by: muthu-mps <[email protected]>
elastic · Apr 29, 2024 · d4e4aa4 · d4e4aa4
1 parent dce5699
commit d4e4aa4
Show file tree

Hide file tree

Showing 13 changed files with 111 additions and 46 deletions.
diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.17.1"
+  changes:
+    - description: Update grok for accepting user-identity.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9632
 - version: "1.17.0"
   changes:
     - description: Limit request tracer log count to five.

diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log
@@ -7,4 +7,5 @@ monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 61
 monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="-"
 89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.0.0.2,10.0.0.1"
 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.225.192.17, 10.2.2.121"
-monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2"
+monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2"
+127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326
diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json
@@ -15,7 +15,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409634501Z",
+                "ingested": "2024-04-26T05:46:25.296250288Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
                 "outcome": "failure"
@@ -63,7 +63,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409644668Z",
+                "ingested": "2024-04-26T05:46:25.296284705Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -124,7 +124,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409645876Z",
+                "ingested": "2024-04-26T05:46:25.296289743Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
                 "outcome": "failure"
@@ -160,7 +160,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409646876Z",
+                "ingested": "2024-04-26T05:46:25.296293311Z",
                 "kind": "event",
                 "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
                 "outcome": "failure"
@@ -221,7 +221,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409647793Z",
+                "ingested": "2024-04-26T05:46:25.296296691Z",
                 "kind": "event",
                 "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"",
                 "outcome": "success"
@@ -282,7 +282,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409648793Z",
+                "ingested": "2024-04-26T05:46:25.296300048Z",
                 "kind": "event",
                 "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"",
                 "outcome": "failure"
@@ -331,7 +331,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409649793Z",
+                "ingested": "2024-04-26T05:46:25.296303835Z",
                 "kind": "event",
                 "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"",
                 "outcome": "success"
@@ -398,7 +398,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409650668Z",
+                "ingested": "2024-04-26T05:46:25.296310193Z",
                 "kind": "event",
                 "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"",
                 "outcome": "success"
@@ -486,7 +486,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409651543Z",
+                "ingested": "2024-04-26T05:46:25.296313609Z",
                 "kind": "event",
                 "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"",
                 "outcome": "success"
@@ -564,7 +564,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.409652876Z",
+                "ingested": "2024-04-26T05:46:25.296316938Z",
                 "kind": "event",
                 "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"",
                 "outcome": "success"
@@ -607,6 +607,55 @@
                 "original": "curl/7.79.1",
                 "version": "7.79.1"
             }
+        },
+        {
+            "@timestamp": "2000-10-10T20:55:36.000Z",
+            "apache": {
+                "access": {
+                    "identity": "user-identity",
+                    "remote_addresses": [
+                        "127.0.0.1"
+                    ]
+                }
+            },
+            "ecs": {
+                "version": "8.5.1"
+            },
+            "event": {
+                "category": "web",
+                "created": "2020-04-28T11:07:58.223Z",
+                "ingested": "2024-04-26T05:46:25.296320274Z",
+                "kind": "event",
+                "original": "127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326",
+                "outcome": "success"
+            },
+            "http": {
+                "request": {
+                    "method": "GET"
+                },
+                "response": {
+                    "body": {
+                        "bytes": 2326
+                    },
+                    "status_code": 200
+                },
+                "version": "1.0"
+            },
+            "source": {
+                "address": "127.0.0.1",
+                "ip": "127.0.0.1"
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "url": {
+                "extension": "gif",
+                "original": "/apache_pb.gif",
+                "path": "/apache_pb.gif"
+            },
+            "user": {
+                "name": "frank"
+            }
         }
     ]
 }
diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json
@@ -15,7 +15,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483539043Z",
+                "ingested": "2024-04-26T05:46:25.447843628Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45",
                 "outcome": "success"
@@ -62,7 +62,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483550209Z",
+                "ingested": "2024-04-26T05:46:25.447895323Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209",
                 "outcome": "failure"
@@ -110,7 +110,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483551501Z",
+                "ingested": "2024-04-26T05:46:25.447905030Z",
                 "kind": "event",
                 "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -",
                 "outcome": "failure"
@@ -146,7 +146,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483552501Z",
+                "ingested": "2024-04-26T05:46:25.447912585Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45",
                 "outcome": "success"
@@ -211,7 +211,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483553418Z",
+                "ingested": "2024-04-26T05:46:25.447919912Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206",
                 "outcome": "failure"
@@ -276,7 +276,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.483554501Z",
+                "ingested": "2024-04-26T05:46:25.447927217Z",
                 "kind": "event",
                 "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201",
                 "outcome": "failure"

diff --git a/...es/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/...es/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json
@@ -19,7 +19,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.533303168Z",
+                "ingested": "2024-04-26T05:46:25.568940509Z",
                 "kind": "event",
                 "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375"
             },
@@ -72,7 +72,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.533318376Z",
+                "ingested": "2024-04-26T05:46:25.568967013Z",
                 "kind": "event",
                 "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -"
             },

diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json
@@ -15,7 +15,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577647543Z",
+                "ingested": "2024-04-26T05:46:25.677798830Z",
                 "kind": "event",
                 "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"",
                 "outcome": "success"
@@ -74,7 +74,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577659626Z",
+                "ingested": "2024-04-26T05:46:25.677841187Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
                 "outcome": "success"
@@ -135,7 +135,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577660959Z",
+                "ingested": "2024-04-26T05:46:25.677846439Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"",
                 "outcome": "failure"
@@ -197,7 +197,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577662126Z",
+                "ingested": "2024-04-26T05:46:25.677850179Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "success"
@@ -258,7 +258,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577663126Z",
+                "ingested": "2024-04-26T05:46:25.677853779Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -320,7 +320,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577664043Z",
+                "ingested": "2024-04-26T05:46:25.677857034Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -382,7 +382,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577664918Z",
+                "ingested": "2024-04-26T05:46:25.677860896Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -443,7 +443,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577665918Z",
+                "ingested": "2024-04-26T05:46:25.677864153Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"
@@ -504,7 +504,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.577666793Z",
+                "ingested": "2024-04-26T05:46:25.677867406Z",
                 "kind": "event",
                 "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"

diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json
@@ -18,7 +18,7 @@
             "event": {
                 "category": "web",
                 "created": "2020-04-28T11:07:58.223Z",
-                "ingested": "2022-12-08T15:09:52.634020126Z",
+                "ingested": "2024-04-26T05:46:25.815837486Z",
                 "kind": "event",
                 "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"",
                 "outcome": "failure"

diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml
@@ -18,7 +18,7 @@ processors:
   - grok:
       field: event.original
       patterns:
-      - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
+      - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} %{DATA:apache.access.identity} %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\]
         "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?"
         %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)(
         "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?( X-Forwarded-For="%{ADDRESS_LIST:apache.access.remote_addresses}")?'
@@ -197,6 +197,11 @@ processors:
       if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
       ignore_failure: true
       ignore_missing: true
+  - remove:
+      field: apache.access.identity
+      if: ctx.apache?.access?.identity == "-"
+      ignore_failure: true
+      ignore_missing: true
 on_failure:
   - set:
       field: error.message

diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml
@@ -14,3 +14,7 @@
       type: keyword
       description: |
         An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`.
+    - name: identity
+      type: keyword
+      description: |
+        The client's identity, as specified in RFC 1413, determined by the identd on the client's machine.