From 7a73f077b60622f9aa53097c2b2178ab1c73135e Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 17 Apr 2024 17:31:48 +0530 Subject: [PATCH 1/6] update grok pattern --- packages/apache/changelog.yml | 5 ++ .../_dev/test/pipeline/test-access-basic.log | 3 +- .../test-access-basic.log-expected.json | 79 ++++++++++++++++--- .../test-access-darwin.log-expected.json | 18 +++-- .../test-access-ssl-request.log-expected.json | 4 +- .../test-access-ubuntu.log-expected.json | 27 ++++--- .../test-access-vhost.log-expected.json | 3 +- .../elasticsearch/ingest_pipeline/default.yml | 2 +- .../data_stream/access/fields/fields.yml | 4 + .../data_stream/access/sample_event.json | 29 +++---- .../apache/data_stream/status/manifest.yml | 2 +- packages/apache/docs/README.md | 1 + packages/apache/manifest.yml | 2 +- 13 files changed, 133 insertions(+), 46 deletions(-) diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index 1f8dcedb166..45f93174ebf 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.17.1" + changes: + - description: Update grok for accepting user-identity. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 #Fixme - version: "1.17.0" changes: - description: Limit request tracer log count to five. diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log index 6f995c73596..04d6db9c240 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log @@ -7,4 +7,5 @@ monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 61 monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="-" 89.160.20.112 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.0.0.2,10.0.0.1" 2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" X-Forwarded-For="10.225.192.17, 10.2.2.121" -monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2" \ No newline at end of file +monitoring-server - - [17/May/2022:21:41:43 +0000] "GET / HTTP/1.1" 200 45 "-" "curl/7.79.1" X-Forwarded-For="192.168.0.2" +127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 539f364378f..103246721a8 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -4,6 +4,7 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "::1" ] @@ -15,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409634501Z", + "ingested": "2024-04-09T06:11:58.419585881Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -52,6 +53,7 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -63,7 +65,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409644668Z", + "ingested": "2024-04-09T06:11:58.419621774Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -113,6 +115,7 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "::1" ] @@ -124,7 +127,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409645876Z", + "ingested": "2024-04-09T06:11:58.419628526Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -149,6 +152,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "172.17.0.1" ] @@ -160,7 +164,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409646876Z", + "ingested": "2024-04-09T06:11:58.419633501Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -210,6 +214,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "monitoring-server" ] @@ -221,7 +226,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409647793Z", + "ingested": "2024-04-09T06:11:58.419638188Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -271,6 +276,7 @@ "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -282,7 +288,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409648793Z", + "ingested": "2024-04-09T06:11:58.419642845Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -320,6 +326,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "monitoring-server" ] @@ -331,7 +338,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409649793Z", + "ingested": "2024-04-09T06:11:58.419647444Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", "outcome": "success" @@ -382,6 +389,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "10.0.0.2", "10.0.0.1", @@ -398,7 +406,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409650668Z", + "ingested": "2024-04-09T06:11:58.419652022Z", "kind": "event", "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" @@ -470,6 +478,7 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "10.225.192.17", "10.2.2.121", @@ -486,7 +495,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409651543Z", + "ingested": "2024-04-09T06:11:58.419656560Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", "outcome": "success" @@ -549,6 +558,7 @@ "@timestamp": "2022-05-17T21:41:43.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.0.2", "monitoring-server" @@ -564,7 +574,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.409652876Z", + "ingested": "2024-04-09T06:11:58.419661112Z", "kind": "event", "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", "outcome": "success" @@ -607,6 +617,55 @@ "original": "curl/7.79.1", "version": "7.79.1" } + }, + { + "@timestamp": "2000-10-10T20:55:36.000Z", + "apache": { + "access": { + "identity": "user-identity", + "remote_addresses": [ + "127.0.0.1" + ] + } + }, + "ecs": { + "version": "8.5.1" + }, + "event": { + "category": "web", + "created": "2020-04-28T11:07:58.223Z", + "ingested": "2024-04-09T06:11:58.419665782Z", + "kind": "event", + "original": "127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326", + "outcome": "success" + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "body": { + "bytes": 2326 + }, + "status_code": 200 + }, + "version": "1.0" + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "tags": [ + "preserve_original_event" + ], + "url": { + "extension": "gif", + "original": "/apache_pb.gif", + "path": "/apache_pb.gif" + }, + "user": { + "name": "frank" + } } ] } \ No newline at end of file diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 04feb1dc8ae..3f8dc10eec9 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -4,6 +4,7 @@ "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "::1" ] @@ -15,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483539043Z", + "ingested": "2024-04-09T06:11:58.640870130Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -51,6 +52,7 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "::1" ] @@ -62,7 +64,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483550209Z", + "ingested": "2024-04-09T06:11:58.640922672Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -99,6 +101,7 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "::1" ] @@ -110,7 +113,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483551501Z", + "ingested": "2024-04-09T06:11:58.640930188Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -135,6 +138,7 @@ "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -146,7 +150,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483552501Z", + "ingested": "2024-04-09T06:11:58.640936194Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -200,6 +204,7 @@ "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -211,7 +216,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483553418Z", + "ingested": "2024-04-09T06:11:58.640941558Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -265,6 +270,7 @@ "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -276,7 +282,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.483554501Z", + "ingested": "2024-04-09T06:11:58.640946731Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index 6c618cc5793..d799546eab9 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -19,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.533303168Z", + "ingested": "2024-04-09T06:11:58.788662911Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -72,7 +72,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.533318376Z", + "ingested": "2024-04-09T06:11:58.788703961Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index 50a75ea6b31..d0a4be4853e 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -4,6 +4,7 @@ "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -15,7 +16,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577647543Z", + "ingested": "2024-04-09T06:11:58.965243099Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -63,6 +64,7 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -74,7 +76,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577659626Z", + "ingested": "2024-04-09T06:11:58.965287213Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -124,6 +126,7 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -135,7 +138,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577660959Z", + "ingested": "2024-04-09T06:11:58.965294944Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -186,6 +189,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -197,7 +201,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577662126Z", + "ingested": "2024-04-09T06:11:58.965300893Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -247,6 +251,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -258,7 +263,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577663126Z", + "ingested": "2024-04-09T06:11:58.965306208Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -309,6 +314,7 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -320,7 +326,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577664043Z", + "ingested": "2024-04-09T06:11:58.965311416Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -371,6 +377,7 @@ "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -382,7 +389,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577664918Z", + "ingested": "2024-04-09T06:11:58.965316467Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -432,6 +439,7 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -443,7 +451,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577665918Z", + "ingested": "2024-04-09T06:11:58.965321485Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -493,6 +501,7 @@ "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -504,7 +513,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.577666793Z", + "ingested": "2024-04-09T06:11:58.965326509Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index 84b329493a6..e86d94d1f57 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -4,6 +4,7 @@ "@timestamp": "2016-12-26T16:22:14.000Z", "apache": { "access": { + "identity": "-", "remote_addresses": [ "192.168.33.2" ] @@ -18,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2022-12-08T15:09:52.634020126Z", + "ingested": "2024-04-09T06:11:59.183178173Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index 556390d4a14..b96b4592904 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -18,7 +18,7 @@ processors: - grok: field: event.original patterns: - - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] + - '(%{IPORHOST:destination.domain} )?%{IPORHOST:source.address} %{DATA:apache.access.identity} %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?( X-Forwarded-For="%{ADDRESS_LIST:apache.access.remote_addresses}")?' diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index f9dc5e7bd4e..3ef95904c5e 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -14,3 +14,7 @@ type: keyword description: | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. + - name: identity + type: keyword + description: | + The user identity associated with the event, as determined by RFC 1413 identd protocol on the client's machine. diff --git a/packages/apache/data_stream/access/sample_event.json b/packages/apache/data_stream/access/sample_event.json index 24a799813f9..5c766768b10 100644 --- a/packages/apache/data_stream/access/sample_event.json +++ b/packages/apache/data_stream/access/sample_event.json @@ -1,14 +1,15 @@ { - "@timestamp": "2022-12-09T03:54:22.000Z", + "@timestamp": "2024-04-05T06:07:13.000Z", "agent": { - "ephemeral_id": "d14551cd-5fc4-4f19-8a0e-5897ecaefbf7", - "id": "46343e0c-0d8c-464b-a216-cacf63027d6f", + "ephemeral_id": "82b2a2fa-29cf-47c9-855e-ba67bc98260f", + "id": "a4204f31-fc6e-445c-9f5f-273550c03c23", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.5.0" + "version": "8.12.2" }, "apache": { "access": { + "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -23,39 +24,39 @@ "version": "8.5.1" }, "elastic_agent": { - "id": "46343e0c-0d8c-464b-a216-cacf63027d6f", + "id": "a4204f31-fc6e-445c-9f5f-273550c03c23", "snapshot": false, - "version": "8.5.0" + "version": "8.12.2" }, "event": { "agent_id_status": "verified", "category": "web", - "created": "2022-12-09T03:54:39.182Z", + "created": "2024-04-05T06:07:30.313Z", "dataset": "apache.access", - "ingested": "2022-12-09T03:54:40Z", + "ingested": "2024-04-05T06:07:42Z", "kind": "event", "outcome": "success" }, "host": { "architecture": "x86_64", - "containerized": false, + "containerized": true, "hostname": "docker-fleet-agent", - "id": "66392b0697b84641af8006d87aeb89f1", + "id": "009f8d5d825944429c9ae8d252b0019a", "ip": [ - "172.18.0.7" + "192.168.240.8" ], "mac": [ - "02-42-AC-12-00-07" + "02-42-C0-A8-F0-08" ], "name": "docker-fleet-agent", "os": { "codename": "focal", "family": "debian", - "kernel": "5.15.49-linuxkit", + "kernel": "3.10.0-1160.114.2.el7.x86_64", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", - "version": "20.04.5 LTS (Focal Fossa)" + "version": "20.04.6 LTS (Focal Fossa)" } }, "http": { diff --git a/packages/apache/data_stream/status/manifest.yml b/packages/apache/data_stream/status/manifest.yml index 69fcead5c7a..cf869260e2d 100644 --- a/packages/apache/data_stream/status/manifest.yml +++ b/packages/apache/data_stream/status/manifest.yml @@ -20,4 +20,4 @@ streams: title: Apache status metrics description: Collect Apache status metrics elasticsearch: - index_mode: "time_series" \ No newline at end of file + index_mode: "time_series" diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index 55d098ecb52..f22bd08e66b 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,6 +19,7 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | +| apache.access.identity | The user identity associated with the event, as determined by RFC 1413 identd protocol on the client's machine. | keyword | | apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | diff --git a/packages/apache/manifest.yml b/packages/apache/manifest.yml index 991b4cef8f8..f932adfa97d 100644 --- a/packages/apache/manifest.yml +++ b/packages/apache/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: apache title: Apache HTTP Server -version: "1.17.0" +version: "1.17.1" license: basic source: license: Elastic-2.0 From 7afce8949ad813a49cfeef093a66079d3e6ed0be Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Wed, 17 Apr 2024 18:28:44 +0530 Subject: [PATCH 2/6] update changelog --- packages/apache/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apache/changelog.yml b/packages/apache/changelog.yml index 45f93174ebf..7ba6a86de02 100644 --- a/packages/apache/changelog.yml +++ b/packages/apache/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Update grok for accepting user-identity. type: bugfix - link: https://github.com/elastic/integrations/pull/1 #Fixme + link: https://github.com/elastic/integrations/pull/9632 - version: "1.17.0" changes: - description: Limit request tracer log count to five. From 929449d26fa2a1695586bbc8cfbf557d46695e7b Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Fri, 26 Apr 2024 11:26:37 +0530 Subject: [PATCH 3/6] address review comments --- .../test-access-basic.log-expected.json | 32 +++++++------------ .../test-access-darwin.log-expected.json | 18 ++++------- .../test-access-ssl-request.log-expected.json | 4 +-- .../test-access-ubuntu.log-expected.json | 27 ++++++---------- .../test-access-vhost.log-expected.json | 3 +- .../elasticsearch/ingest_pipeline/default.yml | 5 +++ .../data_stream/access/sample_event.json | 25 +++++++-------- 7 files changed, 46 insertions(+), 68 deletions(-) diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json index 103246721a8..a67b4876efa 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-basic.log-expected.json @@ -4,7 +4,6 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "::1" ] @@ -16,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419585881Z", + "ingested": "2024-04-26T05:46:25.296250288Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -53,7 +52,6 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -65,7 +63,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419621774Z", + "ingested": "2024-04-26T05:46:25.296284705Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -115,7 +113,6 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "::1" ] @@ -127,7 +124,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419628526Z", + "ingested": "2024-04-26T05:46:25.296289743Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -152,7 +149,6 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "172.17.0.1" ] @@ -164,7 +160,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419633501Z", + "ingested": "2024-04-26T05:46:25.296293311Z", "kind": "event", "original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "failure" @@ -214,7 +210,6 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "monitoring-server" ] @@ -226,7 +221,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419638188Z", + "ingested": "2024-04-26T05:46:25.296296691Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "outcome": "success" @@ -276,7 +271,6 @@ "@timestamp": "2019-02-02T04:38:45.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -288,7 +282,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419642845Z", + "ingested": "2024-04-26T05:46:25.296300048Z", "kind": "event", "original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "outcome": "failure" @@ -326,7 +320,6 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "monitoring-server" ] @@ -338,7 +331,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419647444Z", + "ingested": "2024-04-26T05:46:25.296303835Z", "kind": "event", "original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"-\"", "outcome": "success" @@ -389,7 +382,6 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "10.0.0.2", "10.0.0.1", @@ -406,7 +398,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419652022Z", + "ingested": "2024-04-26T05:46:25.296310193Z", "kind": "event", "original": "89.160.20.112 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.0.0.2,10.0.0.1\"", "outcome": "success" @@ -478,7 +470,6 @@ "@timestamp": "2017-05-29T19:02:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "10.225.192.17", "10.2.2.121", @@ -495,7 +486,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419656560Z", + "ingested": "2024-04-26T05:46:25.296313609Z", "kind": "event", "original": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6 - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" X-Forwarded-For=\"10.225.192.17, 10.2.2.121\"", "outcome": "success" @@ -558,7 +549,6 @@ "@timestamp": "2022-05-17T21:41:43.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.0.2", "monitoring-server" @@ -574,7 +564,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419661112Z", + "ingested": "2024-04-26T05:46:25.296316938Z", "kind": "event", "original": "monitoring-server - - [17/May/2022:21:41:43 +0000] \"GET / HTTP/1.1\" 200 45 \"-\" \"curl/7.79.1\" X-Forwarded-For=\"192.168.0.2\"", "outcome": "success" @@ -634,7 +624,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.419665782Z", + "ingested": "2024-04-26T05:46:25.296320274Z", "kind": "event", "original": "127.0.0.1 user-identity frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326", "outcome": "success" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json index 3f8dc10eec9..5204fface78 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-darwin.log-expected.json @@ -4,7 +4,6 @@ "@timestamp": "2016-12-26T14:16:28.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "::1" ] @@ -16,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640870130Z", + "ingested": "2024-04-26T05:46:25.447843628Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -52,7 +51,6 @@ "@timestamp": "2016-12-26T14:16:29.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "::1" ] @@ -64,7 +62,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640922672Z", + "ingested": "2024-04-26T05:46:25.447895323Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "outcome": "failure" @@ -101,7 +99,6 @@ "@timestamp": "2016-12-26T14:16:48.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "::1" ] @@ -113,7 +110,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640930188Z", + "ingested": "2024-04-26T05:46:25.447905030Z", "kind": "event", "original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "outcome": "failure" @@ -138,7 +135,6 @@ "@timestamp": "2016-12-26T16:23:35.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -150,7 +146,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640936194Z", + "ingested": "2024-04-26T05:46:25.447912585Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "outcome": "success" @@ -204,7 +200,6 @@ "@timestamp": "2016-12-26T16:23:41.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -216,7 +211,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640941558Z", + "ingested": "2024-04-26T05:46:25.447919912Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "outcome": "failure" @@ -270,7 +265,6 @@ "@timestamp": "2016-12-26T16:23:45.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "89.160.20.156" ] @@ -282,7 +276,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.640946731Z", + "ingested": "2024-04-26T05:46:25.447927217Z", "kind": "event", "original": "89.160.20.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json index d799546eab9..dd593dfa2ed 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ssl-request.log-expected.json @@ -19,7 +19,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.788662911Z", + "ingested": "2024-04-26T05:46:25.568940509Z", "kind": "event", "original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax\u0026amp;opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D\u0026amp;nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375" }, @@ -72,7 +72,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.788703961Z", + "ingested": "2024-04-26T05:46:25.568967013Z", "kind": "event", "original": "[16/Oct/2019:11:53:47 +0200] 89.160.20.156 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax\u0026opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D\u0026nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -" }, diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json index d0a4be4853e..828594c5c61 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-ubuntu.log-expected.json @@ -4,7 +4,6 @@ "@timestamp": "2016-12-26T16:18:09.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -16,7 +15,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965243099Z", + "ingested": "2024-04-26T05:46:25.677798830Z", "kind": "event", "original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "outcome": "success" @@ -64,7 +63,6 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -76,7 +74,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965287213Z", + "ingested": "2024-04-26T05:46:25.677841187Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "success" @@ -126,7 +124,6 @@ "@timestamp": "2016-12-26T16:22:00.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -138,7 +135,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965294944Z", + "ingested": "2024-04-26T05:46:25.677846439Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "outcome": "failure" @@ -189,7 +186,6 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -201,7 +197,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965300893Z", + "ingested": "2024-04-26T05:46:25.677850179Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "success" @@ -251,7 +247,6 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -263,7 +258,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965306208Z", + "ingested": "2024-04-26T05:46:25.677853779Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -314,7 +309,6 @@ "@timestamp": "2016-12-26T16:22:08.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -326,7 +320,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965311416Z", + "ingested": "2024-04-26T05:46:25.677857034Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -377,7 +371,6 @@ "@timestamp": "2016-12-26T16:22:10.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -389,7 +382,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965316467Z", + "ingested": "2024-04-26T05:46:25.677860896Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -439,7 +432,6 @@ "@timestamp": "2016-12-26T16:22:13.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -451,7 +443,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965321485Z", + "ingested": "2024-04-26T05:46:25.677864153Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" @@ -501,7 +493,6 @@ "@timestamp": "2016-12-26T16:22:17.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.1" ] @@ -513,7 +504,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:58.965326509Z", + "ingested": "2024-04-26T05:46:25.677867406Z", "kind": "event", "original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json index e86d94d1f57..67cce22859d 100644 --- a/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json +++ b/packages/apache/data_stream/access/_dev/test/pipeline/test-access-vhost.log-expected.json @@ -4,7 +4,6 @@ "@timestamp": "2016-12-26T16:22:14.000Z", "apache": { "access": { - "identity": "-", "remote_addresses": [ "192.168.33.2" ] @@ -19,7 +18,7 @@ "event": { "category": "web", "created": "2020-04-28T11:07:58.223Z", - "ingested": "2024-04-09T06:11:59.183178173Z", + "ingested": "2024-04-26T05:46:25.815837486Z", "kind": "event", "original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "outcome": "failure" diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index b96b4592904..f7ed8d4d7a5 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -197,6 +197,11 @@ processors: if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" ignore_failure: true ignore_missing: true + - remove: + field: apache.access.identity + if: ctx.apache.access.identity == "-" + ignore_failure: true + ignore_missing: true on_failure: - set: field: error.message diff --git a/packages/apache/data_stream/access/sample_event.json b/packages/apache/data_stream/access/sample_event.json index 5c766768b10..c8763b303e5 100644 --- a/packages/apache/data_stream/access/sample_event.json +++ b/packages/apache/data_stream/access/sample_event.json @@ -1,15 +1,14 @@ { - "@timestamp": "2024-04-05T06:07:13.000Z", + "@timestamp": "2024-04-26T05:43:59.000Z", "agent": { - "ephemeral_id": "82b2a2fa-29cf-47c9-855e-ba67bc98260f", - "id": "a4204f31-fc6e-445c-9f5f-273550c03c23", + "ephemeral_id": "70424ab1-cd93-4ec9-bc96-18818e5a0e4d", + "id": "2d90e456-ced7-419b-89a1-3f9bbbad454f", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.12.2" + "version": "8.13.0" }, "apache": { "access": { - "identity": "-", "remote_addresses": [ "127.0.0.1" ] @@ -24,16 +23,16 @@ "version": "8.5.1" }, "elastic_agent": { - "id": "a4204f31-fc6e-445c-9f5f-273550c03c23", + "id": "2d90e456-ced7-419b-89a1-3f9bbbad454f", "snapshot": false, - "version": "8.12.2" + "version": "8.13.0" }, "event": { "agent_id_status": "verified", "category": "web", - "created": "2024-04-05T06:07:30.313Z", + "created": "2024-04-26T05:44:18.354Z", "dataset": "apache.access", - "ingested": "2024-04-05T06:07:42Z", + "ingested": "2024-04-26T05:44:30Z", "kind": "event", "outcome": "success" }, @@ -41,18 +40,18 @@ "architecture": "x86_64", "containerized": true, "hostname": "docker-fleet-agent", - "id": "009f8d5d825944429c9ae8d252b0019a", + "id": "8259e024976a406e8a54cdbffeb84fec", "ip": [ - "192.168.240.8" + "192.168.248.7" ], "mac": [ - "02-42-C0-A8-F0-08" + "02-42-C0-A8-F8-07" ], "name": "docker-fleet-agent", "os": { "codename": "focal", "family": "debian", - "kernel": "3.10.0-1160.114.2.el7.x86_64", + "kernel": "3.10.0-1160.99.1.el7.x86_64", "name": "Ubuntu", "platform": "ubuntu", "type": "linux", From 759f95c9772aa8b30fed043428ca58ef2c4fb126 Mon Sep 17 00:00:00 2001 From: niraj-elastic <124254029+niraj-elastic@users.noreply.github.com> Date: Mon, 29 Apr 2024 11:06:10 +0530 Subject: [PATCH 4/6] address review comments Co-authored-by: muthu-mps <101238137+muthu-mps@users.noreply.github.com> --- .../access/elasticsearch/ingest_pipeline/default.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml index f7ed8d4d7a5..706702f436a 100644 --- a/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml +++ b/packages/apache/data_stream/access/elasticsearch/ingest_pipeline/default.yml @@ -199,7 +199,7 @@ processors: ignore_missing: true - remove: field: apache.access.identity - if: ctx.apache.access.identity == "-" + if: ctx.apache?.access?.identity == "-" ignore_failure: true ignore_missing: true on_failure: From ca985baffcfd91e08ff4af52017befd7d2dbb7f9 Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Mon, 29 Apr 2024 12:33:01 +0530 Subject: [PATCH 5/6] address review comments --- packages/apache/data_stream/access/fields/fields.yml | 2 +- packages/apache/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index 3ef95904c5e..b306cbf2671 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -17,4 +17,4 @@ - name: identity type: keyword description: | - The user identity associated with the event, as determined by RFC 1413 identd protocol on the client's machine. + The RFC 1413 identity of the client determined by identd on the clients machine. diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index f22bd08e66b..dd07f5558ef 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,7 +19,7 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.identity | The user identity associated with the event, as determined by RFC 1413 identd protocol on the client's machine. | keyword | +| apache.access.identity | The RFC 1413 identity of the client determined by identd on the clients machine. | keyword | | apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword | From de4fa0d32999b574cfdaa9a33a053089856a490b Mon Sep 17 00:00:00 2001 From: niraj-elastic Date: Mon, 29 Apr 2024 12:41:35 +0530 Subject: [PATCH 6/6] address review comment --- packages/apache/data_stream/access/fields/fields.yml | 2 +- packages/apache/docs/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/apache/data_stream/access/fields/fields.yml b/packages/apache/data_stream/access/fields/fields.yml index b306cbf2671..aed9cb815b7 100644 --- a/packages/apache/data_stream/access/fields/fields.yml +++ b/packages/apache/data_stream/access/fields/fields.yml @@ -17,4 +17,4 @@ - name: identity type: keyword description: | - The RFC 1413 identity of the client determined by identd on the clients machine. + The client's identity, as specified in RFC 1413, determined by the identd on the client's machine. diff --git a/packages/apache/docs/README.md b/packages/apache/docs/README.md index dd07f5558ef..039244c8214 100644 --- a/packages/apache/docs/README.md +++ b/packages/apache/docs/README.md @@ -19,7 +19,7 @@ Access logs collects the Apache access logs. | Field | Description | Type | |---|---|---| | @timestamp | Event timestamp. | date | -| apache.access.identity | The RFC 1413 identity of the client determined by identd on the clients machine. | keyword | +| apache.access.identity | The client's identity, as specified in RFC 1413, determined by the identd on the client's machine. | keyword | | apache.access.remote_addresses | An array of remote addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. | keyword | | apache.access.ssl.cipher | SSL cipher name. - name: nginx.access | keyword | | apache.access.ssl.protocol | SSL protocol version. | keyword |