From f1fea95e8d44b5b6c45a7d25026e6276f7248456 Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Tue, 27 Apr 2021 14:31:17 -0500 Subject: [PATCH 01/11] [Filebeat] Add URI Parts Processor to multiple modules (#24699) * Update Nginx pipelines * Update Apache, Nginx, IIS, Traefik pipelines * Update AWS S3 * Update Cisco * Update F5 * Update Fortinet * Update Imperva, Netscout, O365, Sophos, Squid, Suricata, Zscaler * additional fixes * update pipelines * unescape \ * remove urldecodes for url.original * updates after rebase * update zeek SIP * update changelog as requested by @andrewstucki * remove `url_decode` for `http.request.referrer` * update generated data --- CHANGELOG.next.asciidoc | 3 + .../module/apache/access/ingest/pipeline.yml | 47 +- .../test/darwin-2.4.23.log-expected.json | 12 + .../access/test/ssl-request.log-expected.json | 12 +- .../access/test/test-vhost.log-expected.json | 3 + filebeat/module/apache/access/test/test.log | 1 + .../apache/access/test/test.log-expected.json | 42 + .../test/ubuntu-2.2.22.log-expected.json | 21 + .../module/apache/error/ingest/pipeline.yml | 40 +- .../test/darwin-2.4.23.log-expected.json | 2 + .../apache/error/test/test.log-expected.json | 6 + .../test/ubuntu-2.2.22.log-expected.json | 14 + .../module/iis/access/ingest/pipeline.yml | 63 +- .../test/test-iis-7.2.log-expected.json | 19 +- .../test/test-iis-7.5.log-expected.json | 8 + .../test/test-ipv6zone.log-expected.json | 3 + ...t-x-forward-for-extended.log-expected.json | 24 + .../test/test-x-forward-for.log-expected.json | 22 +- .../iis/access/test/test.log-expected.json | 15 +- filebeat/module/iis/error/ingest/pipeline.yml | 45 +- .../test/iis_error_url.log-expected.json | 26 +- .../error/test/ipv6_zone_id.log-expected.json | 1 + .../iis/error/test/test.log-expected.json | 15 +- .../module/nginx/access/ingest/pipeline.yml | 45 +- filebeat/module/nginx/access/test/access.log | 2 + .../access/test/access.log-expected.json | 113 ++ .../test/test-with-host.log-expected.json | 29 +- .../nginx/access/test/test.log-expected.json | 20 + .../module/nginx/error/ingest/pipeline.yml | 5 +- .../nginx/error/test/error.log-expected.json | 4 + .../ingress_controller/ingest/pipeline.yml | 38 +- .../nginx/ingress_controller/test/test.log | 1 + .../test/test.log-expected.json | 125 +- .../module/traefik/access/ingest/pipeline.yml | 47 +- .../access/test/test.log-expected.json | 19 + .../module/aws/s3access/ingest/pipeline.yml | 42 +- .../aws/s3access/test/s3_server_access.log | 2 +- .../test/s3_server_access.log-expected.json | 10 +- .../aws/s3access/test/test.log-expected.json | 2 + .../additional_messages.log-expected.json | 20 +- .../cisco/asa/test/sample.log-expected.json | 14 +- .../cisco/ftd/test/sample.log-expected.json | 14 +- .../security-connection.log-expected.json | 15 +- .../security-file-malware.log-expected.json | 45 + .../security-malware-site.log-expected.json | 8 +- .../module/cisco/meraki/ingest/pipeline.yml | 19 + .../meraki/test/generated.log-expected.json | 256 ++- .../cisco/shared/ingest/asa-ftd-pipeline.yml | 41 + .../module/cisco/umbrella/ingest/pipeline.yml | 7 + ...brella-cloudfirewalllogs.log-expected.json | 2 + .../test/umbrella-proxylogs.log-expected.json | 16 +- .../module/f5/bigipafm/ingest/pipeline.yml | 24 + .../bigipafm/test/generated.log-expected.json | 410 ++--- .../module/f5/bigipapm/ingest/pipeline.yml | 24 + .../bigipapm/test/generated.log-expected.json | 28 +- .../test/generated.log-expected.json | 192 +-- .../test/generated.log-expected.json | 40 +- .../fortinet/fortimanager/ingest/pipeline.yml | 24 + .../test/generated.log-expected.json | 876 +++++++--- .../admin-sites-test.json.log-expected.json | 12 +- .../module/google_workspace/ingest/common.yml | 4 + .../imperva/securesphere/ingest/pipeline.yml | 24 + .../test/generated.log-expected.json | 716 +++++--- .../defender_atp/ingest/pipeline.yml | 4 + .../m365_defender/ingest/pipeline.yml | 5 +- .../netscout/sightline/ingest/pipeline.yml | 24 + .../test/generated.log-expected.json | 160 +- .../module/o365/audit/ingest/pipeline.yml | 24 + .../06-sharepointfileop.log-expected.json | 44 + .../test/11-dlp-sharepoint.log-expected.json | 28 + .../test/13-dlp-exchange.log-expected.json | 4 + .../test/25-ms-teams-groups.log-expected.json | 6 + .../module/sophos/utm/ingest/pipeline.yml | 24 + .../utm/test/generated.log-expected.json | 130 +- .../module/sophos/xg/ingest/antivirus.yml | 23 + .../filebeat/module/sophos/xg/ingest/atp.yml | 27 + .../module/sophos/xg/ingest/cfilter.yml | 38 +- .../filebeat/module/sophos/xg/ingest/waf.yml | 38 +- .../xg/test/anti-virus.log-expected.json | 6 + .../sophos/xg/test/atp.log-expected.json | 12 +- .../sophos/xg/test/cfilter.log-expected.json | 35 +- .../sophos/xg/test/waf.log-expected.json | 13 +- .../module/squid/log/ingest/pipeline.yml | 28 + .../squid/log/test/access1.log-expected.json | 647 ++++--- .../log/test/generated.log-expected.json | 1488 ++++++++++++----- .../module/suricata/eve/ingest/http.yml | 83 + .../module/suricata/eve/ingest/pipeline.yml | 56 +- .../filebeat/module/suricata/eve/manifest.yml | 1 + .../eve/test/eve-6.0.log-expected.json | 1 + .../module/suricata/eve/test/eve-small.log | 2 +- .../eve/test/eve-small.log-expected.json | 14 +- .../module/zeek/sip/ingest/pipeline.yml | 29 + .../zeek/sip/test/sip-json.log-expected.json | 9 + .../module/zoom/webhook/ingest/meeting.yml | 16 +- .../module/zoom/webhook/ingest/phone.yml | 4 + .../module/zoom/webhook/ingest/recording.yml | 4 + .../test/meeting.ndjson.log-expected.json | 8 + .../test/phone.ndjson.log-expected.json | 5 + .../test/recording.ndjson.log-expected.json | 20 + .../module/zscaler/zia/ingest/pipeline.yml | 26 +- .../zia/test/generated.log-expected.json | 980 ++++++++--- 101 files changed, 5939 insertions(+), 1906 deletions(-) create mode 100644 x-pack/filebeat/module/suricata/eve/ingest/http.yml diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 492d6f21092..c8b400a05b5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -107,6 +107,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] +- All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699] *Heartbeat* @@ -406,6 +407,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928] - Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038] - Change `checkpoint.source_object` from Long to Keyword. {issue}25124[25124] {pull}25145[25145] +- Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* @@ -849,6 +851,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] +- Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] *Heartbeat* diff --git a/filebeat/module/apache/access/ingest/pipeline.yml b/filebeat/module/apache/access/ingest/pipeline.yml index a9f23eb4a10..37543d119e3 100644 --- a/filebeat/module/apache/access/ingest/pipeline.yml +++ b/filebeat/module/apache/access/ingest/pipeline.yml @@ -4,25 +4,37 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{IPORHOST:destination.domain} %{IPORHOST:source.ip} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] - "(?:%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}|-)?" + "(?:%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}|-)?" %{NUMBER:http.response.status_code:long} (?:%{NUMBER:http.response.body.bytes:long}|-)( "%{DATA:http.request.referrer}")?( "%{DATA:user_agent.original}")?' - '%{IPORHOST:source.address} - %{DATA:user.name} \[%{HTTPDATE:apache.access.time}\] "-" %{NUMBER:http.response.status_code:long} -' - \[%{HTTPDATE:apache.access.time}\] %{IPORHOST:source.address} %{DATA:apache.access.ssl.protocol} - %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:url.original} + %{DATA:apache.access.ssl.cipher} "%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}" (-|%{NUMBER:http.response.body.bytes:long}) ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: message + field: + - _tmp.url_orig + ignore_missing: true - set: field: event.kind value: event @@ -97,6 +109,31 @@ processors: } ctx.tls.version_protocol = parts[0]; +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: error.message diff --git a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json index 1f3600f2e09..468bfb9fdc9 100644 --- a/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/access/test/darwin-2.4.23.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:28 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -17,6 +18,7 @@ "source.address": "::1", "source.ip": "::1", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -25,6 +27,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -36,7 +39,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -45,6 +50,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -61,6 +67,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:35 +0200] \"GET / HTTP/1.1\" 200 45", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -83,6 +90,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user.name": "-" }, { @@ -91,6 +99,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:41 +0200] \"GET /notfound HTTP/1.1\" 404 206", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -113,6 +122,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/notfound", + "url.path": "/notfound", "user.name": "-" }, { @@ -121,6 +131,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "77.179.66.156 - - [26/Dec/2016:18:23:45 +0200] \"GET /hmm HTTP/1.1\" 404 201", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -143,6 +154,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/hmm", + "url.path": "/hmm", "user.name": "-" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ssl-request.log-expected.json b/filebeat/module/apache/access/test/ssl-request.log-expected.json index 3eb3e283b19..6b4288724b1 100644 --- a/filebeat/module/apache/access/test/ssl-request.log-expected.json +++ b/filebeat/module/apache/access/test/ssl-request.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[10/Aug/2018:09:45:56 +0200] 172.30.0.119 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21 HTTP/1.1\" 1375", "fileset.name": "access", "http.request.method": "GET", "http.response.body.bytes": 1375, @@ -19,7 +20,10 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" + "url.extension": "php", + "url.original": "/nagiosxi/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_admin_tasks_html%22%2C%22args%22%3A%22%22%7D&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21", + "url.path": "/nagiosxi/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_admin_tasks_html\",\"args\":\"\"}&nsp=b5c7d5d4b6f7d0cf0c92f9cbdf737f6a5c838218425e6ae21" }, { "@timestamp": "2019-10-16T09:53:47.000Z", @@ -29,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "[16/Oct/2019:11:53:47 +0200] 11.19.0.217 TLSv1.2 ECDHE-RSA-AES128-GCM-SHA256 \"GET /appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d HTTP/1.1\" -", "fileset.name": "access", "http.request.method": "GET", "http.version": "1.1", @@ -45,6 +50,9 @@ "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", "tls.version": "1.2", "tls.version_protocol": "tls", - "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" + "url.extension": "php", + "url.original": "/appl/ajaxhelper.php?cmd=getxicoreajax&opts=%7B%22func%22%3A%22get_pagetop_alert_content_html%22%2C%22args%22%3A%22%22%7D&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d", + "url.path": "/appl/ajaxhelper.php", + "url.query": "cmd=getxicoreajax&opts={\"func\":\"get_pagetop_alert_content_html\",\"args\":\"\"}&nsp=c2700eab9797eda8a9f65a3ab17a6adbceccd60a6cca7708650a5923950d" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/test-vhost.log-expected.json b/filebeat/module/apache/access/test/test-vhost.log-expected.json index b332788ad2b..64f4926d586 100644 --- a/filebeat/module/apache/access/test/test-vhost.log-expected.json +++ b/filebeat/module/apache/access/test/test-vhost.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "vhost1.domaine.fr 192.168.33.2 - - [26/Dec/2016:16:22:14 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -17,7 +18,9 @@ "log.offset": 0, "service.type": "apache", "source.ip": "192.168.33.2", + "url.domain": "vhost1.domaine.fr", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/access/test/test.log b/filebeat/module/apache/access/test/test.log index 7422d4415bd..362543c91a8 100644 --- a/filebeat/module/apache/access/test/test.log +++ b/filebeat/module/apache/access/test/test.log @@ -4,3 +4,4 @@ 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /status HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] "-" 408 152 "-" "-" +monitoring-server - - [29/May/2017:19:02:48 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" diff --git a/filebeat/module/apache/access/test/test.log-expected.json b/filebeat/module/apache/access/test/test.log-expected.json index ebe88847586..84bf4ea81a1 100644 --- a/filebeat/module/apache/access/test/test.log-expected.json +++ b/filebeat/module/apache/access/test/test.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -16,7 +17,9 @@ "service.type": "apache", "source.address": "::1", "source.ip": "::1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-" }, { @@ -25,6 +28,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -38,6 +42,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -53,6 +58,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "event.outcome": "failure", "fileset.name": "access", "http.response.status_code": 408, @@ -69,6 +75,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -82,6 +89,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -97,6 +105,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /status HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -110,6 +119,7 @@ "source.address": "monitoring-server", "source.domain": "monitoring-server", "url.original": "/status", + "url.path": "/status", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", @@ -125,6 +135,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [02/Feb/2019:05:38:45 +0100] \"-\" 408 152 \"-\" \"-\"", "event.outcome": "failure", "fileset.name": "access", "http.request.referrer": "-", @@ -139,5 +150,36 @@ "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "-" + }, + { + "@timestamp": "2017-05-29T19:02:48.000Z", + "event.category": "web", + "event.dataset": "apache.access", + "event.kind": "event", + "event.module": "apache", + "event.original": "monitoring-server - - [29/May/2017:19:02:48 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", + "event.outcome": "success", + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.body.bytes": 612, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 666, + "service.type": "apache", + "source.address": "monitoring-server", + "source.domain": "monitoring-server", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user.name": "-", + "user_agent.device.name": "Other", + "user_agent.name": "Firefox Alpha", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "user_agent.os.full": "Windows 7", + "user_agent.os.name": "Windows", + "user_agent.os.version": "7", + "user_agent.version": "15.0.a2" } ] \ No newline at end of file diff --git a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json index e9680e5b7fb..bcff5697692 100644 --- a/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/access/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "127.0.0.1 - - [26/Dec/2016:16:18:09 +0000] \"GET / HTTP/1.1\" 200 491 \"-\" \"Wget/1.13.4 (linux-gnu)\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -18,6 +19,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Wget", @@ -31,6 +33,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -44,6 +47,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -59,6 +63,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:00 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"http://192.168.33.72/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -71,7 +76,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -87,6 +94,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET / HTTP/1.1\" 200 484 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "success", "fileset.name": "access", "http.request.method": "GET", @@ -100,6 +108,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -115,6 +124,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -127,7 +137,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -143,6 +155,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:08 +0000] \"GET /favicon.ico HTTP/1.1\" 404 504 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -155,7 +168,9 @@ "service.type": "apache", "source.address": "192.168.33.1", "source.ip": "192.168.33.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -171,6 +186,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:10 +0000] \"GET /test HTTP/1.1\" 404 498 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -184,6 +200,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/test", + "url.path": "/test", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -199,6 +216,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -212,6 +230,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/hello", + "url.path": "/hello", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", @@ -227,6 +246,7 @@ "event.dataset": "apache.access", "event.kind": "event", "event.module": "apache", + "event.original": "192.168.33.1 - - [26/Dec/2016:16:22:17 +0000] \"GET /crap HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "event.outcome": "failure", "fileset.name": "access", "http.request.method": "GET", @@ -240,6 +260,7 @@ "source.address": "192.168.33.1", "source.ip": "192.168.33.1", "url.original": "/crap", + "url.path": "/crap", "user.name": "-", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", diff --git a/filebeat/module/apache/error/ingest/pipeline.yml b/filebeat/module/apache/error/ingest/pipeline.yml index aad4c3f4a5f..4b8495dd9c8 100644 --- a/filebeat/module/apache/error/ingest/pipeline.yml +++ b/filebeat/module/apache/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - \[%{APACHE_TIME:apache.error.timestamp}\] \[%{LOGLEVEL:log.level}\]( \[client %{IPORHOST:source.address}(:%{POSINT:source.port})?\])? %{GREEDYDATA:message} @@ -14,6 +17,13 @@ processors: pattern_definitions: APACHE_TIME: '%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}' ignore_missing: true +- grok: + field: message + patterns: + - "File does not exist: %{URIPATH:file.path}, referer: %{URI:http.request.referrer}" + - "File does not exist: %{URIPATH:file.path}" + ignore_missing: true + ignore_failure: true - date: if: ctx.event.timezone == null field: apache.error.timestamp @@ -38,7 +48,9 @@ processors: field: error.message value: '{{ _ingest.on_failure_message }}' - remove: - field: apache.error.timestamp + field: + - apache.error.timestamp + - _tmp.url_orig ignore_failure: true - set: field: event.kind @@ -82,6 +94,30 @@ processors: field: source.as.organization_name target_field: source.as.organization.name ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json index 4453f2bebec..90990fc95a4 100644 --- a/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json +++ b/filebeat/module/apache/error/test/darwin-2.4.23.log-expected.json @@ -6,6 +6,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103522 2016] [mpm_prefork:notice] [pid 11379] AH00163: Apache/2.4.23 (Unix) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -23,6 +24,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/test.log-expected.json b/filebeat/module/apache/error/test/test.log-expected.json index d9f470db46a..6bbb6067469 100644 --- a/filebeat/module/apache/error/test/test.log-expected.json +++ b/filebeat/module/apache/error/test/test.log-expected.json @@ -5,8 +5,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -23,6 +25,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -40,8 +43,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/usr/local/apache2/htdocs/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -70,6 +75,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Thu Jun 27 06:58:09.169510 2019] [include:warn] [pid 15934] [client 123.123.123.123:12345] AH01374: mod_include: Options +Includes (or IncludesNoExec) wasn't set, INCLUDES filter removed: /test.html", "event.timezone": "-02:00", "event.type": "error", "fileset.name": "error", diff --git a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json index 41adf5d7ae5..c946f0c739a 100644 --- a/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json +++ b/filebeat/module/apache/error/test/ubuntu-2.2.22.log-expected.json @@ -5,6 +5,7 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:17:53 2016] [notice] Apache/2.2.22 (Ubuntu) configured -- resuming normal operations", "event.timezone": "-02:00", "event.type": "info", "fileset.name": "error", @@ -20,9 +21,12 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:00 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico, referer: http://192.168.33.72/", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", + "http.request.referrer": "http://192.168.33.72/", "input.type": "log", "log.level": "error", "log.offset": 100, @@ -37,8 +41,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -54,8 +60,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/favicon.ico", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -71,8 +79,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:10 2016] [error] [client 192.168.33.1] File does not exist: /var/www/test", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/test", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -88,8 +98,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:13 2016] [error] [client 192.168.33.1] File does not exist: /var/www/hello", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/hello", "fileset.name": "error", "input.type": "log", "log.level": "error", @@ -105,8 +117,10 @@ "event.dataset": "apache.error", "event.kind": "event", "event.module": "apache", + "event.original": "[Mon Dec 26 16:22:17 2016] [error] [client 192.168.33.1] File does not exist: /var/www/crap", "event.timezone": "-02:00", "event.type": "error", + "file.path": "/var/www/crap", "fileset.name": "error", "input.type": "log", "log.level": "error", diff --git a/filebeat/module/iis/access/ingest/pipeline.yml b/filebeat/module/iis/access/ingest/pipeline.yml index 2e92efd48d5..afbd20c78ba 100644 --- a/filebeat/module/iis/access/ingest/pipeline.yml +++ b/filebeat/module/iis/access/ingest/pipeline.yml @@ -4,42 +4,65 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{NOTSPACE:iis.access.site_name}) (?:-|%{NOTSPACE:iis.access.server_name}) - (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) - (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) + (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|HTTP/%{NUMBER:http.version}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NOTSPACE:iis.access.cookie}) (?:-|%{NOTSPACE:http.request.referrer}) (?:-|%{NOTSPACE:destination.domain}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:http.response.body.bytes:long}) (?:-|%{NUMBER:http.request.body.bytes:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} \[%{IPORHOST:destination.address}\]\(http://%{IPORHOST:destination.address}\) - (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) + (?:-|%{WORD:http.request.method}) (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) \[%{IPORHOST:source.address}\]\(http://%{IPORHOST:source.address}\) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' - '%{TIMESTAMP_ISO8601:iis.access.time} (?:-|%{IPORHOST:destination.address}) (?:-|%{WORD:http.request.method}) - (?:-|%{NOTSPACE:url.path}) (?:-|%{NOTSPACE:url.query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) + (?:-|%{NOTSPACE:_tmp.url_path}) (?:-|%{NOTSPACE:_tmp.url_query}) (?:-|%{NUMBER:destination.port:long}) (?:-|%{NOTSPACE:user.name}) (?:-|%{IPORHOST:source.address}) (?:-|%{NOTSPACE:user_agent.original}) (?:-|%{NUMBER:http.response.status_code:long}) (?:-|%{NUMBER:iis.access.sub_status:long}) (?:-|%{NUMBER:iis.access.win32_status:long}) (?:-|%{NUMBER:temp.duration:long})( (?:-|%{IPORHOST:network.forwarded_ip}))?' ignore_missing: true +- uri_parts: + field: _tmp.url_path + ignore_failure: true +- set: + field: url.original + value: "{{_tmp.url_path}}" + ignore_failure: true + if: ctx?._tmp?.url_path != null && ctx?.url?.original == null +- urldecode: + field: _tmp.url_query + target_field: url.query + ignore_missing: true + ignore_failure: true + if: ctx.url?.query == null +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -133,6 +156,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json index 64ad587bb8b..5f37c7d4e43 100644 --- a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 0", "event.outcome": "failure", "event.type": [ "connection" @@ -30,8 +31,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -53,6 +56,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 46", "event.outcome": "failure", "event.type": [ "connection" @@ -71,8 +75,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -94,6 +100,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /Director - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -112,6 +119,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/Director", "url.path": "/Director", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -134,6 +142,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET / - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -152,6 +161,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -174,6 +184,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe /c+dir+c:\\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 15", "event.outcome": "failure", "event.type": [ "connection" @@ -192,8 +203,10 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.extension": "exe", + "url.original": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", "url.path": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe", - "url.query": "/c+dir+c:\\+/OG", + "url.query": "/c dir c:\\ /OG", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json index 95210536925..df3c9df6d04 100644 --- a/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json +++ b/filebeat/module/iis/access/test/test-iis-7.5.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 792000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-08-28 18:24:25 [10.100.220.70](http://10.100.220.70) GET / - 80 - [10.100.118.31](http://10.100.118.31) Mozilla/4.0+(compatible;+MSIE+7.0;+Windows+NT+6.3;+WOW64;+Trident/7.0;+.NET4.0E;+.NET4.0C;+.NET+CLR+3.5.30729;+.NET+CLR[+2.0.50727](tel:+2050727);+.NET+CLR+3.0.30729) 404 4 2 792", "event.outcome": "failure", "event.type": [ "connection" @@ -30,6 +31,7 @@ "service.type": "iis", "source.address": "10.100.118.31", "source.ip": "10.100.118.31", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "IE", @@ -52,6 +54,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -70,6 +73,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -85,6 +89,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 10.0.140.107 GET /health-monitoring - 80 - 10.0.140.2 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -103,6 +108,7 @@ "service.type": "iis", "source.address": "10.0.140.2", "source.ip": "10.0.140.2", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" }, { @@ -118,6 +124,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2019-03-06 18:43:17 2001:cdba:0000:0000:0000:0000:3257:9652 GET /health-monitoring - 80 - 2001:cdba:0000:0000:0000:0000:3257:9652 - 200 0 0 15", "event.outcome": "success", "event.type": [ "connection" @@ -136,6 +143,7 @@ "service.type": "iis", "source.address": "2001:cdba:0000:0000:0000:0000:3257:9652", "source.ip": "2001:cdba:0000:0000:0000:0000:3257:9652", + "url.original": "/health-monitoring", "url.path": "/health-monitoring" } ] \ No newline at end of file diff --git a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json index c3f4a4932da..35ce856496d 100644 --- a/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json +++ b/filebeat/module/iis/access/test/test-ipv6zone.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME ::1%0 GET / - 80 - ::1%0 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -36,6 +37,8 @@ "service.type": "iis", "source.address": "::1%0", "source.ip": "::1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json index 565bdfca17d..edcefdf8e65 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for-extended.log-expected.json @@ -13,6 +13,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-04 22:00:34 W3SVC2 freca1 10.24.129.162 GET /favicon.ico - 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64;+rv:81.0)+Gecko/20100101+Firefox/81.0 - https://images.hogeschoolrotterdam.nl/Blob/adeec119008c48758c1a6be53aeeb2ac/34ff475072d54117bcb46ea7f023bd87.jpg?width=1200&height=630&mode=crop images.hogeschoolrotterdam.nl 404 0 2 1437 534 0 116.189.86.89", "event.outcome": "failure", "event.type": [ "connection" @@ -38,6 +39,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "ico", + "url.original": "/favicon.ico", "url.path": "/favicon.ico", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -61,6 +65,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:40:30 W3SVC2 freca1 10.24.129.162 GET /robots.txt - 80 - 10.24.136.240 HTTP/1.1 Twitterbot/1.0 - - images.hogeschoolrotterdam.nl 200 0 0 346 306 0 119.16.157.180", "event.outcome": "success", "event.type": [ "connection" @@ -85,6 +90,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "txt", + "url.original": "/robots.txt", "url.path": "/robots.txt", "user_agent.device.name": "Spider", "user_agent.name": "Twitterbot", @@ -105,6 +113,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg width=35&height=38&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMsadfHHYQAGkWEWnyAqAads=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2007 581 15 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -131,6 +140,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.path": "/app_data/cache/9/e/1/c/3/7/9e1c37a203a2a306e8f5d4df1bddb1109dd42e57.jpg", "url.query": "width=35&height=38&mode=crop", "user_agent.device.name": "Other", @@ -155,6 +167,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-05 21:48:33 W3SVC2 freca1 10.24.129.162 GET /app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg width=75&height=40&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36+Edge/18.18362 BIGipServerYAkgvoMHHYQAGkWEWadfsadfnyAqAere=27246369425.20480.0000 https://www.rotterdamuas.com/ images.hogeschoolrotterdam.nl 200 0 0 2926 581 0 119.160.162.213", "event.outcome": "success", "event.type": [ "connection" @@ -181,6 +194,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.path": "/app_data/cache/f/b/7/1/2/7/fb71277260ae26a108c3608ce1a62474a55b2556.jpg", "url.query": "width=75&height=40&mode=crop", "user_agent.device.name": "Other", @@ -205,6 +221,7 @@ "event.duration": 15000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504478161.16021944379;+_gat_UA-155746052-5=1;+BIGipServerYAkgvoMHHYQAGkWsadfsdfEWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYaferu1dRyQoUTVVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 15 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -231,6 +248,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "png", + "url.original": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.path": "/Blob/a9e2fe596ac14a4ab07beb6b6e2c6545/15a3917cacf44de59af9cc899e90a9d4.png", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", @@ -255,6 +275,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-08 22:00:22 W3SVC2 freca1 10.24.129.162 GET /Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg width=60&height=20&mode=crop 80 - 10.24.136.240 HTTP/1.1 Mozilla/5.0+(iPhone;+CPU+iPhone+OS+13_7+like+Mac+OS+X)+AppleWebKit/605.1.15+(KHTML,+like+Gecko)+Version/13.1.2+Mobile/15E148+Safari/604.1 imagedpi=1;+imagequality=30;+_ga=GA1.2.14440223724.16021494379;+_gid=GA1.2.12504748161.16021944379;+_gat_UA-155764052-5=1;+BIGipServerYAkgvoMHHYQAGkWEsadfsdfsWnyAqA=!vbipVwbt0UWJ4QSsPvjjVxViTXMqdXWYu1dRyQoUTVerwerVBGLZPB39PoNX5Tw66+GT0ChJCfnp/xfhM8lI=;+_gcl_au=1.1.1297303538.1602194379 https://www.hogeschoolrotterdam.nl/opleidingen/bachelor/elektrotechniek/voltijd/ images.hogeschoolrotterdam.nl 304 0 0 388 979 0 156.189.143.218", "event.outcome": "success", "event.type": [ "connection" @@ -281,6 +302,9 @@ "service.type": "iis", "source.address": "10.24.136.240", "source.ip": "10.24.136.240", + "url.domain": "images.hogeschoolrotterdam.nl", + "url.extension": "jpg", + "url.original": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.path": "/Blob/ff64cd9efcf4424dbf06b3b8133eeea2/f2e0b2998b1f43cb98e5b31c7faa91f4.jpg", "url.query": "width=60&height=20&mode=crop", "user_agent.device.name": "iPhone", diff --git a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json index 4d8ace5a7fb..8977e920292 100644 --- a/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json +++ b/filebeat/module/iis/access/test/test-x-forward-for.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 26000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 26 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -32,6 +33,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -54,6 +56,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -74,6 +77,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadBatchTotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +100,7 @@ "event.duration": 46000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 POST /Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 46 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -116,6 +121,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "url.path": "/Production-UI/api/finance/legacy/GeneralLedger/LoadJETotals", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -138,6 +144,7 @@ "event.duration": 32000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLAPAprvMaster $filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 32 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -158,8 +165,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLAPAprvMaster", "url.path": "/Production-UI/data/finance/legacy/GLAPAprvMaster", - "url.query": "$filter=BatchId%20eq%20%27FY21HSNG0820%27&$orderby=Subsys,Ref&$skip=0&$top=20", + "url.query": "$filter=BatchId eq 'FY21HSNG0820'&$orderby=Subsys,Ref&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -181,6 +189,7 @@ "event.duration": 166000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:00:17 192.168.16.11 GET /Production-UI/data/finance/legacy/GLATrnsDetail $filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 166 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -201,8 +210,9 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/data/finance/legacy/GLATrnsDetail", "url.path": "/Production-UI/data/finance/legacy/GLATrnsDetail", - "url.query": "$filter=Subsys%20eq%20%27JE%27%20and%20Ref%20eq%20%27HSNG08-MR%27%20and%20BatchId%20eq%20%27FY21HSNG0820%27&$orderby=RecNo&$skip=0&$top=20", + "url.query": "$filter=Subsys eq 'JE' and Ref eq 'HSNG08-MR' and BatchId eq 'FY21HSNG0820'&$orderby=RecNo&$skip=0&$top=20", "user_agent.device.name": "Other", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36", @@ -224,6 +234,7 @@ "event.duration": 60000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 GET /Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 60 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -244,6 +255,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "url.path": "/Production-UI/api/finance/legacy/documents/PendingAttachments/GLJEUB", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -266,6 +278,7 @@ "event.duration": 72000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 72 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -286,6 +299,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLATrnsDetail/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -308,6 +322,7 @@ "event.duration": 88000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:06:42 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/ - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 88 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -328,6 +343,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "url.path": "/Production-UI/api/finance/legacy/documents/GLAPAprvMaster/attachments/", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -350,6 +366,7 @@ "event.duration": 286000000, "event.kind": "event", "event.module": "iis", + "event.original": "2020-10-07 23:07:02 192.168.16.11 POST /Production-UI/api/finance/legacy/documents/attachDoc - 443 - 192.168.7.63 Mozilla/5.0+(Windows+NT+10.0;+Win64;+x64)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/85.0.4183.121+Safari/537.36 https://onesolfarm.ggcity.org/Production-UI/ui/uiscreens/generalledger/GLJEUB 200 0 0 286 192.168.198.23", "event.outcome": "success", "event.type": [ "connection" @@ -370,6 +387,7 @@ "service.type": "iis", "source.address": "192.168.7.63", "source.ip": "192.168.7.63", + "url.original": "/Production-UI/api/finance/legacy/documents/attachDoc", "url.path": "/Production-UI/api/finance/legacy/documents/attachDoc", "user_agent.device.name": "Other", "user_agent.name": "Chrome", diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 786333c1379..cd809c61bc7 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -12,6 +12,7 @@ "event.duration": 123000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 127.0.0.1 GET / q=100 80 - 85.181.35.98 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - 200 0 0 123", "event.outcome": "success", "event.type": [ "connection" @@ -40,6 +41,7 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.original": "/", "url.path": "/", "url.query": "q=100", "user_agent.device.name": "Other", @@ -61,6 +63,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 W3SVC1 GET / - 80 - 127.0.0.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "fileset.name": "access", "http.request.body.bytes": 456, @@ -78,6 +81,8 @@ "service.type": "iis", "source.address": "127.0.0.1", "source.ip": "127.0.0.1", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Other", "user_agent.name": "Firefox", @@ -101,6 +106,7 @@ "event.duration": 789000000, "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789", "event.outcome": "success", "event.type": [ "connection" @@ -134,6 +140,8 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", + "url.original": "/", "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", @@ -156,6 +164,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET / redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()} 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0", "event.outcome": "failure", "event.type": [ "connection" @@ -174,8 +183,9 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", + "url.original": "/", "url.path": "/", - "url.query": "redirect:${%23req%3d%23context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),%23webroot%3d%23req.getSession().getServletContext().getRealPath('/'),%23resp.println(%23webroot),%23resp.flush(),%23resp.close()}", + "url.query": "redirect:${#req=#context.get('com.opensymphony.xwork2.dispatcher.HttpServletRequest'),#webroot=#req.getSession().getServletContext().getRealPath('/'),#resp.println(#webroot),#resp.flush(),#resp.close()}", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", @@ -197,6 +207,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-31 12:52:33 10.44.0.136 GET /${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 0", "event.outcome": "failure", "event.type": [ "connection" @@ -215,7 +226,7 @@ "service.type": "iis", "source.address": "10.50.6.188", "source.ip": "10.50.6.188", - "url.path": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", + "url.original": "/${#context['xwork.MethodAccessor.denyMethodExecution']=!(#_memberAccess['allowStaticMethodAccess']=true),(@java.lang.Runtime@getRuntime()).exec('ipconfig').waitFor()}.action", "user_agent.device.name": "Other", "user_agent.name": "IE", "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)", diff --git a/filebeat/module/iis/error/ingest/pipeline.yml b/filebeat/module/iis/error/ingest/pipeline.yml index a16fde841da..6a09d937fed 100644 --- a/filebeat/module/iis/error/ingest/pipeline.yml +++ b/filebeat/module/iis/error/ingest/pipeline.yml @@ -3,21 +3,34 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER}|-) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' - '%{TIMESTAMP_ISO8601:iis.error.time} %{IPORHOST:source.address} %{NUMBER:source.port:long} %{IPORHOST:destination.address} %{IPORHOST:destination.port:long} (?:HTTP/%{NUMBER:http.version}|-) - (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:url.original}) (?:%{NUMBER:http.response.status_code:long}|-) + (?:%{WORD:http.request.method}|-) (?:-|%{NOTSPACE:_tmp.url_orig}) (?:%{NUMBER:http.response.status_code:long}|-) (?:%{NUMBER}|-) (?:-|%{NOTSPACE:iis.error.reason_phrase}) (?:-|%{NOTSPACE:iis.error.queue_name})' ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.original + value: "{{{_tmp.url_orig}}}" + ignore_failure: true + if: ctx?._tmp?.url_orig != null && ctx?.url?.original == null - remove: - field: message + field: + - _tmp + ignore_missing: true - rename: field: '@timestamp' target_field: event.created @@ -90,6 +103,30 @@ processors: field: event.outcome value: failure if: "ctx?.http?.response?.status_code != null && ctx.http.response.status_code >= 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/iis/error/test/iis_error_url.log-expected.json b/filebeat/module/iis/error/test/iis_error_url.log-expected.json index 8adb2e6d1d0..dcfa5353878 100644 --- a/filebeat/module/iis/error/test/iis_error_url.log-expected.json +++ b/filebeat/module/iis/error/test/iis_error_url.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/0.9 t3 12.2.1 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -35,7 +36,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "12.2.1" + "url.extension": "1", + "url.original": "12.2.1", + "url.path": "12.2.1" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -49,6 +52,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ./././././../../../../../../../../ 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "./././././../../../../../../../../" + "url.extension": "/", + "url.original": "./././././../../../../../../../../", + "url.path": "./././././../../../../../../../../" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /..\\pixfir~1\\how_to_login.html 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -125,6 +132,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET ..\\..\\..\\..\\..\\..\\winnt\\win.ini 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -163,6 +171,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -187,7 +196,9 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" + "url.extension": "ini", + "url.original": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini", + "url.path": "/\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./\ufffd.\ufffd./windows/win.ini" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -201,6 +212,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /nessus\\..\\..\\..\\..\\..\\..\\winnt\\win.ini 403 - Forbidden -", "event.outcome": "failure", "event.type": [ "connection" @@ -239,6 +251,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 OPTIONS * 404 - NotFound -", "event.outcome": "failure", "event.type": [ "connection" @@ -263,7 +276,8 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "*" + "url.original": "*", + "url.path": "*" }, { "@timestamp": "2018-05-05T05:05:55.000Z", @@ -277,6 +291,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-05-05 05:05:55 149.42.83.135 12345 192.168.101.101 443 HTTP/1.1 GET /fee&fie=foe 400 - URL -", "event.outcome": "failure", "event.type": [ "connection" @@ -301,6 +316,7 @@ "source.geo.location.lon": -97.822, "source.ip": "149.42.83.135", "source.port": 12345, - "url.original": "/fee&fie=foe" + "url.original": "/fee&fie=foe", + "url.path": "/fee&fie=foe" } ] \ No newline at end of file diff --git a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json index ebcb2f36b84..e77f92bcc56 100644 --- a/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json +++ b/filebeat/module/iis/error/test/ipv6_zone_id.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-12-30 14:22:07 ::1%0 49958 ::1%0 80 - - - - - - Timer_ConnectionIdle -", "event.type": [ "connection" ], diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json index 506ee6ba2ed..fdf8fa4bdf1 100644 --- a/filebeat/module/iis/error/test/test.log-expected.json +++ b/filebeat/module/iis/error/test/test.log-expected.json @@ -11,6 +11,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 08:09:10 172.31.77.6 2094 172.31.77.6 80 HTTP/1.1 GET /qos/1kbfile.txt 503 - ConnLimit -", "event.outcome": "failure", "event.type": [ "connection" @@ -30,7 +31,9 @@ "source.address": "172.31.77.6", "source.ip": "172.31.77.6", "source.port": 2094, - "url.original": "/qos/1kbfile.txt" + "url.extension": "txt", + "url.original": "/qos/1kbfile.txt", + "url.path": "/qos/1kbfile.txt" }, { "@timestamp": "2018-01-01T09:10:11.000Z", @@ -44,6 +47,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 09:10:11 85.181.35.98 2780 127.0.0.1 80 HTTP/1.1 GET /ThisIsMyUrl.htm 400 - Hostname -", "event.outcome": "failure", "event.type": [ "connection" @@ -73,7 +77,9 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2780, - "url.original": "/ThisIsMyUrl.htm" + "url.extension": "htm", + "url.original": "/ThisIsMyUrl.htm", + "url.path": "/ThisIsMyUrl.htm" }, { "@timestamp": "2018-01-01T10:11:12.000Z", @@ -87,6 +93,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 10:11:12 85.181.35.98 2894 127.0.0.1 80 HTTP/2.0 GET / 505 - Version_N/S -", "event.outcome": "failure", "event.type": [ "connection" @@ -116,7 +123,8 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "source.port": 2894, - "url.original": "/" + "url.original": "/", + "url.path": "/" }, { "@timestamp": "2018-01-01T11:12:13.000Z", @@ -130,6 +138,7 @@ "event.dataset": "iis.error", "event.kind": "event", "event.module": "iis", + "event.original": "2018-01-01 11:12:13 85.181.35.98 64388 127.0.0.1 80 - - - - - Timer_MinBytesPerSecond -", "event.type": [ "connection" ], diff --git a/filebeat/module/nginx/access/ingest/pipeline.yml b/filebeat/module/nginx/access/ingest/pipeline.yml index 57fe9031b55..4a597e30d39 100644 --- a/filebeat/module/nginx/access/ingest/pipeline.yml +++ b/filebeat/module/nginx/access/ingest/pipeline.yml @@ -4,8 +4,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.access.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.access.time}\] "%{DATA:nginx.access.info}" @@ -19,11 +22,21 @@ processors: - grok: field: nginx.access.info patterns: - - '%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}' + - '%{WORD:http.request.method} %{DATA:_tmp.url_orig} HTTP/%{NUMBER:http.version}' - "" ignore_missing: true +- uri_parts: + field: _tmp.url_orig + ignore_failure: true +- set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: - field: nginx.access.info + field: + - nginx.access.info + - _tmp.url_orig + ignore_missing: true - split: field: nginx.access.remote_ip_list separator: '"?,?\s+' @@ -93,8 +106,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true -- remove: - field: message - rename: field: '@timestamp' target_field: event.created @@ -161,6 +172,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/access/test/access.log b/filebeat/module/nginx/access/test/access.log index 58991d26b3b..7acb1428af8 100644 --- a/filebeat/module/nginx/access/test/access.log +++ b/filebeat/module/nginx/access/test/access.log @@ -10,3 +10,5 @@ 127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /taga HTTP/1.1" 404 169 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" +lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] "GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1" 206 7648063 "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4" "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36" diff --git a/filebeat/module/nginx/access/test/access.log-expected.json b/filebeat/module/nginx/access/test/access.log-expected.json index 7981a316c95..6a0af6f499c 100644 --- a/filebeat/module/nginx/access/test/access.log-expected.json +++ b/filebeat/module/nginx/access/test/access.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:33 +0200] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -39,6 +40,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -55,6 +57,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:49:34 +0200] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -87,7 +90,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -104,6 +109,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [25/Oct/2016:14:50:44 +0200] \"GET /adsasd HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -136,6 +142,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/adsasd", + "url.path": "/adsasd", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36", @@ -152,6 +159,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET / HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -184,6 +192,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -200,6 +209,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:34:43 +0100] \"GET /favicon.ico HTTP/1.1\" 404 571 \"http://localhost:8080/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -232,7 +242,9 @@ "source.geo.region_iso_code": "DE-RP", "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -249,6 +261,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:18 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -281,6 +294,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -297,6 +311,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:21 +0100] \"GET /test HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -329,6 +344,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test", + "url.path": "/test", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -345,6 +361,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "77.179.66.156 - - [07/Dec/2016:10:43:23 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -377,6 +394,7 @@ "source.geo.region_name": "Rheinland-Pfalz", "source.ip": "77.179.66.156", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -393,6 +411,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -415,6 +434,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/test1", + "url.path": "/test1", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36", @@ -431,6 +451,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -453,6 +474,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -469,6 +491,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] \"GET / HTTP/1.1\" 304 0 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -491,6 +514,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -507,6 +531,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -529,6 +554,7 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "url.original": "/taga", + "url.path": "/taga", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -536,5 +562,92 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.12", "user_agent.version": "49.0." + }, + { + "@timestamp": "2020-06-09T19:10:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:10:39 -0700] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2346, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" + }, + { + "@timestamp": "2020-06-09T19:15:39.000Z", + "destination.domain": "lessons.example.com", + "event.category": [ + "web" + ], + "event.dataset": "nginx.access", + "event.kind": "event", + "event.module": "nginx", + "event.original": "lessons.example.com 192.168.0.1 - - [09/Jun/2020:12:15:39 -0700] \"GET /%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/ HTTP/1.1\" 206 7648063 \"http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4\" \"Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36\"", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "access" + ], + "fileset.name": "access", + "http.request.method": "GET", + "http.request.referrer": "http://lessons.example.com/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "http.response.body.bytes": 7648063, + "http.response.status_code": 206, + "http.version": "1.1", + "input.type": "log", + "log.offset": 2727, + "nginx.access.remote_ip_list": [ + "192.168.0.1" + ], + "related.ip": [ + "192.168.0.1" + ], + "service.type": "nginx", + "source.address": "192.168.0.1", + "source.ip": "192.168.0.1", + "url.domain": "lessons.example.com", + "url.original": "/%D0%A0%D1%83%D1%81%D1%81%D0%BA%D0%B0%D1%8F%20%D1%88%D0%BA%D0%BE%D0%BB%D0%B0%20-%20InternetUrok%201%D0%BA%D0%BB%D0%B0%D1%81%D1%81/", + "url.path": "/\u0420\u0443\u0441\u0441\u043a\u0430\u044f \u0448\u043a\u043e\u043b\u0430 - InternetUrok 1\u043a\u043b\u0430\u0441\u0441/", + "user_agent.device.name": "Kindle", + "user_agent.name": "Amazon Silk", + "user_agent.original": "Mozilla/5.0 (Linux; Android 5.1.1; KFFOWI) AppleWebKit/537.36 (KHTML, like Gecko) Silk/81.2.16 like Chrome/81.0.4044.138 Safari/537.36", + "user_agent.os.full": "Android 5.1.1", + "user_agent.os.name": "Android", + "user_agent.os.version": "5.1.1", + "user_agent.version": "81.2.16" } ] \ No newline at end of file diff --git a/filebeat/module/nginx/access/test/test-with-host.log-expected.json b/filebeat/module/nginx/access/test/test-with-host.log-expected.json index e07836ce520..85ba8c494f3 100644 --- a/filebeat/module/nginx/access/test/test-with-host.log-expected.json +++ b/filebeat/module/nginx/access/test/test-with-host.log-expected.json @@ -8,6 +8,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -31,7 +32,9 @@ "service.type": "nginx", "source.address": "10.0.0.2", "source.ip": "10.0.0.2", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -49,6 +52,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -70,7 +74,9 @@ "service.type": "nginx", "source.address": "172.17.0.1", "source.ip": "172.17.0.1", + "url.domain": "example.com", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -88,6 +94,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com 10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -121,7 +128,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -140,6 +149,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -171,7 +181,9 @@ "source.geo.region_iso_code": "DE-BE", "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", + "url.domain": "example.com", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -190,6 +202,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 \"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -224,7 +237,10 @@ "source.geo.region_iso_code": "US-IL", "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", + "url.domain": "example.com", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -238,6 +254,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -267,7 +284,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -283,6 +302,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4:80 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -314,6 +334,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com:80 unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -325,7 +346,8 @@ "input.type": "log", "log.offset": 1269, "service.type": "nginx", - "source.address": "unix:" + "source.address": "unix:", + "url.domain": "example.com" }, { "@timestamp": "2017-05-29T19:02:48.000Z", @@ -336,6 +358,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "1.2.3.4 localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -357,6 +380,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -374,6 +398,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "example.com localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -392,7 +417,9 @@ ], "service.type": "nginx", "source.address": "localhost", + "url.domain": "example.com", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index b27c9ccf19b..60731f0cc10 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -31,6 +32,7 @@ "source.address": "10.0.0.2", "source.ip": "10.0.0.2", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -47,6 +49,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -69,6 +72,7 @@ "source.address": "172.17.0.1", "source.ip": "172.17.0.1", "url.original": "/stringpatch", + "url.path": "/stringpatch", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -85,6 +89,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -119,6 +124,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", @@ -135,6 +141,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -167,6 +174,7 @@ "source.geo.region_name": "Land Berlin", "source.ip": "85.181.35.98", "url.original": "/ocelot", + "url.path": "/ocelot", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", @@ -183,6 +191,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -218,6 +227,8 @@ "source.geo.region_name": "Illinois", "source.ip": "199.96.1.1", "url.original": "/assets/xxxx?q=100", + "url.path": "/assets/xxxx", + "url.query": "q=100", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "Amazon CloudFront" @@ -230,6 +241,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -258,7 +270,9 @@ "source.geo.location.lat": 39.5, "source.geo.location.lon": -8.0, "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.extension": "html", "url.original": "/test.html", + "url.path": "/test.html", "user_agent.device.name": "Spider", "user_agent.name": "Facebot", "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", @@ -272,6 +286,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -300,6 +315,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "unix: - - [26/Feb/2019:15:39:42 +0100] \"hello\" 400 173 \"-\" \"-\"", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -321,6 +337,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -339,6 +356,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", @@ -355,6 +373,7 @@ "event.dataset": "nginx.access", "event.kind": "event", "event.module": "nginx", + "event.original": "localhost, localhost - - [29/May/2017:19:02:48 +0000] \"GET /test2 HTTP/1.1\" 200 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -374,6 +393,7 @@ "service.type": "nginx", "source.address": "localhost", "url.original": "/test2", + "url.path": "/test2", "user_agent.device.name": "Other", "user_agent.name": "Firefox Alpha", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", diff --git a/filebeat/module/nginx/error/ingest/pipeline.yml b/filebeat/module/nginx/error/ingest/pipeline.yml index 05691eeb737..51170f6cbbb 100644 --- a/filebeat/module/nginx/error/ingest/pipeline.yml +++ b/filebeat/module/nginx/error/ingest/pipeline.yml @@ -3,8 +3,11 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- grok: +- rename: field: message + target_field: event.original +- grok: + field: event.original patterns: - '%{DATA:nginx.error.time} \[%{DATA:log.level}\] %{NUMBER:process.pid:long}#%{NUMBER:process.thread.id:long}: (\*%{NUMBER:nginx.error.connection_id:long} )?%{GREEDYMULTILINE:message}' diff --git a/filebeat/module/nginx/error/test/error.log-expected.json b/filebeat/module/nginx/error/test/error.log-expected.json index 8896a490705..ac5e5f83caf 100644 --- a/filebeat/module/nginx/error/test/error.log-expected.json +++ b/filebeat/module/nginx/error/test/error.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:49:34 [error] 54053#0: *1 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/favicon.ico\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /favicon.ico HTTP/1.1\", host: \"localhost:8080\", referrer: \"http://localhost:8080/\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -29,6 +30,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2016/10/25 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /adsasd HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" @@ -51,6 +53,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/10/30 23:26:34 [error] 205860#205860: *180289 FastCGI sent in stderr: \"PHP message: PHP Warning: Declaration of FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) should be compatible with FEE_Field_Post::wrap($content, $post_id = 0) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Tags::wrap($content, $before, $sep, $after) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0\nPHP message: PHP Warning: Declaration of FEE_Field_Category::wrap($content, $sep, $parents) should be compatible with FEE_Field_Terms::wrap($content, $taxonomy, $before, $sep, $after) in /var/www/xxx/web/wp-content/plugins/front-end-editor/php/fields/post.php on line 0", "event.timezone": "-02:00", "event.type": [ "error" @@ -76,6 +79,7 @@ "event.dataset": "nginx.error", "event.kind": "event", "event.module": "nginx", + "event.original": "2019/11/05 14:50:44 [error] 54053#0: *3 open() \"/usr/local/Cellar/nginx/1.10.2_1/html/adsasd\" failed (2: No such file or directory), client: 127.0.0.1, server: localhost, request: \"GET /pysio HTTP/1.1\", host: \"localhost:8080\"", "event.timezone": "-02:00", "event.type": [ "error" diff --git a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml index 64fd7567ba1..14a5da9cae9 100644 --- a/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml +++ b/filebeat/module/nginx/ingress_controller/ingest/pipeline.yml @@ -5,8 +5,11 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" - - grok: + - rename: field: message + target_field: event.original + - grok: + field: event.original patterns: - (%{NGINX_HOST} )?"?(?:%{NGINX_ADDRESS_LIST:nginx.ingress_controller.remote_ip_list}|%{NOTSPACE:source.address}) - (-|%{DATA:user.name}) \[%{HTTPDATE:nginx.ingress_controller.time}\] "%{DATA:nginx.ingress_controller.info}" @@ -31,6 +34,13 @@ processors: - "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}" - "" ignore_missing: true + - uri_parts: + field: url.original + ignore_failure: true + - set: + field: url.domain + value: "{{destination.domain}}" + if: ctx.url?.domain == null && ctx.destination?.domain != null - remove: field: nginx.ingress_controller.info - split: @@ -198,8 +208,6 @@ processors: patterns: - ^%{IP:source.ip}$ ignore_failure: true - - remove: - field: message - rename: field: "@timestamp" target_field: event.created @@ -266,6 +274,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null" + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/nginx/ingress_controller/test/test.log b/filebeat/module/nginx/ingress_controller/test/test.log index c8ba580f64d..ef502ebe86b 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log +++ b/filebeat/module/nginx/ingress_controller/test/test.log @@ -21,3 +21,4 @@ 192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] "GET /favicon.ico HTTP/1.1" 200 59 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f 192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402 192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /v2/some HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 +192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] "GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1" 200 61 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402 diff --git a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json index c9cbfb36b7f..7a9e13bf58e 100644 --- a/filebeat/module/nginx/ingress_controller/test/test.log-expected.json +++ b/filebeat/module/nginx/ingress_controller/test/test.log-expected.json @@ -7,6 +7,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:48:51 +0000] \"POST /products HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 89 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 529a007902362a5f51385a5fa7049884", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -52,6 +53,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products", + "url.path": "/products", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -65,6 +67,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:15 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 91 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 68fa971ce4dfce685fdc01c877bfa645", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -110,6 +113,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -123,6 +127,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:30 +0000] \"DELETE /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 94 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 0be411044cb1cb67580e115413b2da60", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -168,6 +173,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -181,6 +187,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:43 +0000] \"PATCH /products/42 HTTP/1.1\" 200 59 \"-\" \"curl/7.54.0\" 93 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 f479ab1d9cc8afcbac9e9f958ff8babc", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -226,6 +233,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "curl", "user_agent.original": "curl/7.54.0", @@ -239,6 +247,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:49:50 +0000] \"PATCHp /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - 4c7d2079340e68353c7d0dfff00b904b", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -266,7 +275,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:50:09.000Z", @@ -276,6 +286,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:50:09 +0000] \"geti /products/42 HTTP/1.1\" 400 163 \"-\" \"-\" 0 0.000 [] [] - - - - efb0c5aa8be6cdeb4a7e7bd090e3d893", "event.outcome": "failure", "event.timezone": "-02:00", "event.type": [ @@ -303,7 +314,8 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", - "url.original": "/products/42" + "url.original": "/products/42", + "url.path": "/products/42" }, { "@timestamp": "2020-02-07T11:55:05.000Z", @@ -313,6 +325,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:05 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Wget/1.20.3 (darwin18.6.0)\" 157 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 457b71c3e1ee1887bb809effd301a0ec", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -358,6 +371,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Other", "user_agent.name": "Wget", "user_agent.original": "Wget/1.20.3 (darwin18.6.0)", @@ -371,6 +385,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 450 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 da29abf31e4d6324cebe5e7bca370709", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -416,6 +431,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -432,6 +448,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:55:57 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 381 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 e983c8cf3d713548baa50c9e2fffeb34", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -477,7 +494,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -494,6 +513,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 441 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 3d7ff18ff4181a7db5013a76f975d900", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -539,6 +559,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -555,6 +576,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:24 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36\" 372 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 d131fe4bcd359cf947f75efca4bfa553", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -600,7 +622,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Chrome", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36", @@ -617,6 +641,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 ef6629fcaaa1ea0d1a843cf2bf40571d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -662,6 +687,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -678,6 +704,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:36 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/products/42\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 325 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 2593a1126588922449c183c8d9ddbbeb", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -723,7 +750,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -740,6 +769,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /products/42 HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 369 0.002 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 0f76ea730f282d5759018eb756b23b14", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -785,6 +815,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42", + "url.path": "/products/42", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -801,6 +832,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET / HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 358 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 21efd18e3a7952fc78c0f2dcc1f05e69", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -846,6 +878,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/", + "url.path": "/", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -862,6 +895,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:54 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 314 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.002 200 e096809c2cb46f004c3b538b23916e5b", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -907,7 +941,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -924,6 +960,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 360 0.002 [default-web2-8080] [] 172.17.0.6:8080 61 0.002 200 0a2a92d080e664dd4e95c85d097c9d3d", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -969,6 +1006,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -985,6 +1023,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:11:56:56 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"http://hello-world.info/v2\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15\" 316 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 4c024ebfc20acfb2d59e542e3ed60789", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1030,7 +1069,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Safari", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.5 Safari/605.1.15", @@ -1047,6 +1088,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:00:28 +0000] \"GET /products/42?address=delhi+technological+university HTTP/1.1\" 200 59 \"-\" \"python-requests/2.22.0\" 197 0.000 [default-web-8080] [] 172.17.0.5:8080 59 0.001 200 9a7babf34ca4ee59d90ac48d452a9214", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1092,6 +1134,8 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/products/42?address=delhi+technological+university", + "url.path": "/products/42", + "url.query": "address=delhi+technological+university", "user_agent.device.name": "Other", "user_agent.name": "Python Requests", "user_agent.original": "python-requests/2.22.0", @@ -1105,6 +1149,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /v2 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 343 0.000 [default-web2-8080] [] 172.17.0.6:8080 61 0.001 200 ba91c30454893c121879396b0a78be79", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1150,6 +1195,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2", + "url.path": "/v2", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1166,6 +1212,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:38 +0000] \"GET /favicon.ico HTTP/1.1\" 200 59 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 262 0.001 [default-web-8080] [] 172.17.0.5:8080 59 0.000 200 98c81aa2d50c67f6fb1fa16d5ce62f8f", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1210,7 +1257,9 @@ "service.type": "nginx", "source.address": "192.168.64.1", "source.ip": "192.168.64.1", + "url.extension": "ico", "url.original": "/favicon.ico", + "url.path": "/favicon.ico", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1227,6 +1276,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.1 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080 61 0.000 200 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1272,6 +1322,7 @@ "source.address": "192.168.64.1", "source.ip": "192.168.64.1", "url.original": "/v2/some", + "url.path": "/v2/some", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -1288,6 +1339,7 @@ "event.dataset": "nginx.ingress_controller", "event.kind": "event", "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /v2/some HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", "event.outcome": "success", "event.timezone": "-02:00", "event.type": [ @@ -1337,6 +1389,75 @@ "source.address": "192.168.64.14", "source.ip": "192.168.64.14", "url.original": "/v2/some", + "url.path": "/v2/some", + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", + "user_agent.os.full": "Mac OS X 10.14", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.14", + "user_agent.version": "72.0." + }, + { + "@timestamp": "2020-02-07T12:02:42.000Z", + "event.category": [ + "web" + ], + "event.dataset": "nginx.ingress_controller", + "event.kind": "event", + "event.module": "nginx", + "event.original": "192.168.64.14 - - [07/Feb/2020:12:02:42 +0000] \"GET /A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4 HTTP/1.1\" 200 61 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0\" 348 0.001 [default-web2-8080] [] 172.17.0.6:8080, 172.17.0.7:8080 61, 100 0.100, 0.004 200, 203 835136ae24486dbb4156dcbe21f5d402", + "event.outcome": "success", + "event.timezone": "-02:00", + "event.type": [ + "info" + ], + "fileset.name": "ingress_controller", + "http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "http.request.method": "GET", + "http.response.body.bytes": 61, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 6026, + "nginx.ingress_controller.http.request.id": "835136ae24486dbb4156dcbe21f5d402", + "nginx.ingress_controller.http.request.length": 348, + "nginx.ingress_controller.http.request.time": 0.001, + "nginx.ingress_controller.remote_ip_list": [ + "192.168.64.14" + ], + "nginx.ingress_controller.upstream.alternative_name": "", + "nginx.ingress_controller.upstream.ip": "172.17.0.7", + "nginx.ingress_controller.upstream.name": "default-web2-8080", + "nginx.ingress_controller.upstream.port": 8080, + "nginx.ingress_controller.upstream.response.length": 100, + "nginx.ingress_controller.upstream.response.length_list": [ + "61", + "100" + ], + "nginx.ingress_controller.upstream.response.status_code": 203, + "nginx.ingress_controller.upstream.response.status_code_list": [ + "200", + "203" + ], + "nginx.ingress_controller.upstream.response.time": 0.104, + "nginx.ingress_controller.upstream.response.time_list": [ + "0.100", + "0.004" + ], + "nginx.ingress_controller.upstream_address_list": [ + "172.17.0.6:8080", + "172.17.0.7:8080" + ], + "related.ip": [ + "192.168.64.14" + ], + "service.type": "nginx", + "source.address": "192.168.64.14", + "source.ip": "192.168.64.14", + "url.extension": "mp4", + "url.original": "/A%20Beka%20G1%20Howe/029_AND_30/15%20reading%20elephants.mp4", + "url.path": "/A Beka G1 Howe/029_AND_30/15 reading elephants.mp4", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:72.0) Gecko/20100101 Firefox/72.0", diff --git a/filebeat/module/traefik/access/ingest/pipeline.yml b/filebeat/module/traefik/access/ingest/pipeline.yml index dd5de1b0b0b..3b63a05fe8d 100644 --- a/filebeat/module/traefik/access/ingest/pipeline.yml +++ b/filebeat/module/traefik/access/ingest/pipeline.yml @@ -4,10 +4,13 @@ processors: - set: field: event.ingested value: '{{_ingest.timestamp}}' -- dissect: +- rename: field: message + target_field: event.original +- dissect: + field: event.original pattern: '%{source.address} %{traefik.access.user_identifier} %{user.name} [%{traefik.access.time}] - "%{http.request.method} %{url.original} HTTP/%{http.version}" %{http.response.status_code} + "%{http.request.method} %{temp.url_orig} HTTP/%{http.version}" %{http.response.status_code} %{traefik.access.message}' - grok: field: traefik.access.message @@ -17,12 +20,9 @@ processors: (?:"%{DATA:traefik.access.frontend_name}"|-)?)?( "%{DATA:traefik.access.backend_url}")?( %{NUMBER:temp.duration:long}ms)?)? ignore_missing: true -- remove: - field: message - ignore_missing: true -- remove: - field: traefik.access.message - ignore_missing: true +- uri_parts: + field: temp.url_orig + ignore_failure: true - rename: field: '@timestamp' target_field: event.created @@ -31,8 +31,6 @@ processors: target_field: '@timestamp' formats: - dd/MMM/yyyy:H:m:s Z -- remove: - field: traefik.access.time - convert: field: http.response.status_code type: long @@ -47,7 +45,10 @@ processors: scale: 1000000 if: ctx.temp?.duration != null - remove: - field: temp.duration + field: + - temp + - traefik.access.message + - traefik.access.time ignore_missing: true - user_agent: field: user_agent.original @@ -103,6 +104,30 @@ processors: field: related.user value: "{{user.name}}" if: "ctx?.user?.name != null && ctx.user.name != '-'" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index 631673fe351..ce695210bef 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -8,6 +8,7 @@ "event.duration": 2000000, "event.kind": "event", "event.module": "traefik", + "event.original": "192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] \"GET /ui/favicons/favicon-16x16.png HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 262 \"Host-host-1\" \"http://172.19.0.3:5601\" 2ms", "event.outcome": "success", "event.type": [ "access" @@ -30,7 +31,9 @@ "traefik.access.frontend_name": "Host-host-1", "traefik.access.request_count": 262, "traefik.access.user_identifier": "-", + "url.extension": "png", "url.original": "/ui/favicons/favicon-16x16.png", + "url.path": "/ui/favicons/favicon-16x16.png", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -47,6 +50,7 @@ "event.duration": 3000000, "event.kind": "event", "event.module": "traefik", + "event.original": "85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] \"GET /ui/favicons/favicon.ico HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 271 \"Host-host1\" \"http://172.19.0.3:5601\" 3ms", "event.outcome": "success", "event.type": [ "access" @@ -79,7 +83,9 @@ "traefik.access.frontend_name": "Host-host1", "traefik.access.request_count": 271, "traefik.access.user_identifier": "-", + "url.extension": "ico", "url.original": "/ui/favicons/favicon.ico", + "url.path": "/ui/favicons/favicon.ico", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "Chrome", @@ -96,6 +102,7 @@ "event.duration": 247000000, "event.kind": "event", "event.module": "traefik", + "event.original": "70.29.80.15 - - [28/Feb/2018:17:30:33 +0000] \"GET /en/ HTTP/2.0\" 200 2814 - \"Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1\" 13 \"Host-host1-com-0\" \"http://172.19.0.6:14008\" 247ms", "event.outcome": "success", "event.type": [ "access" @@ -128,6 +135,7 @@ "traefik.access.request_count": 13, "traefik.access.user_identifier": "-", "url.original": "/en/", + "url.path": "/en/", "user.name": "-", "user_agent.device.name": "iPhone", "user_agent.name": "Mobile Safari", @@ -146,6 +154,7 @@ "event.duration": 0, "event.kind": "event", "event.module": "traefik", + "event.original": "::1 - - [29/Nov/2018:15:03:51 +0000] \"GET / HTTP/1.1\" 404 19 \"-\" \"curl/7.62.0\" 10 \"backend not found\" \"/\" 0ms", "event.outcome": "failure", "event.type": [ "access" @@ -169,6 +178,7 @@ "traefik.access.request_count": 10, "traefik.access.user_identifier": "-", "url.original": "/", + "url.path": "/", "user.name": "-", "user_agent.device.name": "Other", "user_agent.name": "curl", @@ -184,6 +194,7 @@ "event.duration": 13000000, "event.kind": "event", "event.module": "traefik", + "event.original": "94.254.131.115 - - [19/Jan/2018:10:01:02 +0000] \"GET /assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo HTTP/1.1\" 200 85 - \"Android\" 623112 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.9:4140\" 13ms", "event.outcome": "success", "event.type": [ "access" @@ -216,6 +227,8 @@ "traefik.access.request_count": 623112, "traefik.access.user_identifier": "-", "url.original": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", + "url.path": "/assets/52f8f2e711d235d76044799e/owners", + "url.query": "oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -231,6 +244,7 @@ "event.duration": 8000000, "event.kind": "event", "event.module": "traefik", + "event.original": "89.64.35.193 - - [19/Jan/2018:10:01:02 +0000] \"GET /marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM HTTP/1.1\" 200 150 - \"Android\" 623114 \"Host-api-wearerealitygames-com-2\" \"http://172.25.0.6:4140\" 8ms", "event.outcome": "success", "event.type": [ "access" @@ -263,6 +277,8 @@ "traefik.access.request_count": 623114, "traefik.access.user_identifier": "-", "url.original": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", + "url.path": "/marketplace/tax", + "url.query": "oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", "user.name": "-", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Other", @@ -277,6 +293,7 @@ "event.dataset": "traefik.access", "event.kind": "event", "event.module": "traefik", + "event.original": "127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] \"GET /apache_pb.gif HTTP/1.0\" 200 2326", "event.outcome": "success", "event.type": [ "access" @@ -298,7 +315,9 @@ "source.address": "127.0.0.1", "source.ip": "127.0.0.1", "traefik.access.user_identifier": "-", + "url.extension": "gif", "url.original": "/apache_pb.gif", + "url.path": "/apache_pb.gif", "user.name": "frank" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml index 4dea7d027c6..8cfaa7109c0 100644 --- a/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/s3access/ingest/pipeline.yml @@ -34,18 +34,19 @@ processors: field: aws.s3access.request_uri ignore_failure: true patterns: - - '%{NOTSPACE:http.request.method} %{NOTSPACE:url.original} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' + - '%{NOTSPACE:http.request.method} %{NOTSPACE:_temp_.uri_orig} [hH][tT][tT][pP]/%{NOTSPACE:http.version}' - # - # Best-effort parse of url.original in the form /path?query" - # - grok: - field: url.original + field: aws.s3access.host_header ignore_failure: true patterns: - - '^%{ABS_PATH:url.path}(?:\?%{DATA:url.query})?$' - pattern_definitions: - ABS_PATH: '/[^?]*' + - '(s3[a-z-]*).%{NOTSPACE:cloud.region}.%{WORD}.%{WORD}' + if: ctx.aws?.s3access?.host_header != null + + - uri_parts: + field: _temp_.uri_orig + ignore_failure: true + if: ctx?._temp_?.uri_orig != null - append: if: "ctx?.aws?.s3access?.bucket_owner != null" field: related.user @@ -188,6 +189,31 @@ processors: field: _temp_ ignore_missing: true + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + on_failure: - set: field: "error.message" diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log index f96091a7679..72b45f88ef2 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log @@ -3,4 +3,4 @@ 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 4DD6D17D1C5C401C REST.GET.BUCKET - "GET /test-s3-ks/?max-keys=0&encoding-type=url&aws-account=627959692251 HTTP/1.1" 200 - 265 - 2 1 "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - KzvchfojYQnuFC4PABYVJVxIlv/f6r17LRaTSvw7x+bxj4PkkPKT1kX9x8wbqtq40iD4PC881iE= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [01/Aug/2019:00:24:43 +0000] 72.21.217.31 arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9 706992E2F3CC3C3D REST.GET.LOCATION - "GET /test-s3-ks/?location&aws-account=627959692251 HTTP/1.1" 200 - 142 - 4 - "-" "AWS-Support-TrustedAdvisor, aws-internal/3 aws-sdk-java/1.11.590 Linux/4.9.137-0.1.ac.218.74.329.metal1.x86_64 OpenJDK_64-Bit_Server_VM/25.212-b03 java/1.8.0_212 vendor/Oracle_Corporation" - cIN12KTrJwx+uTBZD+opZUPE4iGypi8oG/oXGPzFk9CMuHQGuEpmAeNELdtYKDxf2TDor25Nikg= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 jsoriano-s3-test [10/Sep/2019:15:11:07 +0000] 77.227.156.41 arn:aws:iam::123456:user/test@elastic.co 8CD7A4A71E2E5C9E BATCH.DELETE.OBJECT jolokia-war-1.5.0.war - 204 - - 344017 - - - - - IeDW5I3wefFxU8iHOcAzi5qr+O+1bdRlcQ0AO2WGjFh7JwYM6qCoKq+1TrUshrXMlBxPFtg97Vk= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.eu-central-1.amazonaws.com TLSv1.2 -36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2 +36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2 diff --git a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json index aa9d1bf6938..37345ff30fc 100644 --- a/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/s3_server_access.log-expected.json @@ -22,6 +22,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -96,6 +97,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -171,6 +173,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.BUCKET", "event.category": "web", "event.dataset": "aws.s3access", @@ -245,6 +248,7 @@ "client.ip": "72.21.217.31", "client.user.id": "arn:aws:sts::123456:assumed-role/AWSServiceRoleForTrustedAdvisor/TrustedAdvisor_627959692251_784ab70b-8cc9-4d37-a2ec-2ff4d0c08af9", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "REST.GET.LOCATION", "event.category": "web", "event.dataset": "aws.s3access", @@ -317,6 +321,7 @@ "client.ip": "77.227.156.41", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "eu-central-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", @@ -360,7 +365,7 @@ "aws.s3access.bucket": "test-s3-ks", "aws.s3access.bucket_owner": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2", "aws.s3access.cipher_suite": "ECDHE-RSA-AES128-SHA", - "aws.s3access.host_header": "s3-ap-southeast-1.amazonaws.com", + "aws.s3access.host_header": "s3.ap-southeast-1.amazonaws.com", "aws.s3access.host_id": "LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0=", "aws.s3access.http_status": 204, "aws.s3access.key": "Screen+Shot+2019-09-09+at+9.08.44+AM.png", @@ -375,13 +380,14 @@ "client.ip": "174.29.206.152", "client.user.id": "arn:aws:iam::123456:user/test@elastic.co", "cloud.provider": "aws", + "cloud.region": "ap-southeast-1", "event.action": "BATCH.DELETE.OBJECT", "event.category": "web", "event.dataset": "aws.s3access", "event.id": "6CE38F1312D32BDD", "event.kind": "event", "event.module": "aws", - "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3-ap-southeast-1.amazonaws.com TLSv1.2", + "event.original": "36c1f05b76016b78528454e6e0c60e2b7ff7aa20c0a5e4c748276e5b0a2debd2 test-s3-ks [19/Sep/2019:17:06:39 +0000] 174.29.206.152 arn:aws:iam::123456:user/test@elastic.co 6CE38F1312D32BDD BATCH.DELETE.OBJECT Screen+Shot+2019-09-09+at+9.08.44+AM.png - 204 - - 57138 - - - - - LwRa4w6DbuU48GKQiH3jDbjfTyLCbwasFBsdttugRQ+9lH4jK8lT91+HhGZKMYI3sPyKuQ9LvU0= SigV4 ECDHE-RSA-AES128-SHA AuthHeader s3.ap-southeast-1.amazonaws.com TLSv1.2", "event.outcome": "success", "event.type": [ "access" diff --git a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json index f6ca4d4edf3..94866cfe505 100644 --- a/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json +++ b/x-pack/filebeat/module/aws/s3access/test/test.log-expected.json @@ -305,6 +305,7 @@ "tls.cipher": "ECDHE-RSA-AES128-SHA", "tls.version": "1.1", "tls.version_protocol": "tls", + "url.extension": "pdf", "url.original": "/awsexamplebucket/s3-dg.pdf", "url.path": "/awsexamplebucket/s3-dg.pdf", "user_agent.device.name": "Other", @@ -368,6 +369,7 @@ "tls.version": "1.1", "tls.version_protocol": "tls", "url.original": "*", + "url.path": "*", "user_agent.device.name": "Other", "user_agent.name": "Other", "user_agent.original": "S3Console/0.4" diff --git a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json index 35c5882513c..7f641d9becd 100644 --- a/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/additional_messages.log-expected.json @@ -2499,7 +2499,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2543,7 +2546,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/IOFUHSIU98[0]" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/IOFUHSIU98[0]", + "url.path": "/IOFUHSIU98[0]", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2587,7 +2593,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/some/longer/url-asd-er9789870[0]_=23", + "url.path": "/some/longer/url-asd-er9789870[0]_=23", + "url.scheme": "http" }, { "cisco.asa.message_id": "304001", @@ -2631,7 +2640,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://10.20.30.40/" + "url.domain": "10.20.30.40", + "url.original": "http://10.20.30.40/", + "url.path": "/", + "url.scheme": "http" }, { "cisco.asa.connection_id": "2751765169", diff --git a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json index 6a04d9e08e4..7a8c5d42848 100644 --- a/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/asa/test/sample.log-expected.json @@ -3516,7 +3516,8 @@ "cisco-asa", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3558,7 +3559,10 @@ "cisco-asa", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3602,7 +3606,11 @@ "cisco-asa", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" }, { "@timestamp": "2021-01-13T19:12:37.000-02:00", diff --git a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json index 865c5a2764e..a287b33f252 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/sample.log-expected.json @@ -3502,7 +3502,8 @@ "cisco-ftd", "forwarded" ], - "url.original": "/app" + "url.original": "/app", + "url.path": "/app" }, { "@timestamp": "2009-11-16T14:12:36.000-02:00", @@ -3543,7 +3544,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://example.com" + "url.domain": "example.com", + "url.original": "http://example.com", + "url.path": "", + "url.scheme": "http" }, { "@timestamp": "2009-11-16T14:12:37.000-02:00", @@ -3586,6 +3590,10 @@ "cisco-ftd", "forwarded" ], - "url.original": "http://www.example.net/images/favicon.ico" + "url.domain": "www.example.net", + "url.extension": "ico", + "url.original": "http://www.example.net/images/favicon.ico", + "url.path": "/images/favicon.ico", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json index be1d11ad0af..f65ccf20d71 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-connection.log-expected.json @@ -609,7 +609,10 @@ "forwarded" ], "url.domain": "eu-central-1.ec2.archive.ubuntu.com", + "url.extension": "deb", "url.original": "http://eu-central-1.ec2.archive.ubuntu.com/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.path": "/ubuntu/pool/main/m/manpages/manpages-dev_4.15-1_all.deb", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Debian APT-HTTP/1.3 (1.6.11)" @@ -822,7 +825,10 @@ "forwarded" ], "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" @@ -1014,8 +1020,15 @@ "cisco-ftd", "forwarded" ], - "url.domain": "10.0.100.30:8000", + "url.domain": [ + "10.0.100.30", + "10.0.100.30:8000" + ], + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "curl/7.58.0" diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json index 135a2979210..367c559c2e6 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-file-malware.log-expected.json @@ -69,7 +69,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -143,7 +148,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "exe", "url.original": "http://10.0.100.30:8000/exploit.exe", + "url.path": "/exploit.exe", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -217,7 +227,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "com", "url.original": "http://10.0.100.30:8000/eicar.com", + "url.path": "/eicar.com", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -291,7 +306,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "txt", "url.original": "http://10.0.100.30:8000/eicar.com.txt", + "url.path": "/eicar.com.txt", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -374,7 +394,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -457,7 +482,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -544,7 +574,12 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", + "url.extension": "zip", "url.original": "http://10.0.100.30:8000/eicar_com.zip", + "url.path": "/eicar_com.zip", + "url.port": 8000, + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -640,7 +675,11 @@ "cisco-ftd", "forwarded" ], + "url.domain": "www.eicar.org", + "url.extension": "zip", "url.original": "http://www.eicar.org/download/eicar_com.zip", + "url.path": "/download/eicar_com.zip", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -726,7 +765,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "10.0.100.30", "url.original": "http://10.0.100.30/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" }, @@ -823,7 +865,10 @@ "cisco-ftd", "forwarded" ], + "url.domain": "18.197.225.123", "url.original": "http://18.197.225.123/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.path": "/public/infected/dd3dee576d0cb4abfed00f97f0c71c1d", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required" } diff --git a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json index b23b07b6ac2..e98e7fc90cd 100644 --- a/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json +++ b/x-pack/filebeat/module/cisco/ftd/test/security-malware-site.log-expected.json @@ -117,8 +117,14 @@ "cisco-ftd", "forwarded" ], - "url.domain": "eyedropper-color-pick.info", + "url.domain": [ + "bad-malwaresite-grr.info", + "eyedropper-color-pick.info" + ], + "url.extension": "ico", "url.original": "http://bad-malwaresite-grr.info/favicon.ico", + "url.path": "/favicon.ico", + "url.scheme": "http", "user.id": "No Authentication Required", "user.name": "No Authentication Required", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36" diff --git a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml index a18507659aa..b80d31afad4 100644 --- a/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/meraki/ingest/pipeline.yml @@ -10,6 +10,25 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json index 32182b7f257..826a4f92194 100644 --- a/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cisco/meraki/test/generated.log-expected.json @@ -17,8 +17,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.15.44.253", - "10.193.124.51" + "10.193.124.51", + "10.15.44.253" ], "rsa.internal.event_desc": "olaborissecurity_event tur", "rsa.internal.messageid": "security_event", @@ -40,7 +40,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac" + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "tatemac", + "url.original": "https://example.org/odoco/ria.jpg?ritin=uredolor#tatemac", + "url.path": "/odoco/ria.jpg", + "url.query": "ritin=uredolor", + "url.scheme": "https" }, { "@timestamp": "2016-02-12T13:12:33.000Z", @@ -60,8 +66,8 @@ "appliance" ], "related.ip": [ - "10.15.16.212", - "10.102.218.31" + "10.102.218.31", + "10.15.16.212" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -214,7 +220,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu" + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "niamqu", + "url.original": "https://api.example.org/liqu/lorem.gif?ueipsaqu=uidolore#niamqu", + "url.path": "/liqu/lorem.gif", + "url.query": "ueipsaqu=uidolore", + "url.scheme": "https" }, { "@timestamp": "2016-04-24T00:25:25.000Z", @@ -377,7 +389,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu" + "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "quu", + "url.original": "https://www5.example.net/sun/essecill.html?saute=vel#quu", + "url.path": "/sun/essecill.html", + "url.query": "saute=vel", + "url.scheme": "https" }, { "@timestamp": "2016-07-04T11:38:16.000Z", @@ -569,8 +587,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.187.77.245", - "10.88.231.224" + "10.88.231.224", + "10.187.77.245" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -618,7 +636,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci" + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "xerci", + "url.original": "https://internal.example.com/quae/maccusa.htm?rQuisau=idex#xerci", + "url.path": "/quae/maccusa.htm", + "url.query": "rQuisau=idex", + "url.scheme": "https" }, { "@timestamp": "2016-10-12T12:56:16.000Z", @@ -710,8 +734,8 @@ "appliance" ], "related.ip": [ - "10.182.178.217", - "10.63.194.87" + "10.63.194.87", + "10.182.178.217" ], "rsa.counters.dclass_r1": "fdeFi", "rsa.internal.messageid": "events", @@ -730,7 +754,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate" + "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "uptate", + "url.original": "https://internal.example.net/ris/uamqu.txt?liqui=quioffi#uptate", + "url.path": "/ris/uamqu.txt", + "url.query": "liqui=quioffi", + "url.scheme": "https" }, { "@timestamp": "2016-11-24T10:03:59.000Z", @@ -773,7 +803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt" + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "volupt", + "url.original": "https://mail.example.com/icabo/itatio.jpg?eleum=sintoc#volupt", + "url.path": "/icabo/itatio.jpg", + "url.query": "eleum=sintoc", + "url.scheme": "https" }, { "@timestamp": "2016-12-08T17:06:33.000Z", @@ -926,8 +962,8 @@ "appliance" ], "related.ip": [ - "10.93.68.231", - "10.135.217.12" + "10.135.217.12", + "10.93.68.231" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -993,8 +1029,8 @@ "appliance" ], "related.ip": [ - "10.247.30.212", - "10.66.89.5" + "10.66.89.5", + "10.247.30.212" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1058,8 +1094,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.221.102.245", - "10.173.136.186" + "10.173.136.186", + "10.221.102.245" ], "rsa.internal.event_desc": "idestlab", "rsa.internal.messageid": "security_event", @@ -1097,8 +1133,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.54.37.86", - "10.58.64.108" + "10.58.64.108", + "10.54.37.86" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1216,8 +1252,8 @@ "appliance" ], "related.ip": [ - "10.28.144.180", - "10.148.124.84" + "10.148.124.84", + "10.28.144.180" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1277,7 +1313,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt" + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "avolupt", + "url.original": "https://example.net/ites/isetq.gif?nisiut=tur#avolupt", + "url.path": "/ites/isetq.gif", + "url.query": "nisiut=tur", + "url.scheme": "https" }, { "@timestamp": "2017-06-26T19:42:33.000Z", @@ -1309,7 +1351,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "corpori", + "url.original": "https://www.example.org/ueipsa/tae.html?eriti=atcupi#corpori", + "url.path": "/ueipsa/tae.html", + "url.query": "eriti=atcupi", + "url.scheme": "https" }, { "@timestamp": "2017-07-11T02:45:07.000Z", @@ -1354,8 +1402,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.150.245.88", - "10.242.77.170" + "10.242.77.170", + "10.150.245.88" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1391,8 +1439,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.180.195.43", - "10.247.139.239" + "10.247.139.239", + "10.180.195.43" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1453,8 +1501,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.94.6.140", - "10.147.15.213" + "10.147.15.213", + "10.94.6.140" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -1554,7 +1602,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "ecatcupi", + "url.original": "https://www.example.net/irured/illumqui.txt?tionula=ritqu#ecatcupi", + "url.path": "/irured/illumqui.txt", + "url.query": "tionula=ritqu", + "url.scheme": "https" }, { "@timestamp": "2017-10-19T04:03:07.000Z", @@ -1735,7 +1789,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori" + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "tlabori", + "url.original": "https://mail.example.net/uat/lupta.html?uptassit=ncidi#tlabori", + "url.path": "/uat/lupta.html", + "url.query": "uptassit=ncidi", + "url.scheme": "https" }, { "@timestamp": "2018-01-12T22:18:32.000Z", @@ -1919,8 +1979,8 @@ "appliance" ], "related.ip": [ - "10.39.172.93", - "10.83.131.245" + "10.83.131.245", + "10.39.172.93" ], "rsa.internal.messageid": "flows", "rsa.misc.action": [ @@ -1958,8 +2018,8 @@ "appliance" ], "related.ip": [ - "10.201.168.116", - "10.86.188.179" + "10.86.188.179", + "10.201.168.116" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -1996,8 +2056,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.122.204.151", - "10.148.211.222" + "10.148.211.222", + "10.122.204.151" ], "rsa.internal.event_desc": "umexercisecurity_event duntut", "rsa.internal.messageid": "security_event", @@ -2019,7 +2079,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits" + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "civelits", + "url.original": "https://mail.example.com/prehend/eufug.htm?eufug=est#civelits", + "url.path": "/prehend/eufug.htm", + "url.query": "eufug=est", + "url.scheme": "https" }, { "@timestamp": "2018-05-07T06:39:06.000Z", @@ -2069,8 +2135,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.120.4.9", - "10.97.46.16" + "10.97.46.16", + "10.120.4.9" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2105,8 +2171,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.165.173.162", - "10.171.206.139" + "10.171.206.139", + "10.165.173.162" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -2143,8 +2209,8 @@ "uames4985.mail.localdomain" ], "related.ip": [ - "10.144.57.239", - "10.150.163.151" + "10.150.163.151", + "10.144.57.239" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2317,8 +2383,8 @@ "appliance" ], "related.ip": [ - "10.103.49.129", - "10.2.110.73" + "10.2.110.73", + "10.103.49.129" ], "rsa.counters.dclass_r1": "orumS", "rsa.internal.messageid": "events", @@ -2337,7 +2403,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "tatisetq", + "url.original": "https://www.example.net/doconse/eni.html?mSec=smoditem#tatisetq", + "url.path": "/doconse/eni.html", + "url.query": "mSec=smoditem", + "url.scheme": "https" }, { "@timestamp": "2018-09-12T22:02:15.000Z", @@ -2380,7 +2452,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "urExc", + "url.original": "https://www.example.com/seq/moll.htm?sunt=dquianon#urExc", + "url.path": "/seq/moll.htm", + "url.query": "sunt=dquianon", + "url.scheme": "https" }, { "@timestamp": "2018-09-27T05:04:49.000Z", @@ -2402,8 +2480,8 @@ "lors2232.api.example" ], "related.ip": [ - "10.105.136.146", - "10.46.217.155" + "10.46.217.155", + "10.105.136.146" ], "rsa.internal.messageid": "events", "rsa.misc.event_source": "appliance", @@ -2535,7 +2613,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames" + "url.domain": "www.example.net", + "url.extension": "html", + "url.fragment": "uames", + "url.original": "https://www.example.net/sectetur/edquian.html?turQuis=taevi#uames", + "url.path": "/sectetur/edquian.html", + "url.query": "turQuis=taevi", + "url.scheme": "https" }, { "@timestamp": "2018-11-23T09:15:06.000Z", @@ -2558,8 +2642,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.34.62.190", - "10.246.152.72" + "10.246.152.72", + "10.34.62.190" ], "rsa.internal.event_desc": "Nem", "rsa.internal.messageid": "security_event", @@ -2719,7 +2803,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor" + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", + "url.original": "https://www.example.com/totamrem/aliqu.htm?sBonorum=moenimi#lor", + "url.path": "/totamrem/aliqu.htm", + "url.query": "sBonorum=moenimi", + "url.scheme": "https" }, { "@timestamp": "2019-02-17T03:30:32.000Z", @@ -3117,8 +3207,8 @@ "appliance" ], "related.ip": [ - "10.247.118.132", - "10.254.96.130" + "10.254.96.130", + "10.247.118.132" ], "rsa.counters.dclass_r1": "ectet", "rsa.internal.messageid": "events", @@ -3137,7 +3227,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "taevitae", + "url.original": "https://www5.example.org/eturadi/umS.txt?mSecti=henderi#taevitae", + "url.path": "/eturadi/umS.txt", + "url.query": "mSecti=henderi", + "url.scheme": "https" }, { "@timestamp": "2019-08-07T16:01:23.000Z", @@ -3160,8 +3256,8 @@ "appliance" ], "related.ip": [ - "10.101.13.122", - "10.200.98.243" + "10.200.98.243", + "10.101.13.122" ], "rsa.counters.dclass_r1": "uteirur", "rsa.internal.messageid": "events", @@ -3180,7 +3276,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate" + "url.domain": "internal.example.org", + "url.extension": "htm", + "url.fragment": "tate", + "url.original": "https://internal.example.org/fdeFi/ratv.htm?sequatu=tiumtot#tate", + "url.path": "/fdeFi/ratv.htm", + "url.query": "sequatu=tiumtot", + "url.scheme": "https" }, { "@timestamp": "2019-08-21T23:03:57.000Z", @@ -3213,7 +3315,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec" + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "tiaec", + "url.original": "https://www5.example.net/uov/pariat.htm?litsed=lumd#tiaec", + "url.path": "/uov/pariat.htm", + "url.query": "litsed=lumd", + "url.scheme": "https" }, { "@timestamp": "2019-09-05T06:06:31.000Z", @@ -3299,8 +3407,8 @@ "observer.type": "Wireless", "observer.vendor": "Cisco", "related.ip": [ - "10.137.166.97", - "10.162.202.14" + "10.162.202.14", + "10.137.166.97" ], "rsa.internal.messageid": "ids-alerts", "rsa.misc.event_type": "ids-alerts", @@ -3347,7 +3455,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "itinvolu", + "url.original": "https://www5.example.com/illoinve/etcon.htm?nevolup=erspici#itinvolu", + "url.path": "/illoinve/etcon.htm", + "url.query": "nevolup=erspici", + "url.scheme": "https" }, { "@timestamp": "2019-11-01T10:16:48.000Z", @@ -3446,7 +3560,13 @@ "cisco.meraki", "forwarded" ], - "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco" + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "exeaco", + "url.original": "https://example.com/qui/itse.gif?orsitame=tasn#exeaco", + "url.path": "/qui/itse.gif", + "url.query": "orsitame=tasn", + "url.scheme": "https" }, { "@timestamp": "2019-12-14T07:24:31.000Z", diff --git a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml index b016a5c3fd0..19775fe0be0 100644 --- a/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml +++ b/x-pack/filebeat/module/cisco/shared/ingest/asa-ftd-pipeline.yml @@ -1646,6 +1646,23 @@ processors: ) ) + - set: + field: _temp_.url_domain + value: "{{url.domain}}" + ignore_failure: true + if: ctx?.url?.domain != null + + - uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null + - append: + field: url.domain + value: "{{_temp_.url_domain}}" + ignore_failure: true + allow_duplicates: false + if: ctx?._temp_?.url_domain != null + # # Populate ECS event.code # @@ -1882,6 +1899,30 @@ processors: value: "{{source.domain}}" if: ctx.source?.domain != null && ctx.source?.domain != '' allow_duplicates: false + - script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: # Copy any fields under _temp_.cisco to its final destination. Those can help # with diagnosing the failure. diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml index 2a602ff2331..b659b2fd5ed 100644 --- a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -120,6 +120,11 @@ processors: value: firewall if: ctx?.log?.file?.path.contains('cloudfirewalllogs') +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null + # Identifies is a field that includes any sort of username, device or other asset that is included in the request. # Converting this to an array to make it easier to use in searches and visualizations - split: @@ -198,6 +203,8 @@ processors: - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" ignore_failure: true +- community_id: + ignore_missing: true ###################### # Related ECS Fields # ###################### diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json index 65aabab5a88..6cedb19cced 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json @@ -16,6 +16,7 @@ "input.type": "log", "log.offset": 0, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "outbound", "network.transport": "1", "observer.product": "Umbrella", @@ -52,6 +53,7 @@ "input.type": "log", "log.offset": 128, "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "network.community_id": "1:9s/mgn3WChavy6kpCk25Ed4VCcU=", "network.direction": "inbound", "network.transport": "1", "observer.product": "Umbrella", diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json index fd474d2d029..4f0b9552eb4 100644 --- a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json @@ -34,7 +34,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -72,7 +75,10 @@ "source.address": "192.168.1.1", "source.ip": "192.168.1.1", "source.nat.ip": "1.1.1.1", - "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "url.domain": "elastic.co", + "url.original": "https://elastic.co/blog/ext_id=Anyclip", + "url.path": "/blog/ext_id=Anyclip", + "url.scheme": "https", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" }, { @@ -109,7 +115,11 @@ "source.address": "ActiveDirectoryUserName,ADSite,Network", "source.domain": "ActiveDirectoryUserName,ADSite,Network", "source.nat.ip": "192.192.192.135", - "url.full": "http://google.com/the.js", + "url.domain": "google.com", + "url.extension": "js", + "url.original": "http://google.com/the.js", + "url.path": "/the.js", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml index 1c939d65a22..ae772c9b6a3 100644 --- a/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipafm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json index 0a259d4811c..f2227c07cb1 100644 --- a/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipafm/test/generated.log-expected.json @@ -24,10 +24,10 @@ "tatemac3541.api.corp" ], "related.ip": [ + "10.165.201.71", "10.208.121.85", - "10.11.196.142", "10.228.193.207", - "10.165.201.71" + "10.11.196.142" ], "related.user": [ "billoi" @@ -92,9 +92,9 @@ "enatus2114.mail.home" ], "related.ip": [ + "10.92.202.200", "10.51.132.10", "10.162.9.235", - "10.92.202.200", "10.94.67.230" ], "related.user": [ @@ -161,9 +161,9 @@ ], "related.ip": [ "10.82.56.117", + "10.122.116.161", "10.209.155.149", - "10.191.68.244", - "10.122.116.161" + "10.191.68.244" ], "related.user": [ "seq" @@ -228,8 +228,8 @@ ], "related.ip": [ "10.50.112.141", - "10.131.233.27", "10.12.44.169", + "10.131.233.27", "10.202.66.28" ], "related.user": [ @@ -295,10 +295,10 @@ "emquiavo452.internal.localhost" ], "related.ip": [ - "10.159.182.171", "10.96.35.212", - "10.206.197.113", - "10.151.111.38" + "10.151.111.38", + "10.159.182.171", + "10.206.197.113" ], "related.user": [ "mol" @@ -363,10 +363,10 @@ "sun1403.www.invalid" ], "related.ip": [ - "10.213.113.28", - "10.169.144.147", "10.126.177.162", - "10.89.163.114" + "10.213.113.28", + "10.89.163.114", + "10.169.144.147" ], "related.user": [ "ist" @@ -430,9 +430,9 @@ "ittenbyC7838.api.localdomain" ], "related.ip": [ - "10.18.124.28", "10.101.223.43", "10.103.107.47", + "10.18.124.28", "10.146.88.52" ], "related.user": [ @@ -498,10 +498,10 @@ "ume465.corp" ], "related.ip": [ - "10.150.220.75", - "10.110.99.17", "10.189.109.245", - "10.69.57.206" + "10.110.99.17", + "10.69.57.206", + "10.150.220.75" ], "related.user": [ "onse" @@ -565,10 +565,10 @@ "iciatisu1463.www5.localdomain" ], "related.ip": [ + "10.121.219.204", "10.153.136.222", - "10.19.194.101", "10.199.34.241", - "10.121.219.204" + "10.19.194.101" ], "related.user": [ "temveleu" @@ -632,10 +632,10 @@ "aliqu6801.api.localdomain" ], "related.ip": [ + "10.46.27.57", "10.57.103.192", "10.64.141.105", - "10.182.199.231", - "10.46.27.57" + "10.182.199.231" ], "related.user": [ "ice" @@ -699,10 +699,10 @@ "itame189.domain" ], "related.ip": [ - "10.160.210.31", - "10.164.6.207", "10.3.134.237", - "10.32.67.231" + "10.32.67.231", + "10.164.6.207", + "10.160.210.31" ], "related.user": [ "pic" @@ -768,9 +768,9 @@ ], "related.ip": [ "10.235.101.253", - "10.201.6.10", "10.42.138.192", - "10.182.178.217" + "10.182.178.217", + "10.201.6.10" ], "related.user": [ "giatnu" @@ -836,9 +836,9 @@ ], "related.ip": [ "10.151.161.70", + "10.22.102.198", "10.194.247.171", - "10.86.101.235", - "10.22.102.198" + "10.86.101.235" ], "related.user": [ "nse" @@ -970,9 +970,9 @@ "smo7167.www.test" ], "related.ip": [ - "10.182.191.174", "10.214.249.164", "10.99.249.210", + "10.182.191.174", "10.81.26.208" ], "related.user": [ @@ -1037,10 +1037,10 @@ "sauteiru4554.api.domain" ], "related.ip": [ - "10.101.226.128", "10.220.5.143", - "10.88.101.53", - "10.201.238.90" + "10.201.238.90", + "10.101.226.128", + "10.88.101.53" ], "related.user": [ "porro" @@ -1104,9 +1104,9 @@ "untut4046.internal.domain" ], "related.ip": [ + "10.217.150.196", "10.30.133.66", "10.157.18.252", - "10.217.150.196", "10.243.218.215" ], "related.user": [ @@ -1171,10 +1171,10 @@ "quid3147.mail.home" ], "related.ip": [ - "10.181.133.187", - "10.148.161.250", "10.66.181.6", - "10.167.227.44" + "10.181.133.187", + "10.167.227.44", + "10.148.161.250" ], "related.user": [ "adipisc" @@ -1239,9 +1239,9 @@ "umdolo1029.mail.localhost" ], "related.ip": [ - "10.74.11.43", "10.84.163.178", "10.107.9.163", + "10.74.11.43", "10.54.17.32" ], "related.user": [ @@ -1306,10 +1306,10 @@ "lorsita2019.internal.home" ], "related.ip": [ + "10.184.73.211", "10.192.229.221", "10.112.32.213", - "10.230.129.252", - "10.184.73.211" + "10.230.129.252" ], "related.user": [ "odi" @@ -1375,9 +1375,9 @@ ], "related.ip": [ "10.198.213.189", - "10.161.148.64", + "10.7.200.140", "10.199.216.143", - "10.7.200.140" + "10.161.148.64" ], "related.user": [ "ccaeca" @@ -1441,10 +1441,10 @@ "mex2054.mail.corp" ], "related.ip": [ + "10.65.232.27", "10.206.96.56", "10.128.157.27", - "10.22.187.69", - "10.65.232.27" + "10.22.187.69" ], "related.user": [ "uaeab" @@ -1508,10 +1508,10 @@ "avolupt7576.api.corp" ], "related.ip": [ - "10.71.114.14", - "10.194.210.62", "10.68.253.120", - "10.183.130.225" + "10.71.114.14", + "10.183.130.225", + "10.194.210.62" ], "related.user": [ "admin" @@ -1577,8 +1577,8 @@ ], "related.ip": [ "10.107.45.175", - "10.31.177.226", "10.47.255.237", + "10.31.177.226", "10.45.253.103" ], "related.user": [ @@ -1644,10 +1644,10 @@ "nsequat1971.internal.invalid" ], "related.ip": [ - "10.55.105.113", + "10.213.94.135", "10.225.212.189", - "10.44.58.106", - "10.213.94.135" + "10.55.105.113", + "10.44.58.106" ], "related.user": [ "dquia" @@ -1711,10 +1711,10 @@ "ectiono2241.lan" ], "related.ip": [ - "10.163.209.70", "10.255.74.136", - "10.69.161.78", - "10.2.114.9" + "10.2.114.9", + "10.163.209.70", + "10.69.161.78" ], "related.user": [ "olabor" @@ -1778,10 +1778,10 @@ "umetMal1664.mail.lan" ], "related.ip": [ - "10.184.59.148", + "10.252.102.110", "10.12.129.137", "10.46.115.216", - "10.252.102.110" + "10.184.59.148" ], "related.user": [ "perspici" @@ -1846,10 +1846,10 @@ "derit5270.mail.local" ], "related.ip": [ + "10.199.194.79", "10.81.184.7", "10.155.204.243", - "10.105.52.140", - "10.199.194.79" + "10.105.52.140" ], "related.user": [ "eetd" @@ -1914,10 +1914,10 @@ "orisni5238.mail.lan" ], "related.ip": [ - "10.18.226.72", + "10.251.231.142", "10.177.238.45", "10.110.2.166", - "10.251.231.142" + "10.18.226.72" ], "related.user": [ "taliqui" @@ -1982,9 +1982,9 @@ "iutali7297.www.domain" ], "related.ip": [ - "10.190.122.27", - "10.100.199.226", "10.99.202.229", + "10.100.199.226", + "10.190.122.27", "10.192.98.247" ], "related.user": [ @@ -2050,10 +2050,10 @@ "orumw5960.www5.home" ], "related.ip": [ - "10.172.154.97", "10.248.111.207", - "10.37.193.70", - "10.162.97.197" + "10.172.154.97", + "10.162.97.197", + "10.37.193.70" ], "related.user": [ "culpaq" @@ -2117,10 +2117,10 @@ "oinv5493.internal.domain" ], "related.ip": [ - "10.171.221.230", - "10.45.35.180", "10.36.63.31", - "10.222.165.250" + "10.171.221.230", + "10.222.165.250", + "10.45.35.180" ], "related.user": [ "otamr" @@ -2184,9 +2184,9 @@ "tnonproi195.api.home" ], "related.ip": [ - "10.199.127.211", - "10.83.238.145", "10.238.4.219", + "10.83.238.145", + "10.199.127.211", "10.1.171.61" ], "related.user": [ @@ -2251,9 +2251,9 @@ "edictasu5362.internal.localhost" ], "related.ip": [ - "10.44.226.104", "10.170.252.219", "10.65.141.244", + "10.44.226.104", "10.74.213.42" ], "related.user": [ @@ -2318,9 +2318,9 @@ "uido492.www5.home" ], "related.ip": [ - "10.225.255.211", "10.225.141.172", "10.183.223.149", + "10.225.255.211", "10.180.48.221" ], "related.user": [ @@ -2385,9 +2385,9 @@ "redo6311.api.invalid" ], "related.ip": [ + "10.169.123.103", "10.176.64.28", "10.97.138.181", - "10.169.123.103", "10.205.174.181" ], "related.user": [ @@ -2453,10 +2453,10 @@ "dolorem1698.www.domain" ], "related.ip": [ - "10.53.101.131", - "10.204.4.40", "10.75.120.11", - "10.169.101.161" + "10.204.4.40", + "10.169.101.161", + "10.53.101.131" ], "related.user": [ "tquo" @@ -2523,8 +2523,8 @@ "related.ip": [ "10.28.51.219", "10.6.222.112", - "10.87.120.87", - "10.156.117.169" + "10.156.117.169", + "10.87.120.87" ], "related.user": [ "onsequu" @@ -2588,10 +2588,10 @@ "arc2412.mail.lan" ], "related.ip": [ - "10.57.89.155", - "10.4.126.103", + "10.247.44.59", "10.253.167.17", - "10.247.44.59" + "10.4.126.103", + "10.57.89.155" ], "related.user": [ "ntorever" @@ -2655,10 +2655,10 @@ "olorsi2746.internal.localhost" ], "related.ip": [ - "10.143.183.208", - "10.36.69.125", "10.15.240.220", - "10.248.206.210" + "10.248.206.210", + "10.36.69.125", + "10.143.183.208" ], "related.user": [ "met" @@ -2723,10 +2723,10 @@ "edqu2208.www.localhost" ], "related.ip": [ + "10.34.133.2", "10.69.170.107", - "10.6.32.7", "10.142.186.43", - "10.34.133.2" + "10.6.32.7" ], "related.user": [ "ipitlabo" @@ -2792,9 +2792,9 @@ ], "related.ip": [ "10.142.22.24", - "10.59.103.10", "10.170.165.164", - "10.121.153.197" + "10.121.153.197", + "10.59.103.10" ], "related.user": [ "borumSec" @@ -2859,9 +2859,9 @@ "sis3986.internal.lan" ], "related.ip": [ - "10.247.114.30", - "10.133.10.122", "10.19.99.129", + "10.133.10.122", + "10.247.114.30", "10.176.83.7" ], "related.user": [ @@ -2928,9 +2928,9 @@ ], "related.ip": [ "10.8.29.219", - "10.70.7.23", "10.40.177.138", - "10.64.139.17" + "10.64.139.17", + "10.70.7.23" ], "related.user": [ "rep" @@ -2995,9 +2995,9 @@ ], "related.ip": [ "10.67.173.228", + "10.180.62.222", "10.2.189.20", - "10.67.221.220", - "10.180.62.222" + "10.67.221.220" ], "related.user": [ "uptasnul" @@ -3062,10 +3062,10 @@ "uian521.www.example" ], "related.ip": [ + "10.196.176.243", "10.147.127.181", - "10.209.52.47", "10.56.134.118", - "10.196.176.243" + "10.209.52.47" ], "related.user": [ "tasu" @@ -3130,8 +3130,8 @@ ], "related.ip": [ "10.85.13.237", - "10.231.18.90", "10.248.140.59", + "10.231.18.90", "10.226.24.84" ], "related.user": [ @@ -3265,10 +3265,10 @@ "mexer3864.api.corp" ], "related.ip": [ - "10.255.145.22", "10.73.84.95", - "10.230.38.148", - "10.98.154.146" + "10.98.154.146", + "10.255.145.22", + "10.230.38.148" ], "related.user": [ "sitam" @@ -3332,8 +3332,8 @@ "oluptat6960.www5.test" ], "related.ip": [ - "10.166.142.198", "10.105.120.162", + "10.166.142.198", "10.175.181.138", "10.211.29.187" ], @@ -3400,10 +3400,10 @@ "fugiatnu2498.www.localhost" ], "related.ip": [ - "10.182.213.195", "10.122.133.162", + "10.220.202.102", "10.195.139.25", - "10.220.202.102" + "10.182.213.195" ], "related.user": [ "aquae" @@ -3468,9 +3468,9 @@ "ptat3230.domain" ], "related.ip": [ - "10.156.208.5", "10.53.72.161", "10.33.143.163", + "10.156.208.5", "10.247.144.9" ], "related.user": [ @@ -3535,10 +3535,10 @@ "exer447.internal.localhost" ], "related.ip": [ - "10.113.65.192", "10.241.143.145", "10.35.190.164", - "10.21.58.162" + "10.21.58.162", + "10.113.65.192" ], "related.user": [ "porin" @@ -3672,9 +3672,9 @@ ], "related.ip": [ "10.150.153.61", - "10.125.150.220", + "10.120.50.13", "10.22.213.196", - "10.120.50.13" + "10.125.150.220" ], "related.user": [ "inculpa" @@ -3740,9 +3740,9 @@ ], "related.ip": [ "10.85.52.249", + "10.238.171.184", "10.229.155.171", - "10.113.2.13", - "10.238.171.184" + "10.113.2.13" ], "related.user": [ "tatiset" @@ -3807,10 +3807,10 @@ "tatis7315.mail.home" ], "related.ip": [ - "10.249.174.35", - "10.198.150.185", "10.51.245.225", - "10.220.1.249" + "10.220.1.249", + "10.249.174.35", + "10.198.150.185" ], "related.user": [ "quela" @@ -3875,9 +3875,9 @@ "eosqui3723.api.localdomain" ], "related.ip": [ + "10.152.157.32", "10.190.96.181", "10.38.185.31", - "10.152.157.32", "10.251.82.195" ], "related.user": [ @@ -3942,10 +3942,10 @@ "itaedict199.mail.corp" ], "related.ip": [ - "10.230.112.179", - "10.103.102.242", + "10.190.247.194", "10.211.198.50", - "10.190.247.194" + "10.103.102.242", + "10.230.112.179" ], "related.user": [ "tDuisaut" @@ -4078,9 +4078,9 @@ ], "related.ip": [ "10.21.30.43", - "10.83.136.233", + "10.21.80.157", "10.31.86.83", - "10.21.80.157" + "10.83.136.233" ], "related.user": [ "litsed" @@ -4145,10 +4145,10 @@ "oluptas1637.home" ], "related.ip": [ - "10.195.90.73", - "10.194.197.107", "10.45.152.205", - "10.27.181.27" + "10.27.181.27", + "10.195.90.73", + "10.194.197.107" ], "related.user": [ "datatn" @@ -4213,10 +4213,10 @@ "ididu5505.api.localdomain" ], "related.ip": [ - "10.43.239.97", - "10.129.161.18", "10.222.2.132", - "10.183.90.25" + "10.129.161.18", + "10.183.90.25", + "10.43.239.97" ], "related.user": [ "aedicta" @@ -4280,10 +4280,10 @@ "mqui1099.api.corp" ], "related.ip": [ - "10.231.167.171", - "10.248.156.138", + "10.189.162.131", "10.67.129.100", - "10.189.162.131" + "10.248.156.138", + "10.231.167.171" ], "related.user": [ "sedquia" @@ -4349,9 +4349,9 @@ ], "related.ip": [ "10.63.103.30", - "10.185.107.27", + "10.6.146.184", "10.142.106.66", - "10.6.146.184" + "10.185.107.27" ], "related.user": [ "sequu" @@ -4415,10 +4415,10 @@ "tatiset4191.localdomain" ], "related.ip": [ - "10.119.179.182", + "10.214.93.200", "10.0.202.9", - "10.93.39.237", - "10.214.93.200" + "10.119.179.182", + "10.93.39.237" ], "related.user": [ "tionofd" @@ -4483,10 +4483,10 @@ "aute2433.mail.lan" ], "related.ip": [ - "10.28.145.163", + "10.123.154.140", "10.252.204.162", - "10.30.189.166", - "10.123.154.140" + "10.28.145.163", + "10.30.189.166" ], "related.user": [ "imadmin" @@ -4550,10 +4550,10 @@ "idolo6535.internal.example" ], "related.ip": [ - "10.46.162.198", - "10.79.49.3", + "10.145.128.250", "10.29.122.183", - "10.145.128.250" + "10.79.49.3", + "10.46.162.198" ], "related.user": [ "eni" @@ -4618,10 +4618,10 @@ "one7728.api.localdomain" ], "related.ip": [ + "10.142.235.217", "10.166.169.167", "10.65.174.196", - "10.177.232.136", - "10.142.235.217" + "10.177.232.136" ], "related.user": [ "olors" @@ -4686,10 +4686,10 @@ "uptatem4446.internal.localhost" ], "related.ip": [ - "10.53.188.140", + "10.29.217.44", "10.215.184.154", "10.191.78.86", - "10.29.217.44" + "10.53.188.140" ], "related.user": [ "iarc" @@ -4754,10 +4754,10 @@ "emq2514.api.localhost" ], "related.ip": [ + "10.46.222.149", "10.74.74.129", - "10.76.148.147", "10.135.77.156", - "10.46.222.149" + "10.76.148.147" ], "related.user": [ "urve" @@ -4821,10 +4821,10 @@ "agna5654.www.corp" ], "related.ip": [ - "10.130.203.37", - "10.145.49.29", "10.96.200.223", - "10.11.146.253" + "10.11.146.253", + "10.130.203.37", + "10.145.49.29" ], "related.user": [ "mvele" @@ -4889,9 +4889,9 @@ ], "related.ip": [ "10.162.78.48", - "10.48.75.140", + "10.162.2.180", "10.24.23.209", - "10.162.2.180" + "10.48.75.140" ], "related.user": [ "rumwr" @@ -4955,10 +4955,10 @@ "sequatD163.internal.example" ], "related.ip": [ - "10.151.206.38", - "10.66.92.83", + "10.97.105.115", "10.119.12.186", - "10.97.105.115" + "10.151.206.38", + "10.66.92.83" ], "related.user": [ "nproide" @@ -5090,10 +5090,10 @@ "epr3512.internal.domain" ], "related.ip": [ - "10.35.38.185", "10.111.128.11", "10.200.116.191", - "10.9.236.18" + "10.9.236.18", + "10.35.38.185" ], "related.user": [ "umfug" @@ -5157,10 +5157,10 @@ "uredol2174.home" ], "related.ip": [ - "10.191.27.182", + "10.236.67.227", "10.240.62.238", - "10.134.238.8", - "10.236.67.227" + "10.191.27.182", + "10.134.238.8" ], "related.user": [ "tlabo" @@ -5224,10 +5224,10 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.22.231.91", + "10.65.35.64", "10.109.14.142", - "10.165.66.92", - "10.65.35.64" + "10.22.231.91", + "10.165.66.92" ], "related.user": [ "perna" @@ -5291,10 +5291,10 @@ "inimav5557.www5.test" ], "related.ip": [ + "10.29.230.203", "10.71.112.86", - "10.64.161.215", "10.89.221.90", - "10.29.230.203" + "10.64.161.215" ], "related.user": [ "rnatur" @@ -5358,10 +5358,10 @@ "nonn1650.www.test" ], "related.ip": [ - "10.88.226.76", - "10.221.199.137", + "10.79.208.135", "10.140.118.182", - "10.79.208.135" + "10.221.199.137", + "10.88.226.76" ], "related.user": [ "erspic" @@ -5427,9 +5427,9 @@ ], "related.ip": [ "10.35.73.208", - "10.126.61.230", "10.189.244.22", - "10.133.48.55" + "10.133.48.55", + "10.126.61.230" ], "related.user": [ "tia" @@ -5493,8 +5493,8 @@ "suscipit587.www.localhost" ], "related.ip": [ - "10.35.65.72", "10.240.94.109", + "10.35.65.72", "10.239.194.105", "10.81.154.115" ], @@ -5561,10 +5561,10 @@ "mnisiut6146.internal.local" ], "related.ip": [ - "10.248.72.104", + "10.52.70.192", "10.38.253.213", "10.150.56.227", - "10.52.70.192" + "10.248.72.104" ], "related.user": [ "ionem" @@ -5629,10 +5629,10 @@ "borios1067.www5.home" ], "related.ip": [ + "10.218.15.164", "10.62.218.239", - "10.73.172.186", "10.203.193.134", - "10.218.15.164" + "10.73.172.186" ], "related.user": [ "reh" @@ -5696,10 +5696,10 @@ "msequ323.www.example" ], "related.ip": [ - "10.10.46.43", - "10.131.127.113", "10.136.211.234", - "10.60.20.76" + "10.131.127.113", + "10.60.20.76", + "10.10.46.43" ], "related.user": [ "nev" @@ -5766,8 +5766,8 @@ "related.ip": [ "10.233.181.250", "10.50.177.151", - "10.187.237.220", - "10.248.0.74" + "10.248.0.74", + "10.187.237.220" ], "related.user": [ "ugiatq" @@ -5833,9 +5833,9 @@ ], "related.ip": [ "10.189.43.11", + "10.248.248.120", "10.80.129.81", - "10.96.223.46", - "10.248.248.120" + "10.96.223.46" ], "related.user": [ "iatn" @@ -5902,8 +5902,8 @@ "related.ip": [ "10.91.115.139", "10.102.109.199", - "10.173.114.63", - "10.66.106.186" + "10.66.106.186", + "10.173.114.63" ], "related.user": [ "tNequ" @@ -5968,10 +5968,10 @@ "orpori3334.www.local" ], "related.ip": [ - "10.0.175.17", "10.221.223.127", - "10.198.157.122", - "10.159.155.88" + "10.0.175.17", + "10.159.155.88", + "10.198.157.122" ], "related.user": [ "iquipex" @@ -6035,10 +6035,10 @@ "equu7361.www5.localdomain" ], "related.ip": [ + "10.30.20.187", "10.189.70.237", "10.7.212.201", - "10.252.136.130", - "10.30.20.187" + "10.252.136.130" ], "related.user": [ "ugiat" @@ -6103,9 +6103,9 @@ "tse2979.internal.localhost" ], "related.ip": [ + "10.60.224.93", "10.83.105.69", "10.102.109.194", - "10.60.224.93", "10.242.121.165" ], "related.user": [ @@ -6171,10 +6171,10 @@ "uisnostr2390.mail.domain" ], "related.ip": [ - "10.219.174.45", - "10.17.20.93", + "10.181.134.69", "10.251.167.219", - "10.181.134.69" + "10.219.174.45", + "10.17.20.93" ], "related.user": [ "Uteni" @@ -6240,9 +6240,9 @@ ], "related.ip": [ "10.28.233.253", - "10.37.14.20", + "10.223.99.90", "10.30.117.82", - "10.223.99.90" + "10.37.14.20" ], "related.user": [ "numqua" @@ -6307,9 +6307,9 @@ "lites1614.www.corp" ], "related.ip": [ - "10.50.61.114", - "10.125.20.22", "10.8.32.17", + "10.125.20.22", + "10.50.61.114", "10.57.85.113" ], "related.user": [ @@ -6375,10 +6375,10 @@ "lorinrep7686.mail.corp" ], "related.ip": [ - "10.215.224.27", "10.113.78.101", - "10.181.63.82", - "10.200.28.55" + "10.215.224.27", + "10.200.28.55", + "10.181.63.82" ], "related.user": [ "ficiade" @@ -6443,9 +6443,9 @@ "nderit6272.mail.example" ], "related.ip": [ - "10.139.20.223", "10.169.95.128", "10.177.14.106", + "10.139.20.223", "10.243.43.168" ], "related.user": [ @@ -6511,10 +6511,10 @@ "ntu1279.mail.lan" ], "related.ip": [ - "10.92.168.198", - "10.18.176.44", + "10.39.100.88", "10.90.93.4", - "10.39.100.88" + "10.92.168.198", + "10.18.176.44" ], "related.user": [ "adminima" @@ -6581,8 +6581,8 @@ "related.ip": [ "10.193.43.135", "10.49.68.8", - "10.163.203.191", - "10.173.13.179" + "10.173.13.179", + "10.163.203.191" ], "related.user": [ "tlab" @@ -6646,10 +6646,10 @@ "cipitl2184.localdomain" ], "related.ip": [ - "10.240.47.113", "10.84.64.28", + "10.209.226.7", "10.31.147.51", - "10.209.226.7" + "10.240.47.113" ], "related.user": [ "ull" @@ -6714,9 +6714,9 @@ "item3647.home" ], "related.ip": [ + "10.86.1.244", "10.32.20.4", "10.52.13.192", - "10.86.1.244", "10.225.189.229" ], "related.user": [ diff --git a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml index 895c0723f10..814b2b0f30d 100644 --- a/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/f5/bigipapm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json index 879cbc128a6..244c0a399b2 100644 --- a/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/f5/bigipapm/test/generated.log-expected.json @@ -367,8 +367,8 @@ "observer.vendor": "F5", "process.pid": 2289, "related.ip": [ - "10.225.160.182", - "10.204.123.107" + "10.204.123.107", + "10.225.160.182" ], "rsa.internal.messageid": "01490500", "rsa.misc.log_session_id": "eFinib", @@ -771,7 +771,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco" + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "metco", + "url.original": "https://www5.example.com/olli/rever.html?rsp=oluptat#metco", + "url.path": "/olli/rever.html", + "url.query": "rsp=oluptat", + "url.scheme": "https" }, { "@timestamp": "2017-03-18T20:24:33.000Z", @@ -1476,8 +1482,8 @@ "observer.vendor": "F5", "process.pid": 2943, "related.ip": [ - "10.16.181.60", - "10.142.213.80" + "10.142.213.80", + "10.16.181.60" ], "rsa.internal.messageid": "01490549", "rsa.misc.group": "tationu", @@ -1567,8 +1573,8 @@ "observer.vendor": "F5", "process.pid": 1973, "related.ip": [ - "10.47.99.72", - "10.187.64.126" + "10.187.64.126", + "10.47.99.72" ], "rsa.internal.messageid": "01490500", "rsa.misc.category": "oremipsu", @@ -2131,7 +2137,13 @@ "f5.bigipapm", "forwarded" ], - "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama" + "url.domain": "example.net", + "url.extension": "txt", + "url.fragment": "gnama", + "url.original": "https://example.net/equamn/scipi.txt?eiu=maliquam#gnama", + "url.path": "/equamn/scipi.txt", + "url.query": "eiu=maliquam", + "url.scheme": "https" }, { "@timestamp": "2019-04-01T02:38:14.000Z", diff --git a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json index 160c14c7981..5880bc2c97e 100644 --- a/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/clientendpoint/test/generated.log-expected.json @@ -25,8 +25,8 @@ "boNemoe4402.www.invalid" ], "related.ip": [ - "10.150.92.220", - "10.102.123.34" + "10.102.123.34", + "10.150.92.220" ], "related.user": [ "sumdo" @@ -211,8 +211,8 @@ "tinculp2940.internal.local" ], "related.ip": [ - "10.202.204.154", - "10.134.137.177" + "10.134.137.177", + "10.202.204.154" ], "related.user": [ "orsitame" @@ -335,8 +335,8 @@ "enim5316.www5.local" ], "related.ip": [ - "10.202.72.124", - "10.200.188.142" + "10.200.188.142", + "10.202.72.124" ], "related.user": [ "iusmodt" @@ -397,8 +397,8 @@ "reetdolo2770.www5.local" ], "related.ip": [ - "10.12.44.169", - "10.214.225.125" + "10.214.225.125", + "10.12.44.169" ], "related.user": [ "erep" @@ -644,8 +644,8 @@ "tatno6787.internal.localhost" ], "related.ip": [ - "10.136.252.240", - "10.65.83.160" + "10.65.83.160", + "10.136.252.240" ], "related.user": [ "ender" @@ -767,8 +767,8 @@ "ali6446.localhost" ], "related.ip": [ - "10.200.156.102", - "10.144.82.69" + "10.144.82.69", + "10.200.156.102" ], "related.user": [ "rveli" @@ -828,8 +828,8 @@ "torev7118.internal.domain" ], "related.ip": [ - "10.109.232.112", - "10.72.58.135" + "10.72.58.135", + "10.109.232.112" ], "related.user": [ "xea" @@ -890,8 +890,8 @@ "dolore6103.www5.example" ], "related.ip": [ - "10.38.22.45", - "10.72.29.73" + "10.72.29.73", + "10.38.22.45" ], "related.user": [ "onproide" @@ -952,8 +952,8 @@ "errorsi6996.www.domain" ], "related.ip": [ - "10.76.72.111", - "10.70.95.74" + "10.70.95.74", + "10.76.72.111" ], "related.user": [ "ivelits" @@ -1014,8 +1014,8 @@ "lumquido5839.api.corp" ], "related.ip": [ - "10.73.69.75", - "10.19.201.13" + "10.19.201.13", + "10.73.69.75" ], "related.user": [ "tat" @@ -1075,8 +1075,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.84.105.75", - "10.78.151.178" + "10.78.151.178", + "10.84.105.75" ], "related.user": [ "iquaUten" @@ -1199,8 +1199,8 @@ "eme6710.mail.invalid" ], "related.ip": [ - "10.121.219.204", - "10.104.134.200" + "10.104.134.200", + "10.121.219.204" ], "related.user": [ "uptat" @@ -1323,8 +1323,8 @@ "umexerci1284.internal.localdomain" ], "related.ip": [ - "10.141.44.153", - "10.161.57.8" + "10.161.57.8", + "10.141.44.153" ], "related.user": [ "quisnos" @@ -1385,8 +1385,8 @@ "adol485.example" ], "related.ip": [ - "10.6.167.7", - "10.153.111.103" + "10.153.111.103", + "10.6.167.7" ], "related.user": [ "eumfug" @@ -1446,8 +1446,8 @@ "evita5008.www.localdomain" ], "related.ip": [ - "10.134.148.219", - "10.248.204.182" + "10.248.204.182", + "10.134.148.219" ], "related.user": [ "uioffi" @@ -1507,8 +1507,8 @@ "tsedqu2456.www5.invalid" ], "related.ip": [ - "10.178.77.231", - "10.163.5.243" + "10.163.5.243", + "10.178.77.231" ], "related.user": [ "liquide" @@ -1568,8 +1568,8 @@ "ris3314.mail.invalid" ], "related.ip": [ - "10.221.89.228", - "10.177.194.18" + "10.177.194.18", + "10.221.89.228" ], "related.user": [ "aliquam" @@ -1691,8 +1691,8 @@ "non3341.mail.invalid" ], "related.ip": [ - "10.168.90.81", - "10.101.57.120" + "10.101.57.120", + "10.168.90.81" ], "related.user": [ "eporr" @@ -1815,8 +1815,8 @@ "stquido5705.api.host" ], "related.ip": [ - "10.60.129.15", - "10.248.101.25" + "10.248.101.25", + "10.60.129.15" ], "related.user": [ "evolup" @@ -1938,8 +1938,8 @@ "vita2681.www5.local" ], "related.ip": [ - "10.27.14.168", - "10.66.2.232" + "10.66.2.232", + "10.27.14.168" ], "related.user": [ "uirati" @@ -2185,8 +2185,8 @@ "eumiu765.api.lan" ], "related.ip": [ - "10.4.157.1", - "10.184.18.202" + "10.184.18.202", + "10.4.157.1" ], "related.user": [ "oditem" @@ -2309,8 +2309,8 @@ "idestlab2631.www.lan" ], "related.ip": [ - "10.27.16.118", - "10.83.177.2" + "10.83.177.2", + "10.27.16.118" ], "related.user": [ "borios" @@ -2371,8 +2371,8 @@ "inesci6789.test" ], "related.ip": [ - "10.167.227.44", - "10.38.54.72" + "10.38.54.72", + "10.167.227.44" ], "related.user": [ "riamea" @@ -2494,8 +2494,8 @@ "ima2031.api.corp" ], "related.ip": [ - "10.9.18.237", - "10.9.12.248" + "10.9.12.248", + "10.9.18.237" ], "related.user": [ "uradi" @@ -2680,8 +2680,8 @@ "gnaaliqu3935.api.test" ], "related.ip": [ - "10.142.25.100", - "10.134.18.114" + "10.134.18.114", + "10.142.25.100" ], "related.user": [ "osqui" @@ -2742,8 +2742,8 @@ "nsequat1859.internal.localhost" ], "related.ip": [ - "10.28.118.160", - "10.223.119.218" + "10.223.119.218", + "10.28.118.160" ], "related.user": [ "ntsunt" @@ -2927,8 +2927,8 @@ "rem7043.localhost" ], "related.ip": [ - "10.227.173.252", - "10.65.2.106" + "10.65.2.106", + "10.227.173.252" ], "related.user": [ "itation" @@ -3112,8 +3112,8 @@ "giatquov1918.internal.example" ], "related.ip": [ - "10.180.195.43", - "10.85.185.13" + "10.85.185.13", + "10.180.195.43" ], "related.user": [ "voluptas" @@ -3236,8 +3236,8 @@ "Sedut1775.www.domain" ], "related.ip": [ - "10.248.165.185", - "10.86.11.48" + "10.86.11.48", + "10.248.165.185" ], "related.user": [ "dquiac" @@ -3422,8 +3422,8 @@ "naaliq3710.api.local" ], "related.ip": [ - "10.120.10.211", - "10.28.82.189" + "10.28.82.189", + "10.120.10.211" ], "related.user": [ "rcit" @@ -3484,8 +3484,8 @@ "volupta3552.internal.localhost" ], "related.ip": [ - "10.31.237.225", - "10.6.38.163" + "10.6.38.163", + "10.31.237.225" ], "related.user": [ "olup" @@ -3546,8 +3546,8 @@ "onse380.internal.localdomain" ], "related.ip": [ - "10.226.5.189", - "10.125.165.144" + "10.125.165.144", + "10.226.5.189" ], "related.user": [ "mvolu" @@ -3670,8 +3670,8 @@ "oloreseo5039.test" ], "related.ip": [ - "10.28.105.124", - "10.218.0.197" + "10.218.0.197", + "10.28.105.124" ], "related.user": [ "ntNe" @@ -3732,8 +3732,8 @@ "minim459.mail.local" ], "related.ip": [ - "10.123.199.198", - "10.17.87.79" + "10.17.87.79", + "10.123.199.198" ], "related.user": [ "ratvolu" @@ -3856,8 +3856,8 @@ "aparia1179.www.localdomain" ], "related.ip": [ - "10.193.118.163", - "10.115.174.107" + "10.115.174.107", + "10.193.118.163" ], "related.user": [ "exeacomm" @@ -3918,8 +3918,8 @@ "iatqu6203.mail.corp" ], "related.ip": [ - "10.77.77.208", - "10.37.128.49" + "10.37.128.49", + "10.77.77.208" ], "related.user": [ "moles" @@ -4042,8 +4042,8 @@ "mag1506.internal.domain" ], "related.ip": [ - "10.131.126.109", - "10.182.152.242" + "10.182.152.242", + "10.131.126.109" ], "related.user": [ "dolor" @@ -4351,8 +4351,8 @@ "abor1370.www.domain" ], "related.ip": [ - "10.77.78.180", - "10.97.236.123" + "10.97.236.123", + "10.77.78.180" ], "related.user": [ "nisi" @@ -4536,8 +4536,8 @@ "venia2079.mail.example" ], "related.ip": [ - "10.65.144.51", - "10.5.11.205" + "10.5.11.205", + "10.65.144.51" ], "related.user": [ "uptat" @@ -4660,8 +4660,8 @@ "gelitsed3249.corp" ], "related.ip": [ - "10.138.210.116", - "10.225.255.211" + "10.225.255.211", + "10.138.210.116" ], "related.user": [ "fugiatn" @@ -4968,8 +4968,8 @@ "edq5397.www.test" ], "related.ip": [ - "10.73.28.165", - "10.221.206.74" + "10.221.206.74", + "10.73.28.165" ], "related.user": [ "quas" @@ -5092,8 +5092,8 @@ "rumet6923.www5.lan" ], "related.ip": [ - "10.30.246.132", - "10.208.18.210" + "10.208.18.210", + "10.30.246.132" ], "related.user": [ "veniam" @@ -5216,8 +5216,8 @@ "amc3059.local" ], "related.ip": [ - "10.181.41.154", - "10.29.109.126" + "10.29.109.126", + "10.181.41.154" ], "related.user": [ "labo" @@ -5339,8 +5339,8 @@ "liquipex1155.mail.corp" ], "related.ip": [ - "10.183.189.133", - "10.154.191.225" + "10.154.191.225", + "10.183.189.133" ], "related.user": [ "ita" @@ -5648,8 +5648,8 @@ "reprehen3513.test" ], "related.ip": [ - "10.10.86.55", - "10.61.225.196" + "10.61.225.196", + "10.10.86.55" ], "related.user": [ "eniamqu" @@ -5834,8 +5834,8 @@ "rumSecti111.www5.domain" ], "related.ip": [ - "10.87.90.49", - "10.222.245.80" + "10.222.245.80", + "10.87.90.49" ], "related.user": [ "ptatemse" @@ -6020,8 +6020,8 @@ "uptasnul2751.www5.corp" ], "related.ip": [ - "10.161.64.168", - "10.194.67.223" + "10.194.67.223", + "10.161.64.168" ], "related.user": [ "tion" @@ -6081,8 +6081,8 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.100.154.220", - "10.120.148.241" + "10.120.148.241", + "10.100.154.220" ], "related.user": [ "rsitam" diff --git a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json index 8e79962d5bc..afb294c3814 100644 --- a/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimail/test/generated.log-expected.json @@ -596,8 +596,8 @@ "lamcolab3252.www.invalid" ], "related.ip": [ - "10.179.124.125", - "10.177.36.38" + "10.177.36.38", + "10.179.124.125" ], "rsa.email.email_dst": "ectio", "rsa.email.email_src": "sequine", @@ -1011,8 +1011,8 @@ "lapariat7287.internal.host" ], "related.ip": [ - "10.140.7.83", - "10.68.246.187" + "10.68.246.187", + "10.140.7.83" ], "rsa.email.email_dst": "gna", "rsa.email.email_src": "icabo", @@ -2680,12 +2680,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "taevit4968.mail.local", - "mveni5084.internal.local" + "mveni5084.internal.local", + "taevit4968.mail.local" ], "related.ip": [ - "10.144.111.42", - "10.62.61.1" + "10.62.61.1", + "10.144.111.42" ], "rsa.email.email_dst": "com", "rsa.email.email_src": "lam", @@ -3131,12 +3131,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "modi6930.internal.test", - "taevitae6868.www.corp" + "taevitae6868.www.corp", + "modi6930.internal.test" ], "related.ip": [ - "10.60.164.100", - "10.161.1.146" + "10.161.1.146", + "10.60.164.100" ], "rsa.email.email_dst": "nproiden", "rsa.email.email_src": "etconse", @@ -3235,12 +3235,12 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.hosts": [ - "uradip7802.mail.example", - "tetura7106.www5.corp" + "tetura7106.www5.corp", + "uradip7802.mail.example" ], "related.ip": [ - "10.44.35.57", - "10.93.239.216" + "10.93.239.216", + "10.44.35.57" ], "rsa.email.email_dst": "ciun", "rsa.email.email_src": "vento", @@ -3845,8 +3845,8 @@ "ncu3839.www.localhost" ], "related.ip": [ - "10.201.105.58", - "10.251.183.113" + "10.251.183.113", + "10.201.105.58" ], "rsa.email.email_dst": "ionemu", "rsa.email.email_src": "ent", @@ -3889,8 +3889,8 @@ "observer.type": "Firewall", "observer.vendor": "Fortinet", "related.ip": [ - "10.209.203.156", - "10.132.139.98" + "10.132.139.98", + "10.209.203.156" ], "rsa.email.email_dst": "borisnis", "rsa.email.email_src": "pariat", diff --git a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml index eda458f01de..b423b2203e4 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/fortimanager/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json index bd66027098d..3f078976410 100644 --- a/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json +++ b/x-pack/filebeat/module/fortinet/fortimanager/test/generated.log-expected.json @@ -155,8 +155,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "roinBCS", "url.original": "https://www.example.net/orisn/cca.htm?ofdeF=metcons#roinBCS", - "url.query": "taspe", + "url.path": "/orisn/cca.htm", + "url.query": [ + "taspe", + "ofdeF=metcons" + ], + "url.scheme": "https", "user.name": "oluptas" }, { @@ -188,9 +196,9 @@ "ccaecat" ], "related.ip": [ - "10.15.159.80", + "10.94.103.117", "10.200.188.142", - "10.94.103.117" + "10.15.159.80" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -256,8 +264,8 @@ ], "related.ip": [ "10.50.112.141", - "10.27.88.95", - "10.131.233.27" + "10.131.233.27", + "10.27.88.95" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -322,13 +330,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.5670", "related.hosts": [ - "ntutl", "roinBCSe", + "ntutl", "olo7148.mail.home" ], "related.ip": [ - "10.87.212.179", - "10.157.213.15" + "10.157.213.15", + "10.87.212.179" ], "related.user": [ "rveli" @@ -383,8 +391,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "saute", "url.original": "https://api.example.com/iumto/aboreetd.gif?dun=enim#saute", - "url.query": "nsect", + "url.path": "/iumto/aboreetd.gif", + "url.query": [ + "nsect", + "dun=enim" + ], + "url.scheme": "https", "user.name": "rveli" }, { @@ -415,8 +431,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.152", "related.hosts": [ - "orain", "onse", + "orain", "agna7678.internal.host" ], "related.ip": [ @@ -431,8 +447,8 @@ "rsa.investigations.event_vcat": "mwr", "rsa.misc.OS": "imaven", "rsa.misc.action": [ - "uines", - "accept" + "accept", + "uines" ], "rsa.misc.category": "uidolo", "rsa.misc.client": "emips", @@ -476,8 +492,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "picia", "url.original": "https://www.example.net/tasuntex/sunt.txt?ume=incidi#picia", - "url.query": "nsec", + "url.path": "/tasuntex/sunt.txt", + "url.query": [ + "nsec", + "ume=incidi" + ], + "url.scheme": "https", "user.name": "aperia" }, { @@ -508,8 +532,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4059", "related.hosts": [ - "utla", "tatn", + "utla", "equep5085.mail.domain" ], "related.ip": [ @@ -569,8 +593,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "tlabore", "url.original": "https://www.example.org/inesci/serror.html?mqu=apariat#tlabore", - "url.query": "nsectet", + "url.path": "/inesci/serror.html", + "url.query": [ + "nsectet", + "mqu=apariat" + ], + "url.scheme": "https", "user.name": "nnum" }, { @@ -602,9 +634,9 @@ "labore" ], "related.ip": [ - "10.186.85.3", "10.114.16.155", - "10.176.216.90" + "10.176.216.90", + "10.186.85.3" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -668,13 +700,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3917", "related.hosts": [ - "gnido", "sperna", + "gnido", "eturadi6608.mail.host" ], "related.ip": [ - "10.23.62.94", - "10.61.163.4" + "10.61.163.4", + "10.23.62.94" ], "related.user": [ "non" @@ -684,8 +716,8 @@ "rsa.investigations.event_vcat": "oide", "rsa.misc.OS": "gel", "rsa.misc.action": [ - "cancel", - "luptatem" + "luptatem", + "cancel" ], "rsa.misc.category": "uir", "rsa.misc.client": "ratvolu", @@ -729,8 +761,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "issus", "url.original": "https://api.example.com/nimid/itatione.htm?umwr=oluptate#issus", - "url.query": "atem", + "url.path": "/nimid/itatione.htm", + "url.query": [ + "atem", + "umwr=oluptate" + ], + "url.scheme": "https", "user.name": "non" }, { @@ -766,8 +806,8 @@ "ipsumdol4488.api.localdomain" ], "related.ip": [ - "10.106.31.86", - "10.28.76.42" + "10.28.76.42", + "10.106.31.86" ], "related.user": [ "cons" @@ -822,8 +862,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "pora", "url.original": "https://example.com/tetu/stru.htm?tlabore=Exc#pora", - "url.query": "atno", + "url.path": "/tetu/stru.htm", + "url.query": [ + "atno", + "tlabore=Exc" + ], + "url.scheme": "https", "user.name": "cons" }, { @@ -855,9 +903,9 @@ "diconseq" ], "related.ip": [ - "10.106.162.153", "10.58.214.16", - "10.238.164.74" + "10.238.164.74", + "10.106.162.153" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -922,8 +970,8 @@ "tenimad" ], "related.ip": [ - "10.225.141.20", "10.110.31.190", + "10.225.141.20", "10.217.150.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -988,13 +1036,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3319", "related.hosts": [ - "amc", "mestq", + "amc", "cusant4946.www.domain" ], "related.ip": [ - "10.137.56.173", - "10.69.103.176" + "10.69.103.176", + "10.137.56.173" ], "related.user": [ "proide" @@ -1049,8 +1097,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "isnisi", "url.original": "https://api.example.com/isnostr/umqu.htm?emquia=inesci#isnisi", - "url.query": "uptate", + "url.path": "/isnostr/umqu.htm", + "url.query": [ + "uptate", + "emquia=inesci" + ], + "url.scheme": "https", "user.name": "proide" }, { @@ -1083,8 +1139,8 @@ ], "related.ip": [ "10.30.47.165", - "10.25.212.118", - "10.5.235.217" + "10.5.235.217", + "10.25.212.118" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1209,8 +1265,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "urerepr", "url.original": "https://example.org/tiaec/rumwrit.txt?oconsequ=edquiac#urerepr", - "url.query": "iaeco", + "url.path": "/tiaec/rumwrit.txt", + "url.query": [ + "iaeco", + "oconsequ=edquiac" + ], + "url.scheme": "https", "user.name": "tetur" }, { @@ -1243,8 +1307,8 @@ ], "related.ip": [ "10.233.120.207", - "10.51.213.42", - "10.98.194.212" + "10.98.194.212", + "10.51.213.42" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1309,9 +1373,9 @@ "tla" ], "related.ip": [ - "10.67.132.242", "10.241.132.176", - "10.245.187.229" + "10.245.187.229", + "10.67.132.242" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -1391,8 +1455,8 @@ "rsa.investigations.event_vcat": "mes", "rsa.misc.OS": "riss", "rsa.misc.action": [ - "oin", - "allow" + "allow", + "oin" ], "rsa.misc.category": "idi", "rsa.misc.client": "modocon", @@ -1436,8 +1500,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "itlab", "url.original": "https://mail.example.org/ptassita/its.gif?risnis=uov#itlab", - "url.query": "hil", + "url.path": "/ptassita/its.gif", + "url.query": [ + "hil", + "risnis=uov" + ], + "url.scheme": "https", "user.name": "atv" }, { @@ -1468,8 +1540,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.760", "related.hosts": [ - "dolorsit", "rcit", + "dolorsit", "mve1890.internal.home" ], "related.ip": [ @@ -1484,8 +1556,8 @@ "rsa.investigations.event_vcat": "metcons", "rsa.misc.OS": "ehende", "rsa.misc.action": [ - "umf", - "deny" + "deny", + "umf" ], "rsa.misc.category": "emUte", "rsa.misc.client": "archite", @@ -1529,8 +1601,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "toditaut", "url.original": "https://www.example.org/redol/gnaa.htm?aliquamq=dtempori#toditaut", - "url.query": "quames", + "url.path": "/redol/gnaa.htm", + "url.query": [ + "quames", + "aliquamq=dtempori" + ], + "url.scheme": "https", "user.name": "orese" }, { @@ -1561,8 +1641,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4450", "related.hosts": [ - "saquaea", "billoi", + "saquaea", "eturad6143.www.home" ], "related.ip": [ @@ -1622,8 +1702,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "offici", "url.original": "https://example.net/aparia/tatnon.jpg?rever=ore#offici", - "url.query": "essecill", + "url.path": "/aparia/tatnon.jpg", + "url.query": [ + "essecill", + "rever=ore" + ], + "url.scheme": "https", "user.name": "enim" }, { @@ -1659,8 +1747,8 @@ "orinrep5386.www.corp" ], "related.ip": [ - "10.253.228.140", - "10.208.21.135" + "10.208.21.135", + "10.253.228.140" ], "related.user": [ "inculp" @@ -1670,8 +1758,8 @@ "rsa.investigations.event_vcat": "emagn", "rsa.misc.OS": "oditempo", "rsa.misc.action": [ - "ugitse", - "cancel" + "cancel", + "ugitse" ], "rsa.misc.category": "magnid", "rsa.misc.client": "sci", @@ -1715,8 +1803,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "equamnih", "url.original": "https://www.example.com/tali/BCS.txt?iqu=niamqu#equamnih", - "url.query": "quiineav", + "url.path": "/tali/BCS.txt", + "url.query": [ + "quiineav", + "iqu=niamqu" + ], + "url.scheme": "https", "user.name": "inculp" }, { @@ -1808,8 +1904,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "mexer", "url.original": "https://www.example.com/iame/orroquis.htm?tiumd=ntmoll#mexer", - "url.query": "tec", + "url.path": "/iame/orroquis.htm", + "url.query": [ + "tec", + "tiumd=ntmoll" + ], + "url.scheme": "https", "user.name": "olorem" }, { @@ -1840,13 +1944,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.5380", "related.hosts": [ - "uei", "onse", + "uei", "reseosqu1629.mail.lan" ], "related.ip": [ - "10.106.85.174", - "10.94.242.80" + "10.94.242.80", + "10.106.85.174" ], "related.user": [ "lmo" @@ -1901,8 +2005,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "mnisi", "url.original": "https://www.example.net/iame/laudanti.htm?stquido=rsitvolu#mnisi", - "url.query": "serror", + "url.path": "/iame/laudanti.htm", + "url.query": [ + "serror", + "stquido=rsitvolu" + ], + "url.scheme": "https", "user.name": "lmo" }, { @@ -1935,9 +2047,9 @@ "oluptat" ], "related.ip": [ - "10.117.63.181", + "10.168.20.20", "10.247.53.179", - "10.168.20.20" + "10.117.63.181" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2002,8 +2114,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2208", "related.hosts": [ - "lamcola", "duntut", + "lamcola", "tasnul4179.internal.host" ], "related.ip": [ @@ -2018,8 +2130,8 @@ "rsa.investigations.event_vcat": "illumq", "rsa.misc.OS": "idata", "rsa.misc.action": [ - "emacc", - "block" + "block", + "emacc" ], "rsa.misc.category": "ueporro", "rsa.misc.client": "veli", @@ -2063,8 +2175,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "rQuisaut", "url.original": "https://www.example.org/ess/quiad.jpg?ten=litanim#rQuisaut", - "url.query": "emp", + "url.path": "/ess/quiad.jpg", + "url.query": [ + "emp", + "ten=litanim" + ], + "url.scheme": "https", "user.name": "amqu" }, { @@ -2096,13 +2216,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.3402", "related.hosts": [ - "tur", "imavenia", + "tur", "bore5546.www.local" ], "related.ip": [ - "10.44.198.184", - "10.189.82.19" + "10.189.82.19", + "10.44.198.184" ], "related.user": [ "iconsequ" @@ -2157,8 +2277,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "Finibus", "url.original": "https://example.com/xeac/llitanim.txt?oreverit=scip#Finibus", - "url.query": "atura", + "url.path": "/xeac/llitanim.txt", + "url.query": [ + "atura", + "oreverit=scip" + ], + "url.scheme": "https", "user.name": "iconsequ" }, { @@ -2250,8 +2378,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "iameaque", "url.original": "https://api.example.org/orio/gna.gif?aaliquaU=olu#iameaque", - "url.query": "tatio", + "url.path": "/orio/gna.gif", + "url.query": [ + "tatio", + "aaliquaU=olu" + ], + "url.scheme": "https", "user.name": "iosamni" }, { @@ -2343,8 +2479,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.org/nimadmin/lumqui.txt?iquip=tinculpa#umtota", - "url.query": "iuta", + "url.path": "/nimadmin/lumqui.txt", + "url.query": [ + "iuta", + "iquip=tinculpa" + ], + "url.scheme": "https", "user.name": "eavolup" }, { @@ -2376,9 +2520,9 @@ "uio" ], "related.ip": [ - "10.37.161.101", "10.17.209.252", - "10.111.182.212" + "10.111.182.212", + "10.37.161.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2443,9 +2587,9 @@ "itautfu" ], "related.ip": [ - "10.158.175.98", + "10.170.196.181", "10.153.166.133", - "10.170.196.181" + "10.158.175.98" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2514,8 +2658,8 @@ "con6049.internal.lan" ], "related.ip": [ - "10.63.171.91", - "10.48.25.200" + "10.48.25.200", + "10.63.171.91" ], "related.user": [ "usanti" @@ -2570,8 +2714,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "quisn", "url.original": "https://internal.example.com/temse/caecat.jpg?emeu=tatemac#quisn", - "url.query": "mveniam", + "url.path": "/temse/caecat.jpg", + "url.query": [ + "mveniam", + "emeu=tatemac" + ], + "url.scheme": "https", "user.name": "usanti" }, { @@ -2737,9 +2889,9 @@ "eturadip" ], "related.ip": [ + "10.214.156.161", "10.145.194.12", - "10.66.90.225", - "10.214.156.161" + "10.66.90.225" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -2804,8 +2956,8 @@ "iutal" ], "related.ip": [ - "10.156.208.5", "10.6.242.108", + "10.156.208.5", "10.163.36.101" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -2870,8 +3022,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4713", "related.hosts": [ - "epteurs", "data", + "epteurs", "remeum2641.www5.corp" ], "related.ip": [ @@ -2886,8 +3038,8 @@ "rsa.investigations.event_vcat": "olore", "rsa.misc.OS": "tatem", "rsa.misc.action": [ - "allow", - "itanimi" + "itanimi", + "allow" ], "rsa.misc.category": "psa", "rsa.misc.client": "ugits", @@ -2931,8 +3083,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "isni", "url.original": "https://www5.example.com/elit/sam.htm?nevolu=unt#isni", - "url.query": "onoru", + "url.path": "/elit/sam.htm", + "url.query": [ + "onoru", + "nevolu=unt" + ], + "url.scheme": "https", "user.name": "estiaec" }, { @@ -2968,8 +3128,8 @@ "itaspe3216.localdomain" ], "related.ip": [ - "10.94.177.125", - "10.116.82.108" + "10.116.82.108", + "10.94.177.125" ], "related.user": [ "ecatc" @@ -3024,8 +3184,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "iadeseru", "url.original": "https://example.org/porro/issu.htm?inculpa=ruredol#iadeseru", - "url.query": "olorsit", + "url.path": "/porro/issu.htm", + "url.query": [ + "olorsit", + "inculpa=ruredol" + ], + "url.scheme": "https", "user.name": "ecatc" }, { @@ -3062,8 +3230,8 @@ "mea6298.api.example" ], "related.ip": [ - "10.115.121.243", - "10.113.152.241" + "10.113.152.241", + "10.115.121.243" ], "related.user": [ "norumetM" @@ -3118,8 +3286,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "suntexpl", "url.original": "https://mail.example.org/uamquaer/texplica.gif?sequa=lorum#suntexpl", - "url.query": "udexerci", + "url.path": "/uamquaer/texplica.gif", + "url.query": [ + "udexerci", + "sequa=lorum" + ], + "url.scheme": "https", "user.name": "norumetM" }, { @@ -3150,8 +3326,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3804", "related.hosts": [ - "atcupi", "nder", + "atcupi", "iqu7510.internal.corp" ], "related.ip": [ @@ -3211,8 +3387,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "rsit", "url.original": "https://www5.example.org/aqu/utemvele.gif?serrorsi=tsedquia#rsit", - "url.query": "nihilmol", + "url.path": "/aqu/utemvele.gif", + "url.query": [ + "nihilmol", + "serrorsi=tsedquia" + ], + "url.scheme": "https", "user.name": "dictasun" }, { @@ -3429,8 +3613,8 @@ "deFinibu3940.internal.lan" ], "related.ip": [ - "10.124.71.88", - "10.22.248.52" + "10.22.248.52", + "10.124.71.88" ], "related.user": [ "tcons" @@ -3485,8 +3669,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "adeserun", "url.original": "https://www5.example.com/etcon/chit.txt?erspici=itinvolu#adeserun", - "url.query": "tinvolu", + "url.path": "/etcon/chit.txt", + "url.query": [ + "tinvolu", + "erspici=itinvolu" + ], + "url.scheme": "https", "user.name": "tcons" }, { @@ -3517,8 +3709,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4895", "related.hosts": [ - "ficiade", "ipexeac", + "ficiade", "tatiset4191.localdomain" ], "related.ip": [ @@ -3533,8 +3725,8 @@ "rsa.investigations.event_vcat": "iae", "rsa.misc.OS": "evelite", "rsa.misc.action": [ - "block", - "essequam" + "essequam", + "block" ], "rsa.misc.category": "tmollita", "rsa.misc.client": "uiinea", @@ -3578,8 +3770,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "nseq", "url.original": "https://api.example.org/fug/liquid.txt?ptate=lloi#nseq", - "url.query": "undeo", + "url.path": "/fug/liquid.txt", + "url.query": [ + "undeo", + "ptate=lloi" + ], + "url.scheme": "https", "user.name": "eumiure" }, { @@ -3612,8 +3812,8 @@ ], "related.ip": [ "10.200.12.126", - "10.250.231.196", - "10.14.145.107" + "10.14.145.107", + "10.250.231.196" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3678,9 +3878,9 @@ "atuse" ], "related.ip": [ - "10.21.203.112", + "10.103.36.192", "10.225.34.176", - "10.103.36.192" + "10.21.203.112" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3745,9 +3945,9 @@ "usantiu" ], "related.ip": [ - "10.5.67.140", + "10.118.111.183", "10.140.59.161", - "10.118.111.183" + "10.5.67.140" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -3811,8 +4011,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4493", "related.hosts": [ - "veleumiu", "labor", + "veleumiu", "nimadmi4084.api.home" ], "related.ip": [ @@ -3872,8 +4072,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "litess", "url.original": "https://example.net/aturQui/tquii.html?uiac=squ#litess", - "url.query": "Loremip", + "url.path": "/aturQui/tquii.html", + "url.query": [ + "Loremip", + "uiac=squ" + ], + "url.scheme": "https", "user.name": "itsed" }, { @@ -3920,8 +4128,8 @@ "rsa.investigations.event_vcat": "uep", "rsa.misc.OS": "iatisund", "rsa.misc.action": [ - "nvo", - "block" + "block", + "nvo" ], "rsa.misc.category": "tenima", "rsa.misc.client": "iuntNe", @@ -3965,8 +4173,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "aturvel", "url.original": "https://mail.example.org/mvele/teveli.htm?Nequepor=luptate#aturvel", - "url.query": "lab", + "url.path": "/mvele/teveli.htm", + "url.query": [ + "lab", + "Nequepor=luptate" + ], + "url.scheme": "https", "user.name": "rporis" }, { @@ -3998,9 +4214,9 @@ "sitv" ], "related.ip": [ - "10.251.183.113", "10.217.145.137", - "10.22.149.132" + "10.22.149.132", + "10.251.183.113" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4065,9 +4281,9 @@ "nisi" ], "related.ip": [ + "10.203.66.175", "10.183.16.252", - "10.51.60.203", - "10.203.66.175" + "10.51.60.203" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4131,8 +4347,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.409", "related.hosts": [ - "magnama", "doei", + "magnama", "ursint411.www.lan" ], "related.ip": [ @@ -4147,8 +4363,8 @@ "rsa.investigations.event_vcat": "nisiuta", "rsa.misc.OS": "olu", "rsa.misc.action": [ - "tquov", - "accept" + "accept", + "tquov" ], "rsa.misc.category": "quatD", "rsa.misc.client": "acomm", @@ -4192,8 +4408,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "etc", "url.original": "https://example.org/tem/iadeseru.jpg?olorsita=odoco#etc", - "url.query": "natu", + "url.path": "/tem/iadeseru.jpg", + "url.query": [ + "natu", + "olorsita=odoco" + ], + "url.scheme": "https", "user.name": "nimadmi" }, { @@ -4229,8 +4453,8 @@ "ididunt7607.mail.localhost" ], "related.ip": [ - "10.242.178.15", - "10.217.111.77" + "10.217.111.77", + "10.242.178.15" ], "related.user": [ "nimadmin" @@ -4240,8 +4464,8 @@ "rsa.investigations.event_vcat": "psaqu", "rsa.misc.OS": "nevolu", "rsa.misc.action": [ - "datatno", - "allow" + "allow", + "datatno" ], "rsa.misc.category": "ionu", "rsa.misc.client": "ugiatn", @@ -4285,8 +4509,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "tobe", "url.original": "https://mail.example.net/epteurs/usmodtem.gif?itvo=asi#tobe", - "url.query": "equepor", + "url.path": "/epteurs/usmodtem.gif", + "url.query": [ + "equepor", + "itvo=asi" + ], + "url.scheme": "https", "user.name": "nimadmin" }, { @@ -4333,8 +4565,8 @@ "rsa.investigations.event_vcat": "ora", "rsa.misc.OS": "ommod", "rsa.misc.action": [ - "cancel", - "ant" + "ant", + "cancel" ], "rsa.misc.category": "rehende", "rsa.misc.client": "rehe", @@ -4378,8 +4610,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "eav", "url.original": "https://www.example.com/iadolo/cidu.txt?aliquide=redolori#eav", - "url.query": "tiu", + "url.path": "/iadolo/cidu.txt", + "url.query": [ + "tiu", + "aliquide=redolori" + ], + "url.scheme": "https", "user.name": "msequin" }, { @@ -4410,13 +4650,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.1789", "related.hosts": [ - "dol", "ono", + "dol", "ntex5135.corp" ], "related.ip": [ - "10.234.171.117", - "10.239.194.105" + "10.239.194.105", + "10.234.171.117" ], "related.user": [ "tat" @@ -4426,8 +4666,8 @@ "rsa.investigations.event_vcat": "uia", "rsa.misc.OS": "mquae", "rsa.misc.action": [ - "deny", - "tenatus" + "tenatus", + "deny" ], "rsa.misc.category": "abo", "rsa.misc.client": "umtota", @@ -4471,8 +4711,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "boreetd", "url.original": "https://internal.example.com/isc/umdol.jpg?atn=sectet#boreetd", - "url.query": "odic", + "url.path": "/isc/umdol.jpg", + "url.query": [ + "odic", + "atn=sectet" + ], + "url.scheme": "https", "user.name": "tat" }, { @@ -4506,8 +4754,8 @@ ], "related.ip": [ "10.249.16.201", - "10.107.168.208", - "10.34.41.75" + "10.34.41.75", + "10.107.168.208" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4632,8 +4880,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "txt", + "url.fragment": "ptat", "url.original": "https://www.example.com/imadm/ugiat.txt?Nequepor=nisiu#ptat", - "url.query": "stiaec", + "url.path": "/imadm/ugiat.txt", + "url.query": [ + "stiaec", + "Nequepor=nisiu" + ], + "url.scheme": "https", "user.name": "ommodoc" }, { @@ -4669,8 +4925,8 @@ "ulamc767.internal.lan" ], "related.ip": [ - "10.112.155.228", - "10.47.191.95" + "10.47.191.95", + "10.112.155.228" ], "related.user": [ "aed" @@ -4680,8 +4936,8 @@ "rsa.investigations.event_vcat": "eiusm", "rsa.misc.OS": "emag", "rsa.misc.action": [ - "deny", - "velillu" + "velillu", + "deny" ], "rsa.misc.category": "litseddo", "rsa.misc.client": "aturE", @@ -4725,8 +4981,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sed", "url.original": "https://mail.example.net/ionemu/nul.jpg?volupt=ori#sed", - "url.query": "radipi", + "url.path": "/ionemu/nul.jpg", + "url.query": [ + "radipi", + "volupt=ori" + ], + "url.scheme": "https", "user.name": "aed" }, { @@ -4758,8 +5022,8 @@ "metco" ], "related.ip": [ - "10.140.137.17", "10.103.169.94", + "10.140.137.17", "10.62.241.218" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -4826,8 +5090,8 @@ ], "related.ip": [ "10.251.212.166", - "10.90.229.92", - "10.77.105.160" + "10.77.105.160", + "10.90.229.92" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -4891,8 +5155,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4261", "related.hosts": [ - "dutp", "ipsum", + "dutp", "spici5547.internal.test" ], "related.ip": [ @@ -4907,8 +5171,8 @@ "rsa.investigations.event_vcat": "animi", "rsa.misc.OS": "tisunde", "rsa.misc.action": [ - "aut", - "cancel" + "cancel", + "aut" ], "rsa.misc.category": "lamcorpo", "rsa.misc.client": "com", @@ -4952,8 +5216,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "henderi", "url.original": "https://mail.example.org/asuntex/uovolup.html?amali=uiav#henderi", - "url.query": "eriti", + "url.path": "/asuntex/uovolup.html", + "url.query": [ + "eriti", + "amali=uiav" + ], + "url.scheme": "https", "user.name": "urmag" }, { @@ -5000,8 +5272,8 @@ "rsa.investigations.event_vcat": "uatDuisa", "rsa.misc.OS": "citation", "rsa.misc.action": [ - "utlabore", - "accept" + "accept", + "utlabore" ], "rsa.misc.category": "reeu", "rsa.misc.client": "ntut", @@ -5045,8 +5317,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "dminimve", "url.original": "https://example.org/mnisiut/porinci.htm?norum=emUten#dminimve", - "url.query": "texplica", + "url.path": "/mnisiut/porinci.htm", + "url.query": [ + "texplica", + "norum=emUten" + ], + "url.scheme": "https", "user.name": "lpaquiof" }, { @@ -5078,8 +5358,8 @@ "cons" ], "related.ip": [ - "10.246.41.77", "10.228.61.5", + "10.246.41.77", "10.157.22.21" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5145,8 +5425,8 @@ "llumdo" ], "related.ip": [ - "10.242.119.111", "10.188.131.18", + "10.242.119.111", "10.239.231.168" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -5211,8 +5491,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.979", "related.hosts": [ - "lorem", "iden", + "lorem", "tru3812.mail.lan" ], "related.ip": [ @@ -5272,8 +5552,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "gif", + "url.fragment": "rai", "url.original": "https://api.example.net/nsec/smo.gif?etq=trumexe#rai", - "url.query": "ipsu", + "url.path": "/nsec/smo.gif", + "url.query": [ + "ipsu", + "etq=trumexe" + ], + "url.scheme": "https", "user.name": "ainci" }, { @@ -5351,8 +5639,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4342", "related.hosts": [ - "ten", "onsequ", + "ten", "riaturE1644.www5.example" ], "related.ip": [ @@ -5367,8 +5655,8 @@ "rsa.investigations.event_vcat": "empori", "rsa.misc.OS": "ostru", "rsa.misc.action": [ - "allow", - "quepor" + "quepor", + "allow" ], "rsa.misc.category": "cipitla", "rsa.misc.client": "exeacomm", @@ -5412,8 +5700,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "upta", "url.original": "https://internal.example.com/dolo/exeacom.txt?tlab=eufugiat#upta", - "url.query": "Lor", + "url.path": "/dolo/exeacom.txt", + "url.query": [ + "Lor", + "tlab=eufugiat" + ], + "url.scheme": "https", "user.name": "erspici" }, { @@ -5505,8 +5801,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "utod", "url.original": "https://www5.example.org/orumet/aliqu.txt?tion=sun#utod", - "url.query": "con", + "url.path": "/orumet/aliqu.txt", + "url.query": [ + "con", + "tion=sun" + ], + "url.scheme": "https", "user.name": "Sedutp" }, { @@ -5605,9 +5909,9 @@ "caecatcu" ], "related.ip": [ - "10.7.230.206", "10.249.93.150", - "10.154.151.111" + "10.154.151.111", + "10.7.230.206" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -5676,8 +5980,8 @@ "itse5466.api.example" ], "related.ip": [ - "10.26.4.3", - "10.217.209.221" + "10.217.209.221", + "10.26.4.3" ], "related.user": [ "ciduntut" @@ -5732,8 +6036,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "gif", + "url.fragment": "iscin", "url.original": "https://internal.example.com/nonnumqu/isciveli.gif?wri=aute#iscin", - "url.query": "eca", + "url.path": "/nonnumqu/isciveli.gif", + "url.query": [ + "eca", + "wri=aute" + ], + "url.scheme": "https", "user.name": "ciduntut" }, { @@ -5780,8 +6092,8 @@ "rsa.investigations.event_vcat": "luptatev", "rsa.misc.OS": "emipsu", "rsa.misc.action": [ - "ido", - "accept" + "accept", + "ido" ], "rsa.misc.category": "litse", "rsa.misc.client": "evita", @@ -5825,8 +6137,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "html", + "url.fragment": "upta", "url.original": "https://www5.example.org/utlabo/scip.html?voluptas=inv#upta", - "url.query": "atnu", + "url.path": "/utlabo/scip.html", + "url.query": [ + "atnu", + "voluptas=inv" + ], + "url.scheme": "https", "user.name": "nulapar" }, { @@ -5857,8 +6177,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.2052", "related.hosts": [ - "dat", "asp", + "dat", "amco1592.mail.host" ], "related.ip": [ @@ -5873,8 +6193,8 @@ "rsa.investigations.event_vcat": "atvolupt", "rsa.misc.OS": "riosam", "rsa.misc.action": [ - "ssitasp", - "deny" + "deny", + "ssitasp" ], "rsa.misc.category": "enimadmi", "rsa.misc.client": "uatDui", @@ -5918,8 +6238,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "obeata", "url.original": "https://internal.example.com/ssusci/snostrud.txt?dolo=siutaliq#obeata", - "url.query": "ptat", + "url.path": "/ssusci/snostrud.txt", + "url.query": [ + "ptat", + "dolo=siutaliq" + ], + "url.scheme": "https", "user.name": "moenimi" }, { @@ -5955,8 +6283,8 @@ "dicta7226.mail.example" ], "related.ip": [ - "10.4.244.115", - "10.53.50.77" + "10.53.50.77", + "10.4.244.115" ], "related.user": [ "idolo" @@ -6011,8 +6339,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ptasn", "url.original": "https://api.example.com/temporai/umw.jpg?mveniamq=litsed#ptasn", - "url.query": "lore", + "url.path": "/temporai/umw.jpg", + "url.query": [ + "lore", + "mveniamq=litsed" + ], + "url.scheme": "https", "user.name": "idolo" }, { @@ -6110,8 +6446,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.3052", "related.hosts": [ - "xeacom", "tenima", + "xeacom", "pidatatn2627.www.localdomain" ], "related.ip": [ @@ -6126,8 +6462,8 @@ "rsa.investigations.event_vcat": "lauda", "rsa.misc.OS": "enatuser", "rsa.misc.action": [ - "rios", - "accept" + "accept", + "rios" ], "rsa.misc.category": "aUte", "rsa.misc.client": "iusm", @@ -6171,8 +6507,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "umiurere", "url.original": "https://mail.example.org/ngelit/dipiscin.gif?serro=ctet#umiurere", - "url.query": "diconseq", + "url.path": "/ngelit/dipiscin.gif", + "url.query": [ + "diconseq", + "serro=ctet" + ], + "url.scheme": "https", "user.name": "riatur" }, { @@ -6204,9 +6548,9 @@ "nimides" ], "related.ip": [ - "10.53.251.202", + "10.226.255.3", "10.123.59.69", - "10.226.255.3" + "10.53.251.202" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6271,9 +6615,9 @@ "edut" ], "related.ip": [ - "10.3.85.176", + "10.212.56.26", "10.29.141.252", - "10.212.56.26" + "10.3.85.176" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6342,8 +6686,8 @@ "emveleu4029.api.local" ], "related.ip": [ - "10.126.11.186", - "10.236.175.163" + "10.236.175.163", + "10.126.11.186" ], "related.user": [ "udantiu" @@ -6398,8 +6742,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "lestia", "url.original": "https://internal.example.net/inc/riaturEx.htm?mnihilm=itinvo#lestia", - "url.query": "icaboN", + "url.path": "/inc/riaturEx.htm", + "url.query": [ + "icaboN", + "mnihilm=itinvo" + ], + "url.scheme": "https", "user.name": "udantiu" }, { @@ -6431,9 +6783,9 @@ "mes" ], "related.ip": [ - "10.171.60.173", "10.11.150.136", - "10.83.98.220" + "10.83.98.220", + "10.171.60.173" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6498,9 +6850,9 @@ "datatno" ], "related.ip": [ - "10.238.49.73", "10.74.88.209", - "10.92.3.166" + "10.92.3.166", + "10.238.49.73" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6565,8 +6917,8 @@ "ptate" ], "related.ip": [ - "10.119.248.36", "10.187.107.47", + "10.119.248.36", "10.84.200.121" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -6632,9 +6984,9 @@ "tasu" ], "related.ip": [ - "10.135.213.17", "10.167.128.229", - "10.30.239.222" + "10.30.239.222", + "10.135.213.17" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6714,8 +7066,8 @@ "rsa.investigations.event_vcat": "iumdol", "rsa.misc.OS": "min", "rsa.misc.action": [ - "block", - "eleumiur" + "eleumiur", + "block" ], "rsa.misc.category": "ero", "rsa.misc.client": "gia", @@ -6759,8 +7111,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "rExce", "url.original": "https://api.example.net/ita/esse.txt?amquis=iatquovo#rExce", - "url.query": "ididun", + "url.path": "/ita/esse.txt", + "url.query": [ + "ididun", + "amquis=iatquovo" + ], + "url.scheme": "https", "user.name": "emq" }, { @@ -6791,8 +7151,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.4195", "related.hosts": [ - "aconse", "Except", + "aconse", "rror3870.www5.local" ], "related.ip": [ @@ -6807,8 +7167,8 @@ "rsa.investigations.event_vcat": "ntexp", "rsa.misc.OS": "aconseq", "rsa.misc.action": [ - "block", - "oluptate" + "oluptate", + "block" ], "rsa.misc.category": "edqu", "rsa.misc.client": "ites", @@ -6852,8 +7212,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "rat", "url.original": "https://www5.example.net/culpa/isun.txt?cola=tura#rat", - "url.query": "orumwrit", + "url.path": "/culpa/isun.txt", + "url.query": [ + "orumwrit", + "cola=tura" + ], + "url.scheme": "https", "user.name": "caecatcu" }, { @@ -6885,9 +7253,9 @@ "emaperi" ], "related.ip": [ + "10.35.240.70", "10.224.212.88", - "10.53.82.96", - "10.35.240.70" + "10.53.82.96" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -6952,9 +7320,9 @@ "oeius" ], "related.ip": [ - "10.66.149.234", "10.233.128.7", - "10.186.253.240" + "10.186.253.240", + "10.66.149.234" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7019,9 +7387,9 @@ "irat" ], "related.ip": [ + "10.173.140.201", "10.46.11.114", - "10.227.133.134", - "10.173.140.201" + "10.227.133.134" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7086,9 +7454,9 @@ "emp" ], "related.ip": [ + "10.205.18.11", "10.69.130.207", - "10.170.236.123", - "10.205.18.11" + "10.170.236.123" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7157,8 +7525,8 @@ "velill3821.mail.invalid" ], "related.ip": [ - "10.97.254.192", - "10.124.34.251" + "10.124.34.251", + "10.97.254.192" ], "related.user": [ "epor" @@ -7168,8 +7536,8 @@ "rsa.investigations.event_vcat": "lica", "rsa.misc.OS": "taedi", "rsa.misc.action": [ - "imide", - "deny" + "deny", + "imide" ], "rsa.misc.category": "iurere", "rsa.misc.client": "ollitan", @@ -7213,8 +7581,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "aborios", "url.original": "https://example.com/loremag/mqu.gif?bore=lapari#aborios", - "url.query": "sequa", + "url.path": "/loremag/mqu.gif", + "url.query": [ + "sequa", + "bore=lapari" + ], + "url.scheme": "https", "user.name": "epor" }, { @@ -7361,8 +7737,8 @@ ], "related.ip": [ "10.35.84.125", - "10.37.120.29", - "10.212.208.70" + "10.212.208.70", + "10.37.120.29" ], "rsa.internal.messageid": "generic_fortinetmgr_1", "rsa.misc.action": [ @@ -7427,8 +7803,8 @@ "aturvel" ], "related.ip": [ - "10.143.65.84", "10.199.201.26", + "10.143.65.84", "10.207.207.106" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7494,8 +7870,8 @@ "gni" ], "related.ip": [ - "10.163.236.253", "10.41.61.88", + "10.163.236.253", "10.204.27.48" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7561,8 +7937,8 @@ "dents" ], "related.ip": [ - "10.185.44.26", "10.246.81.164", + "10.185.44.26", "10.53.110.111" ], "rsa.internal.messageid": "generic_fortinetmgr_1", @@ -7628,8 +8004,8 @@ "observer.vendor": "Fortinet", "observer.version": "1.802", "related.hosts": [ - "lam", "proid", + "lam", "cupida6106.www5.local" ], "related.ip": [ @@ -7689,8 +8065,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ido", "url.original": "https://api.example.com/aco/empo.jpg?iumdol=iusm#ido", - "url.query": "ipsamv", + "url.path": "/aco/empo.jpg", + "url.query": [ + "ipsamv", + "iumdol=iusm" + ], + "url.scheme": "https", "user.name": "aquaeab" }, { @@ -7726,8 +8110,8 @@ "unt2122.internal.local" ], "related.ip": [ - "10.38.18.72", - "10.202.250.141" + "10.202.250.141", + "10.38.18.72" ], "related.user": [ "maperia" @@ -7737,8 +8121,8 @@ "rsa.investigations.event_vcat": "rure", "rsa.misc.OS": "iquidexe", "rsa.misc.action": [ - "allow", - "volu" + "volu", + "allow" ], "rsa.misc.category": "ium", "rsa.misc.client": "liquip", @@ -7782,8 +8166,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "ect", "url.original": "https://example.org/uidolor/turve.htm?temporai=uasiarch#ect", - "url.query": "quatDui", + "url.path": "/uidolor/turve.htm", + "url.query": [ + "quatDui", + "temporai=uasiarch" + ], + "url.scheme": "https", "user.name": "maperia" }, { @@ -7814,13 +8206,13 @@ "observer.vendor": "Fortinet", "observer.version": "1.4674", "related.hosts": [ - "oremeu", "ita", + "oremeu", "luptat2613.internal.localhost" ], "related.ip": [ - "10.182.124.88", - "10.139.144.75" + "10.139.144.75", + "10.182.124.88" ], "related.user": [ "modo" @@ -7875,8 +8267,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "odtempo", "url.original": "https://www.example.net/Maloru/lapariat.htm?tlabori=rehender#odtempo", - "url.query": "isiutali", + "url.path": "/Maloru/lapariat.htm", + "url.query": [ + "isiutali", + "tlabori=rehender" + ], + "url.scheme": "https", "user.name": "modo" }, { @@ -7912,8 +8312,8 @@ "neavo4796.internal.domain" ], "related.ip": [ - "10.188.124.185", - "10.35.10.19" + "10.35.10.19", + "10.188.124.185" ], "related.user": [ "dolo" @@ -7923,8 +8323,8 @@ "rsa.investigations.event_vcat": "olupt", "rsa.misc.OS": "rumw", "rsa.misc.action": [ - "tali", - "block" + "block", + "tali" ], "rsa.misc.category": "itsedq", "rsa.misc.client": "esciu", @@ -7968,8 +8368,16 @@ "fortinet.fortimanager", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "html", + "url.fragment": "one", "url.original": "https://www5.example.com/xea/ssecill.html?quianonn=quun#one", - "url.query": "tasper", + "url.path": "/xea/ssecill.html", + "url.query": [ + "tasper", + "quianonn=quun" + ], + "url.scheme": "https", "user.name": "dolo" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json index 23436a2de5f..1e936265994 100644 --- a/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json +++ b/x-pack/filebeat/module/google_workspace/admin/test/admin-sites-test.json.log-expected.json @@ -50,8 +50,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -107,8 +109,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" @@ -219,8 +223,10 @@ "tags": [ "forwarded" ], - "url.full": "http://example.com/path/in/url", + "url.domain": "example.com", + "url.original": "http://example.com/path/in/url", "url.path": "/path/in/url", + "url.scheme": "http", "user.domain": "bar.com", "user.id": "1", "user.name": "foo" diff --git a/x-pack/filebeat/module/google_workspace/ingest/common.yml b/x-pack/filebeat/module/google_workspace/ingest/common.yml index ec7a9f8bbcf..d9ee4e9f86c 100644 --- a/x-pack/filebeat/module/google_workspace/ingest/common.yml +++ b/x-pack/filebeat/module/google_workspace/ingest/common.yml @@ -1,5 +1,9 @@ description: Pipeline for parsing google_workspace logs processors: + - uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null - geoip: field: source.ip target_field: source.geo diff --git a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml index aab53775171..e37b497a0ef 100644 --- a/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml +++ b/x-pack/filebeat/module/imperva/securesphere/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json index 749ef4301f2..271e0db04a3 100644 --- a/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json +++ b/x-pack/filebeat/module/imperva/securesphere/test/generated.log-expected.json @@ -23,13 +23,13 @@ "radipis5408.mail.local" ], "related.ip": [ - "10.70.155.35", - "10.81.122.126" + "10.81.122.126", + "10.70.155.35" ], "related.user": [ + "tatno", "aqui", - "magn", - "tatno" + "magn" ], "rsa.counters.dclass_c1": 5910, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -117,8 +117,8 @@ ], "related.user": [ "qua", - "uradi", - "temUten" + "temUten", + "uradi" ], "rsa.counters.dclass_c1": 3626, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -170,13 +170,13 @@ "elaudant5931.internal.invalid" ], "related.ip": [ - "10.232.27.250", - "10.18.124.28" + "10.18.124.28", + "10.232.27.250" ], "related.user": [ - "modocons", + "lapariat", "mquidol", - "lapariat" + "modocons" ], "rsa.counters.dclass_c1": 6564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -234,13 +234,13 @@ "amest4147.mail.host" ], "related.ip": [ - "10.197.250.10", - "10.6.137.200" + "10.6.137.200", + "10.197.250.10" ], "related.user": [ "oluptas", - "intoc", - "occae" + "occae", + "intoc" ], "rsa.counters.event_counter": 7243, "rsa.db.database": "tNequepo", @@ -273,8 +273,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "eumfug", "url.original": "https://internal.example.net/namali/taevit.html?nsecte=itame#eumfug", - "url.query": "lit", + "url.path": "/namali/taevit.html", + "url.query": [ + "lit", + "nsecte=itame" + ], + "url.scheme": "https", "user.name": "occae" }, { @@ -308,16 +316,16 @@ ], "related.user": [ "ncidid", - "reme", - "acommod" + "acommod", + "reme" ], "rsa.counters.event_counter": 2462, "rsa.db.database": "uaUteni", "rsa.internal.event_desc": "osqui", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "lamcolab" + "lamcolab", + "accept" ], "rsa.misc.category": "xerc", "rsa.misc.disposition": "iutali", @@ -342,8 +350,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "giatnu", "url.original": "https://example.net/tlabo/uames.gif?mpo=offi#giatnu", - "url.query": "ulapa", + "url.path": "/tlabo/uames.gif", + "url.query": [ + "ulapa", + "mpo=offi" + ], + "url.scheme": "https", "user.name": "ncidid" }, { @@ -370,13 +386,13 @@ "didunt1355.corp" ], "related.ip": [ - "10.129.149.43", - "10.211.105.204" + "10.211.105.204", + "10.129.149.43" ], "related.user": [ "labor", - "eveli", - "orema" + "orema", + "eveli" ], "rsa.counters.dclass_c1": 6855, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -432,13 +448,13 @@ "pora6854.www5.home" ], "related.ip": [ - "10.214.191.180", - "10.112.250.193" + "10.112.250.193", + "10.214.191.180" ], "related.user": [ - "ipsumdol", + "ide", "Exc", - "ide" + "ipsumdol" ], "rsa.counters.dclass_c1": 6852, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -498,8 +514,8 @@ ], "related.user": [ "ovol", - "iquipe", - "tnonpro" + "tnonpro", + "iquipe" ], "rsa.counters.dclass_c1": 3645, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -551,13 +567,13 @@ "ptasnu6684.mail.lan" ], "related.ip": [ - "10.59.138.212", - "10.74.105.218" + "10.74.105.218", + "10.59.138.212" ], "related.user": [ - "idunt", "archite", - "boree" + "boree", + "idunt" ], "rsa.counters.dclass_c1": 248, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -613,13 +629,13 @@ "rinre2977.api.corp" ], "related.ip": [ - "10.230.173.4", - "10.168.159.13" + "10.168.159.13", + "10.230.173.4" ], "related.user": [ + "inci", "atemq", - "isnostr", - "inci" + "isnostr" ], "rsa.counters.dclass_c1": 6135, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -675,12 +691,12 @@ "atevelit2450.local" ], "related.ip": [ - "10.41.21.204", - "10.49.167.57" + "10.49.167.57", + "10.41.21.204" ], "related.user": [ - "sau", "tali", + "sau", "ccaeca" ], "rsa.counters.dclass_c1": 6818, @@ -739,12 +755,12 @@ "itla658.api.localhost" ], "related.ip": [ - "10.62.147.186", - "10.216.125.252" + "10.216.125.252", + "10.62.147.186" ], "related.user": [ - "llamco", "dolore", + "llamco", "lorsita" ], "rsa.counters.event_counter": 4603, @@ -752,8 +768,8 @@ "rsa.internal.event_desc": "aquae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "accept", - "quasia" + "quasia", + "accept" ], "rsa.misc.category": "boreetdo", "rsa.misc.disposition": "aturve", @@ -779,8 +795,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "elit", "url.original": "https://example.org/umwrit/uptate.html?ctetura=aveni#elit", - "url.query": "seosqui", + "url.path": "/umwrit/uptate.html", + "url.query": [ + "seosqui", + "ctetura=aveni" + ], + "url.scheme": "https", "user.name": "llamco" }, { @@ -848,8 +872,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "mvele", "url.original": "https://api.example.org/roidents/tem.txt?tametcon=liqua#mvele", - "url.query": "isis", + "url.path": "/roidents/tem.txt", + "url.query": [ + "isis", + "tametcon=liqua" + ], + "url.scheme": "https", "user.name": "rum" }, { @@ -879,8 +911,8 @@ "10.34.148.166" ], "related.user": [ - "icabo", "untutlab", + "icabo", "miu" ], "rsa.counters.dclass_c1": 5427, @@ -937,9 +969,9 @@ "10.134.5.40" ], "related.user": [ + "siu", "licabo", - "conse", - "siu" + "conse" ], "rsa.counters.dclass_c1": 6356, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -999,9 +1031,9 @@ "10.30.98.10" ], "related.user": [ - "olori", "velite", - "dipisci" + "dipisci", + "olori" ], "rsa.counters.dclass_c1": 7717, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1057,13 +1089,13 @@ "nimid372.api.corp" ], "related.ip": [ - "10.190.10.219", - "10.233.120.207" + "10.233.120.207", + "10.190.10.219" ], "related.user": [ - "quamnih", + "item", "accusant", - "item" + "quamnih" ], "rsa.counters.dclass_c1": 3278, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1147,13 +1179,13 @@ "maliquam2147.internal.home" ], "related.ip": [ - "10.100.98.56", - "10.248.184.200" + "10.248.184.200", + "10.100.98.56" ], "related.user": [ - "proident", "boru", - "ritati" + "ritati", + "proident" ], "rsa.counters.dclass_c1": 5923, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1213,8 +1245,8 @@ "10.82.28.220" ], "related.user": [ - "oluptat", "aecatcup", + "oluptat", "dtempo" ], "rsa.counters.dclass_c1": 3071, @@ -1275,9 +1307,9 @@ "10.167.252.183" ], "related.user": [ - "redol", + "ationul", "asnu", - "ationul" + "redol" ], "rsa.counters.dclass_c1": 6606, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1348,8 +1380,8 @@ "rsa.internal.event_desc": "iae", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "illu", - "deny" + "deny", + "illu" ], "rsa.misc.category": "quido", "rsa.misc.disposition": "emip", @@ -1375,8 +1407,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "tetur", "url.original": "https://mail.example.com/lorsi/repreh.gif?sitamet=utlabo#tetur", - "url.query": "tionula", + "url.path": "/lorsi/repreh.gif", + "url.query": [ + "tionula", + "sitamet=utlabo" + ], + "url.scheme": "https", "user.name": "undeomni" }, { @@ -1403,13 +1443,13 @@ "dolorem6882.api.local" ], "related.ip": [ - "10.29.119.245", - "10.214.3.140" + "10.214.3.140", + "10.29.119.245" ], "related.user": [ - "taliqui", "scipitl", - "edolorin" + "edolorin", + "taliqui" ], "rsa.counters.dclass_c1": 5140, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1467,8 +1507,8 @@ "temaccu5302.test" ], "related.ip": [ - "10.218.123.234", - "10.110.133.7" + "10.110.133.7", + "10.218.123.234" ], "related.user": [ "caboNem", @@ -1480,8 +1520,8 @@ "rsa.internal.event_desc": "liquid", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "vitaed", - "allow" + "allow", + "vitaed" ], "rsa.misc.category": "enim", "rsa.misc.disposition": "Finibus", @@ -1507,8 +1547,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "idolo", "url.original": "https://example.org/etconsec/elillum.htm?mporinc=onsectet#idolo", - "url.query": "atemUte", + "url.path": "/etconsec/elillum.htm", + "url.query": [ + "atemUte", + "mporinc=onsectet" + ], + "url.scheme": "https", "user.name": "pta" }, { @@ -1540,8 +1588,8 @@ ], "related.user": [ "doeiu", - "mquisn", - "litan" + "litan", + "mquisn" ], "rsa.counters.dclass_c1": 3474, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1603,9 +1651,9 @@ "10.123.166.197" ], "related.user": [ - "min", "emUte", - "liquam" + "liquam", + "min" ], "rsa.counters.event_counter": 7102, "rsa.db.database": "oluptat", @@ -1638,8 +1686,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.query": "tan", + "url.path": "/atnula/ditautf.jpg", + "url.query": [ + "tan", + "iquidex=olup" + ], + "url.scheme": "https", "user.name": "emUte" }, { @@ -1732,8 +1788,8 @@ "10.58.133.175" ], "related.user": [ - "nde", "oco", + "nde", "mfu" ], "rsa.counters.dclass_c1": 3795, @@ -1794,9 +1850,9 @@ "10.70.29.203" ], "related.user": [ - "pta", "mquisnos", - "veniamq" + "veniamq", + "pta" ], "rsa.counters.dclass_c1": 2358, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1852,13 +1908,13 @@ "lesti6939.api.local" ], "related.ip": [ - "10.137.85.123", - "10.165.182.111" + "10.165.182.111", + "10.137.85.123" ], "related.user": [ - "ames", + "Bonorum", "sis", - "Bonorum" + "ames" ], "rsa.counters.dclass_c1": 6401, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -1944,21 +2000,21 @@ "upt6017.api.localdomain" ], "related.ip": [ - "10.173.178.109", - "10.64.184.196" + "10.64.184.196", + "10.173.178.109" ], "related.user": [ - "tam", "nesci", - "uian" + "uian", + "tam" ], "rsa.counters.event_counter": 4493, "rsa.db.database": "sin", "rsa.internal.event_desc": "orin", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "lamco", - "block" + "block", + "lamco" ], "rsa.misc.category": "enia", "rsa.misc.disposition": "iavol", @@ -1984,8 +2040,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.query": "tlabore", + "url.path": "/hender/ptatemU.htm", + "url.query": [ + "tlabore", + "mquisnos=tnulapa" + ], + "url.scheme": "https", "user.name": "uian" }, { @@ -2012,13 +2076,13 @@ "turQuis4046.api.test" ], "related.ip": [ - "10.90.50.149", - "10.168.225.209" + "10.168.225.209", + "10.90.50.149" ], "related.user": [ + "olupta", "olu", - "aUtenima", - "olupta" + "aUtenima" ], "rsa.counters.dclass_c1": 1127, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2074,13 +2138,13 @@ "con6049.internal.lan" ], "related.ip": [ - "10.18.150.82", - "10.59.182.36" + "10.59.182.36", + "10.18.150.82" ], "related.user": [ - "luptat", + "mtota", "qua", - "mtota" + "luptat" ], "rsa.counters.dclass_c1": 6112, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2167,9 +2231,9 @@ "10.151.240.35" ], "related.user": [ + "lam", "ama", - "ametcons", - "lam" + "ametcons" ], "rsa.counters.dclass_c1": 4325, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2221,13 +2285,13 @@ "tium3542.internal.invalid" ], "related.ip": [ - "10.242.48.203", - "10.147.142.242" + "10.147.142.242", + "10.242.48.203" ], "related.user": [ - "ese", + "quasi", "quisn", - "quasi" + "ese" ], "rsa.counters.dclass_c1": 3970, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2285,13 +2349,13 @@ "radipis3991.mail.invalid" ], "related.ip": [ - "10.213.165.165", - "10.254.10.98" + "10.254.10.98", + "10.213.165.165" ], "related.user": [ - "ttenb", + "civeli", "eufugia", - "civeli" + "ttenb" ], "rsa.counters.event_counter": 7365, "rsa.db.database": "utlabore", @@ -2325,8 +2389,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "temqui", "url.original": "https://example.net/itati/oidentsu.gif?eporroqu=aturve#temqui", - "url.query": "lup", + "url.path": "/itati/oidentsu.gif", + "url.query": [ + "lup", + "eporroqu=aturve" + ], + "url.scheme": "https", "user.name": "ttenb" }, { @@ -2383,8 +2455,8 @@ "ihi7294.www5.localhost" ], "related.ip": [ - "10.169.28.157", - "10.116.1.130" + "10.116.1.130", + "10.169.28.157" ], "related.user": [ "eturadip", @@ -2423,8 +2495,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "pteurs", "url.original": "https://api.example.org/uamqua/Neq.gif?eumiu=nim#pteurs", - "url.query": "ercitati", + "url.path": "/uamqua/Neq.gif", + "url.query": [ + "ercitati", + "eumiu=nim" + ], + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2451,13 +2531,13 @@ "caecat4920.api.host" ], "related.ip": [ - "10.29.138.31", - "10.45.69.152" + "10.45.69.152", + "10.29.138.31" ], "related.user": [ - "volupta", "umq", - "tsunt" + "tsunt", + "volupta" ], "rsa.counters.dclass_c1": 744, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2513,13 +2593,13 @@ "setquas6188.internal.local" ], "related.ip": [ - "10.100.113.11", - "10.152.213.228" + "10.152.213.228", + "10.100.113.11" ], "related.user": [ "itationu", - "velillum", - "ptatev" + "ptatev", + "velillum" ], "rsa.counters.dclass_c1": 7245, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2603,8 +2683,8 @@ "nibusBo3674.www5.localhost" ], "related.ip": [ - "10.248.102.129", - "10.208.33.55" + "10.208.33.55", + "10.248.102.129" ], "related.user": [ "inimv", @@ -2669,8 +2749,8 @@ "10.203.164.132" ], "related.user": [ - "ectobea", "mporin", + "ectobea", "ibus" ], "rsa.counters.dclass_c1": 547, @@ -2727,12 +2807,12 @@ "idents7231.mail.home" ], "related.ip": [ - "10.117.81.75", - "10.151.203.60" + "10.151.203.60", + "10.117.81.75" ], "related.user": [ - "dol", "iconsequ", + "dol", "exeac" ], "rsa.counters.dclass_c1": 484, @@ -2789,13 +2869,13 @@ "tat50.mail.host" ], "related.ip": [ - "10.45.152.205", - "10.224.217.153" + "10.224.217.153", + "10.45.152.205" ], "related.user": [ + "utlabo", "eriti", - "imav", - "utlabo" + "imav" ], "rsa.counters.dclass_c1": 922, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2856,8 +2936,8 @@ "10.1.193.187" ], "related.user": [ - "adipis", "hite", + "adipis", "ugi" ], "rsa.counters.event_counter": 508, @@ -2865,8 +2945,8 @@ "rsa.internal.event_desc": "epteurs", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "taevitae", - "allow" + "allow", + "taevitae" ], "rsa.misc.category": "itse", "rsa.misc.disposition": "rever", @@ -2892,8 +2972,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "emipsumq", "url.original": "https://api.example.net/quam/saute.htm?nostru=docons#emipsumq", - "url.query": "orinr", + "url.path": "/quam/saute.htm", + "url.query": [ + "orinr", + "nostru=docons" + ], + "url.scheme": "https", "user.name": "ugi" }, { @@ -2923,9 +3011,9 @@ "10.248.244.203" ], "related.user": [ - "mquamei", "sum", - "eiusm" + "eiusm", + "mquamei" ], "rsa.counters.dclass_c1": 3058, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -2982,8 +3070,8 @@ ], "related.user": [ "ine", - "consecte", - "nimv" + "nimv", + "consecte" ], "rsa.counters.dclass_c1": 2771, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3039,12 +3127,12 @@ "agnama5013.internal.example" ], "related.ip": [ - "10.201.223.119", - "10.204.223.184" + "10.204.223.184", + "10.201.223.119" ], "related.user": [ - "teni", "rcit", + "teni", "tuserror" ], "rsa.counters.dclass_c1": 4113, @@ -3101,13 +3189,13 @@ "edictas4693.home" ], "related.ip": [ - "10.200.12.126", - "10.223.56.33" + "10.223.56.33", + "10.200.12.126" ], "related.user": [ + "Nequepo", "elitsedd", - "magnido", - "Nequepo" + "magnido" ], "rsa.counters.dclass_c1": 3243, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3170,8 +3258,8 @@ ], "related.user": [ "tuserror", - "emquel", - "citation" + "citation", + "emquel" ], "rsa.counters.event_counter": 2513, "rsa.db.database": "rspiciat", @@ -3205,8 +3293,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "asia", "url.original": "https://internal.example.com/imide/uiineav.htm?lloinve=eni#asia", - "url.query": "edquiac", + "url.path": "/imide/uiineav.htm", + "url.query": [ + "edquiac", + "lloinve=eni" + ], + "url.scheme": "https", "user.name": "tuserror" }, { @@ -3232,13 +3328,13 @@ "tsun7120.home" ], "related.ip": [ - "10.191.184.105", - "10.65.174.196" + "10.65.174.196", + "10.191.184.105" ], "related.user": [ - "tione", + "iin", "uta", - "iin" + "tione" ], "rsa.counters.dclass_c1": 5836, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3292,12 +3388,12 @@ "lumquid6940.mail.localdomain" ], "related.ip": [ - "10.41.181.179", - "10.224.148.48" + "10.224.148.48", + "10.41.181.179" ], "related.user": [ - "niam", "iosamn", + "niam", "equepor" ], "rsa.counters.event_counter": 7468, @@ -3305,8 +3401,8 @@ "rsa.internal.event_desc": "ibusB", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "deny", - "rumwr" + "rumwr", + "deny" ], "rsa.misc.category": "rporis", "rsa.misc.disposition": "etco", @@ -3332,8 +3428,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iumd", "url.original": "https://api.example.com/ptatem/mporain.gif?corpo=commod#iumd", - "url.query": "ntore", + "url.path": "/ptatem/mporain.gif", + "url.query": [ + "ntore", + "corpo=commod" + ], + "url.scheme": "https", "user.name": "niam" }, { @@ -3360,13 +3464,13 @@ "amcorp7299.api.example" ], "related.ip": [ - "10.21.208.103", - "10.21.61.134" + "10.21.61.134", + "10.21.208.103" ], "related.user": [ "ostr", - "imidest", - "mipsa" + "mipsa", + "imidest" ], "rsa.counters.dclass_c1": 7766, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3486,21 +3590,21 @@ "tionevol3157.mail.invalid" ], "related.ip": [ - "10.240.62.238", - "10.191.142.143" + "10.191.142.143", + "10.240.62.238" ], "related.user": [ - "animide", + "nofde", "modtempo", - "nofde" + "animide" ], "rsa.counters.event_counter": 7580, "rsa.db.database": "Lore", "rsa.internal.event_desc": "nto", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "cancel", - "ali" + "ali", + "cancel" ], "rsa.misc.category": "sciv", "rsa.misc.disposition": "tlabo", @@ -3525,8 +3629,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "onsequa", "url.original": "https://www5.example.net/aUten/edutpers.gif?apariatu=mnisis#onsequa", - "url.query": "sunt", + "url.path": "/aUten/edutpers.gif", + "url.query": [ + "sunt", + "apariatu=mnisis" + ], + "url.scheme": "https", "user.name": "modtempo" }, { @@ -3559,8 +3671,8 @@ "10.178.79.217" ], "related.user": [ - "tqui", "inibusBo", + "tqui", "ccusan" ], "rsa.counters.event_counter": 3538, @@ -3594,8 +3706,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "iaeco", "url.original": "https://www.example.net/mvolup/pidat.jpg?ents=nsec#iaeco", - "url.query": "ommodoco", + "url.path": "/mvolup/pidat.jpg", + "url.query": [ + "ommodoco", + "ents=nsec" + ], + "url.scheme": "https", "user.name": "ccusan" }, { @@ -3687,9 +3807,9 @@ "10.186.133.184" ], "related.user": [ - "boriosa", "sci", - "acons" + "acons", + "boriosa" ], "rsa.counters.dclass_c1": 1578, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3744,9 +3864,9 @@ "10.160.147.230" ], "related.user": [ + "nimvenia", "ndeomnis", - "illoin", - "nimvenia" + "illoin" ], "rsa.counters.dclass_c1": 5988, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3860,13 +3980,13 @@ "itte6905.mail.invalid" ], "related.ip": [ - "10.249.13.159", - "10.108.130.106" + "10.108.130.106", + "10.249.13.159" ], "related.user": [ - "uisautei", "exeacomm", - "colab" + "colab", + "uisautei" ], "rsa.counters.dclass_c1": 1044, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -3924,13 +4044,13 @@ "caboNemo274.www.host" ], "related.ip": [ - "10.64.94.174", - "10.39.244.49" + "10.39.244.49", + "10.64.94.174" ], "related.user": [ - "estiae", + "iunt", "Sedut", - "iunt" + "estiae" ], "rsa.counters.event_counter": 7128, "rsa.db.database": "eFinibu", @@ -3963,8 +4083,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "rautod", "url.original": "https://internal.example.org/aev/uovolup.txt?aqueip=aqueip#rautod", - "url.query": "tur", + "url.path": "/aev/uovolup.txt", + "url.query": [ + "tur", + "aqueip=aqueip" + ], + "url.scheme": "https", "user.name": "Sedut" }, { @@ -4047,13 +4175,13 @@ "qui5978.api.test" ], "related.ip": [ - "10.115.203.143", - "10.134.135.22" + "10.134.135.22", + "10.115.203.143" ], "related.user": [ - "orpori", "involu", - "utoditau" + "utoditau", + "orpori" ], "rsa.counters.dclass_c1": 7868, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4109,12 +4237,12 @@ "iamq2577.internal.corp" ], "related.ip": [ - "10.251.212.166", - "10.43.244.252" + "10.43.244.252", + "10.251.212.166" ], "related.user": [ - "inculp", "gnido", + "inculp", "uptat" ], "rsa.counters.dclass_c1": 6947, @@ -4199,13 +4327,13 @@ "usB4127.localhost" ], "related.ip": [ - "10.20.231.188", - "10.88.189.164" + "10.88.189.164", + "10.20.231.188" ], "related.user": [ - "mqu", + "uatDuisa", "tesseq", - "uatDuisa" + "mqu" ], "rsa.counters.dclass_c1": 1623, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4289,8 +4417,8 @@ "abor3266.mail.home" ], "related.ip": [ - "10.225.11.197", - "10.231.77.26" + "10.231.77.26", + "10.225.11.197" ], "related.user": [ "rehe", @@ -4353,9 +4481,9 @@ "10.106.166.105" ], "related.user": [ + "olupt", "avolup", - "usa", - "olupt" + "usa" ], "rsa.counters.dclass_c1": 2658, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4411,9 +4539,9 @@ "10.57.169.205" ], "related.user": [ - "ipsu", + "ctas", "iuta", - "ctas" + "ipsu" ], "rsa.counters.dclass_c1": 392, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4473,9 +4601,9 @@ "10.129.234.200" ], "related.user": [ + "tisundeo", "dquia", - "tevelit", - "tisundeo" + "tevelit" ], "rsa.counters.dclass_c1": 6709, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4531,13 +4659,13 @@ "ididu5928.www5.local" ], "related.ip": [ - "10.111.132.221", - "10.76.121.224" + "10.76.121.224", + "10.111.132.221" ], "related.user": [ - "scive", "ali", - "oloremi" + "oloremi", + "scive" ], "rsa.counters.dclass_c1": 6155, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4598,8 +4726,8 @@ ], "related.user": [ "enimip", - "dolo", - "ota" + "ota", + "dolo" ], "rsa.counters.dclass_c1": 469, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4655,12 +4783,12 @@ "ssusc1892.internal.host" ], "related.ip": [ - "10.179.60.167", - "10.173.13.179" + "10.173.13.179", + "10.179.60.167" ], "related.user": [ - "apar", "isn", + "apar", "ptasn" ], "rsa.counters.dclass_c1": 758, @@ -4717,13 +4845,13 @@ "iatisund424.mail.localdomain" ], "related.ip": [ - "10.42.135.34", - "10.178.190.123" + "10.178.190.123", + "10.42.135.34" ], "related.user": [ - "orsi", + "ore", "tiset", - "ore" + "orsi" ], "rsa.counters.dclass_c1": 2290, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4811,9 +4939,9 @@ "10.207.198.239" ], "related.user": [ + "aUteni", "Loremips", - "incididu", - "aUteni" + "incididu" ], "rsa.counters.dclass_c1": 3043, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4873,8 +5001,8 @@ ], "related.user": [ "nseq", - "litesseq", - "oNe" + "oNe", + "litesseq" ], "rsa.counters.dclass_c1": 3218, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -4926,13 +5054,13 @@ "nibusBo1864.domain" ], "related.ip": [ - "10.253.127.130", - "10.86.180.150" + "10.86.180.150", + "10.253.127.130" ], "related.user": [ - "itasper", + "etconsec", "mnisis", - "etconsec" + "itasper" ], "rsa.counters.dclass_c1": 4564, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5028,8 +5156,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "amnisiu", "url.original": "https://internal.example.com/architec/incul.txt?aborios=mco#amnisiu", - "url.query": "suntincu", + "url.path": "/architec/incul.txt", + "url.query": [ + "suntincu", + "aborios=mco" + ], + "url.scheme": "https", "user.name": "dolo" }, { @@ -5088,9 +5224,9 @@ "10.248.16.82" ], "related.user": [ - "ditautf", + "tuserror", "res", - "tuserror" + "ditautf" ], "rsa.counters.dclass_c1": 4367, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5150,9 +5286,9 @@ "10.173.19.140" ], "related.user": [ - "orsi", + "Except", "olo", - "Except" + "orsi" ], "rsa.counters.dclass_c1": 5844, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5211,8 +5347,8 @@ "10.69.5.227" ], "related.user": [ - "rumw", "doloreme", + "rumw", "ntocc" ], "rsa.counters.dclass_c1": 5201, @@ -5269,9 +5405,9 @@ "10.253.175.129" ], "related.user": [ - "ate", + "epteurs", "nrep", - "epteurs" + "ate" ], "rsa.counters.dclass_c1": 6260, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5333,9 +5469,9 @@ "10.89.26.170" ], "related.user": [ + "orumetMa", "atus", - "aboris", - "orumetMa" + "aboris" ], "rsa.counters.event_counter": 5863, "rsa.db.database": "inventor", @@ -5368,8 +5504,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "edut", "url.original": "https://www.example.com/sit/ugi.gif?sitametc=rur#edut", - "url.query": "sitametc", + "url.path": "/sit/ugi.gif", + "url.query": [ + "sitametc", + "sitametc=rur" + ], + "url.scheme": "https", "user.name": "aboris" }, { @@ -5402,9 +5546,9 @@ "10.81.108.232" ], "related.user": [ - "neavolup", + "uaturve", "aco", - "uaturve" + "neavolup" ], "rsa.counters.event_counter": 5098, "rsa.db.database": "lapa", @@ -5438,8 +5582,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "amrem", "url.original": "https://internal.example.net/obeatae/sedqui.jpg?nulap=onseq#amrem", - "url.query": "plicab", + "url.path": "/obeatae/sedqui.jpg", + "url.query": [ + "plicab", + "nulap=onseq" + ], + "url.scheme": "https", "user.name": "aco" }, { @@ -5468,12 +5620,12 @@ "par3605.internal.localdomain" ], "related.ip": [ - "10.223.10.28", - "10.230.48.97" + "10.230.48.97", + "10.223.10.28" ], "related.user": [ - "untex", "erit", + "untex", "usmodte" ], "rsa.counters.event_counter": 4029, @@ -5481,8 +5633,8 @@ "rsa.internal.event_desc": "itatiset", "rsa.internal.messageid": "Imperva", "rsa.misc.action": [ - "tconse", - "deny" + "deny", + "tconse" ], "rsa.misc.category": "uaerat", "rsa.misc.disposition": "met", @@ -5508,8 +5660,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "itinvo", "url.original": "https://mail.example.com/iaturE/inc.htm?uisaut=mnihilm#itinvo", - "url.query": "lestia", + "url.path": "/iaturE/inc.htm", + "url.query": [ + "lestia", + "uisaut=mnihilm" + ], + "url.scheme": "https", "user.name": "erit" }, { @@ -5536,13 +5696,13 @@ "isau4356.www.home" ], "related.ip": [ - "10.161.212.150", - "10.115.42.231" + "10.115.42.231", + "10.161.212.150" ], "related.user": [ - "sequamn", "tasnul", - "res" + "res", + "sequamn" ], "rsa.counters.dclass_c1": 4846, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5600,13 +5760,13 @@ "labo3477.www5.domain" ], "related.ip": [ - "10.247.108.144", - "10.226.75.20" + "10.226.75.20", + "10.247.108.144" ], "related.user": [ - "fugia", "maccusan", - "tema" + "tema", + "fugia" ], "rsa.counters.event_counter": 3711, "rsa.db.database": "psa", @@ -5640,8 +5800,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "eip", "url.original": "https://www5.example.org/elaud/temsequ.htm?dolo=iciatisu#eip", - "url.query": "iquaUte", + "url.path": "/elaud/temsequ.htm", + "url.query": [ + "iquaUte", + "dolo=iciatisu" + ], + "url.scheme": "https", "user.name": "tema" }, { @@ -5671,9 +5839,9 @@ "10.192.15.65" ], "related.user": [ - "rExcep", "illumd", - "nimides" + "nimides", + "rExcep" ], "rsa.counters.dclass_c1": 4173, "rsa.counters.dclass_c1_str": "Affected Rows", @@ -5727,12 +5895,12 @@ "duntutl3396.api.host" ], "related.ip": [ - "10.116.76.161", - "10.197.254.133" + "10.197.254.133", + "10.116.76.161" ], "related.user": [ - "idu", "ide", + "idu", "trudex" ], "rsa.counters.event_counter": 2608, @@ -5766,8 +5934,16 @@ "imperva.securesphere", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "ema", "url.original": "https://mail.example.net/teturad/nimide.htm?ueporroq=writ#ema", - "url.query": "ioffici", + "url.path": "/teturad/nimide.htm", + "url.query": [ + "ioffici", + "ueporroq=writ" + ], + "url.scheme": "https", "user.name": "ide" }, { @@ -5794,8 +5970,8 @@ "colabo6686.internal.invalid" ], "related.ip": [ - "10.28.77.79", - "10.144.14.15" + "10.144.14.15", + "10.28.77.79" ], "related.user": [ "rspic", @@ -5855,13 +6031,13 @@ "tsunti1164.www.example" ], "related.ip": [ - "10.18.15.43", - "10.248.177.182" + "10.248.177.182", + "10.18.15.43" ], "related.user": [ - "quaturve", "quei", - "caecat" + "caecat", + "quaturve" ], "rsa.counters.dclass_c1": 983, "rsa.counters.dclass_c1_str": "Affected Rows", diff --git a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml index b04d4b5d67f..aa8e3f97692 100644 --- a/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/defender_atp/ingest/pipeline.yml @@ -232,6 +232,10 @@ processors: ignore_missing: true if: ctx?.json?.evidence?.url != null +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS Host Mapping ## ###################### diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml index ae33c77d6d5..5103d12de9d 100644 --- a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -221,7 +221,10 @@ processors: target_field: url.full ignore_missing: true if: ctx?.json?.entities?.url != null - +- uri_parts: + field: url.full + ignore_failure: true + if: ctx?.url?.full != null ###################### ## ECS User Mapping ## ###################### diff --git a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml index 8a25a657a0a..22f32138c27 100644 --- a/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml +++ b/x-pack/filebeat/module/netscout/sightline/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json index 7e500a27094..66df7548ac2 100644 --- a/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json +++ b/x-pack/filebeat/module/netscout/sightline/test/generated.log-expected.json @@ -323,7 +323,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tamremap", + "url.original": "https://mail.example.net/uam/untutl.jpg?llu=uptassi#tamremap", + "url.path": "/uam/untutl.jpg", + "url.query": "llu=uptassi", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -365,8 +371,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.155.162.162", - "10.66.171.247" + "10.66.171.247", + "10.155.162.162" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -379,7 +385,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis" + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "uatDuis", + "url.original": "https://www5.example.org/seq/olorema.jpg?quid=fug#uatDuis", + "url.path": "/seq/olorema.jpg", + "url.query": "quid=fug", + "url.scheme": "https" }, { "event.action": "Fault Occured", @@ -458,7 +470,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc" + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "suntinc", + "url.original": "https://example.org/isiu/nimadmi.gif?ari=equun#suntinc", + "url.path": "/isiu/nimadmi.gif", + "url.query": "ari=equun", + "url.scheme": "https" }, { "event.code": "Hardware", @@ -848,7 +866,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae" + "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "psaquae", + "url.original": "https://www.example.net/nse/sinto.gif?CSed=lupt#psaquae", + "url.path": "/nse/sinto.gif", + "url.query": "CSed=lupt", + "url.scheme": "https" }, { "event.code": "BGP", @@ -980,7 +1004,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup" + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "lup", + "url.original": "https://mail.example.net/corp/modtemp.jpg?oluptas=tNequepo#lup", + "url.path": "/corp/modtemp.jpg", + "url.query": "oluptas=tNequepo", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1036,7 +1066,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu" + "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "pariatu", + "url.original": "https://example.net/temqu/edol.jpg?ipi=reseos#pariatu", + "url.path": "/temqu/edol.jpg", + "url.query": "ipi=reseos", + "url.scheme": "https" }, { "destination.ip": [ @@ -1147,8 +1183,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.83.23.104", - "10.163.161.165" + "10.163.161.165", + "10.83.23.104" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1161,7 +1197,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte" + "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "nsecte", + "url.original": "https://www5.example.org/atem/gnido.txt?tmollita=fde#nsecte", + "url.path": "/atem/gnido.txt", + "url.query": "tmollita=fde", + "url.scheme": "https" }, { "destination.ip": [ @@ -1402,7 +1444,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse" + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "tse", + "url.original": "https://api.example.com/eufugi/pici.html?ccaecat=tquiin#tse", + "url.path": "/eufugi/pici.html", + "url.query": "ccaecat=tquiin", + "url.scheme": "https" }, { "event.code": "Change_Log", @@ -1752,8 +1800,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.28.226.128", - "10.122.76.148" + "10.122.76.148", + "10.28.226.128" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -1766,7 +1814,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "remips", + "url.original": "https://mail.example.org/idunt/luptat.txt?ica=lillum#remips", + "url.path": "/idunt/luptat.txt", + "url.query": "ica=lillum", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1790,7 +1844,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt" + "url.domain": "internal.example.com", + "url.extension": "html", + "url.fragment": "amvolupt", + "url.original": "https://internal.example.com/ineavol/iosa.html?usc=rem#amvolupt", + "url.path": "/ineavol/iosa.html", + "url.query": "usc=rem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1814,7 +1874,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation" + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "itation", + "url.original": "https://internal.example.org/quaUten/nisiut.txt?teturad=perspici#itation", + "url.path": "/quaUten/nisiut.txt", + "url.query": "teturad=perspici", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -1838,7 +1904,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt" + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "quunt", + "url.original": "https://mail.example.org/nofd/dipisci.txt?ilmol=eri#quunt", + "url.path": "/nofd/dipisci.txt", + "url.query": "ilmol=eri", + "url.scheme": "https" }, { "destination.ip": [ @@ -1870,7 +1942,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo" + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "labo", + "url.original": "https://www.example.org/ptateve/enderi.html?toccaec=fugi#labo", + "url.path": "/ptateve/enderi.html", + "url.query": "toccaec=fugi", + "url.scheme": "https" }, { "destination.ip": [ @@ -1902,7 +1980,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo" + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", + "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https" }, { "event.code": "configuration", @@ -2169,8 +2253,8 @@ "observer.type": "DDOS", "observer.vendor": "Netscout", "related.ip": [ - "10.151.129.181", - "10.55.156.64" + "10.55.156.64", + "10.151.129.181" ], "rsa.internal.messageid": "Blocked_Host", "rsa.misc.msgIdPart1": "Blocked", @@ -2183,7 +2267,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae" + "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "iae", + "url.original": "https://www.example.net/itanim/nesciun.txt?mollita=tatem#iae", + "url.path": "/itanim/nesciun.txt", + "url.query": "mollita=tatem", + "url.scheme": "https" }, { "event.code": "Protection_Mode", @@ -2207,7 +2297,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom" + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "acom", + "url.original": "https://www.example.net/rever/ore.jpg?oluptat=metco#acom", + "url.path": "/rever/ore.jpg", + "url.query": "oluptat=metco", + "url.scheme": "https" }, { "event.code": "Host", @@ -2314,7 +2410,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu" + "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "apariatu", + "url.original": "https://api.example.org/eosquira/pta.htm?econs=lmolesti#apariatu", + "url.path": "/eosquira/pta.htm", + "url.query": "econs=lmolesti", + "url.scheme": "https" }, { "event.action": "Script mitigation", @@ -2362,7 +2464,13 @@ "netscout.sightline", "forwarded" ], - "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "taliqui", + "url.original": "https://www5.example.com/atcupida/tessequa.htm?dolores=equamnih#taliqui", + "url.path": "/atcupida/tessequa.htm", + "url.query": "dolores=equamnih", + "url.scheme": "https" }, { "event.code": "Change_Log", diff --git a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml index 49371346925..c2d1a74c030 100644 --- a/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/o365/audit/ingest/pipeline.yml @@ -7,6 +7,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json index 4a6f14974fc..41256959a5c 100644 --- a/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/06-sharepointfileop.log-expected.json @@ -64,7 +64,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -142,7 +146,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -220,7 +228,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -298,7 +310,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "aspx", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Forms/All.aspx", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -377,7 +393,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -455,7 +475,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -533,7 +557,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot 2020-01-27 at 11.30.48.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -612,7 +640,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -690,7 +722,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -768,7 +804,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", @@ -846,7 +886,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem-my.sharepoint.com", + "url.extension": "png", "url.original": "https://testsiem-my.sharepoint.com/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.path": "/personal/asr_testsiem_onmicrosoft_com/Documents/Screenshot.png", + "url.scheme": "https", "user.domain": "testsiem.onmicrosoft.com", "user.email": "asr@testsiem.onmicrosoft.com", "user.id": "asr@testsiem.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json index 75f16b9867e..c440645b360 100644 --- a/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/11-dlp-sharepoint.log-expected.json @@ -80,7 +80,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -176,7 +180,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -268,7 +276,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data%20Copy.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data Copy.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -364,7 +376,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers%20Financial%20Data.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/Customers Financial Data.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -460,7 +476,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2-my.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2-my.sharepoint.com/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL%20CREDIT%20CARD%20NUMBERS.docx", + "url.path": "/personal/asr_testsiem2_onmicrosoft_com/Documents/INTERNAL CREDIT CARD NUMBERS.docx", + "url.scheme": "https", "user.domain": "TESTSIEM2.ONMICROSOFT.COM", "user.email": "ASR@TESTSIEM2.ONMICROSOFT.COM", "user.id": "ASR@TESTSIEM2.ONMICROSOFT.COM", @@ -556,7 +576,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", @@ -652,7 +676,11 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem2.sharepoint.com", + "url.extension": "docx", "url.original": "https://testsiem2.sharepoint.com/sites/Internalcommunications/Shared%20Documents/Document.docx", + "url.path": "/sites/Internalcommunications/Shared Documents/Document.docx", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json index 941adb1e1ac..7996a2d808d 100644 --- a/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/13-dlp-exchange.log-expected.json @@ -791,7 +791,11 @@ "tags": [ "forwarded" ], + "url.domain": "example.net", + "url.extension": "com/sharepoint", "url.original": "https://example.net/testsiem2.onmicrosoft.com/sharepoint", + "url.path": "/testsiem2.onmicrosoft.com/sharepoint", + "url.scheme": "https", "user.domain": "testsiem2.onmicrosoft.com", "user.email": "alice@testsiem2.onmicrosoft.com", "user.id": "alice@testsiem2.onmicrosoft.com", diff --git a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json index 372b29d8c2c..89a70bc165f 100644 --- a/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json +++ b/x-pack/filebeat/module/o365/audit/test/25-ms-teams-groups.log-expected.json @@ -249,7 +249,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", @@ -1547,7 +1550,10 @@ "tags": [ "forwarded" ], + "url.domain": "testsiem4.sharepoint.com", "url.original": "https://testsiem4.sharepoint.com/sites/users/Shared Documents/General", + "url.path": "/sites/users/Shared Documents/General", + "url.scheme": "https", "user.domain": "testsiem4.onmicrosoft.com", "user.email": "root@testsiem4.onmicrosoft.com", "user.id": "root@testsiem4.onmicrosoft.com", diff --git a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml index 3c41092be69..66d787b27ce 100644 --- a/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml +++ b/x-pack/filebeat/module/sophos/utm/ingest/pipeline.yml @@ -10,6 +10,30 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json index e0caf6d7abb..bf35e865916 100644 --- a/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json +++ b/x-pack/filebeat/module/sophos/utm/test/generated.log-expected.json @@ -70,8 +70,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "block", - "ugiatnu" + "ugiatnu", + "block" ], "rsa.misc.comments": "colabo", "rsa.misc.content_type": "sedd", @@ -103,7 +103,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "iciatisu", "url.original": "https://mail.example.net/tati/utaliqu.html?iquaUten=santium#iciatisu", + "url.path": "/tati/utaliqu.html", + "url.query": "iquaUten=santium", + "url.scheme": "https", "user.name": "sunt" }, { @@ -163,8 +169,8 @@ "10.106.239.55" ], "related.user": [ - "eaq", - "itquiin" + "itquiin", + "eaq" ], "rsa.identity.logon_type": "stquidol", "rsa.internal.event_desc": "bor", @@ -974,8 +980,8 @@ "10.232.108.32" ], "related.user": [ - "rsp", - "llum" + "llum", + "rsp" ], "rsa.identity.logon_type": "ntut", "rsa.internal.event_desc": "ittenb", @@ -1038,8 +1044,8 @@ ], "related.user": [ "tcustatuscode=eumiu", - "tio", - "pteurs" + "pteurs", + "tio" ], "rsa.db.index": "eavolupt", "rsa.identity.logon_type": "ursintoc", @@ -1048,8 +1054,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "iuntN", - "deny" + "deny", + "iuntN" ], "rsa.misc.comments": "onorume", "rsa.misc.content_type": "lapa", @@ -1084,7 +1090,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ssequamn", "url.original": "https://mail.example.net/velillu/ecatcupi.txt?rsitamet=leumiur#ssequamn", + "url.path": "/velillu/ecatcupi.txt", + "url.query": "rsitamet=leumiur", + "url.scheme": "https", "user.name": "tio" }, { @@ -1237,7 +1249,13 @@ "sophos.utm", "forwarded" ], - "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure" + "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "eumiure", + "url.original": "https://www5.example.com/dantium/ors.htm?sinto=edi#eumiure", + "url.path": "/dantium/ors.htm", + "url.query": "sinto=edi", + "url.scheme": "https" }, { "@timestamp": "2017-05-29T07:37:24.000Z", @@ -1610,8 +1628,8 @@ "10.244.96.61" ], "related.user": [ - "itsedqui", - "iumt" + "iumt", + "itsedqui" ], "rsa.identity.logon_type": "psamvolu", "rsa.internal.event_desc": "orroqui", @@ -1851,8 +1869,8 @@ "tenbyCi4371.www5.localdomain" ], "related.ip": [ - "10.214.167.164", - "10.98.126.206" + "10.98.126.206", + "10.214.167.164" ], "related.user": [ "amremapstatuscode=dolorsit", @@ -1899,7 +1917,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "enbyCi", "url.original": "https://api.example.org/iono/secillum.txt?apariat=tse#enbyCi", + "url.path": "/iono/secillum.txt", + "url.query": "apariat=tse", + "url.scheme": "https", "user.name": "hen" }, { @@ -2029,14 +2053,14 @@ "iusmo901.www.home" ], "related.ip": [ - "10.2.24.156", - "10.92.93.236" + "10.92.93.236", + "10.2.24.156" ], "related.user": [ - "dolorsistatuscode=acc", - "Sedutper", + "ulpaq", "ntoccae", - "ulpaq" + "Sedutper", + "dolorsistatuscode=acc" ], "rsa.db.index": "snisiut", "rsa.identity.logon_type": "umdol", @@ -2082,8 +2106,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "tru", "url.original": "https://www.example.org/imipsamv/doeiu.jpg?nderit=ficia#tru", - "url.query": "exerc", + "url.path": "/imipsamv/doeiu.jpg", + "url.query": [ + "exerc", + "nderit=ficia" + ], + "url.scheme": "https", "user.name": "Sedutper" }, { @@ -2124,8 +2156,8 @@ "10.180.169.49" ], "related.user": [ - "atatno", "tasu", + "atatno", "iscivelistatuscode=urve" ], "rsa.db.index": "amrem", @@ -2135,8 +2167,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "mquisn", - "cancel" + "cancel", + "mquisn" ], "rsa.misc.comments": "architec", "rsa.misc.content_type": "atisetqu", @@ -2168,7 +2200,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "adm", "url.original": "https://mail.example.net/isqua/deF.html?iameaq=orainci#adm", + "url.path": "/isqua/deF.html", + "url.query": "iameaq=orainci", + "url.scheme": "https", "user.name": "atatno" }, { @@ -2314,13 +2352,13 @@ "obea2960.mail.corp" ], "related.ip": [ - "10.45.12.53", - "10.33.138.154" + "10.33.138.154", + "10.45.12.53" ], "related.user": [ - "eturadip", + "umqustatuscode=ntexpli", "porincid", - "umqustatuscode=ntexpli" + "eturadip" ], "rsa.db.index": "dolor", "rsa.identity.logon_type": "eturadi", @@ -2362,7 +2400,13 @@ "sophos.utm", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "oditautf", "url.original": "https://www5.example.org/etcons/aecatc.jpg?ditem=tut#oditautf", + "url.path": "/etcons/aecatc.jpg", + "url.query": "ditem=tut", + "url.scheme": "https", "user.name": "eturadip" }, { @@ -2481,8 +2525,8 @@ "10.32.85.21" ], "related.user": [ - "antium", - "etconsec" + "etconsec", + "antium" ], "rsa.identity.logon_type": "umiurere", "rsa.internal.event_desc": "serro", @@ -2634,8 +2678,8 @@ "related.user": [ "Loremi", "inimastatuscode=emipsum", - "reetd", - "rExce" + "rExce", + "reetd" ], "rsa.db.index": "apa", "rsa.identity.logon_type": "sedquia", @@ -2644,8 +2688,8 @@ "rsa.investigations.event_cat": 1901000000, "rsa.investigations.event_cat_name": "Other.Default", "rsa.misc.action": [ - "odte", - "cancel" + "cancel", + "odte" ], "rsa.misc.comments": "emquia", "rsa.misc.content_type": "sauteir", @@ -2678,8 +2722,16 @@ "sophos.utm", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "agnaaliq", "url.original": "https://internal.example.com/ritat/dipi.jpg?aliquide=aliqui#agnaaliq", - "url.query": "iatquovo", + "url.path": "/ritat/dipi.jpg", + "url.query": [ + "iatquovo", + "aliquide=aliqui" + ], + "url.scheme": "https", "user.name": "reetd" }, { @@ -2707,8 +2759,8 @@ "itametc1599.api.test" ], "related.ip": [ - "10.115.166.48", - "10.133.45.45" + "10.133.45.45", + "10.115.166.48" ], "rsa.internal.event_desc": "Authentication", "rsa.internal.messageid": "ulogd", @@ -3463,8 +3515,8 @@ "imv1805.api.host" ], "related.ip": [ - "10.248.62.55", - "10.96.243.231" + "10.96.243.231", + "10.248.62.55" ], "rsa.internal.event_desc": "ICMP", "rsa.internal.messageid": "ulogd", diff --git a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml index 845a93c945e..6eb60e460bb 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/antivirus.yml @@ -192,6 +192,29 @@ processors: target_field: url.domain ignore_missing: true if: "ctx.sophos?.xg?.src_domainname != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true ############################ ## ECS User Agent Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml index c659264d633..64c3e52bf03 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/atp.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/atp.yml @@ -120,6 +120,33 @@ processors: target_field: url.original ignore_missing: true if: "ctx.sophos?.xg?.url != null" +- uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null && ctx?.url?.original != ctx?.destination?.ip +- script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } +- remove: + field: _temp_ + ignore_missing: true +- urldecode: + field: url.original + ignore_missing: true + ignore_failure: true ############################# ## ECS Network/Geo Mapping ## diff --git a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml index 0b5f92c1e69..7b4853ce5bf 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/cfilter.yml @@ -122,16 +122,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -268,7 +271,32 @@ processors: - sophos.xg.src_port - sophos.xg.recv_bytes - sophos.xg.sent_bytes + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml index e0ea89c41dc..6462ac5d6bf 100644 --- a/x-pack/filebeat/module/sophos/xg/ingest/waf.yml +++ b/x-pack/filebeat/module/sophos/xg/ingest/waf.yml @@ -118,16 +118,19 @@ processors: ##################### ## ECS URL Mapping ## ##################### +- uri_parts: + field: sophos.xg.url + ignore_failure: true + if: "ctx.sophos?.xg?.url != null" - rename: field: sophos.xg.url target_field: url.full ignore_missing: true if: "ctx.sophos?.xg?.url != null" -- rename: - field: sophos.xg.domain - target_field: url.domain - ignore_missing: true - if: "ctx.sophos?.xg?.domain != null" +- set: + field: url.domain + value: "{{sophos.xg.domain}}" + ignore_empty_value: true ############################ ## ECS User Agent Mapping ## @@ -279,7 +282,32 @@ processors: - sophos.xg.bytesrcv - sophos.xg.bytessent - sophos.xg.httpstatus + - sophos.xg.domain ignore_missing: true +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json index 4afefcee9b4..8ab666cc94b 100644 --- a/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/anti-virus.log-expected.json @@ -77,7 +77,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "pdf", "url.original": "http://sophostest.com/Sandstorm/SBTestFile1.pdf", + "url.path": "/Sandstorm/SBTestFile1.pdf", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { @@ -158,7 +161,10 @@ "forwarded" ], "url.domain": "sophostest.com", + "url.extension": "html", "url.original": "http://sophostest.com/eicar/index.html", + "url.path": "/eicar/index.html", + "url.scheme": "http", "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36" }, { diff --git a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json index a0230cb1dc4..969ca99d793 100644 --- a/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/atp.log-expected.json @@ -140,7 +140,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2020-05-18T14:38:35.000-02:00", @@ -211,7 +215,11 @@ "sophos-xg", "forwarded" ], - "url.original": "http://sophostest.com/callhome/index.html" + "url.domain": "sophostest.com", + "url.extension": "html", + "url.original": "http://sophostest.com/callhome/index.html", + "url.path": "/callhome/index.html", + "url.scheme": "http" }, { "@timestamp": "2018-06-05T08:49:00.000-02:00", diff --git a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json index c8bb6001058..3194c309b5b 100644 --- a/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/cfilter.log-expected.json @@ -71,7 +71,10 @@ "forwarded" ], "url.domain": "r8---sn-ci5gup-qxas.googlevideo.com", - "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/" + "url.full": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.original": "https://r8---sn-ci5gup-qxas.googlevideo.com/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2017-02-01T18:20:21.000-02:00", @@ -152,7 +155,10 @@ "forwarded" ], "url.domain": "hanuman.com", - "url.full": "http://hanuman.com/" + "url.full": "http://hanuman.com/", + "url.original": "http://hanuman.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2017-02-01T18:13:29.000-02:00", @@ -309,7 +315,10 @@ "forwarded" ], "url.domain": "his-eur1-neur1.servicebus.windows.net", - "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket" + "url.full": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.original": "https://his-eur1-neur1.servicebus.windows.net/$servicebus/websocket", + "url.path": "/$servicebus/websocket", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:52.000-02:00", @@ -386,7 +395,10 @@ "forwarded" ], "url.domain": "40.90.137.127", - "url.full": "https://40.90.137.127/" + "url.full": "https://40.90.137.127/", + "url.original": "https://40.90.137.127/", + "url.path": "/", + "url.scheme": "https" }, { "@timestamp": "2020-05-18T14:38:53.000-02:00", @@ -462,7 +474,11 @@ "forwarded" ], "url.domain": "update.eset.com", + "url.extension": "signed", "url.full": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.original": "http://update.eset.com/eset_upd/ep7/dll/update.ver.signed", + "url.path": "/eset_upd/ep7/dll/update.ver.signed", + "url.scheme": "http", "user_agent.original": "EFSW Update (Windows; U; 64bit; BPC 7.1.12010.0; OS: 10.0.17763 SP 0.0 NT; TDB 45511; CL 1.1.1; x64s; APP efsw; PX 1; PUA 1; CD 1; RA 1; PEV 0; UNS 1; UBR 1158; HVCI 0; SHA256 1; WU 3; HWF: 01009DAA-757A-D666-EFD2-92DD0D501284; PLOC de_de; PCODE 211.0.0; " }, { @@ -592,7 +608,10 @@ "forwarded" ], "url.domain": "www.google.com", - "url.full": "http://www.google.com/" + "url.full": "http://www.google.com/", + "url.original": "http://www.google.com/", + "url.path": "/", + "url.scheme": "http" }, { "@timestamp": "2016-12-02T18:50:22.000-02:00", @@ -669,6 +688,10 @@ "forwarded" ], "url.domain": "www.google.ca", - "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw" + "url.full": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.original": "http://www.google.ca/?gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.path": "/", + "url.query": "gfe_rd=cr&ei=ojxHWP3WC4WN8QeRioDABw", + "url.scheme": "http" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json index 055f255a15a..9950baa2a62 100644 --- a/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json +++ b/x-pack/filebeat/module/sophos/xg/test/waf.log-expected.json @@ -81,6 +81,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 10.0; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -166,6 +168,8 @@ "forwarded" ], "url.full": "/mapi/nspi/", + "url.original": "/mapi/nspi/", + "url.path": "/mapi/nspi/", "user_agent.original": "Microsoft Office/16.0 (Windows NT 6.2; Microsoft Outlook 16.0.4954; Pro)" }, { @@ -236,6 +240,8 @@ "forwarded" ], "url.full": "/", + "url.original": "/", + "url.path": "/", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -308,7 +314,10 @@ "sophos-xg", "forwarded" ], + "url.extension": "zip", "url.full": "/download/eicarcom2.zip", + "url.original": "/download/eicarcom2.zip", + "url.path": "/download/eicarcom2.zip", "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:50.0) Gecko/20100101 Firefox/50.0" }, { @@ -392,6 +401,8 @@ "sophos-xg", "forwarded" ], - "url.full": "/" + "url.full": "/", + "url.original": "/", + "url.path": "/" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml index d2bf90581b8..f9f6ddaa027 100644 --- a/x-pack/filebeat/module/squid/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/squid/log/ingest/pipeline.yml @@ -10,6 +10,34 @@ processors: - user_agent: field: user_agent.original ignore_missing: true + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true + - remove: + field: url.scheme + ignore_missing: true + if: ctx?.url?.scheme == ctx?.url?.domain # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json index bcced9c85a3..f70c9ad8ae4 100644 --- a/x-pack/filebeat/module/squid/log/test/access1.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/access1.log-expected.json @@ -26,8 +26,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -37,8 +37,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -97,8 +97,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -109,8 +109,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -134,7 +134,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -170,8 +172,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -182,8 +184,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -206,8 +208,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -241,8 +246,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -265,8 +270,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "css", "url.original": "http://www.goonernews.com/styles.css", + "url.path": "/styles.css", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -324,8 +332,11 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "js", "url.original": "http://www.google-analytics.com/urchin.js", + "url.path": "/urchin.js", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -360,8 +371,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -397,7 +408,9 @@ ], "url.domain": "www.goonernews.com", "url.original": "http://www.goonernews.com/", + "url.path": "/", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -442,8 +455,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -466,8 +479,12 @@ "forwarded" ], "url.domain": "www.google-analytics.com", + "url.extension": "gif", "url.original": "http://www.google-analytics.com/__utm.gif?", + "url.path": "/__utm.gif", + "url.query": "", "url.registered_domain": "google-analytics.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -503,8 +520,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -539,8 +556,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/newslogo.gif", + "url.path": "/graphics/newslogo.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -588,8 +608,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -612,8 +632,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "jpg", "url.original": "http://www.goonernews.com/shop/arsenal_shop_ad.jpg", + "url.path": "/shop/arsenal_shop_ad.jpg", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -649,8 +672,8 @@ "www.goonernews.com" ], "related.ip": [ - "10.105.21.199", - "207.58.145.61" + "207.58.145.61", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -661,8 +684,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -685,8 +708,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FUS.gif", + "url.path": "/flags/FUS.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -720,8 +746,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -744,8 +770,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FGB.gif", + "url.path": "/flags/FGB.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -781,8 +810,8 @@ "as.casalemedia.com" ], "related.ip": [ - "10.105.21.199", - "209.85.16.38" + "209.85.16.38", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -818,7 +847,10 @@ ], "url.domain": "as.casalemedia.com", "url.original": "http://as.casalemedia.com/s?", + "url.path": "/s", + "url.query": "", "url.registered_domain": "casalemedia.com", + "url.scheme": "http", "url.subdomain": "as", "url.top_level_domain": "com", "user.name": "badeyek" @@ -848,8 +880,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "68.142.213.132", - "10.105.21.199" + "10.105.21.199", + "68.142.213.132" ], "related.user": [ "badeyek" @@ -859,8 +891,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -917,8 +949,8 @@ "impgb.tradedoubler.com" ], "related.ip": [ - "10.105.21.199", - "217.212.240.172" + "217.212.240.172", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -929,8 +961,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -954,7 +986,9 @@ ], "url.domain": "impgb.tradedoubler.com", "url.original": "http://impgb.tradedoubler.com/imp/img/16349696/992098", + "url.path": "/imp/img/16349696/992098", "url.registered_domain": "tradedoubler.com", + "url.scheme": "http", "url.subdomain": "impgb", "url.top_level_domain": "com", "user.name": "badeyek" @@ -990,8 +1024,8 @@ "4.adbrite.com" ], "related.ip": [ - "10.105.21.199", - "206.169.136.22" + "206.169.136.22", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1002,8 +1036,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -1026,8 +1060,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1061,8 +1099,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -1085,8 +1123,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FFR.gif", + "url.path": "/flags/FFR.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1134,8 +1175,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1158,8 +1199,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/flags/FAU.gif", + "url.path": "/flags/FAU.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1195,8 +1239,8 @@ "www.goonernews.com" ], "related.ip": [ - "207.58.145.61", - "10.105.21.199" + "10.105.21.199", + "207.58.145.61" ], "related.user": [ "badeyek" @@ -1207,8 +1251,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -1231,8 +1275,11 @@ "forwarded" ], "url.domain": "www.goonernews.com", + "url.extension": "gif", "url.original": "http://www.goonernews.com/graphics/spacer.gif", + "url.path": "/graphics/spacer.gif", "url.registered_domain": "goonernews.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1263,8 +1310,8 @@ "4.adbrite.com" ], "related.ip": [ - "64.127.126.178", - "10.105.21.199" + "10.105.21.199", + "64.127.126.178" ], "related.user": [ "badeyek" @@ -1299,8 +1346,12 @@ "forwarded" ], "url.domain": "4.adbrite.com", + "url.extension": "php", "url.original": "http://4.adbrite.com/mb/text_group.php?", + "url.path": "/mb/text_group.php", + "url.query": "", "url.registered_domain": "adbrite.com", + "url.scheme": "http", "url.subdomain": "4", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1336,8 +1387,8 @@ "ff.connextra.com" ], "related.ip": [ - "10.105.21.199", - "213.160.98.161" + "213.160.98.161", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -1348,8 +1399,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "302", @@ -1373,7 +1424,10 @@ ], "url.domain": "ff.connextra.com", "url.original": "http://ff.connextra.com/Ladbrokes/selector/image?", + "url.path": "/Ladbrokes/selector/image", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "ff", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1446,7 +1500,10 @@ ], "url.domain": "dd.connextra.com", "url.original": "http://dd.connextra.com/servlet/controller?", + "url.path": "/servlet/controller", + "url.query": "", "url.registered_domain": "connextra.com", + "url.scheme": "http", "url.subdomain": "dd", "url.top_level_domain": "com", "user.name": "badeyek" @@ -1503,7 +1560,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "-" }, @@ -1534,8 +1593,8 @@ "login.yahoo.com" ], "related.ip": [ - "209.73.177.115", - "10.105.21.199" + "10.105.21.199", + "209.73.177.115" ], "related.user": [ "badeyek" @@ -1604,8 +1663,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -1628,8 +1687,11 @@ "forwarded" ], "url.domain": "update.messenger.yahoo.com", + "url.extension": "html", "url.original": "http://update.messenger.yahoo.com/msgrcli7.html", + "url.path": "/msgrcli7.html", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "update.messenger", "url.top_level_domain": "com", "user.name": "-" @@ -1697,7 +1759,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1768,7 +1832,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1803,8 +1869,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -1815,8 +1881,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -1838,8 +1904,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/homepage.css", + "url.path": "/friend/styles/homepage.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -1870,8 +1939,8 @@ "shttp.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.155.194.239" + "216.155.194.239", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -1881,8 +1950,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -1906,7 +1975,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -1964,8 +2035,11 @@ "forwarded" ], "url.domain": "rms.adobe.com", + "url.extension": "xml", "url.original": "http://rms.adobe.com/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", + "url.path": "/read/0600/win_/ENU/read0600win_ENUadbe0000.xml", "url.registered_domain": "adobe.com", + "url.scheme": "http", "url.subdomain": "rms", "url.top_level_domain": "com", "user.name": "-" @@ -2023,8 +2097,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/styles/style.css", + "url.path": "/styles/style.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2058,8 +2135,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "304", @@ -2082,8 +2159,11 @@ "forwarded" ], "url.domain": "images.hi5.com", + "url.extension": "css", "url.original": "http://images.hi5.com/friend/styles/buttons_en_us.css", + "url.path": "/friend/styles/buttons_en_us.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.subdomain": "images", "url.top_level_domain": "com", "user.name": "nazsoau" @@ -2118,8 +2198,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2154,7 +2234,9 @@ ], "url.domain": "hi5.com", "url.original": "http://hi5.com/", + "url.path": "/", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2189,8 +2271,8 @@ "hi5.com" ], "related.ip": [ - "10.105.47.218", - "204.13.51.238" + "204.13.51.238", + "10.105.47.218" ], "related.user": [ "nazsoau" @@ -2201,8 +2283,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/css", "rsa.misc.result_code": "200", @@ -2224,8 +2306,11 @@ "forwarded" ], "url.domain": "hi5.com", + "url.extension": "css", "url.original": "http://hi5.com/friend/styles/headernav.css", + "url.path": "/friend/styles/headernav.css", "url.registered_domain": "hi5.com", + "url.scheme": "http", "url.top_level_domain": "com", "user.name": "nazsoau" }, @@ -2292,7 +2377,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2322,8 +2409,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.194.14" + "68.142.194.14", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2359,7 +2446,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/?", + "url.path": "/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2392,8 +2482,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -2428,8 +2518,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/playmessenger.asp", + "url.path": "/radio/play/playmessenger.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2497,7 +2590,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2565,7 +2660,10 @@ ], "url.domain": "address.yahoo.com", "url.original": "http://address.yahoo.com/yab/us?", + "url.path": "/yab/us", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "address", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2637,8 +2735,11 @@ "forwarded" ], "url.domain": "fxfeeds.mozilla.org", + "url.extension": "xml", "url.original": "http://fxfeeds.mozilla.org/rss20.xml", + "url.path": "/rss20.xml", "url.registered_domain": "mozilla.org", + "url.scheme": "http", "url.subdomain": "fxfeeds", "url.top_level_domain": "org", "user.name": "badeyek" @@ -2682,8 +2783,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2707,7 +2808,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2749,8 +2853,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/xml", "rsa.misc.result_code": "200", @@ -2774,7 +2878,10 @@ ], "url.domain": "insider.msg.yahoo.com", "url.original": "http://insider.msg.yahoo.com/ycontent/?", + "url.path": "/ycontent/", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -2806,8 +2913,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "CONNECT" + "CONNECT", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -2864,8 +2971,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_DENIED" + "TCP_DENIED", + "POST" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -2888,8 +2995,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -2946,8 +3056,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -2979,8 +3092,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_DENIED" + "TCP_DENIED", + "CONNECT" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3049,8 +3162,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3073,8 +3186,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations.gif", + "url.path": "/radio/clientdata/538/images/btn_stations.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3164,8 +3280,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -3200,8 +3316,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_stations_over.gif", + "url.path": "/radio/clientdata/538/images/btn_stations_over.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3244,8 +3363,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "TCP_MISS" + "TCP_MISS", + "POST" ], "rsa.misc.content_type": "text/plain", "rsa.misc.result_code": "200", @@ -3269,7 +3388,9 @@ ], "url.domain": "shttp.msg.yahoo.com", "url.original": "http://shttp.msg.yahoo.com/notify/", + "url.path": "/notify/", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "shttp.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3338,8 +3459,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_left.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_left.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3372,8 +3496,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -3384,8 +3508,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3408,8 +3532,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/launchcast_radio.gif", + "url.path": "/radio/clientdata/538/images/launchcast_radio.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3442,8 +3569,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "POST" + "POST", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -3466,8 +3593,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appinstru.asp", + "url.path": "/apps/agent/submgr/appinstru.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3524,8 +3654,11 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/submgr/appsync.asp", + "url.path": "/apps/agent/submgr/appsync.asp", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -3570,8 +3703,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3594,8 +3727,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_right.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_right.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3640,8 +3776,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3664,8 +3800,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_center.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_center.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3710,8 +3849,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_REFRESH_HIT", - "GET" + "GET", + "TCP_REFRESH_HIT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3734,8 +3873,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_off.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_off.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3780,8 +3922,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_REFRESH_HIT" + "TCP_REFRESH_HIT", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -3804,8 +3946,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/radio/common_radio/resources/images/t.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3863,8 +4008,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/images/btn_off_state_station.gif", + "url.path": "/radio/clientdata/538/images/btn_off_state_station.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3922,8 +4070,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "gif", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", + "url.path": "/radio/clientdata/538/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -3957,8 +4108,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -3981,8 +4132,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/toolbar50x50.gif", + "url.path": "/us.yimg.com/i/us/toolbar50x50.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4051,8 +4205,11 @@ "forwarded" ], "url.domain": "newsrss.bbc.co.uk", + "url.extension": "xml", "url.original": "http://newsrss.bbc.co.uk/rss/newsonline_world_edition/front_page/rss.xml", + "url.path": "/rss/newsonline_world_edition/front_page/rss.xml", "url.registered_domain": "bbc.co.uk", + "url.scheme": "http", "url.subdomain": "newsrss", "url.top_level_domain": "co.uk", "user.name": "badeyek" @@ -4085,8 +4242,8 @@ "insider.msg.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.231.252" + "68.142.231.252", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4097,8 +4254,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -4121,8 +4278,11 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php", + "url.path": "/ycontent/beacon.php", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4180,8 +4340,11 @@ "forwarded" ], "url.domain": "us.ent1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.ent1.yimg.com/images.launch.yahoo.com/000/032/457/32457654.jpg", + "url.path": "/images.launch.yahoo.com/000/032/457/32457654.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.ent1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4215,8 +4378,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_HIT", - "GET" + "GET", + "TCP_HIT" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4239,8 +4402,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", + "url.path": "/us.yimg.com/p/ap/20060906/thumb.71d29ded334347c48ac88433d033c9a9.pakistan_bin_laden_nyol440.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4273,8 +4439,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4308,8 +4474,11 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp", + "url.path": "/radio/play/authplay.asp", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4357,8 +4526,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/jpeg", "rsa.misc.result_code": "200", @@ -4381,8 +4550,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", + "url.path": "/us.yimg.com/p/ap/20060908/thumb.443f57762d7349669f609fbf0c97a5f1.academy_awards_host_cacp101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4415,8 +4587,8 @@ "radio.music.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.219.132" + "68.142.219.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -4451,8 +4623,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4521,8 +4697,12 @@ "forwarded" ], "url.domain": "radio.music.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.music.yahoo.com/radio/player/ymsgr/initstationfeed.asp?", + "url.path": "/radio/player/ymsgr/initstationfeed.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.music", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4555,8 +4735,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4567,8 +4747,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "200", @@ -4591,8 +4771,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/play/authplay.asp?", + "url.path": "/radio/play/authplay.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4651,7 +4835,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/iesocks?", + "url.path": "/iesocks", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4685,8 +4873,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_DENIED" + "TCP_DENIED", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -4710,7 +4898,11 @@ ], "url.domain": "natrocket.kmip.net", "url.original": "http://natrocket.kmip.net:5288/return?", + "url.path": "/return", + "url.port": 5288, + "url.query": "", "url.registered_domain": "kmip.net", + "url.scheme": "http", "url.subdomain": "natrocket", "url.top_level_domain": "net", "user.name": "-" @@ -4746,8 +4938,8 @@ "us.news1.yimg.com" ], "related.ip": [ - "213.160.98.159", - "10.105.33.214" + "10.105.33.214", + "213.160.98.159" ], "related.user": [ "adeolaegbedokun" @@ -4782,8 +4974,11 @@ "forwarded" ], "url.domain": "us.news1.yimg.com", + "url.extension": "jpg", "url.original": "http://us.news1.yimg.com/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", + "url.path": "/us.yimg.com/p/ap/20060907/thumb.1caf18e56db54eafb16da58356eb3382.amazon_com_online_video_watw101.jpg", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.news1", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4828,8 +5023,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "302", @@ -4852,8 +5047,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/clientdata/515/starter.asp?", + "url.path": "/radio/clientdata/515/starter.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4886,8 +5085,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -4922,8 +5121,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/default.asp?", + "url.path": "/radio/player/default.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -4971,8 +5174,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "application/x-shockwave-flash", "rsa.misc.result_code": "200", @@ -4995,8 +5198,12 @@ "forwarded" ], "url.domain": "us.a2.yimg.com", + "url.extension": "swf", "url.original": "http://us.a2.yimg.com/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf?", + "url.path": "/us.yimg.com/a/ya/yahoo_messenger/081106_lrec_msgr_interophitchhiker.swf", + "url.query": "", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.a2", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5029,8 +5236,8 @@ "radio.launch.yahoo.com" ], "related.ip": [ - "68.142.219.132", - "10.105.33.214" + "10.105.33.214", + "68.142.219.132" ], "related.user": [ "adeolaegbedokun" @@ -5065,8 +5272,12 @@ "forwarded" ], "url.domain": "radio.launch.yahoo.com", + "url.extension": "asp", "url.original": "http://radio.launch.yahoo.com/radio/player/stickwall.asp?", + "url.path": "/radio/player/stickwall.asp", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "radio.launch", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5097,8 +5308,8 @@ "us.bc.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "68.142.213.132" + "68.142.213.132", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5109,8 +5320,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -5134,7 +5345,10 @@ ], "url.domain": "us.bc.yahoo.com", "url.original": "http://us.bc.yahoo.com/b?", + "url.path": "/b", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "us.bc", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5201,8 +5415,12 @@ "forwarded" ], "url.domain": "insider.msg.yahoo.com", + "url.extension": "php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", "url.original": "http://insider.msg.yahoo.com/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw?", + "url.path": "/ycontent/beacon.php;_ylc=X1MDNTcwMzAyODMEX3IDMgRldnQDdDAEaW50bAN1cwR2ZXIDNywwLDIsMTIw", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "insider.msg", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -5243,8 +5461,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "CONNECT" + "CONNECT", + "TCP_MISS" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5302,8 +5520,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_IMS_HIT", - "GET" + "GET", + "TCP_IMS_HIT" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "304", @@ -5326,8 +5544,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "js", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", + "url.path": "/7/1568/1600/20051025184124/radio.launch.yahoo.com/radioapi/includes/js/compVersionedJS/rapiBridge_1_4.js", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5363,8 +5584,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5399,8 +5620,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "css", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", + "url.path": "/7/1568/1600/20040405222754/radio.launch.yahoo.com/radio/clientdata/515/other.css", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5448,8 +5672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_MISS", - "GET" + "GET", + "TCP_MISS" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5472,8 +5696,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_left.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5505,8 +5732,8 @@ "login.yahoo.com" ], "related.ip": [ - "10.105.21.199", - "209.73.177.115" + "209.73.177.115", + "10.105.21.199" ], "related.user": [ "badeyek" @@ -5516,8 +5743,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "TCP_MISS" + "TCP_MISS", + "CONNECT" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "200", @@ -5577,8 +5804,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "213.160.98.167", - "10.105.33.214" + "10.105.33.214", + "213.160.98.167" ], "related.user": [ "adeolaegbedokun" @@ -5589,8 +5816,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "-", "rsa.misc.result_code": "304", @@ -5613,8 +5840,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", + "url.path": "/7/1568/1600/20050829181418/radio.launch.yahoo.com/radio/common_radio/resources/images/noaccess_msgr_uk.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5650,8 +5880,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.159" + "213.160.98.159", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5662,8 +5892,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5686,8 +5916,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_right.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5721,8 +5954,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_IMS_HIT" + "TCP_IMS_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5745,8 +5978,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", + "url.path": "/7/1568/1600/20040405222807/radio.launch.yahoo.com/radio/common_radio/resources/images/t.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5818,8 +6054,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_off.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5891,8 +6130,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", + "url.path": "/7/1568/1600/20040405222756/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_center.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5928,8 +6170,8 @@ "a1568.g.akamai.net" ], "related.ip": [ - "10.105.33.214", - "213.160.98.167" + "213.160.98.167", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -5940,8 +6182,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "304", @@ -5964,8 +6206,11 @@ "forwarded" ], "url.domain": "a1568.g.akamai.net", + "url.extension": "gif", "url.original": "http://a1568.g.akamai.net/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", + "url.path": "/7/1568/1600/20040405222757/radio.launch.yahoo.com/radio/clientdata/515/skins/1/images/bg_controls_fill.gif", "url.registered_domain": "akamai.net", + "url.scheme": "http", "url.subdomain": "a1568.g", "url.top_level_domain": "net", "user.name": "adeolaegbedokun" @@ -5999,8 +6244,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_DENIED", - "GET" + "GET", + "TCP_DENIED" ], "rsa.misc.content_type": "text/html", "rsa.misc.result_code": "407", @@ -6024,7 +6269,9 @@ ], "url.domain": "www.google.com", "url.original": "http://www.google.com/supported_domains", + "url.path": "/supported_domains", "url.registered_domain": "google.com", + "url.scheme": "http", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "-" @@ -6082,8 +6329,12 @@ "forwarded" ], "url.domain": "us.mcafee.com", + "url.extension": "asp", "url.original": "http://us.mcafee.com/apps/agent/en-us/agent5/chknews.asp?", + "url.path": "/apps/agent/en-us/agent5/chknews.asp", + "url.query": "", "url.registered_domain": "mcafee.com", + "url.scheme": "http", "url.subdomain": "us", "url.top_level_domain": "com", "user.name": "-" @@ -6114,8 +6365,8 @@ "launch.adserver.yahoo.com" ], "related.ip": [ - "10.105.33.214", - "216.109.125.112" + "216.109.125.112", + "10.105.33.214" ], "related.user": [ "adeolaegbedokun" @@ -6126,8 +6377,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_MISS" + "TCP_MISS", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -6151,7 +6402,10 @@ ], "url.domain": "launch.adserver.yahoo.com", "url.original": "http://launch.adserver.yahoo.com/l?", + "url.path": "/l", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "launch.adserver", "url.top_level_domain": "com", "user.name": "adeolaegbedokun" @@ -6221,7 +6475,10 @@ ], "url.domain": "uk.f250.mail.yahoo.com", "url.original": "http://uk.f250.mail.yahoo.com/dc/launch?", + "url.path": "/dc/launch", + "url.query": "", "url.registered_domain": "yahoo.com", + "url.scheme": "http", "url.subdomain": "uk.f250.mail", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6314,8 +6571,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6326,8 +6583,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "TCP_SWAPFAIL_MISS", - "GET" + "GET", + "TCP_SWAPFAIL_MISS" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", @@ -6350,8 +6607,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/77cf3e56414f974dfd8616f56f0f632c_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6409,8 +6669,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "css", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/hdr/ygma5.css", + "url.path": "/us.yimg.com/lib/hdr/ygma5.css", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6446,8 +6709,8 @@ "us.js2.yimg.com" ], "related.ip": [ - "213.160.98.169", - "10.105.21.199" + "10.105.21.199", + "213.160.98.169" ], "related.user": [ "badeyek" @@ -6482,8 +6745,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", + "url.path": "/us.js.yimg.com/lib/pim/r/dclient/d/js/uk/f7fc76100697c9c2d25dd0ec35e563b0_1.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6541,8 +6807,11 @@ "forwarded" ], "url.domain": "us.js1.yimg.com", + "url.extension": "js", "url.original": "http://us.js1.yimg.com/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", + "url.path": "/us.yimg.com/lib/pim/r/medici/13_15/mail/ac.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js1", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6576,8 +6845,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "application/x-javascript", "rsa.misc.result_code": "200", @@ -6600,8 +6869,11 @@ "forwarded" ], "url.domain": "us.js2.yimg.com", + "url.extension": "js", "url.original": "http://us.js2.yimg.com/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", + "url.path": "/us.js.yimg.com/lib/common/utils/2/yahoo_2.0.0-b4.js", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.js2", "url.top_level_domain": "com", "user.name": "badeyek" @@ -6635,8 +6907,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "GET", - "TCP_HIT" + "TCP_HIT", + "GET" ], "rsa.misc.content_type": "image/gif", "rsa.misc.result_code": "200", @@ -6659,8 +6931,11 @@ "forwarded" ], "url.domain": "us.i1.yimg.com", + "url.extension": "gif", "url.original": "http://us.i1.yimg.com/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", + "url.path": "/us.yimg.com/i/us/pim/dclient/d/img/liam_ball_1.gif", "url.registered_domain": "yimg.com", + "url.scheme": "http", "url.subdomain": "us.i1", "url.top_level_domain": "com", "user.name": "badeyek" diff --git a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json index 69a23c8a571..070d7b72259 100644 --- a/x-pack/filebeat/module/squid/log/test/generated.log-expected.json +++ b/x-pack/filebeat/module/squid/log/test/generated.log-expected.json @@ -18,8 +18,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "example.net" + "example.net", + "example.org" ], "related.ip": [ "10.234.224.44", @@ -55,9 +55,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "min", "url.original": "https://example.org/exercita/der.htm?odoco=ria#min", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/exercita/der.htm" + ], + "url.query": "odoco=ria", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "tation", "user_agent.device.name": "Samsung SM-A715F", @@ -87,8 +94,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www.example.org" + "www.example.org", + "example.com" ], "related.ip": [ "10.102.123.34", @@ -125,9 +132,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "nidolor", "url.original": "https://www.example.org/enderitq/sperna.txt?billoi=oreetdol#nidolor", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/enderitq/sperna.txt" + ], + "url.query": "billoi=oreetdol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "doeiu", @@ -158,8 +172,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "example.net" + "example.net", + "internal.example.com" ], "related.ip": [ "10.142.172.64", @@ -196,9 +210,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "iin", "url.original": "https://internal.example.com/aqu/utper.jpg?eFinib=omm#iin", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/aqu/utper.jpg" + ], + "url.query": "eFinib=omm", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tia", @@ -228,8 +249,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ "10.44.134.153", @@ -240,8 +261,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "GET" + "GET", + "allow" ], "rsa.misc.content_type": "taev", "rsa.misc.result_code": "quiavo", @@ -267,9 +288,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "con", "url.original": "https://api.example.org/ceroinBC/ratvolup.gif?iatu=ionofde#con", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ceroinBC/ratvolup.gif" + ], + "url.query": "iatu=ionofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "5024D_RU", @@ -337,9 +365,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eos", "url.original": "https://api.example.org/isetq/estqui.gif?magn=equuntu#eos", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/isetq/estqui.gif" + ], + "url.query": "magn=equuntu", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "onev", @@ -374,8 +409,8 @@ "api.example.com" ], "related.ip": [ - "10.175.107.139", - "10.12.195.60" + "10.12.195.60", + "10.175.107.139" ], "related.user": [ "mrema" @@ -385,8 +420,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "accept" + "accept", + "HEAD" ], "rsa.misc.content_type": "aboris", "rsa.misc.result_code": "natura", @@ -411,9 +446,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "ect", "url.original": "https://www.example.org/inesci/rsitvolu.txt?pori=occ#ect", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/inesci/rsitvolu.txt" + ], + "url.query": "pori=occ", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mrema", @@ -482,9 +524,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "atat", "url.original": "https://www5.example.com/ari/eataevit.txt?iam=mqua#atat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/ari/eataevit.txt" + ], + "url.query": "iam=mqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "tsed", @@ -515,8 +564,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.org", - "internal.example.net" + "internal.example.net", + "example.org" ], "related.ip": [ "10.116.120.216", @@ -527,8 +576,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "mol", "rsa.misc.result_code": "apariat", @@ -552,9 +601,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "orsitame", "url.original": "https://example.org/tatno/imav.htm?ofdeF=tion#orsitame", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/tatno/imav.htm" + ], + "url.query": "ofdeF=tion", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "umdo", "user_agent.device.name": "Notepad_K10", @@ -588,8 +644,8 @@ "example.org" ], "related.ip": [ - "10.90.131.186", - "10.30.216.41" + "10.30.216.41", + "10.90.131.186" ], "related.user": [ "saute" @@ -624,9 +680,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "iumto", "url.original": "https://example.org/uaera/sitas.txt?aedic=atquovo#iumto", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/uaera/sitas.txt" + ], + "url.query": "aedic=atquovo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "saute", "user_agent.device.name": "STK-L21", @@ -696,9 +759,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "nsequu", "url.original": "https://mail.example.org/edquiano/loru.htm?end=enia#nsequu", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edquiano/loru.htm" + ], + "url.query": "end=enia", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "Nemoeni", @@ -732,16 +802,16 @@ "www.example.org" ], "related.ip": [ - "10.158.185.163", - "10.71.34.9" + "10.71.34.9", + "10.158.185.163" ], "related.user": [ "aliq" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "allow", - "UNLOCK" + "UNLOCK", + "allow" ], "rsa.misc.content_type": "adm", "rsa.misc.result_code": "quam", @@ -766,9 +836,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "xercitat", "url.original": "https://www.example.org/iqui/etc.txt?tatiset=eprehen#xercitat", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/iqui/etc.txt" + ], + "url.query": "tatiset=eprehen", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aliq", @@ -799,8 +876,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.net" + "internal.example.net", + "example.net" ], "related.ip": [ "10.210.74.24", @@ -811,8 +888,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "OPTIONS", - "accept" + "accept", + "OPTIONS" ], "rsa.misc.content_type": "emips", "rsa.misc.result_code": "onse", @@ -837,9 +914,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "txt", + "url.fragment": "ugiatnu", "url.original": "https://internal.example.net/ommod/sequatur.txt?tlabo=suntexp#ugiatnu", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/ommod/sequatur.txt" + ], + "url.query": "tlabo=suntexp", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "uines", @@ -866,8 +950,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www.example.net" + "www.example.net", + "api.example.org" ], "related.ip": [ "10.206.136.206", @@ -904,9 +988,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "eporroqu", "url.original": "https://api.example.org/oriosamn/deFinibu.gif?iciatisu=rehender#eporroqu", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/oriosamn/deFinibu.gif" + ], + "url.query": "iciatisu=rehender", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "xeac", @@ -937,8 +1028,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.org", - "www.example.com" + "www.example.com", + "mail.example.org" ], "related.ip": [ "10.200.199.166", @@ -975,9 +1066,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "civeli", "url.original": "https://mail.example.org/rehend/tio.html?numqu=qui#civeli", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/rehend/tio.html" + ], + "url.query": "numqu=qui", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "ipitla", @@ -1008,8 +1106,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "www5.example.com" + "www5.example.com", + "www.example.org" ], "related.ip": [ "10.122.46.71", @@ -1020,8 +1118,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "allow" + "allow", + "NONE" ], "rsa.misc.content_type": "aaliquaU", "rsa.misc.result_code": "mpori", @@ -1046,9 +1144,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "txt", + "url.fragment": "iamquis", "url.original": "https://www.example.org/eav/ionevo.txt?siar=orev#iamquis", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/eav/ionevo.txt" + ], + "url.query": "siar=orev", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "aturve", @@ -1083,16 +1188,16 @@ "api.example.com" ], "related.ip": [ - "10.249.213.83", - "10.164.250.63" + "10.164.250.63", + "10.249.213.83" ], "related.user": [ "itame" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "accept" + "accept", + "PROPFIND" ], "rsa.misc.content_type": "asun", "rsa.misc.result_code": "lit", @@ -1117,9 +1222,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "leumiu", "url.original": "https://internal.example.net/wri/bor.jpg?hitect=dol#leumiu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/wri/bor.jpg" + ], + "url.query": "hitect=dol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "itame", @@ -1150,12 +1262,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.net" + "api.example.net", + "www5.example.net" ], "related.ip": [ - "10.236.248.65", - "10.61.242.75" + "10.61.242.75", + "10.236.248.65" ], "related.user": [ "iquidex" @@ -1191,9 +1303,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "eriamea", "url.original": "https://api.example.net/equat/doloreme.htm?ione=ihilmole#eriamea", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/equat/doloreme.htm" + ], + "url.query": "ione=ihilmole", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "iquidex", @@ -1224,20 +1343,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www5.example.com" + "www5.example.com", + "internal.example.net" ], "related.ip": [ - "10.13.59.31", - "10.214.7.83" + "10.214.7.83", + "10.13.59.31" ], "related.user": [ "etdol" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "block", - "PUT" + "PUT", + "block" ], "rsa.misc.content_type": "eprehend", "rsa.misc.result_code": "boN", @@ -1262,9 +1381,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "xerc", "url.original": "https://www5.example.com/estia/tper.gif?volupt=osqui#xerc", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/estia/tper.gif" + ], + "url.query": "volupt=osqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "etdol", @@ -1310,8 +1436,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "GET" + "GET", + "accept" ], "rsa.misc.content_type": "tcons", "rsa.misc.result_code": "tsu", @@ -1336,9 +1462,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "isnis", "url.original": "https://internal.example.net/sin/rvel.htm?nimid=itatione#isnis", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/sin/rvel.htm" + ], + "url.query": "nimid=itatione", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "isnisiu", @@ -1369,8 +1502,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "mail.example.net" + "mail.example.net", + "api.example.org" ], "related.ip": [ "10.90.86.89", @@ -1407,9 +1540,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "gif", + "url.fragment": "equat", "url.original": "https://api.example.org/abo/veniamqu.gif?aliquide=ofde#equat", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/abo/veniamqu.gif" + ], + "url.query": "aliquide=ofde", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "lapar", @@ -1440,20 +1580,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.org" + "api.example.org", + "www5.example.net" ], "related.ip": [ - "10.14.211.43", - "10.14.48.16" + "10.14.48.16", + "10.14.211.43" ], "related.user": [ "volupt" ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "PROPFIND", - "cancel" + "cancel", + "PROPFIND" ], "rsa.misc.content_type": "Utenima", "rsa.misc.result_code": "uiinea", @@ -1478,9 +1618,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "icabo", "url.original": "https://api.example.org/autfu/gnaaliq.jpg?olupta=litse#icabo", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/autfu/gnaaliq.jpg" + ], + "url.query": "olupta=litse", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "volupt", @@ -1515,8 +1662,8 @@ "example.com" ], "related.ip": [ - "10.47.25.230", - "10.93.123.174" + "10.93.123.174", + "10.47.25.230" ], "related.user": [ "reetdolo" @@ -1525,8 +1672,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iusmodi", "rsa.misc.result_code": "etcons", @@ -1551,9 +1698,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ptatemq/luptatev.html" + ], + "url.query": "Nequepo=ipsumd", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "reetdolo", @@ -1584,8 +1738,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "www5.example.com" + "www5.example.com", + "www5.example.net" ], "related.ip": [ "10.233.48.103", @@ -1622,9 +1776,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "cons", "url.original": "https://www5.example.net/quiavol/rrorsi.gif?iatisu=sec#cons", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/quiavol/rrorsi.gif" + ], + "url.query": "iatisu=sec", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "tlab", @@ -1655,12 +1816,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ - "10.93.220.10", - "10.27.58.92" + "10.27.58.92", + "10.93.220.10" ], "related.user": [ "qui" @@ -1693,9 +1854,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://api.example.net/orain/tiumt.jpg?litessec=itas#edquia", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/orain/tiumt.jpg" + ], + "url.query": "litessec=itas", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "qui", @@ -1730,8 +1898,8 @@ "example.net" ], "related.ip": [ - "10.213.144.249", - "10.135.217.12" + "10.135.217.12", + "10.213.144.249" ], "related.user": [ "ntexplic" @@ -1764,9 +1932,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "untut", "url.original": "https://www.example.net/str/idolore.txt?eetdolo=cteturad#untut", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/str/idolore.txt" + ], + "url.query": "eetdolo=cteturad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ntexplic", @@ -1801,8 +1976,8 @@ "internal.example.com" ], "related.ip": [ - "10.13.226.57", - "10.233.239.112" + "10.233.239.112", + "10.13.226.57" ], "related.user": [ "mquelau" @@ -1835,9 +2010,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "mestq", "url.original": "https://mail.example.net/velitse/oditem.html?torever=oremi#mestq", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/velitse/oditem.html" + ], + "url.query": "torever=oremi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "mquelau", @@ -1872,8 +2054,8 @@ "example.com" ], "related.ip": [ - "10.161.203.252", - "10.21.169.127" + "10.21.169.127", + "10.161.203.252" ], "related.user": [ "ice" @@ -1908,9 +2090,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "uisa", "url.original": "https://internal.example.org/isnisi/ritatise.gif?tamet=quatur#uisa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/isnisi/ritatise.gif" + ], + "url.query": "tamet=quatur", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ice", @@ -1941,20 +2130,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.net", - "api.example.com" + "api.example.com", + "www.example.net" ], "related.ip": [ - "10.17.215.111", - "10.69.139.26" + "10.69.139.26", + "10.17.215.111" ], "related.user": [ "edqui" ], "rsa.internal.messageid": "LOCK", "rsa.misc.action": [ - "LOCK", - "block" + "block", + "LOCK" ], "rsa.misc.content_type": "volupta", "rsa.misc.result_code": "veli", @@ -1979,9 +2168,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "ano", "url.original": "https://www.example.net/ianon/tsed.htm?ameiusm=proide#ano", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/ianon/tsed.htm" + ], + "url.query": "ameiusm=proide", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "edqui", @@ -2024,8 +2220,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "accept", - "COPY" + "COPY", + "accept" ], "rsa.misc.content_type": "onulam", "rsa.misc.result_code": "ugiat", @@ -2050,9 +2246,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "apariat", "url.original": "https://www5.example.org/ncididun/umSe.jpg?ise=itau#apariat", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/ncididun/umSe.jpg" + ], + "url.query": "ise=itau", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "onsecte", @@ -2095,8 +2298,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "block", - "UNLOCK" + "UNLOCK", + "block" ], "rsa.misc.content_type": "emUteni", "rsa.misc.result_code": "utlab", @@ -2120,9 +2323,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "tem", "url.original": "https://example.org/uatur/adminimv.gif?exeacom=roidents#tem", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/uatur/adminimv.gif" + ], + "url.query": "exeacom=roidents", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "isis", "user_agent.device.name": "Generic Smartphone", @@ -2154,16 +2364,16 @@ "mail.example.com" ], "related.ip": [ - "10.26.96.202", - "10.119.90.128" + "10.119.90.128", + "10.26.96.202" ], "related.user": [ "oraincid" ], "rsa.internal.messageid": "ICP_QUERY", "rsa.misc.action": [ - "ICP_QUERY", - "cancel" + "cancel", + "ICP_QUERY" ], "rsa.misc.content_type": "ntsunt", "rsa.misc.result_code": "amcorp", @@ -2188,9 +2398,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "reprehe", "url.original": "https://api.example.com/ven/rQu.html?doloreme=dun#reprehe", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ven/rQu.html" + ], + "url.query": "doloreme=dun", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "oraincid", @@ -2221,12 +2438,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.net" + "api.example.net", + "mail.example.com" ], "related.ip": [ - "10.0.98.205", - "10.76.110.144" + "10.76.110.144", + "10.0.98.205" ], "related.user": [ "upt" @@ -2262,9 +2479,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ate", "url.original": "https://api.example.net/eseru/quamest.html?qua=rsita#ate", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/eseru/quamest.html" + ], + "url.query": "qua=rsita", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "upt", @@ -2299,16 +2523,16 @@ "www.example.org" ], "related.ip": [ - "10.224.11.165", - "10.135.46.242" + "10.135.46.242", + "10.224.11.165" ], "related.user": [ "xeaco" ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "MOVE", - "deny" + "deny", + "MOVE" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "oremi", @@ -2333,9 +2557,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "isiu", "url.original": "https://internal.example.org/mvolu/conse.txt?aincidu=nimadmin#isiu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/mvolu/conse.txt" + ], + "url.query": "aincidu=nimadmin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "xeaco", @@ -2377,8 +2608,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "TRACE", - "accept" + "accept", + "TRACE" ], "rsa.misc.content_type": "plica", "rsa.misc.result_code": "cidunt", @@ -2403,9 +2634,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "ilmol", "url.original": "https://internal.example.org/isciv/rroqu.html?uisa=tametco#ilmol", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/isciv/rroqu.html" + ], + "url.query": "uisa=tametco", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "autodit", @@ -2440,16 +2678,16 @@ "example.org" ], "related.ip": [ - "10.93.39.140", - "10.150.245.88" + "10.150.245.88", + "10.93.39.140" ], "related.user": [ "reetd" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "cancel", - "COPY" + "COPY", + "cancel" ], "rsa.misc.content_type": "iusmodte", "rsa.misc.result_code": "ntutlabo", @@ -2474,9 +2712,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "pid", "url.original": "https://www5.example.net/uaeratv/isa.txt?periam=dqu#pid", - "url.path": "https://example.org", + "url.path": [ + "https://example.org", + "/uaeratv/isa.txt" + ], + "url.query": "periam=dqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "reetd", @@ -2507,12 +2752,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www5.example.com" + "www5.example.com", + "mail.example.net" ], "related.ip": [ - "10.61.92.2", - "10.73.207.70" + "10.73.207.70", + "10.61.92.2" ], "related.user": [ "atu" @@ -2545,9 +2790,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "remagn", "url.original": "https://www5.example.com/orroq/vitaedic.txt?orisni=ons#remagn", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/orroq/vitaedic.txt" + ], + "url.query": "orisni=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "atu", @@ -2577,12 +2829,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "internal.example.com" + "internal.example.com", + "example.net" ], "related.ip": [ - "10.84.32.178", - "10.50.124.116" + "10.50.124.116", + "10.84.32.178" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -2615,9 +2867,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "gif", + "url.fragment": "ruredo", "url.original": "https://example.net/mven/olorsit.gif?oremag=illu#ruredo", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/mven/olorsit.gif" + ], + "url.query": "oremag=illu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -2646,8 +2905,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "www5.example.net" + "www5.example.net", + "api.example.org" ], "related.ip": [ "10.173.222.131", @@ -2658,8 +2917,8 @@ ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "accept", - "TRACE" + "TRACE", + "accept" ], "rsa.misc.content_type": "boNem", "rsa.misc.result_code": "aUt", @@ -2684,9 +2943,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "html", + "url.fragment": "oin", "url.original": "https://www5.example.net/rum/eataevi.html?ulla=iqu#oin", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/rum/eataevi.html" + ], + "url.query": "ulla=iqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "Duisa", @@ -2717,12 +2983,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "mail.example.net" + "mail.example.net", + "www5.example.net" ], "related.ip": [ - "10.0.157.225", - "10.11.83.126" + "10.11.83.126", + "10.0.157.225" ], "related.user": [ "atu" @@ -2755,9 +3021,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "tinvolup", "url.original": "https://mail.example.net/osquir/mod.txt?fugitse=imad#tinvolup", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/osquir/mod.txt" + ], + "url.query": "fugitse=imad", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "atu", @@ -2788,20 +3061,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.com", - "api.example.com" + "api.example.com", + "www5.example.com" ], "related.ip": [ - "10.228.77.21", - "10.92.237.93" + "10.92.237.93", + "10.228.77.21" ], "related.user": [ "onse" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "cancel" + "cancel", + "PUT" ], "rsa.misc.content_type": "mod", "rsa.misc.result_code": "gnaa", @@ -2826,9 +3099,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "mve", "url.original": "https://api.example.com/asper/umq.txt?itasper=uae#mve", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/asper/umq.txt" + ], + "url.query": "itasper=uae", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "onse", @@ -2859,12 +3139,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www5.example.net" + "www5.example.net", + "api.example.com" ], "related.ip": [ - "10.102.215.23", - "10.20.28.92" + "10.20.28.92", + "10.102.215.23" ], "related.user": [ "ntexpl" @@ -2899,9 +3179,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "veniamqu", "url.original": "https://www5.example.net/quatD/isqua.jpg?oloreseo=iruredol#veniamqu", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quatD/isqua.jpg" + ], + "url.query": "oloreseo=iruredol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "ntexpl", @@ -2934,16 +3221,16 @@ "api.example.net" ], "related.ip": [ - "10.45.28.159", - "10.17.87.79" + "10.17.87.79", + "10.45.28.159" ], "related.user": [ "tionula" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "NONE", - "block" + "block", + "NONE" ], "rsa.misc.content_type": "uamei", "rsa.misc.result_code": "ecatcupi", @@ -2968,9 +3255,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "repreh", "url.original": "https://api.example.net/ation/luptas.html?iatqu=lorsi#repreh", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/ation/luptas.html" + ], + "url.query": "iatqu=lorsi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "tionula", @@ -3001,8 +3295,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "api.example.com" + "api.example.com", + "internal.example.com" ], "related.ip": [ "10.177.238.45", @@ -3039,9 +3333,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "eumi", "url.original": "https://internal.example.com/mpo/unte.jpg?ueipsa=scipitl#eumi", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/mpo/unte.jpg" + ], + "url.query": "ueipsa=scipitl", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "rsp", @@ -3072,8 +3373,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "example.com" + "example.com", + "www.example.org" ], "related.ip": [ "10.101.85.169", @@ -3113,9 +3414,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "olup", "url.original": "https://www.example.org/uov/quaeab.jpg?moles=dipiscin#olup", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uov/quaeab.jpg" + ], + "url.query": "moles=dipiscin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "liquid", @@ -3146,12 +3454,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.net", - "www5.example.org" + "www5.example.org", + "example.net" ], "related.ip": [ - "10.24.54.129", - "10.231.7.209" + "10.231.7.209", + "10.24.54.129" ], "related.user": [ "eavol" @@ -3161,8 +3469,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "HEAD", - "block" + "block", + "HEAD" ], "rsa.misc.content_type": "scipitl", "rsa.misc.result_code": "temaccu", @@ -3186,9 +3494,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "snos", "url.original": "https://example.net/edolo/ugiatquo.jpg?eosquira=pta#snos", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/edolo/ugiatquo.jpg" + ], + "url.query": "eosquira=pta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "eavol", "user_agent.device.name": "Asus X01BDA", @@ -3222,8 +3537,8 @@ "www.example.com" ], "related.ip": [ - "10.121.163.5", - "10.77.129.175" + "10.77.129.175", + "10.121.163.5" ], "related.user": [ "BCS" @@ -3232,8 +3547,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "CONNECT" + "CONNECT", + "allow" ], "rsa.misc.content_type": "incidid", "rsa.misc.result_code": "ugiatquo", @@ -3258,9 +3573,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nrepreh", "url.original": "https://api.example.org/isci/dolor.htm?orinrep=quiavol#nrepreh", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/isci/dolor.htm" + ], + "url.query": "orinrep=quiavol", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "BCS", @@ -3329,9 +3651,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "culpaqui", "url.original": "https://mail.example.com/roide/tem.gif?rerepre=nculpaq#culpaqui", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/roide/tem.gif" + ], + "url.query": "rerepre=nculpaq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "obea", @@ -3374,8 +3703,8 @@ ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "tobeatae", "rsa.misc.result_code": "tion", @@ -3400,9 +3729,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "edquia", "url.original": "https://internal.example.net/rautod/olest.jpg?lapar=ritati#edquia", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/rautod/olest.jpg" + ], + "url.query": "lapar=ritati", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "amvolu", @@ -3471,9 +3807,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "iame", "url.original": "https://api.example.org/iusmodt/enim.txt?aquio=ersp#iame", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/iusmodt/enim.txt" + ], + "url.query": "aquio=ersp", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "mexer", @@ -3508,8 +3851,8 @@ "example.org" ], "related.ip": [ - "10.45.54.107", - "10.45.114.111" + "10.45.114.111", + "10.45.54.107" ], "related.user": [ "nse" @@ -3518,8 +3861,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "POST", - "accept" + "accept", + "POST" ], "rsa.misc.content_type": "mdolors", "rsa.misc.result_code": "edictasu", @@ -3543,9 +3886,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "eetdo", "url.original": "https://example.org/abillo/undeom.html?oraincid=quaer#eetdo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/abillo/undeom.html" + ], + "url.query": "oraincid=quaer", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "nse", "user_agent.device.name": "Samsung SM-A715F", @@ -3575,8 +3925,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "mail.example.net" + "mail.example.net", + "www5.example.net" ], "related.ip": [ "10.205.28.24", @@ -3613,9 +3963,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "sse", "url.original": "https://mail.example.net/uam/orumSec.jpg?isnisiu=suntincu#sse", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/uam/orumSec.jpg" + ], + "url.query": "isnisiu=suntincu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "untutlab", @@ -3661,8 +4018,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "HEAD" + "HEAD", + "deny" ], "rsa.misc.content_type": "elites", "rsa.misc.result_code": "itseddoe", @@ -3687,9 +4044,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "iamea", "url.original": "https://www5.example.org/umiurer/rere.txt?mnisi=usmo#iamea", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/umiurer/rere.txt" + ], + "url.query": "mnisi=usmo", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "odoco", @@ -3716,20 +4080,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "internal.example.net" + "internal.example.net", + "internal.example.org" ], "related.ip": [ - "10.81.140.173", - "10.88.172.222" + "10.88.172.222", + "10.81.140.173" ], "related.user": [ "etdol" ], "rsa.internal.messageid": "MOVE", "rsa.misc.action": [ - "cancel", - "MOVE" + "MOVE", + "cancel" ], "rsa.misc.content_type": "tura", "rsa.misc.result_code": "nturmag", @@ -3754,9 +4118,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "remipsu", "url.original": "https://internal.example.net/atnula/ditautf.jpg?iquidex=olup#remipsu", - "url.path": "https://internal.example.org", + "url.path": [ + "https://internal.example.org", + "/atnula/ditautf.jpg" + ], + "url.query": "iquidex=olup", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "etdol", @@ -3787,8 +4158,8 @@ "example.com" ], "related.ip": [ - "10.247.53.179", - "10.162.129.196" + "10.162.129.196", + "10.247.53.179" ], "related.user": [ "identsu" @@ -3821,9 +4192,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "xeacommo", "url.original": "https://api.example.net/uscip/umS.txt?quiacons=uisa#xeacommo", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uscip/umS.txt" + ], + "url.query": "quiacons=uisa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "identsu", @@ -3854,16 +4232,16 @@ "www5.example.net" ], "related.ip": [ - "10.110.86.230", - "10.172.148.223" + "10.172.148.223", + "10.110.86.230" ], "related.user": [ "enimadm" ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "accept", - "UNLOCK" + "UNLOCK", + "accept" ], "rsa.misc.content_type": "atc", "rsa.misc.result_code": "upta", @@ -3888,9 +4266,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "temporai", "url.original": "https://mail.example.com/mrema/ullamc.txt?eufug=roquisq#temporai", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/mrema/ullamc.txt" + ], + "url.query": "eufug=roquisq", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "enimadm", @@ -3962,9 +4347,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "orsitam", "url.original": "https://www5.example.com/isau/itinvol.txt?saquaea=ons#orsitam", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/isau/itinvol.txt" + ], + "url.query": "saquaea=ons", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "riame", @@ -4033,9 +4425,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "onemulla", "url.original": "https://api.example.net/veli/venia.htm?etdolor=uat#onemulla", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/veli/venia.htm" + ], + "url.query": "etdolor=uat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "asp", @@ -4066,12 +4465,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "internal.example.net" + "internal.example.net", + "www5.example.net" ], "related.ip": [ - "10.89.73.240", - "10.41.156.88" + "10.41.156.88", + "10.89.73.240" ], "related.user": [ "ntorever" @@ -4104,9 +4503,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "expli", "url.original": "https://internal.example.net/ainci/osqu.jpg?sus=imavenia#expli", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ainci/osqu.jpg" + ], + "url.query": "sus=imavenia", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "ntorever", @@ -4137,8 +4543,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.net", - "www.example.org" + "www.example.org", + "internal.example.net" ], "related.ip": [ "10.54.44.231", @@ -4151,8 +4557,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "allow" + "allow", + "CONNECT" ], "rsa.misc.content_type": "oreverit", "rsa.misc.result_code": "abor", @@ -4177,9 +4583,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "seos", "url.original": "https://www.example.org/runtm/eturadip.htm?psumd=oloree#seos", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/runtm/eturadip.htm" + ], + "url.query": "psumd=oloree", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "mcorpo", @@ -4206,8 +4619,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.org", - "www.example.net" + "www.example.net", + "internal.example.org" ], "related.ip": [ "10.181.177.74", @@ -4244,9 +4657,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "html", + "url.fragment": "umwri", "url.original": "https://internal.example.org/liquipex/uisnos.html?ventor=lupt#umwri", - "url.path": "https://www.example.net", + "url.path": [ + "https://www.example.net", + "/liquipex/uisnos.html" + ], + "url.query": "ventor=lupt", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "nvo", @@ -4281,8 +4701,8 @@ "api.example.net" ], "related.ip": [ - "10.83.130.95", - "10.76.220.3" + "10.76.220.3", + "10.83.130.95" ], "related.user": [ "userror" @@ -4318,9 +4738,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "veniamq", "url.original": "https://api.example.org/mquisnos/lore.txt?siar=isn#veniamq", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/mquisnos/lore.txt" + ], + "url.query": "siar=isn", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "userror", @@ -4351,12 +4778,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.com", - "www.example.com" + "www.example.com", + "api.example.com" ], "related.ip": [ - "10.166.160.217", - "10.219.245.58" + "10.219.245.58", + "10.166.160.217" ], "related.user": [ "radip" @@ -4389,9 +4816,16 @@ "forwarded" ], "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "uptatem", "url.original": "https://www.example.com/quas/occaeca.htm?ender=dico#uptatem", - "url.path": "https://api.example.com", + "url.path": [ + "https://api.example.com", + "/quas/occaeca.htm" + ], + "url.query": "ender=dico", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "com", "user.name": "radip", @@ -4420,8 +4854,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.org" + "www5.example.org", + "example.com" ], "related.ip": [ "10.121.121.153", @@ -4432,8 +4866,8 @@ ], "rsa.internal.messageid": "UNLOCK", "rsa.misc.action": [ - "cancel", - "UNLOCK" + "UNLOCK", + "cancel" ], "rsa.misc.content_type": "magnam", "rsa.misc.result_code": "modoc", @@ -4458,9 +4892,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "sit", "url.original": "https://www5.example.org/uidolo/umdolore.jpg?oquisq=abori#sit", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/uidolo/umdolore.jpg" + ], + "url.query": "oquisq=abori", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "tatio", @@ -4529,9 +4970,16 @@ "forwarded" ], "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "tisu", "url.original": "https://internal.example.net/eniamqu/inimav.htm?imadm=uta#tisu", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/eniamqu/inimav.htm" + ], + "url.query": "imadm=uta", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "net", "user.name": "aturv", @@ -4562,8 +5010,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "mail.example.com" + "mail.example.com", + "mail.example.net" ], "related.ip": [ "10.72.99.69", @@ -4600,9 +5048,16 @@ "forwarded" ], "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "giatquo", "url.original": "https://mail.example.net/sintocca/mipsumqu.htm?tnulapar=ico#giatquo", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/sintocca/mipsumqu.htm" + ], + "url.query": "tnulapar=ico", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "net", "user.name": "uatu", @@ -4633,20 +5088,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.org", - "internal.example.net" + "internal.example.net", + "api.example.org" ], "related.ip": [ - "10.245.240.47", - "10.142.130.227" + "10.142.130.227", + "10.245.240.47" ], "related.user": [ "odic" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "allow", - "DELETE" + "DELETE", + "allow" ], "rsa.misc.content_type": "scivelit", "rsa.misc.result_code": "liquaUte", @@ -4671,9 +5126,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "eleumi", "url.original": "https://api.example.org/rep/remap.html?siarc=fdeFin#eleumi", - "url.path": "https://internal.example.net", + "url.path": [ + "https://internal.example.net", + "/rep/remap.html" + ], + "url.query": "siarc=fdeFin", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user.name": "odic", @@ -4708,8 +5170,8 @@ "api.example.net" ], "related.ip": [ - "10.62.188.193", - "10.61.110.7" + "10.61.110.7", + "10.62.188.193" ], "related.user": [ "quaU" @@ -4742,9 +5204,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "osa", "url.original": "https://api.example.net/eturad/tDuis.htm?enimadmi=tateveli#osa", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/eturad/tDuis.htm" + ], + "url.query": "enimadmi=tateveli", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "quaU", @@ -4771,8 +5240,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.net", - "www.example.net" + "www.example.net", + "mail.example.net" ], "related.ip": [ "10.68.198.188", @@ -4783,8 +5252,8 @@ ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "COPY", - "block" + "block", + "COPY" ], "rsa.misc.content_type": "animid", "rsa.misc.result_code": "inea", @@ -4809,9 +5278,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "htm", + "url.fragment": "madmi", "url.original": "https://www.example.net/hender/ptatemU.htm?mquisnos=tnulapa#madmi", - "url.path": "https://mail.example.net", + "url.path": [ + "https://mail.example.net", + "/hender/ptatemU.htm" + ], + "url.query": "mquisnos=tnulapa", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "onsectet", @@ -4856,8 +5332,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "teturad", "rsa.misc.result_code": "avolu", @@ -4882,9 +5358,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "oid", "url.original": "https://www.example.org/iduntutl/rsitam.htm?ntor=oinBCSed#oid", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/iduntutl/rsitam.htm" + ], + "url.query": "ntor=oinBCSed", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "midestl", @@ -4917,16 +5400,16 @@ "api.example.org" ], "related.ip": [ - "10.62.10.137", - "10.32.98.109" + "10.32.98.109", + "10.62.10.137" ], "related.user": [ "deomnisi" ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "cancel" + "cancel", + "PURGE" ], "rsa.misc.content_type": "laboree", "rsa.misc.result_code": "oll", @@ -4950,9 +5433,16 @@ "forwarded" ], "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "dat", "url.original": "https://example.org/itessequ/porissu.html?uip=ectobea#dat", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/itessequ/porissu.html" + ], + "url.query": "uip=ectobea", "url.registered_domain": "example.org", + "url.scheme": "https", "url.top_level_domain": "org", "user.name": "deomnisi", "user_agent.device.name": "LG-$2", @@ -4986,8 +5476,8 @@ "www5.example.com" ], "related.ip": [ - "10.176.62.146", - "10.255.40.12" + "10.255.40.12", + "10.176.62.146" ], "related.user": [ "oeiusmo" @@ -5020,9 +5510,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "nimide", "url.original": "https://api.example.net/enimad/tis.txt?mipsumq=ident#nimide", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/enimad/tis.txt" + ], + "url.query": "mipsumq=ident", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oeiusmo", @@ -5056,8 +5553,8 @@ "internal.example.com" ], "related.ip": [ - "10.194.198.46", - "10.88.98.31" + "10.88.98.31", + "10.194.198.46" ], "rsa.internal.messageid": "GET", "rsa.investigations.ec_activity": "Request", @@ -5091,9 +5588,16 @@ "forwarded" ], "url.domain": "api.example.org", + "url.extension": "htm", + "url.fragment": "nse", "url.original": "https://api.example.org/taspe/yCiceroi.htm?cti=ommodoc#nse", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/taspe/yCiceroi.htm" + ], + "url.query": "cti=ommodoc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "org", "user_agent.device.name": "iPhone", @@ -5123,8 +5627,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "www5.example.com" + "www5.example.com", + "example.com" ], "related.ip": [ "10.1.27.133", @@ -5135,8 +5639,8 @@ ], "rsa.internal.messageid": "OPTIONS", "rsa.misc.action": [ - "block", - "OPTIONS" + "OPTIONS", + "block" ], "rsa.misc.content_type": "ntutlab", "rsa.misc.result_code": "olore", @@ -5160,9 +5664,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "iarchite", "url.original": "https://example.com/beat/rro.jpg?uisau=qua#iarchite", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/beat/rro.jpg" + ], + "url.query": "uisau=qua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "tationu", "user_agent.device.name": "Lenovo A2016a40 ", @@ -5192,12 +5703,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "internal.example.com", - "www5.example.org" + "www5.example.org", + "internal.example.com" ], "related.ip": [ - "10.11.73.145", - "10.70.244.155" + "10.70.244.155", + "10.11.73.145" ], "related.user": [ "caboNemo" @@ -5206,8 +5717,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "accept", - "POST" + "POST", + "accept" ], "rsa.misc.content_type": "ons", "rsa.misc.result_code": "temaccus", @@ -5232,9 +5743,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "txt", + "url.fragment": "dexea", "url.original": "https://www5.example.org/loremq/turmagni.txt?emUtenim=ende#dexea", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/loremq/turmagni.txt" + ], + "url.query": "emUtenim=ende", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "caboNemo", @@ -5303,9 +5821,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "ecte", "url.original": "https://www5.example.net/deomnisi/ddoe.txt?oremi=ectobeat#ecte", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/deomnisi/ddoe.txt" + ], + "url.query": "oremi=ectobeat", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "cillumdo", @@ -5374,9 +5899,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "lors", "url.original": "https://api.example.net/tiset/sci.jpg?rauto=doloreeu#lors", - "url.path": "https://www.example.com", + "url.path": [ + "https://www.example.com", + "/tiset/sci.jpg" + ], + "url.query": "rauto=doloreeu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "roquisq", @@ -5407,8 +5939,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "mail.example.com" + "mail.example.com", + "api.example.net" ], "related.ip": [ "10.191.220.1", @@ -5419,8 +5951,8 @@ ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "deny" + "deny", + "DELETE" ], "rsa.misc.content_type": "tdolo", "rsa.misc.result_code": "ntu", @@ -5445,9 +5977,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "tconsect", "url.original": "https://api.example.net/tDuisau/aturve.htm?tper=pisciv#tconsect", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/tDuisau/aturve.htm" + ], + "url.query": "tper=pisciv", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "isi", @@ -5482,8 +6021,8 @@ "www5.example.org" ], "related.ip": [ - "10.254.10.98", - "10.109.88.27" + "10.109.88.27", + "10.254.10.98" ], "related.user": [ "aparia" @@ -5516,9 +6055,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "jpg", + "url.fragment": "scip", "url.original": "https://internal.example.com/oluptate/todi.jpg?tdolo=ident#scip", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/oluptate/todi.jpg" + ], + "url.query": "tdolo=ident", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "aparia", @@ -5587,9 +6133,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "utlabore", "url.original": "https://mail.example.com/radipis/lore.html?civeli=eufugia#utlabore", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/radipis/lore.html" + ], + "url.query": "civeli=eufugia", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "onemul", @@ -5620,8 +6173,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "internal.example.org" + "internal.example.org", + "example.com" ], "related.ip": [ "10.18.199.203", @@ -5632,8 +6185,8 @@ ], "rsa.internal.messageid": "PROPFIND", "rsa.misc.action": [ - "allow", - "PROPFIND" + "PROPFIND", + "allow" ], "rsa.misc.content_type": "llum", "rsa.misc.result_code": "ntut", @@ -5658,9 +6211,16 @@ "forwarded" ], "url.domain": "internal.example.org", + "url.extension": "jpg", + "url.fragment": "emp", "url.original": "https://internal.example.org/olupta/tio.jpg?idestl=litani#emp", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/olupta/tio.jpg" + ], + "url.query": "idestl=litani", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "org", "user.name": "ittenb", @@ -5695,8 +6255,8 @@ "www5.example.org" ], "related.ip": [ - "10.1.220.47", - "10.73.80.251" + "10.73.80.251", + "10.1.220.47" ], "related.user": [ "ercitati" @@ -5729,9 +6289,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "Neq", "url.original": "https://www5.example.com/sequines/cto.gif?temaccu=uamqua#Neq", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/sequines/cto.gif" + ], + "url.query": "temaccu=uamqua", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ercitati", @@ -5766,16 +6333,16 @@ "api.example.org" ], "related.ip": [ - "10.153.109.61", - "10.22.34.206" + "10.22.34.206", + "10.153.109.61" ], "related.user": [ "mve" ], "rsa.internal.messageid": "PURGE", "rsa.misc.action": [ - "PURGE", - "block" + "block", + "PURGE" ], "rsa.misc.content_type": "velites", "rsa.misc.result_code": "uasiarch", @@ -5800,9 +6367,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "gif", + "url.fragment": "onorume", "url.original": "https://www.example.net/periam/ain.gif?iquipex=mqu#onorume", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/periam/ain.gif" + ], + "url.query": "iquipex=mqu", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mve", @@ -5873,9 +6447,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "Duis", "url.original": "https://www.example.net/ecatc/quovolu.jpg?dexe=nemul#Duis", - "url.path": "https://mail.example.com", + "url.path": [ + "https://mail.example.com", + "/ecatc/quovolu.jpg" + ], + "url.query": "dexe=nemul", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "ipsa", @@ -5910,16 +6491,16 @@ "www5.example.com" ], "related.ip": [ - "10.128.84.27", - "10.97.33.56" + "10.97.33.56", + "10.128.84.27" ], "related.user": [ "ptate" ], "rsa.internal.messageid": "COPY", "rsa.misc.action": [ - "block", - "COPY" + "COPY", + "block" ], "rsa.misc.content_type": "utla", "rsa.misc.result_code": "umfu", @@ -5943,9 +6524,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "cipitla", "url.original": "https://example.com/tqui/ssequ.gif?emse=emqui#cipitla", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/tqui/ssequ.gif" + ], + "url.query": "emse=emqui", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ptate", "user_agent.device.name": "Asus X01BDA", @@ -6015,9 +6603,16 @@ "forwarded" ], "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "quisnost", "url.original": "https://example.com/caboN/imipsam.jpg?catcupid=ritquiin#quisnost", - "url.path": "https://www5.example.com", + "url.path": [ + "https://www5.example.com", + "/caboN/imipsam.jpg" + ], + "url.query": "catcupid=ritquiin", "url.registered_domain": "example.com", + "url.scheme": "https", "url.top_level_domain": "com", "user.name": "ore", "user_agent.device.name": "Samsung SM-A260G", @@ -6047,8 +6642,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www.example.org", - "internal.example.com" + "internal.example.com", + "www.example.org" ], "related.ip": [ "10.33.112.100", @@ -6085,9 +6680,16 @@ "forwarded" ], "url.domain": "www.example.org", + "url.extension": "htm", + "url.fragment": "osqu", "url.original": "https://www.example.org/oinvento/ali.htm?utaliqui=isciv#osqu", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/oinvento/ali.htm" + ], + "url.query": "utaliqui=isciv", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "org", "user.name": "enimad", @@ -6132,8 +6734,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "cancel", - "CONNECT" + "CONNECT", + "cancel" ], "rsa.misc.content_type": "urau", "rsa.misc.result_code": "lla", @@ -6158,9 +6760,16 @@ "forwarded" ], "url.domain": "api.example.net", + "url.extension": "htm", + "url.fragment": "rch", "url.original": "https://api.example.net/quido/llo.htm?tpersp=assi#rch", - "url.path": "https://example.net", + "url.path": [ + "https://example.net", + "/quido/llo.htm" + ], + "url.query": "tpersp=assi", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "net", "user.name": "oremeu", @@ -6206,8 +6815,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "allow", - "HEAD" + "HEAD", + "allow" ], "rsa.misc.content_type": "tevel", "rsa.misc.result_code": "taevitae", @@ -6232,9 +6841,16 @@ "forwarded" ], "url.domain": "www.example.net", + "url.extension": "txt", + "url.fragment": "deriti", "url.original": "https://www.example.net/tvolu/imve.txt?gnaaliq=quam#deriti", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/tvolu/imve.txt" + ], + "url.query": "gnaaliq=quam", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www", "url.top_level_domain": "net", "user.name": "mSecti", @@ -6265,20 +6881,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "api.example.net", - "www5.example.com" + "www5.example.com", + "api.example.net" ], "related.ip": [ - "10.82.148.126", - "10.60.56.205" + "10.60.56.205", + "10.82.148.126" ], "related.user": [ "ita" ], "rsa.internal.messageid": "NONE", "rsa.misc.action": [ - "block", - "NONE" + "NONE", + "block" ], "rsa.misc.content_type": "nder", "rsa.misc.result_code": "nihilmol", @@ -6303,9 +6919,16 @@ "forwarded" ], "url.domain": "www5.example.com", + "url.extension": "htm", + "url.fragment": "tquii", "url.original": "https://www5.example.com/ugitsed/dminimve.htm?onse=uiac#tquii", - "url.path": "https://api.example.net", + "url.path": [ + "https://api.example.net", + "/ugitsed/dminimve.htm" + ], + "url.query": "onse=uiac", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "com", "user.name": "ita", @@ -6336,20 +6959,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "www5.example.net", - "api.example.com" + "api.example.com", + "www5.example.net" ], "related.ip": [ - "10.245.251.98", - "10.6.11.124" + "10.6.11.124", + "10.245.251.98" ], "related.user": [ "tvolu" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "DELETE", - "accept" + "accept", + "DELETE" ], "rsa.misc.content_type": "onsequ", "rsa.misc.result_code": "strud", @@ -6374,9 +6997,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "usan", "url.original": "https://api.example.com/ntium/ide.htm?tamrema=isautem#usan", - "url.path": "https://www5.example.net", + "url.path": [ + "https://www5.example.net", + "/ntium/ide.htm" + ], + "url.query": "tamrema=isautem", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "tvolu", @@ -6411,8 +7041,8 @@ "www5.example.org" ], "related.ip": [ - "10.99.55.115", - "10.145.25.55" + "10.145.25.55", + "10.99.55.115" ], "related.user": [ "lumd" @@ -6421,8 +7051,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "CONNECT", - "block" + "block", + "CONNECT" ], "rsa.misc.content_type": "iamquisn", "rsa.misc.result_code": "lorem", @@ -6447,9 +7077,16 @@ "forwarded" ], "url.domain": "www5.example.org", + "url.extension": "jpg", + "url.fragment": "utaliqui", "url.original": "https://www5.example.org/iad/ngelits.jpg?mporin=orissusc#utaliqui", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/iad/ngelits.jpg" + ], + "url.query": "mporin=orissusc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "org", "user.name": "lumd", @@ -6484,8 +7121,8 @@ "internal.example.com" ], "related.ip": [ - "10.187.86.64", - "10.6.88.105" + "10.6.88.105", + "10.187.86.64" ], "related.user": [ "rem" @@ -6518,9 +7155,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "eprehe", "url.original": "https://mail.example.com/iatnulap/roi.htm?uine=loreeu#eprehe", - "url.path": "https://internal.example.com", + "url.path": [ + "https://internal.example.com", + "/iatnulap/roi.htm" + ], + "url.query": "uine=loreeu", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "rem", @@ -6551,12 +7195,12 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "mail.example.org" + "mail.example.org", + "example.com" ], "related.ip": [ - "10.252.146.132", - "10.163.9.35" + "10.163.9.35", + "10.252.146.132" ], "related.user": [ "umq" @@ -6591,9 +7235,16 @@ "forwarded" ], "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "illoin", "url.original": "https://mail.example.org/turv/use.jpg?mtot=macc#illoin", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/turv/use.jpg" + ], + "url.query": "mtot=macc", "url.registered_domain": "example.org", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "org", "user.name": "umq", @@ -6624,20 +7275,20 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "mail.example.com", - "api.example.org" + "api.example.org", + "mail.example.com" ], "related.ip": [ - "10.235.160.245", - "10.249.101.177" + "10.249.101.177", + "10.235.160.245" ], "related.user": [ "upta" ], "rsa.internal.messageid": "DELETE", "rsa.misc.action": [ - "deny", - "DELETE" + "DELETE", + "deny" ], "rsa.misc.content_type": "uameiu", "rsa.misc.result_code": "porinc", @@ -6662,9 +7313,16 @@ "forwarded" ], "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "ineavol", "url.original": "https://mail.example.com/umdol/rerepr.txt?emipsumq=orinr#ineavol", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/umdol/rerepr.txt" + ], + "url.query": "emipsumq=orinr", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "mail", "url.top_level_domain": "com", "user.name": "upta", @@ -6699,16 +7357,16 @@ "mail.example.org" ], "related.ip": [ - "10.73.218.58", - "10.140.170.171" + "10.140.170.171", + "10.73.218.58" ], "related.user": [ "tinv" ], "rsa.internal.messageid": "TRACE", "rsa.misc.action": [ - "block", - "TRACE" + "TRACE", + "block" ], "rsa.misc.content_type": "umq", "rsa.misc.result_code": "nse", @@ -6733,9 +7391,16 @@ "forwarded" ], "url.domain": "internal.example.com", + "url.extension": "htm", + "url.fragment": "nevolup", "url.original": "https://internal.example.com/rautod/onorumet.htm?mvo=agnidol#nevolup", - "url.path": "https://mail.example.org", + "url.path": [ + "https://mail.example.org", + "/rautod/onorumet.htm" + ], + "url.query": "mvo=agnidol", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "internal", "url.top_level_domain": "com", "user.name": "tinv", @@ -6803,9 +7468,16 @@ "forwarded" ], "url.domain": "example.net", + "url.extension": "jpg", + "url.fragment": "asiar", "url.original": "https://example.net/dun/xce.jpg?nsequat=mvol#asiar", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/dun/xce.jpg" + ], + "url.query": "nsequat=mvol", "url.registered_domain": "example.net", + "url.scheme": "https", "url.top_level_domain": "net", "user.name": "squamest", "user_agent.device.name": "Generic Smartphone", @@ -6837,8 +7509,8 @@ "www.example.org" ], "related.ip": [ - "10.83.154.75", - "10.37.33.179" + "10.37.33.179", + "10.83.154.75" ], "related.user": [ "eatae" @@ -6871,9 +7543,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "sequu", "url.original": "https://api.example.com/teiru/mquamei.jpg?pta=uradi#sequu", - "url.path": "https://www.example.org", + "url.path": [ + "https://www.example.org", + "/teiru/mquamei.jpg" + ], + "url.query": "pta=uradi", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "eatae", @@ -6916,8 +7595,8 @@ ], "rsa.internal.messageid": "MKOL", "rsa.misc.action": [ - "deny", - "MKOL" + "MKOL", + "deny" ], "rsa.misc.content_type": "itse", "rsa.misc.result_code": "qui", @@ -6942,9 +7621,16 @@ "forwarded" ], "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "umdolo", "url.original": "https://www5.example.net/dtempor/rroquisq.gif?liquid=uidex#umdolo", - "url.path": "https://www5.example.org", + "url.path": [ + "https://www5.example.org", + "/dtempor/rroquisq.gif" + ], + "url.query": "liquid=uidex", "url.registered_domain": "example.net", + "url.scheme": "https", "url.subdomain": "www5", "url.top_level_domain": "net", "user.name": "usmod", @@ -6975,8 +7661,8 @@ "observer.type": "Proxies", "observer.vendor": "Squid", "related.hosts": [ - "example.com", - "api.example.com" + "api.example.com", + "example.com" ], "related.ip": [ "10.221.86.133", @@ -6989,8 +7675,8 @@ "rsa.investigations.ec_subject": "NetworkComm", "rsa.investigations.ec_theme": "ALM", "rsa.misc.action": [ - "deny", - "POST" + "POST", + "deny" ], "rsa.misc.content_type": "rerepr", "rsa.misc.result_code": "mcorpor", @@ -7015,9 +7701,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "rchite", "url.original": "https://api.example.com/ore/adeser.htm?pre=aute#rchite", - "url.path": "https://example.com", + "url.path": [ + "https://example.com", + "/ore/adeser.htm" + ], + "url.query": "pre=aute", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "ptasnul", @@ -7052,16 +7745,16 @@ "api.example.com" ], "related.ip": [ - "10.195.4.70", - "10.229.39.190" + "10.229.39.190", + "10.195.4.70" ], "related.user": [ "edictas" ], "rsa.internal.messageid": "PUT", "rsa.misc.action": [ - "PUT", - "deny" + "deny", + "PUT" ], "rsa.misc.content_type": "exeaco", "rsa.misc.result_code": "rmagnido", @@ -7086,9 +7779,16 @@ "forwarded" ], "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "aer", "url.original": "https://api.example.com/liqu/dolor.htm?ess=umdo#aer", - "url.path": "https://api.example.org", + "url.path": [ + "https://api.example.org", + "/liqu/dolor.htm" + ], + "url.query": "ess=umdo", "url.registered_domain": "example.com", + "url.scheme": "https", "url.subdomain": "api", "url.top_level_domain": "com", "user.name": "edictas", diff --git a/x-pack/filebeat/module/suricata/eve/ingest/http.yml b/x-pack/filebeat/module/suricata/eve/ingest/http.yml new file mode 100644 index 00000000000..de8c1710ed7 --- /dev/null +++ b/x-pack/filebeat/module/suricata/eve/ingest/http.yml @@ -0,0 +1,83 @@ +--- +description: Pipeline for Suricata DNS Events + +processors: + - set: + value: "{{suricata.eve.http.http_method}}" + field: http.request.method + if: "ctx?.suricata?.eve?.http?.http_method != null" + - rename: + field: suricata.eve.http.status + target_field: http.response.status_code + ignore_missing: true + - rename: + field: suricata.eve.http.hostname + target_field: destination.domain + ignore_missing: true + # - grok: + # field: suricata.eve.http.url + # patterns: + # - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' + # ignore_missing: true + # pattern_definitions: + # PATH: '[^?#]*' + # QUERY: '[^#]*' + # ANY: '.*' + - rename: + field: suricata.eve.http.url + target_field: url.original + ignore_missing: true + - rename: + field: suricata.eve.http.http_port + target_field: url.port + ignore_missing: true + + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - set: + field: url.domain + value: '{{destination.domain}}' + ignore_empty_value: true + if: ctx?.url?.domain == null && ctx?.destination?.domain != null + - remove: + field: _temp_ + ignore_missing: true + - rename: + field: suricata.eve.http.http_refer + target_field: http.request.referrer + ignore_missing: true + - rename: + field: suricata.eve.http.length + target_field: http.response.body.bytes + ignore_missing: true + - user_agent: + field: suricata.eve.http.http_user_agent + ignore_missing: true + +on_failure: + - append: + field: error.message + value: >- + error in HTTP pipeline: + error in [{{_ingest.on_failure_processor_type}}] processor{{#_ingest.on_failure_processor_tag}} + with tag [{{_ingest.on_failure_processor_tag }}]{{/_ingest.on_failure_processor_tag}} + {{ _ingest.on_failure_message }} diff --git a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml index a24a5df3ff6..e957d177f38 100644 --- a/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml +++ b/x-pack/filebeat/module/suricata/eve/ingest/pipeline.yml @@ -152,6 +152,11 @@ processors: - pipeline: if: ctx?.network?.protocol == "tls" name: '{< IngestPipeline "tls" >}' + ## HTTP + - pipeline: + if: >- + ctx?.network?.protocol == "http" + name: '{< IngestPipeline "http" >}' ## Flow - append: if: ctx?.suricata?.eve?.flow?.state == "new" @@ -163,24 +168,6 @@ processors: field: event.type value: - end - - - - set: - value: "{{suricata.eve.http.http_method}}" - field: http.request.method - if: "ctx?.suricata?.eve?.http?.http_method != null" - - rename: - field: suricata.eve.http.status - target_field: http.response.status_code - ignore_missing: true - - append: - if: ctx.suricata?.eve?.http?.hostname != null - value: '{{suricata.eve.http.hostname}}' - field: destination.domain - allow_duplicates: false - - remove: - field: suricata.eve.http.hostname - ignore_failure: true - script: lang: painless tag: suricata_deduplicate_dest_domain @@ -194,36 +181,6 @@ processors: ctx.destination.domain = domain; } ignore_failure: true - - set: - if: ctx?.network?.protocol == 'http' - field: url.domain - value: '{{destination.domain}}' - ignore_empty_value: true - - grok: - field: suricata.eve.http.url - patterns: - - '%{PATH:url.path}(?:\?%{QUERY:url.query})?(?:#%{ANY:url.fragment})?' - ignore_missing: true - pattern_definitions: - PATH: '[^?#]*' - QUERY: '[^#]*' - ANY: '.*' - - rename: - field: suricata.eve.http.url - target_field: url.original - ignore_missing: true - - rename: - field: suricata.eve.http.http_port - target_field: url.port - ignore_missing: true - - rename: - field: suricata.eve.http.http_refer - target_field: http.request.referrer - ignore_missing: true - - rename: - field: suricata.eve.http.length - target_field: http.response.body.bytes - ignore_missing: true - rename: field: suricata.eve.fileinfo.filename target_field: file.path @@ -342,9 +299,6 @@ processors: field: suricata.eve.proto target_field: network.transport ignore_missing: true - - user_agent: - field: suricata.eve.http.http_user_agent - ignore_missing: true - geoip: if: ctx?.source?.geo == null field: source.ip diff --git a/x-pack/filebeat/module/suricata/eve/manifest.yml b/x-pack/filebeat/module/suricata/eve/manifest.yml index 8d7f87d8373..a712ab66336 100644 --- a/x-pack/filebeat/module/suricata/eve/manifest.yml +++ b/x-pack/filebeat/module/suricata/eve/manifest.yml @@ -19,6 +19,7 @@ ingest_pipeline: - ingest/dns-answer-v1.yml - ingest/dns-answer-v2.yml - ingest/tls.yml + - ingest/http.yml input: config/eve.yml requires.processors: diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json index e8f77f9033a..b294e0cbdca 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-6.0.log-expected.json @@ -78,6 +78,7 @@ "suricata" ], "url.domain": "testmynids.org", + "url.extension": "html", "url.original": "/uid/index.html", "url.path": "/uid/index.html", "user_agent.device.name": "Other", diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log b/x-pack/filebeat/module/suricata/eve/test/eve-small.log index 9cc157a9e75..28104b077ec 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log @@ -1,7 +1,7 @@ {"timestamp":"2018-07-05T15:01:09.820360-0400","flow_id":298824096901438,"in_iface":"en0","event_type":"ssh","src_ip":"192.168.86.85","src_port":55406,"dest_ip":"192.168.253.112","dest_port":22,"proto":"TCP","ssh":{"client":{"proto_version":"2.0","software_version":"OpenSSH_7.6"},"server":{"proto_version":"2.0","software_version":"libssh_0.7.0"}}} {"timestamp":"2018-07-05T15:07:20.910626-0400","flow_id":904992230150281,"in_iface":"en0","event_type":"alert","src_ip":"192.168.86.85","src_port":55641,"dest_ip":"192.168.156.70","dest_port":443,"proto":"TCP","tx_id":0,"alert":{"action":"allowed","gid":1,"signature_id":2024833,"rev":3,"signature":"ET POLICY Observed IP Lookup Domain (l2 .io in TLS SNI)","category":"Potential Corporate Privacy Violation","severity":1},"tls":{"session_resumed":true,"sni":"l2.io","version":"TLS 1.2"},"app_proto":"tls","flow":{"pkts_toserver":4,"pkts_toclient":3,"bytes_toserver":793,"bytes_toclient":343,"start":"2018-07-05T15:07:19.659593-0400"}} {"timestamp":"2018-07-05T15:43:47.690014-0400","flow_id":2115002772430095,"in_iface":"en0","event_type":"http","src_ip":"192.168.86.85","src_port":56119,"dest_ip":"192.168.86.28","dest_port":63963,"proto":"TCP","tx_id":0,"http":{"hostname":"192.168.86.28","url":"\/dd.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"text\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1155}} -{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.28","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} +{"timestamp":"2018-07-05T15:44:33.222441-0400","flow_id":2211411903323127,"in_iface":"en0","event_type":"fileinfo","src_ip":"192.168.86.28","src_port":8008,"dest_ip":"192.168.86.85","dest_port":56118,"proto":"TCP","http":{"hostname":"192.168.86.85","url":"\/ssdp\/device-desc.xml","http_user_agent":"Mozilla\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/67.0.3396.99 Safari\/537.36","http_content_type":"application\/xml","http_method":"GET","protocol":"HTTP\/1.1","status":200,"length":1071},"app_proto":"http","fileinfo":{"filename":"\/ssdp\/device-desc.xml","gaps":false,"state":"CLOSED","md5":"427b7337ff37eeb24d74f47d8e04cf21","sha1":"313573490192c685e9e53abef25453ed0d5e2aee","sha256":"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b","stored":false,"size":1071,"tx_id":0}} {"timestamp":"2018-07-05T15:51:20.213418-0400","flow_id":1684780223079543,"in_iface":"en0","event_type":"dns","src_ip":"192.168.86.1","src_port":53,"dest_ip":"192.168.86.85","dest_port":39464,"proto":"UDP","dns":{"type":"answer","id":12308,"rcode":"NOERROR","rrname":"clients.l.google.com","rrtype":"A","ttl":299,"rdata":"172.217.13.110"}} {"timestamp":"2018-07-05T15:51:23.009510-0400","event_type":"stats","stats":{"uptime":5400,"capture":{"kernel_packets":430313,"kernel_drops":0,"kernel_ifdrops":0},"decoder":{"pkts":430313,"bytes":335138381,"invalid":2,"ipv4":425873,"ipv6":3785,"ethernet":430313,"raw":0,"null":0,"sll":0,"tcp":370093,"udp":58337,"sctp":0,"icmpv4":186,"icmpv6":1019,"ppp":0,"pppoe":0,"gre":0,"vlan":0,"vlan_qinq":0,"ieee8021ah":0,"teredo":1,"ipv4_in_ipv6":0,"ipv6_in_ipv6":0,"mpls":0,"avg_pkt_size":778,"max_pkt_size":1514,"erspan":0,"ipraw":{"invalid_ip_version":0},"ltnull":{"pkt_too_small":0,"unsupported_type":0},"dce":{"pkt_too_small":0}},"flow":{"memcap":0,"tcp":1113,"udp":1881,"icmpv4":0,"icmpv6":677,"spare":10000,"emerg_mode_entered":0,"emerg_mode_over":0,"tcp_reuse":0,"memuse":11537312},"defrag":{"ipv4":{"fragments":0,"reassembled":0,"timeouts":0},"ipv6":{"fragments":0,"reassembled":0,"timeouts":0},"max_frag_hits":0},"tcp":{"sessions":842,"ssn_memcap_drop":0,"pseudo":0,"pseudo_failed":0,"invalid_checksum":0,"no_flow":0,"syn":1138,"synack":656,"rst":1165,"segment_memcap_drop":0,"stream_depth_reached":63,"reassembly_gap":0,"overlap":5979,"overlap_diff_data":0,"insert_data_normal_fail":0,"insert_data_overlap_fail":0,"insert_list_fail":0,"memuse":4587520,"reassembly_memuse":768000},"detect":{"alert":2},"app_layer":{"flow":{"http":22,"ftp":0,"smtp":0,"tls":560,"ssh":4,"imap":0,"msn":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"failed_tcp":2,"dcerpc_udp":0,"dns_udp":762,"failed_udp":1119},"tx":{"http":25,"ftp":0,"smtp":0,"tls":0,"ssh":0,"smb":0,"dcerpc_tcp":0,"dns_tcp":0,"dcerpc_udp":0,"dns_udp":762}},"flow_mgr":{"closed_pruned":729,"new_pruned":1879,"est_pruned":975,"bypassed_pruned":0,"flows_checked":8,"flows_notimeout":8,"flows_timeout":0,"flows_timeout_inuse":0,"flows_removed":0,"rows_checked":65536,"rows_skipped":65530,"rows_empty":0,"rows_busy":0,"rows_maxlen":2},"file_store":{"open_files":0},"dns":{"memuse":7749,"memcap_state":0,"memcap_global":0},"http":{"memuse":17861,"memcap":0}}} {"timestamp":"2018-07-05T15:51:50.666597-0400","flow_id":89751777876473,"in_iface":"en0","event_type":"tls","src_ip":"192.168.86.85","src_port":56187,"dest_ip":"17.142.164.13","dest_port":443,"proto":"TCP","tls":{"subject":"CN=*.icloud.com, OU=management:idms.group.506364, O=Apple Inc., ST=California, C=US","issuerdn":"CN=Apple IST CA 2 - G1, OU=Certification Authority, O=Apple Inc., C=US","serial":"5C:9C:E1:09:78:87:F8:07","fingerprint":"6a:ff:ac:a6:5f:8a:05:e7:a9:8c:76:29:b9:08:c7:69:ad:dc:72:47","sni":"p33-btmmdns.icloud.com.","version":"TLS 1.2","notbefore":"2017-02-27T17:54:31","notafter":"2019-03-29T17:54:31"}} diff --git a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json index 50125bc3f3c..cfd6fa5ff83 100644 --- a/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json +++ b/x-pack/filebeat/module/suricata/eve/test/eve-small.log-expected.json @@ -151,6 +151,7 @@ "suricata" ], "url.domain": "192.168.86.28", + "url.extension": "xml", "url.original": "/dd.xml", "url.path": "/dd.xml", "user_agent.device.name": "Mac", @@ -164,7 +165,7 @@ { "@timestamp": "2018-07-05T19:44:33.222Z", "destination.address": "192.168.86.85", - "destination.domain": "192.168.86.28", + "destination.domain": "192.168.86.85", "destination.ip": "192.168.86.85", "destination.port": 56118, "event.category": [ @@ -173,7 +174,7 @@ "event.dataset": "suricata.eve", "event.kind": "event", "event.module": "suricata", - "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.28\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", + "event.original": "{\"timestamp\":\"2018-07-05T15:44:33.222441-0400\",\"flow_id\":2211411903323127,\"in_iface\":\"en0\",\"event_type\":\"fileinfo\",\"src_ip\":\"192.168.86.28\",\"src_port\":8008,\"dest_ip\":\"192.168.86.85\",\"dest_port\":56118,\"proto\":\"TCP\",\"http\":{\"hostname\":\"192.168.86.85\",\"url\":\"\\/ssdp\\/device-desc.xml\",\"http_user_agent\":\"Mozilla\\/5.0 (Macintosh; Intel Mac OS X 10_13_5) AppleWebKit\\/537.36 (KHTML, like Gecko) Chrome\\/67.0.3396.99 Safari\\/537.36\",\"http_content_type\":\"application\\/xml\",\"http_method\":\"GET\",\"protocol\":\"HTTP\\/1.1\",\"status\":200,\"length\":1071},\"app_proto\":\"http\",\"fileinfo\":{\"filename\":\"\\/ssdp\\/device-desc.xml\",\"gaps\":false,\"state\":\"CLOSED\",\"md5\":\"427b7337ff37eeb24d74f47d8e04cf21\",\"sha1\":\"313573490192c685e9e53abef25453ed0d5e2aee\",\"sha256\":\"f610428ebddf6f8cf9e39322e672583c45fcdcf885efad0ab48fd53a3dfc2c4b\",\"stored\":false,\"size\":1071,\"tx_id\":0}}", "file.path": "/ssdp/device-desc.xml", "file.size": 1071, "fileset.name": "eve", @@ -186,7 +187,7 @@ "network.protocol": "http", "network.transport": "tcp", "related.hosts": [ - "192.168.86.28" + "192.168.86.85" ], "related.ip": [ "192.168.86.28", @@ -211,7 +212,8 @@ "tags": [ "suricata" ], - "url.domain": "192.168.86.28", + "url.domain": "192.168.86.85", + "url.extension": "xml", "url.original": "/ssdp/device-desc.xml", "url.path": "/ssdp/device-desc.xml", "user_agent.device.name": "Mac", @@ -582,9 +584,11 @@ "suricata" ], "url.domain": "ctldl.windowsupdate.com", + "url.extension": "cab", "url.original": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab?111111111111", - "url.path": "http://ctldl.windowsupdate.com/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", + "url.path": "/msdownload/update/v3/static/trustedr/en/pinrulesstl.cab", "url.query": "111111111111", + "url.scheme": "http", "user_agent.device.name": "Other", "user_agent.name": "Microsoft-CryptoAPI", "user_agent.original": "Microsoft-CryptoAPI/10.0", diff --git a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml index d8408c51133..045d5afe760 100644 --- a/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zeek/sip/ingest/pipeline.yml @@ -20,6 +20,11 @@ processors: - remove: field: zeek.sip.seq ignore_missing: true +- grok: + field: url.full + patterns: + - '%{WORD:url.scheme}:(%{USERNAME:url.username}@)?%{IPORHOST:url.domain}(:%{POSINT:url.port:long})?' + ignore_missing: true - geoip: field: destination.ip target_field: destination.geo @@ -80,6 +85,30 @@ processors: field: event.outcome value: success if: "ctx?.zeek?.sip?.status?.code != null && ctx.zeek.sip.status.code < 400" +- script: + lang: painless + description: This script processor iterates over the whole document to remove fields with null values. + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json index 71061cd293b..a9e15731ebc 100644 --- a/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json +++ b/x-pack/filebeat/module/zeek/sip/test/sip-json.log-expected.json @@ -42,7 +42,10 @@ "tags": [ "zeek.sip" ], + "url.domain": "newyork.voip.ms", "url.full": "sip:newyork.voip.ms:5060", + "url.port": 5060, + "url.scheme": "sip", "zeek.session_id": "CPRLCB4eWHdjP852Bk", "zeek.sip.call_id": "8694cd7e-976e4fc3-d76f6e38@172.16.133.19", "zeek.sip.request.body_length": 0, @@ -120,7 +123,11 @@ "tags": [ "zeek.sip" ], + "url.domain": "bestel.com", "url.full": "sip:francisco@bestel.com:55060", + "url.port": 55060, + "url.scheme": "sip", + "url.username": "francisco", "zeek.session_id": "ComJz236lSOcuOmix3", "zeek.sip.call_id": "12013223@200.57.7.195", "zeek.sip.request.body_length": 229, @@ -201,7 +208,9 @@ "tags": [ "zeek.sip" ], + "url.domain": "Verso.com", "url.full": "sip:Verso.com", + "url.scheme": "sip", "zeek.session_id": "CJZDWgixtwqXctWEg", "zeek.sip.call_id": "46E1C3CB36304F84A020CF6DD3F96461@Verso.com", "zeek.sip.request.body_length": 0, diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml index 9291add3593..62f05fa073f 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/meeting.yml @@ -38,14 +38,22 @@ processors: ignore_missing: true - rename: field: zoom.meeting.join_url - target_field: url.full + target_field: url.original ignore_missing: true - rename: field: zoom.registrant.join_url - target_field: url.full + target_field: url.original ignore_missing: true - if: ctx?.url?.full == null - + if: ctx?.original?.full == null +- uri_parts: + field: url.original + ignore_failure: true + if: ctx?.url?.original != null +- set: + field: url.full + copy_from: url.original + ignore_failure: true + if: ctx?.url?.original != null # # Set user.* from participant, if any. # diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml index b836cd9c96c..e5fca522f33 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/phone.yml @@ -19,6 +19,10 @@ processors: field: zoom.object target_field: zoom.phone ignore_missing: true +- uri_parts: + field: zoom.phone.download_url + ignore_failure: true + if: ctx?.zoom?.phone?.download_url != null - rename: field: zoom.phone.download_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml index 715f46bcbd9..f9e5b1c3042 100644 --- a/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml +++ b/x-pack/filebeat/module/zoom/webhook/ingest/recording.yml @@ -35,6 +35,10 @@ processors: field: zoom.object target_field: zoom.recording ignore_missing: true +- uri_parts: + field: zoom.recording.share_url + ignore_failure: true + if: ctx?.zoom?.recording?.share_url != null - rename: field: zoom.recording.share_url target_field: url.full diff --git a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json index 123de911c51..723a37a326f 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/meeting.ndjson.log-expected.json @@ -99,7 +99,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/j/00000000", + "url.original": "https://zoom.us/j/00000000", + "url.path": "/j/00000000", + "url.scheme": "https", "user.email": "someemail@email.com", "user.id": "BBBBBBBBBB", "zoom.account_id": "AAAAAAAAAAA", @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/w/someendpointhere", + "url.original": "https://zoom.us/w/someendpointhere", + "url.path": "/w/someendpointhere", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.meeting.host_id": "uLobbbbbbbbbb_qQsQ", diff --git a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json index 507943735b2..ee24b279b3c 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/phone.ndjson.log-expected.json @@ -320,7 +320,12 @@ "zoom-webhook", "forwarded" ], + "url.domain": "testurl.com", + "url.extension": "mp4", "url.full": "https://testurl.com/file.mp4", + "url.original": "https://testurl.com/file.mp4", + "url.path": "/file.mp4", + "url.scheme": "https", "zoom.account_id": "test", "zoom.phone.callee.extension_type": "2", "zoom.phone.callee.id": "1234", diff --git a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json index f9be7349ab4..a0296121e1a 100644 --- a/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json +++ b/x-pack/filebeat/module/zoom/webhook/test/recording.ndjson.log-expected.json @@ -175,7 +175,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -254,7 +258,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -293,7 +301,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -332,7 +344,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, @@ -371,7 +387,11 @@ "zoom-webhook", "forwarded" ], + "url.domain": "zoom.us", "url.full": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.original": "https://zoom.us/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.path": "/recording/share/aaaaaannnnnldglrkgmrmhh", + "url.scheme": "https", "user.id": "uLobbbbbbbbbb_qQsQ", "zoom.account_id": "lAAAAAAAAAAAAA", "zoom.recording.duration": 1, diff --git a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml index b6105a0fddd..abf179e94b3 100644 --- a/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml +++ b/x-pack/filebeat/module/zscaler/zia/ingest/pipeline.yml @@ -9,7 +9,31 @@ processors: # User agent - user_agent: field: user_agent.original - ignore_missing: true + ignore_missing: + # URL + - uri_parts: + field: url.original + target_field: _temp_.url + ignore_failure: true + if: ctx?.url?.original != null + - script: + lang: painless + description: Updates the URL ECS fields from the results of the URI parts processor to not overwrite the RSA mappings + if: ctx?._temp_?.url != null + source: | + for (entry in ctx._temp_.url.entrySet()) { + if (entry != null && entry.getValue() != null) { + if(ctx.url[entry.getKey()] == null) { + ctx.url[entry.getKey()] = entry.getValue(); + } else if (!ctx.url[entry.getKey()].contains(entry.getValue())) { + ctx.url[entry.getKey()] = [ctx.url[entry.getKey()]]; + ctx.url[entry.getKey()].add(entry.getValue()); + } + } + } + - remove: + field: _temp_ + ignore_missing: true # IP Geolocation Lookup - geoip: field: source.ip diff --git a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json index 54841870df7..5c4bd9fc36c 100644 --- a/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json +++ b/x-pack/filebeat/module/zscaler/zia/test/generated.log-expected.json @@ -26,8 +26,8 @@ "rci737.www5.example" ], "related.ip": [ - "10.206.191.17", - "10.176.10.114" + "10.176.10.114", + "10.206.191.17" ], "related.user": [ "sumdo" @@ -41,8 +41,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntium", "rsa.misc.action": [ - "Blocked", - "pisciv" + "pisciv", + "Blocked" ], "rsa.misc.category": "umq", "rsa.misc.filter": "oremi", @@ -65,7 +65,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "htm", + "url.fragment": "amremap", "url.original": "https://api.example.com/ivelitse/ritin.htm?utl=vol#amremap", + "url.path": "/ivelitse/ritin.htm", + "url.query": "utl=vol", + "url.scheme": "https", "user.name": "sumdo", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -139,7 +145,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "mquia", "url.original": "https://internal.example.net/isiutal/moenimi.jpg?gnaali=enatus#mquia", + "url.path": "/isiutal/moenimi.jpg", + "url.query": "gnaali=enatus", + "url.scheme": "https", "user.name": "eataevi", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -215,7 +227,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "iquaUte", "url.original": "https://example.com/taspe/mvolu.gif?atcup=snos#iquaUte", + "url.path": "/taspe/mvolu.gif", + "url.query": "atcup=snos", + "url.scheme": "https", "user.name": "tenima", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -291,7 +309,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "uid", "url.original": "https://api.example.org/doloreeu/pori.jpg?itati=mfu#uid", + "url.path": "/doloreeu/pori.jpg", + "url.query": "itati=mfu", + "url.scheme": "https", "user.name": "equun", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -328,8 +352,8 @@ "ore2933.www.test" ], "related.ip": [ - "10.136.153.149", - "10.61.78.108" + "10.61.78.108", + "10.136.153.149" ], "related.user": [ "ercit" @@ -367,7 +391,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iinea", "url.original": "https://api.example.com/ele/tenbyCic.gif?porainc=amquisno#iinea", + "url.path": "/ele/tenbyCic.gif", + "url.query": "porainc=amquisno", + "url.scheme": "https", "user.name": "ercit", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -404,8 +434,8 @@ "ollit4105.mail.localdomain" ], "related.ip": [ - "10.183.16.166", - "10.66.250.92" + "10.66.250.92", + "10.183.16.166" ], "related.user": [ "tessec" @@ -419,8 +449,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "avol", "rsa.misc.action": [ - "Allowed", - "ist" + "ist", + "Allowed" ], "rsa.misc.category": "lorema", "rsa.misc.filter": "sun", @@ -443,7 +473,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "aboreetd", "url.original": "https://mail.example.org/sitas/ehenderi.jpg?atquovo=iumto#aboreetd", + "url.path": "/sitas/ehenderi.jpg", + "url.query": "atquovo=iumto", + "url.scheme": "https", "user.name": "tessec", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -519,7 +555,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "txt", + "url.fragment": "ctetura", "url.original": "https://mail.example.net/aborumSe/luptat.txt?antiumto=strude#ctetura", + "url.path": "/aborumSe/luptat.txt", + "url.query": "antiumto=strude", + "url.scheme": "https", "user.name": "xercitat", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -595,7 +637,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "jpg", + "url.fragment": "des", "url.original": "https://www5.example.net/ntutla/equa.jpg?civeli=errorsi#des", + "url.path": "/ntutla/equa.jpg", + "url.query": "civeli=errorsi", + "url.scheme": "https", "user.name": "erc", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -632,8 +680,8 @@ "aperia4409.www5.invalid" ], "related.ip": [ - "10.78.151.178", - "10.25.192.202" + "10.25.192.202", + "10.78.151.178" ], "related.user": [ "quip" @@ -671,7 +719,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "txt", + "url.fragment": "oriosamn", "url.original": "https://api.example.net/atvol/umiur.txt?tati=utaliqu#oriosamn", + "url.path": "/atvol/umiur.txt", + "url.query": "tati=utaliqu", + "url.scheme": "https", "user.name": "quip", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -708,8 +762,8 @@ "sitvolup368.internal.host" ], "related.ip": [ - "10.71.170.37", - "10.135.225.244" + "10.135.225.244", + "10.71.170.37" ], "related.user": [ "atu" @@ -723,8 +777,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ihilm", "rsa.misc.action": [ - "Allowed", - "psaquae" + "psaquae", + "Allowed" ], "rsa.misc.category": "eFinib", "rsa.misc.filter": "inesci", @@ -747,7 +801,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "uipe", "url.original": "https://mail.example.net/equep/iavolu.gif?aqu=rpo#uipe", + "url.path": "/equep/iavolu.gif", + "url.query": "aqu=rpo", + "url.scheme": "https", "user.name": "atu", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -784,8 +844,8 @@ "ite2026.www.invalid" ], "related.ip": [ - "10.223.247.86", - "10.19.145.131" + "10.19.145.131", + "10.223.247.86" ], "related.user": [ "tNequepo" @@ -823,7 +883,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "namali", "url.original": "https://example.org/bor/occa.htm?dol=leumiu#namali", + "url.path": "/bor/occa.htm", + "url.query": "dol=leumiu", + "url.scheme": "https", "user.name": "tNequepo", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -899,7 +965,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "htm", + "url.fragment": "reseos", "url.original": "https://internal.example.net/oru/temqu.htm?etMalor=ipi#reseos", + "url.path": "/oru/temqu.htm", + "url.query": "etMalor=ipi", + "url.scheme": "https", "user.name": "ihilmo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -951,8 +1023,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "catc", "rsa.misc.action": [ - "Allowed", - "veni" + "veni", + "Allowed" ], "rsa.misc.category": "sBono", "rsa.misc.filter": "isnisiu", @@ -975,7 +1047,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "html", + "url.fragment": "osamn", "url.original": "https://mail.example.net/itatione/isnis.html?oluptate=issus#osamn", + "url.path": "/itatione/isnis.html", + "url.query": "oluptate=issus", + "url.scheme": "https", "user.name": "ratvolu", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1012,8 +1090,8 @@ "piscin6866.internal.host" ], "related.ip": [ - "10.135.160.125", - "10.0.55.9" + "10.0.55.9", + "10.135.160.125" ], "related.user": [ "volupta" @@ -1051,7 +1129,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "rau", "url.original": "https://www.example.org/eporr/xeacomm.html?aturQui=utlabor#rau", + "url.path": "/eporr/xeacomm.html", + "url.query": "aturQui=utlabor", + "url.scheme": "https", "user.name": "volupta", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -1103,8 +1187,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nnum", "rsa.misc.action": [ - "Allowed", - "ntoccae" + "ntoccae", + "Allowed" ], "rsa.misc.category": "tium", "rsa.misc.filter": "uteirure", @@ -1127,7 +1211,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "html", + "url.fragment": "ntocc", "url.original": "https://internal.example.net/ptatemq/luptatev.html?Nequepo=ipsumd#ntocc", + "url.path": "/ptatemq/luptatev.html", + "url.query": "Nequepo=ipsumd", + "url.scheme": "https", "user.name": "saute", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1203,7 +1293,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "ipsumd", "url.original": "https://www5.example.com/tateve/itinvol.txt?tenatus=cipitlab#ipsumd", + "url.path": "/tateve/itinvol.txt", + "url.query": "tenatus=cipitlab", + "url.scheme": "https", "user.name": "inibusB", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1279,7 +1375,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "taedicta", "url.original": "https://api.example.net/tquiin/tse.jpg?ovol=ptasn#taedicta", + "url.path": "/tquiin/tse.jpg", + "url.query": "ovol=ptasn", + "url.scheme": "https", "user.name": "exercita", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1355,7 +1457,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "amcol", "url.original": "https://mail.example.org/olor/ineavo.gif?mquelau=iadolor#amcol", + "url.path": "/olor/ineavo.gif", + "url.query": "mquelau=iadolor", + "url.scheme": "https", "user.name": "str", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -1407,8 +1515,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "oNemoeni", "rsa.misc.action": [ - "Blocked", - "nre" + "nre", + "Blocked" ], "rsa.misc.category": "labo", "rsa.misc.filter": "tutlab", @@ -1431,7 +1539,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "seq", "url.original": "https://www.example.com/its/ender.gif?oles=edic#seq", + "url.path": "/its/ender.gif", + "url.query": "oles=edic", + "url.scheme": "https", "user.name": "aturve", "user_agent.device.name": "Samsung SM-S337TL", "user_agent.name": "Chrome Mobile", @@ -1468,8 +1582,8 @@ "eacommod1930.internal.lan" ], "related.ip": [ - "10.229.83.165", - "10.29.155.171" + "10.29.155.171", + "10.229.83.165" ], "related.user": [ "ulapar" @@ -1507,7 +1621,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "psum", "url.original": "https://www5.example.org/oeni/tdol.gif?llamco=nea#psum", + "url.path": "/oeni/tdol.gif", + "url.query": "llamco=nea", + "url.scheme": "https", "user.name": "ulapar", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -1559,8 +1679,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uaUten", "rsa.misc.action": [ - "amcorp", - "Blocked" + "Blocked", + "amcorp" ], "rsa.misc.category": "umdolor", "rsa.misc.filter": "velillu", @@ -1583,7 +1703,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "lor", "url.original": "https://www.example.com/uasiar/utlab.htm?loremqu=dantium#lor", + "url.path": "/uasiar/utlab.htm", + "url.query": "loremqu=dantium", + "url.scheme": "https", "user.name": "lor", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -1635,8 +1761,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tdol", "rsa.misc.action": [ - "Allowed", - "nte" + "nte", + "Allowed" ], "rsa.misc.category": "adeseru", "rsa.misc.filter": "mac", @@ -1659,7 +1785,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "quamest", "url.original": "https://api.example.org/icabo/gna.html?urerepr=eseru#quamest", + "url.path": "/icabo/gna.html", + "url.query": "urerepr=eseru", + "url.scheme": "https", "user.name": "snost", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -1735,7 +1867,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "onsequ", "url.original": "https://api.example.org/oremi/elites.html?iosa=boNemoe#onsequ", + "url.path": "/oremi/elites.html", + "url.query": "iosa=boNemoe", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -1787,8 +1925,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ulpa", "rsa.misc.action": [ - "Allowed", - "gnaal" + "gnaal", + "Allowed" ], "rsa.misc.category": "nte", "rsa.misc.filter": "pid", @@ -1811,7 +1949,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "dqu", "url.original": "https://example.com/luptatem/uaeratv.gif?dat=periam#dqu", + "url.path": "/luptatem/uaeratv.gif", + "url.query": "dat=periam", + "url.scheme": "https", "user.name": "fugi", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -1848,8 +1992,8 @@ "sitam5077.internal.host" ], "related.ip": [ - "10.179.210.218", - "10.32.39.220" + "10.32.39.220", + "10.179.210.218" ], "related.user": [ "boreetdo" @@ -1887,7 +2031,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "umetMalo", "url.original": "https://www.example.org/tanimi/rumSecti.jpg?emporain=ntiumto#umetMalo", + "url.path": "/tanimi/rumSecti.jpg", + "url.query": "emporain=ntiumto", + "url.scheme": "https", "user.name": "boreetdo", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -1939,8 +2089,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ntNeq", "rsa.misc.action": [ - "dtempo", - "Blocked" + "Blocked", + "dtempo" ], "rsa.misc.category": "ipsu", "rsa.misc.filter": "iqu", @@ -1963,7 +2113,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "html", + "url.fragment": "ulla", "url.original": "https://api.example.com/ori/tconsect.html?ercit=eporroq#ulla", + "url.path": "/ori/tconsect.html", + "url.query": "ercit=eporroq", + "url.scheme": "https", "user.name": "agnaaliq", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -2000,8 +2156,8 @@ "lloin4019.www.localhost" ], "related.ip": [ - "10.130.241.232", - "10.238.224.49" + "10.238.224.49", + "10.130.241.232" ], "related.user": [ "onse" @@ -2015,8 +2171,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mnisiut", "rsa.misc.action": [ - "mod", - "Allowed" + "Allowed", + "mod" ], "rsa.misc.category": "uiinea", "rsa.misc.filter": "aturQu", @@ -2039,7 +2195,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "txt", + "url.fragment": "utfug", "url.original": "https://api.example.org/rure/asiarchi.txt?loremeu=aturve#utfug", + "url.path": "/rure/asiarchi.txt", + "url.query": "loremeu=aturve", + "url.scheme": "https", "user.name": "onse", "user_agent.device.name": "POCOPHONE F1", "user_agent.name": "Chrome Mobile", @@ -2076,8 +2238,8 @@ "tamet6317.www.host" ], "related.ip": [ - "10.2.67.127", - "10.115.53.31" + "10.115.53.31", + "10.2.67.127" ], "related.user": [ "Cic" @@ -2091,8 +2253,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "quatD", "rsa.misc.action": [ - "Allowed", - "tatem" + "tatem", + "Allowed" ], "rsa.misc.category": "aincidun", "rsa.misc.filter": "uela", @@ -2115,7 +2277,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "htm", + "url.fragment": "labore", "url.original": "https://example.com/emUte/molestia.htm?orroqu=elitsed#labore", + "url.path": "/emUte/molestia.htm", + "url.query": "orroqu=elitsed", + "url.scheme": "https", "user.name": "Cic", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2191,7 +2359,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "tionula", "url.original": "https://mail.example.net/repreh/plic.jpg?utlabo=tetur#tionula", + "url.path": "/repreh/plic.jpg", + "url.query": "utlabo=tetur", + "url.scheme": "https", "user.name": "ueipsa", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -2228,8 +2402,8 @@ "utaliqu4248.www.localhost" ], "related.ip": [ - "10.18.226.72", - "10.101.85.169" + "10.101.85.169", + "10.18.226.72" ], "related.user": [ "rroqu" @@ -2267,7 +2441,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "ema", "url.original": "https://api.example.com/tcu/iatqu.jpg?quovo=urExcep#ema", + "url.path": "/tcu/iatqu.jpg", + "url.query": "quovo=urExcep", + "url.scheme": "https", "user.name": "rroqu", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -2304,8 +2484,8 @@ "mdolore473.internal.test" ], "related.ip": [ - "10.87.100.240", - "10.242.182.193" + "10.242.182.193", + "10.87.100.240" ], "related.user": [ "stenatus" @@ -2319,8 +2499,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mag", "rsa.misc.action": [ - "Allowed", - "tali" + "tali", + "Allowed" ], "rsa.misc.category": "oconse", "rsa.misc.filter": "npr", @@ -2343,7 +2523,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "gif", + "url.fragment": "lupta", "url.original": "https://www5.example.com/apariatu/lorsita.gif?msequ=uat#lupta", + "url.path": "/apariatu/lorsita.gif", + "url.query": "msequ=uat", + "url.scheme": "https", "user.name": "stenatus", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -2419,7 +2605,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.net", + "url.extension": "jpg", + "url.fragment": "tesseq", "url.original": "https://internal.example.net/ende/abor.jpg?riameaqu=ame#tesseq", + "url.path": "/ende/abor.jpg", + "url.query": "riameaqu=ame", + "url.scheme": "https", "user.name": "itasp", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -2456,8 +2648,8 @@ "lapar1599.www.lan" ], "related.ip": [ - "10.106.77.138", - "10.193.66.155" + "10.193.66.155", + "10.106.77.138" ], "related.user": [ "iusmodt" @@ -2495,7 +2687,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "txt", + "url.fragment": "mvolupta", "url.original": "https://example.com/ame/amvolu.txt?equaturv=lamc#mvolupta", + "url.path": "/ame/amvolu.txt", + "url.query": "equaturv=lamc", + "url.scheme": "https", "user.name": "iusmodt", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2532,8 +2730,8 @@ "aquioff3853.www.localdomain" ], "related.ip": [ - "10.236.230.136", - "10.54.159.1" + "10.54.159.1", + "10.236.230.136" ], "related.user": [ "mUteni" @@ -2571,7 +2769,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "txt", + "url.fragment": "nisi", "url.original": "https://mail.example.org/uisnostr/reetdol.txt?ugi=niamquis#nisi", + "url.path": "/uisnostr/reetdol.txt", + "url.query": "ugi=niamquis", + "url.scheme": "https", "user.name": "mUteni", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -2608,8 +2812,8 @@ "ura675.mail.localdomain" ], "related.ip": [ - "10.49.242.174", - "10.131.246.134" + "10.131.246.134", + "10.49.242.174" ], "related.user": [ "umdolo" @@ -2623,8 +2827,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tvolup", "rsa.misc.action": [ - "utemvel", - "Allowed" + "Allowed", + "utemvel" ], "rsa.misc.category": "untutlab", "rsa.misc.filter": "dol", @@ -2647,7 +2851,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "stiaecon", "url.original": "https://api.example.com/radipis/cive.gif?orumSec=nisiuta#stiaecon", + "url.path": "/radipis/cive.gif", + "url.query": "orumSec=nisiuta", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -2699,8 +2909,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ende", "rsa.misc.action": [ - "Blocked", - "doconse" + "doconse", + "Blocked" ], "rsa.misc.category": "uovolupt", "rsa.misc.filter": "litesse", @@ -2723,7 +2933,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "eursinto", "url.original": "https://mail.example.org/oin/itseddoe.html?citati=uamei#eursinto", + "url.path": "/oin/itseddoe.html", + "url.query": "citati=uamei", + "url.scheme": "https", "user.name": "olori", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2760,8 +2976,8 @@ "eaque6543.api.domain" ], "related.ip": [ - "10.138.188.201", - "10.128.184.241" + "10.128.184.241", + "10.138.188.201" ], "related.user": [ "etur" @@ -2775,8 +2991,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issu", "rsa.misc.action": [ - "sed", - "Allowed" + "Allowed", + "sed" ], "rsa.misc.category": "atur", "rsa.misc.filter": "iciadese", @@ -2799,7 +3015,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "html", + "url.fragment": "umS", "url.original": "https://mail.example.com/eseruntm/lpaquiof.html?magnaal=uscip#umS", + "url.path": "/eseruntm/lpaquiof.html", + "url.query": "magnaal=uscip", + "url.scheme": "https", "user.name": "etur", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -2851,8 +3073,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ese", "rsa.misc.action": [ - "litanim", - "Allowed" + "Allowed", + "litanim" ], "rsa.misc.category": "idata", "rsa.misc.filter": "urerepre", @@ -2875,7 +3097,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.net", + "url.extension": "html", + "url.fragment": "ine", "url.original": "https://example.net/snulap/enimadm.html?writte=sitvo#ine", + "url.path": "/snulap/enimadm.html", + "url.query": "writte=sitvo", + "url.scheme": "https", "user.name": "isau", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -2951,7 +3179,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "ents", "url.original": "https://internal.example.org/etcon/onsequu.gif?Bonoru=madminim#ents", + "url.path": "/etcon/onsequu.gif", + "url.query": "Bonoru=madminim", + "url.scheme": "https", "user.name": "eiusmo", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -2988,8 +3222,8 @@ "pariatur7238.www5.invalid" ], "related.ip": [ - "10.202.224.79", - "10.33.144.10" + "10.33.144.10", + "10.202.224.79" ], "related.user": [ "rios" @@ -3027,7 +3261,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "seos", "url.original": "https://www.example.org/rur/itse.gif?pisciv=fugiatqu#seos", + "url.path": "/rur/itse.gif", + "url.query": "pisciv=fugiatqu", + "url.scheme": "https", "user.name": "rios", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -3079,8 +3319,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Loremip", "rsa.misc.action": [ - "Allowed", - "quid" + "quid", + "Allowed" ], "rsa.misc.category": "mini", "rsa.misc.filter": "uisnos", @@ -3103,7 +3343,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "liquipex", "url.original": "https://mail.example.com/qui/equeporr.jpg?itsedd=texpli#liquipex", + "url.path": "/qui/equeporr.jpg", + "url.query": "itsedd=texpli", + "url.scheme": "https", "user.name": "CSe", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -3140,8 +3386,8 @@ "mquisnos7453.home" ], "related.ip": [ - "10.134.128.27", - "10.118.177.136" + "10.118.177.136", + "10.134.128.27" ], "related.user": [ "Utenima" @@ -3179,7 +3425,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "eacommo", "url.original": "https://api.example.net/lup/iumtotam.html?ipitlabo=userror#eacommo", + "url.path": "/lup/iumtotam.html", + "url.query": "ipitlabo=userror", + "url.scheme": "https", "user.name": "Utenima", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3231,8 +3483,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "amni", "rsa.misc.action": [ - "Allowed", - "edutp" + "edutp", + "Allowed" ], "rsa.misc.category": "ames", "rsa.misc.filter": "dmi", @@ -3255,7 +3507,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "htm", + "url.fragment": "umdolore", "url.original": "https://example.org/onproide/uamnih.htm?tatisetq=uidolo#umdolore", + "url.path": "/onproide/uamnih.htm", + "url.query": "tatisetq=uidolo", + "url.scheme": "https", "user.name": "reet", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -3331,7 +3589,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "emip", "url.original": "https://www5.example.org/obeataev/umf.htm?moll=quaeabil#emip", + "url.path": "/obeataev/umf.htm", + "url.query": "moll=quaeabil", + "url.scheme": "https", "user.name": "orissus", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -3368,8 +3632,8 @@ "etdolore4227.internal.corp" ], "related.ip": [ - "10.30.87.51", - "10.156.177.53" + "10.156.177.53", + "10.30.87.51" ], "related.user": [ "psaquaea" @@ -3383,8 +3647,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatno", "rsa.misc.action": [ - "ptatev", - "Blocked" + "Blocked", + "ptatev" ], "rsa.misc.category": "udexerc", "rsa.misc.filter": "ptatemse", @@ -3407,7 +3671,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "gif", + "url.fragment": "leumiur", "url.original": "https://mail.example.org/consequa/eaqueip.gif?aevitaed=byCic#leumiur", + "url.path": "/consequa/eaqueip.gif", + "url.query": "aevitaed=byCic", + "url.scheme": "https", "user.name": "psaquaea", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -3459,8 +3729,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tatemse", "rsa.misc.action": [ - "Blocked", - "upta" + "upta", + "Blocked" ], "rsa.misc.category": "tlabo", "rsa.misc.filter": "aliqui", @@ -3483,7 +3753,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "uptasnul", "url.original": "https://example.org/tmo/onofdeF.txt?oremip=its#uptasnul", + "url.path": "/tmo/onofdeF.txt", + "url.query": "oremip=its", + "url.scheme": "https", "user.name": "dentsunt", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -3535,8 +3811,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "itesse", "rsa.misc.action": [ - "Allowed", - "uip" + "uip", + "Allowed" ], "rsa.misc.category": "teturad", "rsa.misc.filter": "roquisqu", @@ -3559,7 +3835,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "htm", + "url.fragment": "smodtem", "url.original": "https://mail.example.com/orsitvol/ntor.htm?itqu=minimav#smodtem", + "url.path": "/orsitvol/ntor.htm", + "url.query": "itqu=minimav", + "url.scheme": "https", "user.name": "taliq", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -3596,8 +3878,8 @@ "laboree3880.api.invalid" ], "related.ip": [ - "10.166.195.20", - "10.255.40.12" + "10.255.40.12", + "10.166.195.20" ], "related.user": [ "lamcolab" @@ -3611,8 +3893,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "mipsumq", "rsa.misc.action": [ - "citation", - "Allowed" + "Allowed", + "citation" ], "rsa.misc.category": "usant", "rsa.misc.filter": "Nem", @@ -3635,7 +3917,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "gna", "url.original": "https://internal.example.org/rumexe/xerci.gif?olor=quiav#gna", + "url.path": "/rumexe/xerci.gif", + "url.query": "olor=quiav", + "url.scheme": "https", "user.name": "lamcolab", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3685,8 +3973,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ento", "rsa.misc.action": [ - "Bonoru", - "Blocked" + "Blocked", + "Bonoru" ], "rsa.misc.category": "luptasnu", "rsa.misc.filter": "quamni", @@ -3709,7 +3997,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "equuntu", "url.original": "https://example.org/tvolu/dutper.html?nbyCicer=scipit#equuntu", + "url.path": "/tvolu/dutper.html", + "url.query": "nbyCicer=scipit", + "url.scheme": "https", "user.name": "ute", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -3746,8 +4040,8 @@ "ine3181.www.invalid" ], "related.ip": [ - "10.119.53.68", - "10.121.9.5" + "10.121.9.5", + "10.119.53.68" ], "related.user": [ "ssec" @@ -3761,8 +4055,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "dexea", "rsa.misc.action": [ - "tinvolup", - "Blocked" + "Blocked", + "tinvolup" ], "rsa.misc.category": "ende", "rsa.misc.filter": "onse", @@ -3785,7 +4079,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "nul", "url.original": "https://www.example.com/uiavo/uisaut.htm?paq=uianon#nul", + "url.path": "/uiavo/uisaut.htm", + "url.query": "paq=uianon", + "url.scheme": "https", "user.name": "ssec", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -3822,8 +4122,8 @@ "tsunt3403.www5.test" ], "related.ip": [ - "10.31.153.177", - "10.237.0.173" + "10.237.0.173", + "10.31.153.177" ], "related.user": [ "sci" @@ -3837,8 +4137,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "eritqui", "rsa.misc.action": [ - "Blocked", - "dolor" + "dolor", + "Blocked" ], "rsa.misc.category": "taspe", "rsa.misc.filter": "oremipsu", @@ -3861,7 +4161,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "upt", "url.original": "https://mail.example.com/uasiarch/Malor.jpg?iinea=snos#upt", + "url.path": "/uasiarch/Malor.jpg", + "url.query": "iinea=snos", + "url.scheme": "https", "user.name": "sci", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -3911,8 +4217,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "epor", "rsa.misc.action": [ - "etquasia", - "Allowed" + "Allowed", + "etquasia" ], "rsa.misc.category": "iaturE", "rsa.misc.filter": "rep", @@ -3935,7 +4241,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "jpg", + "url.fragment": "com", "url.original": "https://api.example.org/ntiumt/sumquia.jpg?lam=asnu#com", + "url.path": "/ntiumt/sumquia.jpg", + "url.query": "lam=asnu", + "url.scheme": "https", "user.name": "duntut", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -3983,8 +4295,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "adipisc", "rsa.misc.action": [ - "exer", - "Blocked" + "Blocked", + "exer" ], "rsa.misc.category": "remagna", "rsa.misc.filter": "emvel", @@ -4007,7 +4319,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "todi", "url.original": "https://example.com/itsedqu/paq.jpg?hilmol=oluptate#todi", + "url.path": "/itsedqu/paq.jpg", + "url.query": "hilmol=oluptate", + "url.scheme": "https", "user.name": "picia", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -4044,8 +4362,8 @@ "tamr1693.api.home" ], "related.ip": [ - "10.53.191.49", - "10.133.102.57" + "10.133.102.57", + "10.53.191.49" ], "related.user": [ "onsec" @@ -4083,7 +4401,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.org", + "url.extension": "html", + "url.fragment": "ctionofd", "url.original": "https://api.example.org/remeum/etur.html?Quisa=quiav#ctionofd", + "url.path": "/remeum/etur.html", + "url.query": "Quisa=quiav", + "url.scheme": "https", "user.name": "onsec", "user_agent.device.name": "Asus X01BDA", "user_agent.name": "Chrome Mobile", @@ -4159,7 +4483,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "nostrum", "url.original": "https://internal.example.org/ree/itten.gif?rsp=imipsa#nostrum", + "url.path": "/ree/itten.gif", + "url.query": "rsp=imipsa", + "url.scheme": "https", "user.name": "tem", "user_agent.device.name": "Samsung SM-A260G", "user_agent.name": "Chrome Mobile WebView", @@ -4235,7 +4565,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.net", + "url.extension": "jpg", + "url.fragment": "onemulla", "url.original": "https://www.example.net/ritquiin/reseo.jpg?ari=umtot#onemulla", + "url.path": "/ritquiin/reseo.jpg", + "url.query": "ari=umtot", + "url.scheme": "https", "user.name": "uasiarch", "user_agent.device.name": "Meizu M6", "user_agent.name": "Chrome Mobile", @@ -4272,8 +4608,8 @@ "pici1525.www5.corp" ], "related.ip": [ - "10.155.252.123", - "10.178.148.188" + "10.178.148.188", + "10.155.252.123" ], "related.user": [ "inrepreh" @@ -4311,7 +4647,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "jpg", + "url.fragment": "eavolupt", "url.original": "https://mail.example.com/dexe/nemul.jpg?yCicero=inimave#eavolupt", + "url.path": "/dexe/nemul.jpg", + "url.query": "yCicero=inimave", + "url.scheme": "https", "user.name": "inrepreh", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -4363,8 +4705,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uamquaer", "rsa.misc.action": [ - "Blocked", - "aerat" + "aerat", + "Blocked" ], "rsa.misc.category": "quela", "rsa.misc.filter": "qui", @@ -4387,7 +4729,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "sistena", "url.original": "https://mail.example.org/caecat/uel.html?enim=umq#sistena", + "url.path": "/caecat/uel.html", + "url.query": "enim=umq", + "url.scheme": "https", "user.name": "olup", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4422,8 +4770,8 @@ "imveni193.www5.host" ], "related.ip": [ - "10.112.190.154", - "10.55.38.153" + "10.55.38.153", + "10.112.190.154" ], "related.user": [ "oremeu" @@ -4437,8 +4785,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tin", "rsa.misc.action": [ - "urau", - "Allowed" + "Allowed", + "urau" ], "rsa.misc.category": "isiut", "rsa.misc.filter": "cons", @@ -4461,7 +4809,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "rna", "url.original": "https://mail.example.com/runtmoll/busBon.txt?ionev=vitaedi#rna", + "url.path": "/runtmoll/busBon.txt", + "url.query": "ionev=vitaedi", + "url.scheme": "https", "user.name": "oremeu", "user_agent.device.name": "XiaoMi Redmi 4X", "user_agent.name": "MiuiBrowser", @@ -4537,7 +4891,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "jpg", + "url.fragment": "quam", "url.original": "https://api.example.com/lits/tvolu.jpg?squir=gnaaliq#quam", + "url.path": "/lits/tvolu.jpg", + "url.query": "squir=gnaaliq", + "url.scheme": "https", "user.name": "tsedquia", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -4574,8 +4934,8 @@ "remips1499.www.local" ], "related.ip": [ - "10.60.52.219", - "10.252.164.230" + "10.252.164.230", + "10.60.52.219" ], "related.user": [ "gnamali" @@ -4613,7 +4973,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "nder", "url.original": "https://mail.example.net/loremi/queporro.jpg?ade=nihilmol#nder", + "url.path": "/loremi/queporro.jpg", + "url.query": "ade=nihilmol", + "url.scheme": "https", "user.name": "gnamali", "user_agent.device.name": "Other", "user_agent.name": "Other", @@ -4646,8 +5012,8 @@ "mdoloree96.domain" ], "related.ip": [ - "10.187.16.73", - "10.122.102.156" + "10.122.102.156", + "10.187.16.73" ], "related.user": [ "emoen" @@ -4685,7 +5051,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "gif", + "url.fragment": "iad", "url.original": "https://api.example.com/nge/psum.gif?exerci=isnostru#iad", + "url.path": "/nge/psum.gif", + "url.query": "exerci=isnostru", + "url.scheme": "https", "user.name": "emoen", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -4722,8 +5094,8 @@ "iatnulap7662.internal.local" ], "related.ip": [ - "10.120.215.174", - "10.248.108.55" + "10.248.108.55", + "10.120.215.174" ], "related.user": [ "prehend" @@ -4761,7 +5133,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "txt", + "url.fragment": "onorumet", "url.original": "https://internal.example.org/ddoeiusm/apa.txt?uptatemU=rem#onorumet", + "url.path": "/ddoeiusm/apa.txt", + "url.query": "uptatemU=rem", + "url.scheme": "https", "user.name": "prehend", "user_agent.device.name": "Generic Smartphone", "user_agent.name": "Opera Mini", @@ -4835,7 +5213,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "htm", + "url.fragment": "quam", "url.original": "https://www5.example.net/yCice/uinesci.htm?taevitae=dminimv#quam", + "url.path": "/yCice/uinesci.htm", + "url.query": "taevitae=dminimv", + "url.scheme": "https", "user.name": "abo", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -4911,7 +5295,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.com", + "url.extension": "txt", + "url.fragment": "nse", "url.original": "https://api.example.com/itinvolu/adeserun.txt?tinv=Utenima#nse", + "url.path": "/itinvolu/adeserun.txt", + "url.query": "tinv=Utenima", + "url.scheme": "https", "user.name": "equamn", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -4963,8 +5353,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "orinrep", "rsa.misc.action": [ - "Blocked", - "squirat" + "squirat", + "Blocked" ], "rsa.misc.category": "sequa", "rsa.misc.filter": "orainci", @@ -4987,7 +5377,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "loreeuf", "url.original": "https://www.example.org/sci/isquames.gif?tlabor=itecto#loreeuf", + "url.path": "/sci/isquames.gif", + "url.query": "tlabor=itecto", + "url.scheme": "https", "user.name": "evelite", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5024,8 +5420,8 @@ "beataevi7552.api.test" ], "related.ip": [ - "10.215.63.248", - "10.138.0.214" + "10.138.0.214", + "10.215.63.248" ], "related.user": [ "eavolupt" @@ -5039,8 +5435,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "odita", "rsa.misc.action": [ - "Blocked", - "dqu" + "dqu", + "Blocked" ], "rsa.misc.category": "ipex", "rsa.misc.filter": "ine", @@ -5063,7 +5459,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "usmod", "url.original": "https://mail.example.org/umdolo/nimv.htm?equunt=tutla#usmod", + "url.path": "/umdolo/nimv.htm", + "url.query": "equunt=tutla", + "url.scheme": "https", "user.name": "eavolupt", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", @@ -5100,8 +5502,8 @@ "rvelill1981.www.invalid" ], "related.ip": [ - "10.26.115.88", - "10.12.130.224" + "10.12.130.224", + "10.26.115.88" ], "related.user": [ "Nequepo" @@ -5139,7 +5541,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "htm", + "url.fragment": "byCice", "url.original": "https://mail.example.net/tvol/ostru.htm?oei=iquipex#byCice", + "url.path": "/tvol/ostru.htm", + "url.query": "oei=iquipex", + "url.scheme": "https", "user.name": "Nequepo", "user_agent.device.name": "STK-L21", "user_agent.name": "Chrome Mobile", @@ -5176,8 +5584,8 @@ "quia7214.example" ], "related.ip": [ - "10.193.152.42", - "10.91.20.27" + "10.91.20.27", + "10.193.152.42" ], "related.user": [ "edict" @@ -5215,7 +5623,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "html", + "url.fragment": "atemacc", "url.original": "https://mail.example.org/pariatur/cita.html?equuntur=rve#atemacc", + "url.path": "/pariatur/cita.html", + "url.query": "equuntur=rve", + "url.scheme": "https", "user.name": "edict", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -5252,8 +5666,8 @@ "aturExc7343.invalid" ], "related.ip": [ - "10.146.69.38", - "10.55.192.102" + "10.55.192.102", + "10.146.69.38" ], "related.user": [ "quia" @@ -5291,7 +5705,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "gif", + "url.fragment": "elillumq", "url.original": "https://example.org/aturE/aaliqu.gif?nvol=doloreeu#elillumq", + "url.path": "/aturE/aaliqu.gif", + "url.query": "nvol=doloreeu", + "url.scheme": "https", "user.name": "quia", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -5367,7 +5787,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.org", + "url.extension": "gif", + "url.fragment": "dese", "url.original": "https://internal.example.org/olorin/orisnisi.gif?eritquii=atevelit#dese", + "url.path": "/olorin/orisnisi.gif", + "url.query": "eritquii=atevelit", + "url.scheme": "https", "user.name": "isciveli", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -5443,7 +5869,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "html", + "url.fragment": "deo", "url.original": "https://example.org/vel/preh.html?sequamni=edutpers#deo", + "url.path": "/vel/preh.html", + "url.query": "sequamni=edutpers", + "url.scheme": "https", "user.name": "estla", "user_agent.device.name": "iPhone", "user_agent.name": "Facebook", @@ -5480,8 +5912,8 @@ "agna5654.www.corp" ], "related.ip": [ - "10.200.74.101", - "10.203.47.23" + "10.203.47.23", + "10.200.74.101" ], "related.user": [ "litesse" @@ -5495,8 +5927,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "nde", "rsa.misc.action": [ - "Allowed", - "iqu" + "iqu", + "Allowed" ], "rsa.misc.category": "ametco", "rsa.misc.filter": "ntincul", @@ -5519,7 +5951,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "jpg", + "url.fragment": "aUtenim", "url.original": "https://example.com/nonproi/dolor.jpg?molli=oeiusm#aUtenim", + "url.path": "/nonproi/dolor.jpg", + "url.query": "molli=oeiusm", + "url.scheme": "https", "user.name": "litesse", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5571,8 +6009,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "ereprehe", "rsa.misc.action": [ - "Blocked", - "tutl" + "tutl", + "Blocked" ], "rsa.misc.category": "mip", "rsa.misc.filter": "umSecti", @@ -5595,7 +6033,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.com", + "url.extension": "gif", + "url.fragment": "uepor", "url.original": "https://example.com/sedqui/iuntNe.gif?epteu=nvent#uepor", + "url.path": "/sedqui/iuntNe.gif", + "url.query": "epteu=nvent", + "url.scheme": "https", "user.name": "ntore", "user_agent.device.name": "U307AS", "user_agent.name": "Chrome Mobile", @@ -5647,8 +6091,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "diconseq", "rsa.misc.action": [ - "umet", - "Allowed" + "Allowed", + "umet" ], "rsa.misc.category": "ciad", "rsa.misc.filter": "oeiusmod", @@ -5671,7 +6115,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "txt", + "url.fragment": "imidest", "url.original": "https://www5.example.net/lits/Nemoen.txt?elillu=seruntmo#imidest", + "url.path": "/lits/Nemoen.txt", + "url.query": "elillu=seruntmo", + "url.scheme": "https", "user.name": "squir", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5723,8 +6173,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "iamquisn", "rsa.misc.action": [ - "lupta", - "Blocked" + "Blocked", + "lupta" ], "rsa.misc.category": "uasiarch", "rsa.misc.filter": "usBonor", @@ -5747,7 +6197,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "jpg", + "url.fragment": "ipis", "url.original": "https://example.org/eius/evo.jpg?iarchit=volupt#ipis", + "url.path": "/eius/evo.jpg", + "url.query": "iarchit=volupt", + "url.scheme": "https", "user.name": "mes", "user_agent.device.name": "G8142", "user_agent.name": "Chrome Mobile", @@ -5784,8 +6240,8 @@ "tiumtot3611.internal.localdomain" ], "related.ip": [ - "10.107.68.114", - "10.84.9.150" + "10.84.9.150", + "10.107.68.114" ], "related.user": [ "sequatDu" @@ -5823,7 +6279,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.net", + "url.extension": "gif", + "url.fragment": "uiineavo", "url.original": "https://www5.example.net/equun/veli.gif?tem=iadeseru#uiineavo", + "url.path": "/equun/veli.gif", + "url.query": "tem=iadeseru", + "url.scheme": "https", "user.name": "sequatDu", "user_agent.device.name": "LG-$2", "user_agent.name": "Chrome Mobile", @@ -5899,7 +6361,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "internal.example.com", + "url.extension": "txt", + "url.fragment": "rese", "url.original": "https://internal.example.com/ecatcu/tMalo.txt?nse=rauto#rese", + "url.path": "/ecatcu/tMalo.txt", + "url.query": "nse=rauto", + "url.scheme": "https", "user.name": "nre", "user_agent.device.name": "Samsung SM-A305FN", "user_agent.name": "YandexSearch", @@ -5936,8 +6404,8 @@ "psaqu6066.www5.localhost" ], "related.ip": [ - "10.223.11.164", - "10.164.190.2" + "10.164.190.2", + "10.223.11.164" ], "related.user": [ "ten" @@ -5951,8 +6419,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "officiad", "rsa.misc.action": [ - "Allowed", - "antium" + "antium", + "Allowed" ], "rsa.misc.category": "emoeni", "rsa.misc.filter": "itvo", @@ -5975,7 +6443,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "tionevol", "url.original": "https://mail.example.org/ntutlabo/leumiure.htm?eacommo=amqua#tionevol", + "url.path": "/ntutlabo/leumiure.htm", + "url.query": "eacommo=amqua", + "url.scheme": "https", "user.name": "ten", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6012,8 +6486,8 @@ "iavol5202.api.example" ], "related.ip": [ - "10.14.37.8", - "10.121.181.243" + "10.121.181.243", + "10.14.37.8" ], "related.user": [ "umwr" @@ -6027,8 +6501,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "vitaedic", "rsa.misc.action": [ - "Blocked", - "rinc" + "rinc", + "Blocked" ], "rsa.misc.category": "prehende", "rsa.misc.filter": "rume", @@ -6051,7 +6525,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "jpg", + "url.fragment": "mquis", "url.original": "https://www.example.org/ugitsed/ritatis.jpg?xplic=stenat#mquis", + "url.path": "/ugitsed/ritatis.jpg", + "url.query": "xplic=stenat", + "url.scheme": "https", "user.name": "umwr", "user_agent.device.name": "Lenovo A2016a40 ", "user_agent.name": "Chrome Mobile", @@ -6088,8 +6568,8 @@ "uame1361.api.local" ], "related.ip": [ - "10.10.93.133", - "10.90.20.202" + "10.90.20.202", + "10.10.93.133" ], "related.user": [ "evita" @@ -6127,7 +6607,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "gif", + "url.fragment": "cidu", "url.original": "https://mail.example.com/aute/dictasu.gif?ptas=iadolo#cidu", + "url.path": "/aute/dictasu.gif", + "url.query": "ptas=iadolo", + "url.scheme": "https", "user.name": "evita", "user_agent.device.name": "ZTE Blade V1000RU", "user_agent.name": "Chrome Mobile", @@ -6164,8 +6650,8 @@ "rsitame4049.internal.corp" ], "related.ip": [ - "10.77.102.206", - "10.34.98.144" + "10.34.98.144", + "10.77.102.206" ], "related.user": [ "tectobe" @@ -6179,8 +6665,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "Exce", "rsa.misc.action": [ - "ulapa", - "Allowed" + "Allowed", + "ulapa" ], "rsa.misc.category": "reprehen", "rsa.misc.filter": "itsedqui", @@ -6203,7 +6689,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "psamvolu", "url.original": "https://mail.example.net/enbyCic/aturau.gif?orroqui=sci#psamvolu", + "url.path": "/enbyCic/aturau.gif", + "url.query": "orroqui=sci", + "url.scheme": "https", "user.name": "tectobe", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -6240,8 +6732,8 @@ "elit912.www5.test" ], "related.ip": [ - "10.176.233.249", - "10.75.144.118" + "10.75.144.118", + "10.176.233.249" ], "related.user": [ "isnos" @@ -6279,7 +6771,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "example.org", + "url.extension": "txt", + "url.fragment": "aeratvol", "url.original": "https://example.org/olu/mqua.txt?mdolore=ita#aeratvol", + "url.path": "/olu/mqua.txt", + "url.query": "mdolore=ita", + "url.scheme": "https", "user.name": "isnos", "user_agent.device.name": "VS996", "user_agent.name": "Chrome Mobile", @@ -6331,8 +6829,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "uis", "rsa.misc.action": [ - "mvele", - "Allowed" + "Allowed", + "mvele" ], "rsa.misc.category": "vitaedi", "rsa.misc.filter": "ndeomni", @@ -6355,7 +6853,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "jpg", + "url.fragment": "ccusa", "url.original": "https://api.example.net/mnisiut/eabil.jpg?psumqui=trude#ccusa", + "url.path": "/mnisiut/eabil.jpg", + "url.query": "psumqui=trude", + "url.scheme": "https", "user.name": "redolo", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6392,8 +6896,8 @@ "uis5050.www.local" ], "related.ip": [ - "10.97.202.149", - "10.13.125.101" + "10.13.125.101", + "10.97.202.149" ], "related.user": [ "colab" @@ -6407,8 +6911,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "atcupi", "rsa.misc.action": [ - "Blocked", - "uaUten" + "uaUten", + "Blocked" ], "rsa.misc.category": "modt", "rsa.misc.filter": "magnidol", @@ -6431,7 +6935,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "ntNeque", "url.original": "https://api.example.net/uamestq/eetdol.html?ctionofd=uianonnu#ntNeque", + "url.path": "/uamestq/eetdol.html", + "url.query": "ctionofd=uianonnu", + "url.scheme": "https", "user.name": "colab", "user_agent.device.name": "Micromax P410i", "user_agent.name": "Chrome Mobile", @@ -6468,8 +6978,8 @@ "ficiad1312.api.host" ], "related.ip": [ - "10.141.66.163", - "10.230.61.102" + "10.230.61.102", + "10.141.66.163" ], "related.user": [ "umdolo" @@ -6507,7 +7017,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "jpg", + "url.fragment": "gnido", "url.original": "https://mail.example.net/ius/msequ.jpg?ptat=tionula#gnido", + "url.path": "/ius/msequ.jpg", + "url.query": "ptat=tionula", + "url.scheme": "https", "user.name": "umdolo", "user_agent.device.name": "ZTE BLADE V7", "user_agent.name": "Chrome Mobile", @@ -6544,8 +7060,8 @@ "itaspe921.mail.invalid" ], "related.ip": [ - "10.10.25.145", - "10.224.249.228" + "10.224.249.228", + "10.10.25.145" ], "related.user": [ "mnisiuta" @@ -6559,8 +7075,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "issuscip", "rsa.misc.action": [ - "remap", - "Blocked" + "Blocked", + "remap" ], "rsa.misc.category": "eetdolo", "rsa.misc.filter": "rsitam", @@ -6583,7 +7099,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "html", + "url.fragment": "umqu", "url.original": "https://www.example.org/iat/acom.html?umdolo=oluptass#umqu", + "url.path": "/iat/acom.html", + "url.query": "umdolo=oluptass", + "url.scheme": "https", "user.name": "mnisiuta", "user_agent.device.name": "LM-V350", "user_agent.name": "Chrome Mobile", @@ -6659,7 +7181,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "trumexer", "url.original": "https://www.example.com/onorum/umiure.gif?lites=admini#trumexer", + "url.path": "/onorum/umiure.gif", + "url.query": "lites=admini", + "url.scheme": "https", "user.name": "aeabillo", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -6735,7 +7263,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "htm", + "url.fragment": "dquiac", "url.original": "https://mail.example.org/veni/rspi.htm?ntium=imadmi#dquiac", + "url.path": "/veni/rspi.htm", + "url.query": "ntium=imadmi", + "url.scheme": "https", "user.name": "tNequ", "user_agent.device.name": "Pixel 3", "user_agent.name": "Chrome Mobile", @@ -6772,8 +7306,8 @@ "Bonoru7444.www5.example" ], "related.ip": [ - "10.166.205.159", - "10.154.188.132" + "10.154.188.132", + "10.166.205.159" ], "related.user": [ "uptat" @@ -6787,8 +7321,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "proid", "rsa.misc.action": [ - "onevolu", - "Allowed" + "Allowed", + "onevolu" ], "rsa.misc.category": "iratio", "rsa.misc.filter": "odita", @@ -6811,7 +7345,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "beat", "url.original": "https://www.example.com/tem/litsedq.htm?ium=utfugit#beat", + "url.path": "/tem/litsedq.htm", + "url.query": "ium=utfugit", + "url.scheme": "https", "user.name": "uptat", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6883,7 +7423,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "gif", + "url.fragment": "laborisn", "url.original": "https://www.example.com/amcola/eumiurer.gif?stiaeco=equu#laborisn", + "url.path": "/amcola/eumiurer.gif", + "url.query": "stiaeco=equu", + "url.scheme": "https", "user.name": "sintocca", "user_agent.device.name": "Spider", "user_agent.name": "Other", @@ -6916,8 +7462,8 @@ "oloremeu5047.www5.invalid" ], "related.ip": [ - "10.254.119.31", - "10.172.159.251" + "10.172.159.251", + "10.254.119.31" ], "related.user": [ "usm" @@ -6955,7 +7501,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "api.example.net", + "url.extension": "html", + "url.fragment": "veleum", "url.original": "https://api.example.net/sedquian/lamcorpo.html?sequatD=Nequepo#veleum", + "url.path": "/sedquian/lamcorpo.html", + "url.query": "sequatD=Nequepo", + "url.scheme": "https", "user.name": "usm", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -6992,8 +7544,8 @@ "edutpe1255.internal.lan" ], "related.ip": [ - "10.195.62.230", - "10.98.126.206" + "10.98.126.206", + "10.195.62.230" ], "related.user": [ "ptassit" @@ -7031,7 +7583,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.com", + "url.extension": "txt", + "url.fragment": "uisnostr", "url.original": "https://www5.example.com/ictasun/iumto.txt?erro=admin#uisnostr", + "url.path": "/ictasun/iumto.txt", + "url.query": "erro=admin", + "url.scheme": "https", "user.name": "ptassit", "user_agent.device.name": "Samsung SM-A715F", "user_agent.name": "Facebook", @@ -7107,7 +7665,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "htm", + "url.fragment": "isi", "url.original": "https://www5.example.org/oriosa/ssusc.htm?atemacc=rsitvolu#isi", + "url.path": "/oriosa/ssusc.htm", + "url.query": "atemacc=rsitvolu", + "url.scheme": "https", "user.name": "eroi", "user_agent.device.name": "Mac", "user_agent.name": "Yandex Browser", @@ -7159,8 +7723,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "tquovo", "rsa.misc.action": [ - "qua", - "Allowed" + "Allowed", + "qua" ], "rsa.misc.category": "ectet", "rsa.misc.filter": "lites", @@ -7183,7 +7747,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.net", + "url.extension": "gif", + "url.fragment": "ame", "url.original": "https://mail.example.net/tseddoei/byCi.gif?assitas=nul#ame", + "url.path": "/tseddoei/byCi.gif", + "url.query": "assitas=nul", + "url.scheme": "https", "user.name": "unt", "user_agent.device.name": "Android", "user_agent.name": "Chrome Mobile", @@ -7235,8 +7805,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "rrorsi", "rsa.misc.action": [ - "exe", - "Allowed" + "Allowed", + "exe" ], "rsa.misc.category": "mnihi", "rsa.misc.filter": "consequa", @@ -7259,7 +7829,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www5.example.org", + "url.extension": "gif", + "url.fragment": "suntin", "url.original": "https://www5.example.org/liquipe/rehe.gif?niamqu=uioffi#suntin", + "url.path": "/liquipe/rehe.gif", + "url.query": "niamqu=uioffi", + "url.scheme": "https", "user.name": "hende", "user_agent.device.name": "Samsung GT-P3100 ", "user_agent.name": "Android", @@ -7296,8 +7872,8 @@ "ueip6097.api.host" ], "related.ip": [ - "10.152.217.174", - "10.128.43.71" + "10.128.43.71", + "10.152.217.174" ], "related.user": [ "mquiado" @@ -7311,8 +7887,8 @@ "rsa.investigations.ec_theme": "Communication", "rsa.investigations.event_vcat": "olupt", "rsa.misc.action": [ - "Blocked", - "temvele" + "temvele", + "Blocked" ], "rsa.misc.category": "natuser", "rsa.misc.filter": "amnihil", @@ -7335,7 +7911,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.org", + "url.extension": "gif", + "url.fragment": "siuta", "url.original": "https://www.example.org/erit/asiarch.gif?tdolor=oremagna#siuta", + "url.path": "/erit/asiarch.gif", + "url.query": "tdolor=oremagna", + "url.scheme": "https", "user.name": "mquiado", "user_agent.device.name": "Notepad_K10", "user_agent.name": "Chrome", @@ -7411,7 +7993,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.org", + "url.extension": "jpg", + "url.fragment": "ainci", "url.original": "https://mail.example.org/maven/tectob.jpg?litsedd=mnis#ainci", + "url.path": "/maven/tectob.jpg", + "url.query": "litsedd=mnis", + "url.scheme": "https", "user.name": "uisa", "user_agent.device.name": "QMobile X700 PRO II", "user_agent.name": "Chrome Mobile", @@ -7448,8 +8036,8 @@ "onsequ3168.www.corp" ], "related.ip": [ - "10.109.192.53", - "10.172.17.6" + "10.172.17.6", + "10.109.192.53" ], "related.user": [ "eprehen" @@ -7487,7 +8075,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "www.example.com", + "url.extension": "htm", + "url.fragment": "tNe", "url.original": "https://www.example.com/siarch/oloremi.htm?one=iduntutl#tNe", + "url.path": "/siarch/oloremi.htm", + "url.query": "one=iduntutl", + "url.scheme": "https", "user.name": "eprehen", "user_agent.device.name": "U20", "user_agent.name": "Chrome Mobile", @@ -7524,8 +8118,8 @@ "oremquel3120.internal.localhost" ], "related.ip": [ - "10.135.38.213", - "10.119.106.108" + "10.119.106.108", + "10.135.38.213" ], "related.user": [ "ore" @@ -7563,7 +8157,13 @@ "zscaler.zia", "forwarded" ], + "url.domain": "mail.example.com", + "url.extension": "txt", + "url.fragment": "umtota", "url.original": "https://mail.example.com/ostr/liqu.txt?niam=mullamc#umtota", + "url.path": "/ostr/liqu.txt", + "url.query": "niam=mullamc", + "url.scheme": "https", "user.name": "ore", "user_agent.device.name": "5024D_RU", "user_agent.name": "Chrome Mobile", From 362d485834c593d7aabb6c04644b086132d6a52d Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 07:58:10 +0200 Subject: [PATCH 02/11] [Elastic Agent] Fix picking correct policy id for fleet-server (#25352) When running the container command, the policy id configured in the fleet-setup.yml file was only picked up, when fleet-server was not enabled. If a fleet-server policy is set, this will be set as the default. Later on this setup call should be improved that not both setups are possible. Currently it could be that the fleet enroll and fleet server setup policy are not identical. To partially test this PR, I did run the elastic agent locally as following: ``` sudo STATE_PATH=${GOPATH}/src/github.com/elastic/beats/x-pack/elastic-agent/state ./elastic-agent container ``` Adjust the path to your needs. In addition, I put the currently checked in fleet-setup.yml file into state directory. In parallel, an elastic-stack setup was running. --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 3 ++- x-pack/elastic-agent/pkg/agent/cmd/container.go | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index c3dc8b38ce2..403f0a106c6 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -55,7 +55,8 @@ - Fixed: limit for retries to Kibana configurable {issue}25063[25063] - Fix issue with status and inspect inside of container {pull}25204[25204] - Remove FLEET_SERVER_POLICY_NAME env variable as it was not used {pull}25149[25149] -- Reduce log level for listener cleanup to debug {pull}25274 +- Passing in policy id to container command works {pull}25352[25352] +- Reduce log level for listener cleanup to debug {pull}25274[25274] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index 35437e68376..a370c4f3106 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -139,8 +139,8 @@ func logError(streams *cli.IOStreams, err error) { fmt.Fprintf(streams.Err, "Error: %v\n", err) } -func logInfo(streams *cli.IOStreams, msg string) { - fmt.Fprintln(streams.Out, msg) +func logInfo(streams *cli.IOStreams, a ...interface{}) { + fmt.Fprintln(streams.Out, a...) } func logContainerCmd(streams *cli.IOStreams, cmd *cobra.Command) error { @@ -292,10 +292,11 @@ func runContainerCmd(streams *cli.IOStreams, cmd *cobra.Command, cfg setupConfig return err } } - policyID := "" + policyID := cfg.FleetServer.PolicyID if policy != nil { policyID = policy.ID } + logInfo(streams, "Policy selected for enrollment: ", policyID) cmdArgs, err := buildEnrollArgs(cfg, token, policyID) if err != nil { return err From 7a0b14e326e14f3121cc8f50c58700abf1255e33 Mon Sep 17 00:00:00 2001 From: Andres Rodriguez Date: Wed, 28 Apr 2021 09:40:48 +0200 Subject: [PATCH 03/11] Forward-port 7.12.1 changelog to master (#25328) * Forward-port 7.12.1 changelog to 7.x (#25327) * docs: Prepare Changelog for 7.12.1 (#25325) * docs: Close changelog for 7.12.1 * Cleanup * Cleanup Co-authored-by: Andres Rodriguez (cherry picked from commit e28c59c469ec910764dc5bd826fbc5c4a18fb7f1) * Missing removals * Missing removals * Missing removals * Missing removals Co-authored-by: Elastic Machine (cherry picked from commit 1341cff15c57e2113ab197aa82e05f0a04918da4) * Cleanup * Cleanup * Cleanup * Cleanup --- CHANGELOG.asciidoc | 55 +++++++++++++++++++++++++++++++++++ CHANGELOG.next.asciidoc | 27 +---------------- libbeat/docs/release.asciidoc | 1 + 3 files changed, 57 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 269a69546ca..55a0594e80b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,61 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-7.12.1]] +=== Beats version 7.12.1 +https://github.com/elastic/beats/compare/v7.12.0...v7.12.1[View commits] + +==== Breaking changes + +*Filebeat* + +- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] + +==== Bugfixes + +*Affecting all Beats* + +- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332] +- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223] +- Fix inode removal tracking code when files are replaced by files with the same name {pull}25002[25002] +- Fix `mage GenerateCustomBeat` instructions for a new beat {pull}17679[17679] +- Fix bug with annotations dedot config on k8s not used {pull}25111[25111] +- Fix negative Kafka partition bug {pull}25048[25048] + +*Filebeat* + +- Properly update offset in case of unparasable line. {pull}22685[22685] +- Fix Cisco ASA parser for message 722051. {pull}24410[24410] +- Fix `google_workspace` pagination. {pull}24668[24668] +- Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270] +- Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697] +- Fix Cisco AMP `@metadata._id` calculation {issue}24717[24717] {pull}24718[24718] +- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694] +- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719] +- Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] +- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] +- Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967] +- Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038] + +*Heartbeat* + +- Fix panic when initialization of ICMP monitors fail twice. {pull}25073[25073] + +*Metricbeat* + +- Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502] + + +==== Added + +*Filebeat* + +- Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661] +- Add support for upper case field names in Sophos XG module {pull}24693[24693] +- Add `fail_on_template_error` option for httpjson input. {pull}24784[24784] + + + [[release-notes-7.12.0]] === Beats version 7.12.0 https://github.com/elastic/beats/compare/v7.11.2...v7.12.0[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c8b400a05b5..291b6379129 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -104,7 +104,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571] - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975] - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] -- Possible values for Netflow's locality fields (source.locality, destination.locality and flow.locality) are now `internal` and `external`, instead of `private` and `public`. {issue}24272[24272] {pull}24295[24295] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] - All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699] @@ -166,8 +165,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Affecting all Beats* - Fix events being dropped if they contain a floating point value of NaN or Inf. {pull}25051[25051] -- Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332] -- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223] - Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808] - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] - Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146] @@ -175,7 +172,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945] - Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338] - Update replicaset group to apps/v1 {pull}15854[15802] -- Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223] - Fix missing output in dockerlogbeat {pull}15719[15719] - Do not load dashboards where not available. {pull}15802[15802] - Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516[15516] @@ -248,10 +244,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `expand_keys` to the list of permitted config fields for `decode_json_fields` {24862}[24862] - Fix 'make setup' instructions for a new beat {pull}24944[24944] - Fix discovery of short-living and failing pods in Kubernetes autodiscover {issue}22718[22718] {pull}24742[24742] -- Fix inode removal tracking code when files are replaced by files with the same name {pull}25002[25002] -- Fix `mage GenerateCustomBeat` instructions for a new beat {pull}17679[17679] -- Fix bug with annotations dedot config on k8s not used {pull}25111[25111] -- Fix negative Kafka partition bug {pull}25048[25048] - Fix panic when overwriting metadata {pull}24741[24741] *Auditbeat* @@ -289,7 +281,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix a connection error in httpjson input. {pull}16123[16123] - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523] - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277] -- Strip Azure Eventhub connection string in debug logs. {pulll}25066[25066] +- Strip Azure Eventhub connection string in debug logs. {pull}25066[25066] - Fix o365 module config when client_secret contains special characters. {issue}25058[25058] *Filebeat* @@ -377,7 +369,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix handing missing eventtime and assignip field being set to N/A for fortinet module. {pull}22361[22361] - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696] - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377] -- Properly update offset in case of unparasable line. {pull}22685[22685] - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716] - Fix cisco umbrella module config by adding input variable. {pull}22892[22892] - Fix network.direction logic in zeek connection fileset. {pull}22967[22967] @@ -390,22 +381,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273] - Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534] - Fix handling of ModifiedProperties field in Office 365. {pull}23777[23777] -- Fix netflow module ignoring detect_sequence_reset flag. {issue}24268[24268] {pull}24270[24270] -- Fix Cisco ASA parser for message 722051. {pull}24410[24410] -- Fix `google_workspace` pagination. {pull}24668[24668] -- Fix Cisco ASA parser for message 302022. {issue}24405[24405] {pull}24697[24697] -- Fix Cisco AMP `@metadata._id` calculation {issue}24717[24717] {pull}24718[24718] - Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719] -- Fix date parsing in GSuite/login and Google Workspace/login filesets. {issue}24694[24694] -- Fix date parsing in GSuite/login fileset. {issue}24694[24694] - Improve Cisco ASA/FTD parsing of messages - better support for identity FW messages. Change network.bytes, source.bytes, and destination.bytes to long from integer since value can exceed integer capacity. Add descriptions for various processors for easier pipeline editing in Kibana UI. {pull}23766[23766] - Fix usage of unallowed ECS event.outcome values in Cisco ASA/FTD pipeline. {pull}24744[24744]. - Updating Oauth2 flow for m365_defender fileset. {pull}24829[24829] -- Improve PanOS parsing and ingest pipeline. {issue}22413[22413] {issue}22748[22748] {pull}24799[24799] -- Fix S3 input validation for non amazonaws.com domains. {issue}24420[24420] {pull}24861[24861] -- Fix google_workspace and okta modules pagination when next page template is empty. {pull}24967[24967] - Fix IPtables Pipeline and Ubiquiti dashboard. {issue}24878[24878] {pull}24928[24928] -- Fix gcp module field names to use gcp instead of googlecloud. {pull}25038[25038] - Change `checkpoint.source_object` from Long to Keyword. {issue}25124[25124] {pull}25145[25145] - Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699] @@ -413,7 +393,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639] - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549] -- Fix panic when initialization of ICMP monitors fail twice. {pull}25073[25073] *Journalbeat* @@ -525,7 +504,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505] - Fix GCP not able to request Cloudfunctions metrics if a region filter was set {pull}24218[24218] - Fix type of `uwsgi.status.worker.rss` type. {pull}24468[24468] -- Ignore unsupported derive types for filesystem metricset. {issue}22501[22501] {pull}24502[24502] - Accept text/plain type by default for prometheus client scraping. {pull}24622[24622] *Packetbeat* @@ -841,12 +819,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added `alternative_host` option to google pubsub input {pull}23215[23215] - Support X-Forwarder-For in IIS logs. {pull}19142[192142] - Add support for logs generated by servers configured with `log_statement` and `log_duration` in PostgreSQL module. {pull}24607[24607] -- Updating field mappings for Cisco AMP module, fixing certain fields. {pull}24661[24661] - Added fifteen new message IDs to Cisco ASA/FTD pipeline. {pull}24744[24744] - Added NTP fileset to Zeek module {pull}24224[24224] - Add `proxy_url` config for httpjson v2 input. {issue}24615[24615] {pull}24662[24662] -- Add support for upper case field names in Sophos XG module {pull}24693[24693] -- Add `fail_on_template_error` option for httpjson input. {pull}24784[24784] - Change `okta.target` to `flattened` field type. {issue}24354[24354] {pull}24636[24636] - Added `http.request.id` to `nginx/ingress_controller` and `elasticsearch/audit`. {pull}24994[24994] - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index a53bf859bc3..16c5f75d2bf 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From 4c9fd79f7d7d195b3294309cd26c2fb1b850f739 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 28 Apr 2021 12:13:09 +0200 Subject: [PATCH 04/11] Cyberarkpas: Map interesting fields out of flattened field. (#25315) This updates the mappings on the Cyberarkpas module so that the `extra_details` and `ca_properties` objects contain known fields. All other fields are mapped to the `other` subobject as a flattened field. --- filebeat/docs/fields.asciidoc | 324 +++++++++++++++++- .../module/cyberarkpas/audit/_meta/fields.yml | 98 +++++- .../cyberarkpas/audit/ingest/pipeline.yml | 66 ++++ ...130_cpm_disable_password.log-expected.json | 4 +- .../22_cpm_verify_password.log-expected.json | 4 +- .../24_cpm_change_password.log-expected.json | 8 +- .../test/294_store_password.log-expected.json | 12 +- .../295_retrieve_password.log-expected.json | 8 +- .../test/300_psm_connect.log-expected.json | 40 +-- .../test/302_psm_disconnect.log-expected.json | 36 +- ...304_psm_upload_recording.log-expected.json | 2 +- .../test/308_use_password.log-expected.json | 4 +- ...1_cpm_reconcile_password.log-expected.json | 2 +- .../test/359_sql_command.log-expected.json | 80 ++--- .../361_keystroke_logging.log-expected.json | 46 +-- ...m_verify_password_failed.log-expected.json | 50 +-- .../test/411_window_title.log-expected.json | 6 +- .../412_keystroke_logging.log-expected.json | 8 +- .../414_cpm_verify_ssh_key.log-expected.json | 8 +- .../test/459_general_audit.log-expected.json | 12 +- .../test/52_delete_file.log-expected.json | 10 +- ...m_change_password_failed.log-expected.json | 10 +- ...econcile_password_failed.log-expected.json | 32 +- x-pack/filebeat/module/cyberarkpas/fields.go | 2 +- 24 files changed, 674 insertions(+), 198 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index c2c06e39ecd..31d8b4491d1 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -34224,11 +34224,211 @@ type: keyword -- -*`cyberarkpas.audit.ca_properties`*:: +[float] +=== ca_properties + +Account metadata. + + +*`cyberarkpas.audit.ca_properties.address`*:: + -- -Account metadata. +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.cpm_disabled`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.cpm_error_details`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.cpm_status`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.creation_method`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.customer`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.database`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.device_type`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.dual_account_status`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.group_name`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.in_process`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.index`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.last_fail_date`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.last_success_change`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.last_success_reconciliation`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.last_success_verification`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.last_task`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.logon_domain`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.policy_id`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.port`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.privcloud`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.reset_immediately`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.retries_count`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.sequence_id`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.tags`*:: ++ +-- +type: keyword +-- + +*`cyberarkpas.audit.ca_properties.user_dn`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.user_name`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.virtual_username`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.ca_properties.other`*:: ++ +-- type: flattened -- @@ -34251,11 +34451,127 @@ type: keyword -- -*`cyberarkpas.audit.extra_details`*:: +[float] +=== extra_details + +Specific extra details of the audit records. + + +*`cyberarkpas.audit.extra_details.ad_process_id`*:: + -- -Specific extra details of the audit records. +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.ad_process_name`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.application_type`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.command`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.connection_component_id`*:: ++ +-- +type: keyword + +-- +*`cyberarkpas.audit.extra_details.dst_host`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.logon_account`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.managed_account`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.process_id`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.process_name`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.protocol`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.psmid`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.session_duration`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.session_id`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.src_host`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.username`*:: ++ +-- +type: keyword + +-- + +*`cyberarkpas.audit.extra_details.other`*:: ++ +-- type: flattened -- diff --git a/x-pack/filebeat/module/cyberarkpas/audit/_meta/fields.yml b/x-pack/filebeat/module/cyberarkpas/audit/_meta/fields.yml index 9dcb53669fd..14f2c9d172d 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/_meta/fields.yml +++ b/x-pack/filebeat/module/cyberarkpas/audit/_meta/fields.yml @@ -8,8 +8,67 @@ type: keyword description: A description of the audit record. - name: ca_properties - type: flattened + type: group description: Account metadata. + fields: + - name: address + type: keyword + - name: cpm_disabled + type: keyword + - name: cpm_error_details + type: keyword + - name: cpm_status + type: keyword + - name: creation_method + type: keyword + - name: customer + type: keyword + - name: database + type: keyword + - name: device_type + type: keyword + - name: dual_account_status + type: keyword + - name: group_name + type: keyword + - name: in_process + type: keyword + - name: index + type: keyword + - name: last_fail_date + type: keyword + - name: last_success_change + type: keyword + - name: last_success_reconciliation + type: keyword + - name: last_success_verification + type: keyword + - name: last_task + type: keyword + - name: logon_domain + type: keyword + - name: policy_id + type: keyword + - name: port + type: keyword + - name: privcloud + type: keyword + - name: reset_immediately + type: keyword + - name: retries_count + type: keyword + - name: sequence_id + type: keyword + - name: tags + type: keyword + - name: user_dn + type: keyword + - name: user_name + type: keyword + - name: virtual_username + type: keyword + - name: other + type: flattened - name: category type: keyword description: The category name (for category-related operations). @@ -17,8 +76,43 @@ type: keyword description: A static value that displays a description of the audit codes. - name: extra_details - type: flattened + type: group description: Specific extra details of the audit records. + fields: + - name: ad_process_id + type: keyword + - name: ad_process_name + type: keyword + - name: application_type + type: keyword + - name: command + type: keyword + - name: connection_component_id + type: keyword + - name: dst_host + type: keyword + - name: logon_account + type: keyword + - name: managed_account + type: keyword + - name: process_id + type: keyword + - name: process_name + type: keyword + - name: protocol + type: keyword + - name: psmid + type: keyword + - name: session_duration + type: keyword + - name: session_id + type: keyword + - name: src_host + type: keyword + - name: username + type: keyword + - name: other + type: flattened - name: file type: keyword description: The name of the target file. diff --git a/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml b/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml index 2a27502a365..32e9c221f2e 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cyberarkpas/audit/ingest/pipeline.yml @@ -1088,6 +1088,72 @@ processors: - private - unspecified + # + # Save only interesting fields under extra_fields and ca_properties + # to prevent mapping explosion. Keep the rest under .other (type flattened). + # + - script: + lang: painless + description: Map interesting fields from ca_properties and extra_details. + params: + ca_properties: + - address + - cpm_disabled + - cpm_error_details + - cpm_status + - creation_method + - customer + - database + - device_type + - dual_account_status + - group_name + - in_process + - index + - last_fail_date + - last_success_change + - last_success_reconciliation + - last_success_verification + - last_task + - logon_domain + - policy_id + - port + - privcloud + - reset_immediately + - retries_count + - sequence_id + - tags + - user_dn + - user_name + - virtual_username + extra_details: + - ad_process_id + - ad_process_name + - application_type + - command + - connection_component_id + - dst_host + - logon_account + - managed_account + - process_id + - process_name + - protocol + - psmid + - session_duration + - session_id + - src_host + - username + source: > + Map audit = ctx.cyberarkpas.audit; + params.entrySet().stream().filter(e -> audit.containsKey(e.getKey())).forEach(lst -> { + Map base = audit[lst.getKey()], + selected = new HashMap(); + lst.getValue().stream().filter(fld -> base.containsKey(fld)).forEach(fld -> { + selected[fld] = base.remove(fld); + }); + selected['other'] = base; + audit[lst.getKey()] = selected; + }); + # # Cleanup # diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/130_cpm_disable_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/130_cpm_disable_password.log-expected.json index 0f598e7e3f3..2882ace09bb 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/130_cpm_disable_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/130_cpm_disable_password.log-expected.json @@ -16,8 +16,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "5", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Disable Password", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "5", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "5", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T12:57:13Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/22_cpm_verify_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/22_cpm_verify_password.log-expected.json index a549886a098..90d668dbefb 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/22_cpm_verify_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/22_cpm_verify_password.log-expected.json @@ -12,7 +12,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "-1", "cyberarkpas.audit.ca_properties.user_name": "test12", "cyberarkpas.audit.desc": "CPM Verify Password", - "cyberarkpas.audit.extra_details.address": "radiussrv.cyberark.local", + "cyberarkpas.audit.extra_details.other.address": "radiussrv.cyberark.local", "cyberarkpas.audit.extra_details.username": "test12", "cyberarkpas.audit.file": "Root\\Operating System-LINUX-SSH-radiussrv.cyberark.local-test12", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", @@ -81,7 +81,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "-1", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "CPM Verify Password", - "cyberarkpas.audit.extra_details.address": "34.123.103.115", + "cyberarkpas.audit.extra_details.other.address": "34.123.103.115", "cyberarkpas.audit.extra_details.username": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:22:44Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/24_cpm_change_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/24_cpm_change_password.log-expected.json index 3cf879a9996..ccc17d039a9 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/24_cpm_change_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/24_cpm_change_password.log-expected.json @@ -13,7 +13,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "-1", "cyberarkpas.audit.ca_properties.user_name": "test12", "cyberarkpas.audit.desc": "CPM Change Password", - "cyberarkpas.audit.extra_details.address": "radiussrv.cyberark.local", + "cyberarkpas.audit.extra_details.other.address": "radiussrv.cyberark.local", "cyberarkpas.audit.extra_details.username": "test12", "cyberarkpas.audit.file": "Root\\Operating System-LINUX-SSH-radiussrv.cyberark.local-test12", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", @@ -84,7 +84,7 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "CPM Change Password", - "cyberarkpas.audit.extra_details.address": "components", + "cyberarkpas.audit.extra_details.other.address": "components", "cyberarkpas.audit.extra_details.username": "x_accountA", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountA", "cyberarkpas.audit.iso_timestamp": "2021-03-08T19:20:05Z", @@ -158,7 +158,7 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountB", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "CPM Change Password", - "cyberarkpas.audit.extra_details.address": "components", + "cyberarkpas.audit.extra_details.other.address": "components", "cyberarkpas.audit.extra_details.username": "x_accountB", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountB", "cyberarkpas.audit.iso_timestamp": "2021-03-10T23:39:28Z", @@ -232,7 +232,7 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "CPM Change Password", - "cyberarkpas.audit.extra_details.address": "components", + "cyberarkpas.audit.extra_details.other.address": "components", "cyberarkpas.audit.extra_details.username": "x_accountA", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountA", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:12:24Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/294_store_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/294_store_password.log-expected.json index c3afc5ec8df..b26bdf2e009 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/294_store_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/294_store_password.log-expected.json @@ -3,10 +3,10 @@ "@timestamp": "2021-03-08T10:19:42.000Z", "cyberarkpas.audit.action": "Store password", "cyberarkpas.audit.ca_properties.cpm_status": "failure", - "cyberarkpas.audit.ca_properties.curr_ind": "2", "cyberarkpas.audit.ca_properties.in_process": "ChangeTask", "cyberarkpas.audit.ca_properties.last_success_change": "1615198782", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.curr_ind": "2", "cyberarkpas.audit.ca_properties.policy_id": "WindowsDesktopLocalAccountsRotationalPolicy", "cyberarkpas.audit.desc": "Store password", "cyberarkpas.audit.file": "Root\\Groups\\WindowsGroup", @@ -105,11 +105,11 @@ "cyberarkpas.audit.ca_properties.index": "1", "cyberarkpas.audit.ca_properties.last_success_change": "1614785704", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.start_change_not_before": "1615231182", "cyberarkpas.audit.ca_properties.policy_id": "WinDesktopLocal", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.sequence_id": "26", - "cyberarkpas.audit.ca_properties.start_change_not_before": "1615231182", "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "Store password", @@ -154,10 +154,10 @@ "@timestamp": "2021-03-10T14:38:57.000Z", "cyberarkpas.audit.action": "Store password", "cyberarkpas.audit.ca_properties.cpm_status": "failure", - "cyberarkpas.audit.ca_properties.curr_ind": "1", "cyberarkpas.audit.ca_properties.in_process": "ChangeTask", "cyberarkpas.audit.ca_properties.last_success_change": "1615387136", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.curr_ind": "1", "cyberarkpas.audit.ca_properties.policy_id": "WindowsDesktopLocalAccountsRotationalPolicy", "cyberarkpas.audit.desc": "Store password", "cyberarkpas.audit.file": "Root\\Groups\\WindowsGroup", @@ -307,11 +307,11 @@ "cyberarkpas.audit.ca_properties.index": "2", "cyberarkpas.audit.ca_properties.last_success_change": "1614868762", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.start_change_not_before": "1615419536", "cyberarkpas.audit.ca_properties.policy_id": "WinDesktopLocal", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.sequence_id": "24", - "cyberarkpas.audit.ca_properties.start_change_not_before": "1615419536", "cyberarkpas.audit.ca_properties.user_name": "x_accountB", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "Store password", @@ -356,10 +356,10 @@ "@timestamp": "2021-03-14T11:48:26.000Z", "cyberarkpas.audit.action": "Store password", "cyberarkpas.audit.ca_properties.cpm_status": "failure", - "cyberarkpas.audit.ca_properties.curr_ind": "2", "cyberarkpas.audit.ca_properties.in_process": "ChangeTask", "cyberarkpas.audit.ca_properties.last_success_change": "1615722505", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.curr_ind": "2", "cyberarkpas.audit.ca_properties.policy_id": "WindowsDesktopLocalAccountsRotationalPolicy", "cyberarkpas.audit.desc": "Store password", "cyberarkpas.audit.file": "Root\\Groups\\WindowsGroup", @@ -413,11 +413,11 @@ "cyberarkpas.audit.ca_properties.index": "1", "cyberarkpas.audit.ca_properties.last_success_change": "1615231204", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.start_change_not_before": "1615754905", "cyberarkpas.audit.ca_properties.policy_id": "WinDesktopLocal", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.sequence_id": "27", - "cyberarkpas.audit.ca_properties.start_change_not_before": "1615754905", "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "Store password", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/295_retrieve_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/295_retrieve_password.log-expected.json index e3afb5cf05a..a7e89b58fcc 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/295_retrieve_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/295_retrieve_password.log-expected.json @@ -214,11 +214,11 @@ "cyberarkpas.audit.ca_properties.index": "1", "cyberarkpas.audit.ca_properties.last_success_change": "1614785704", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.start_change_not_before": "1615231182", "cyberarkpas.audit.ca_properties.policy_id": "WinDesktopLocal", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.sequence_id": "26", - "cyberarkpas.audit.ca_properties.start_change_not_before": "1615231182", "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "Retrieve password", @@ -282,9 +282,9 @@ "@timestamp": "2021-03-08T19:20:02.000Z", "cyberarkpas.audit.action": "Retrieve password", "cyberarkpas.audit.ca_properties.cpm_status": "success", - "cyberarkpas.audit.ca_properties.curr_ind": "2", "cyberarkpas.audit.ca_properties.last_success_change": "1615198782", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.curr_ind": "2", "cyberarkpas.audit.ca_properties.policy_id": "WindowsDesktopLocalAccountsRotationalPolicy", "cyberarkpas.audit.desc": "Retrieve password", "cyberarkpas.audit.file": "Root\\Groups\\WindowsGroup", @@ -556,11 +556,11 @@ "cyberarkpas.audit.ca_properties.index": "2", "cyberarkpas.audit.ca_properties.last_success_change": "1614868762", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.start_change_not_before": "1615419536", "cyberarkpas.audit.ca_properties.policy_id": "WinDesktopLocal", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.sequence_id": "24", - "cyberarkpas.audit.ca_properties.start_change_not_before": "1615419536", "cyberarkpas.audit.ca_properties.user_name": "x_accountB", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "Retrieve password", @@ -624,9 +624,9 @@ "@timestamp": "2021-03-10T23:39:25.000Z", "cyberarkpas.audit.action": "Retrieve password", "cyberarkpas.audit.ca_properties.cpm_status": "success", - "cyberarkpas.audit.ca_properties.curr_ind": "1", "cyberarkpas.audit.ca_properties.last_success_change": "1615387136", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.curr_ind": "1", "cyberarkpas.audit.ca_properties.policy_id": "WindowsDesktopLocalAccountsRotationalPolicy", "cyberarkpas.audit.desc": "Retrieve password", "cyberarkpas.audit.file": "Root\\Groups\\WindowsGroup", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/300_psm_connect.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/300_psm_connect.log-expected.json index f8e788c087e..d8d5c95ffe8 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/300_psm_connect.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/300_psm_connect.log-expected.json @@ -13,11 +13,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "radiussrv.cyberark.local", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "admin2", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "35fac41e-22b5-11eb-83ca-000c297aae88", "cyberarkpas.audit.extra_details.src_host": "10.2.0.6", - "cyberarkpas.audit.extra_details.user": "admin2", "cyberarkpas.audit.file": "Root\\Operating System-LINUX-SSH-radiussrv.cyberark.local-admin2", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", "cyberarkpas.audit.issuer": "Administrator", @@ -82,11 +82,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "87012dcc-8290-11eb-949e-080027efd402", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:38:20Z", "cyberarkpas.audit.issuer": "Administrator", @@ -162,11 +162,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "ba22b012-8291-11eb-b981-080027efd402", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:46:56Z", "cyberarkpas.audit.issuer": "Administrator", @@ -242,11 +242,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "f6acbf00-8291-11eb-b9ba-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:48:34Z", "cyberarkpas.audit.issuer": "Administrator", @@ -322,11 +322,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "d8ff4d32-8292-11eb-b962-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:54:56Z", "cyberarkpas.audit.issuer": "Administrator", @@ -402,11 +402,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "173dd46a-8293-11eb-afcb-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:56:37Z", "cyberarkpas.audit.issuer": "Administrator", @@ -482,11 +482,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "988b22e8-82a7-11eb-83b9-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T20:23:25Z", "cyberarkpas.audit.issuer": "Administrator", @@ -568,11 +568,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "d284c268-2ba0-4366-af52-e33459b073a1", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:49:37Z", "cyberarkpas.audit.issuer": "Administrator", @@ -662,11 +662,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "47747796-03e1-4a11-af39-ab56c00e7732", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:50:43Z", "cyberarkpas.audit.issuer": "Administrator", @@ -754,11 +754,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "29f340df-89e9-405a-beae-0216390cda42", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:31:56Z", "cyberarkpas.audit.issuer": "Administrator", @@ -846,11 +846,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "f1654cf8-8ce5-472a-8205-ba731b0fab46", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:33:39Z", "cyberarkpas.audit.issuer": "Administrator", @@ -938,11 +938,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "8b3d0b38-aef5-49d9-bdd7-d57706887d8b", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:35:00Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1026,11 +1026,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "692fe25f-f940-4170-8ea4-5241b35173f0", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:18:31Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1114,11 +1114,11 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "f5725611-ca57-4a2a-a089-f45b3174a358", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:08:06Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1201,20 +1201,20 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615814025", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "0", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "PSM Connect", "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "7db90436-8a1a-4203-9a96-65137625ab2d", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:08:28Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1297,20 +1297,20 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615814025", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "0", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "PSM Connect", "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "27f74dce-f5d5-4c94-bf99-ca6aafe2c518", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:11:09Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1393,20 +1393,20 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615888216", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "4", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "PSM Connect", "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "8b222ac9-c2ad-49ea-9c4e-6829940f58d4", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-16T10:04:51Z", "cyberarkpas.audit.issuer": "Administrator", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/302_psm_disconnect.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/302_psm_disconnect.log-expected.json index 8aa327ff1a4..d305dbfcd0d 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/302_psm_disconnect.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/302_psm_disconnect.log-expected.json @@ -13,12 +13,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "radiussrv.cyberark.local", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "admin2", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:07", "cyberarkpas.audit.extra_details.session_id": "35fac41e-22b5-11eb-83ca-000c297aae88", "cyberarkpas.audit.extra_details.src_host": "10.2.0.6", - "cyberarkpas.audit.extra_details.user": "admin2", "cyberarkpas.audit.file": "Root\\Operating System-LINUX-SSH-radiussrv.cyberark.local-admin2", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", "cyberarkpas.audit.issuer": "Administrator", @@ -84,12 +84,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:13", "cyberarkpas.audit.extra_details.session_id": "87012dcc-8290-11eb-949e-080027efd402", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:38:26Z", "cyberarkpas.audit.issuer": "Administrator", @@ -166,12 +166,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:11", "cyberarkpas.audit.extra_details.session_id": "ba22b012-8291-11eb-b981-080027efd402", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:47:01Z", "cyberarkpas.audit.issuer": "Administrator", @@ -248,12 +248,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:12", "cyberarkpas.audit.extra_details.session_id": "f6acbf00-8291-11eb-b9ba-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:48:40Z", "cyberarkpas.audit.issuer": "Administrator", @@ -330,12 +330,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:12", "cyberarkpas.audit.extra_details.session_id": "d8ff4d32-8292-11eb-b962-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:55:02Z", "cyberarkpas.audit.issuer": "Administrator", @@ -412,12 +412,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:12", "cyberarkpas.audit.extra_details.session_id": "173dd46a-8293-11eb-afcb-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T17:56:42Z", "cyberarkpas.audit.issuer": "Administrator", @@ -494,12 +494,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "ssh", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:12", "cyberarkpas.audit.extra_details.session_id": "988b22e8-82a7-11eb-83b9-080027efd402", "cyberarkpas.audit.extra_details.src_host": "10.0.2.2", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-11T20:23:30Z", "cyberarkpas.audit.issuer": "Administrator", @@ -582,12 +582,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:18", "cyberarkpas.audit.extra_details.session_id": "d284c268-2ba0-4366-af52-e33459b073a1", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:49:54Z", "cyberarkpas.audit.issuer": "Administrator", @@ -678,12 +678,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:54", "cyberarkpas.audit.extra_details.session_id": "47747796-03e1-4a11-af39-ab56c00e7732", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:51:35Z", "cyberarkpas.audit.issuer": "Administrator", @@ -772,12 +772,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:01:35", "cyberarkpas.audit.extra_details.session_id": "29f340df-89e9-405a-beae-0216390cda42", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:33:30Z", "cyberarkpas.audit.issuer": "Administrator", @@ -866,12 +866,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:01:13", "cyberarkpas.audit.extra_details.session_id": "f1654cf8-8ce5-472a-8205-ba731b0fab46", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:34:50Z", "cyberarkpas.audit.issuer": "Administrator", @@ -960,12 +960,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:37:10", "cyberarkpas.audit.extra_details.session_id": "8b3d0b38-aef5-49d9-bdd7-d57706887d8b", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T11:12:09Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1050,12 +1050,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:05", "cyberarkpas.audit.extra_details.session_id": "692fe25f-f940-4170-8ea4-5241b35173f0", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:18:36Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1140,12 +1140,12 @@ "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "adrian", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:06", "cyberarkpas.audit.extra_details.session_id": "f5725611-ca57-4a2a-a089-f45b3174a358", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "adrian", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-34.123.103.115-adrian", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:08:11Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1229,21 +1229,21 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615814025", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "0", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "PSM Disconnect", "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:09", "cyberarkpas.audit.extra_details.session_id": "7db90436-8a1a-4203-9a96-65137625ab2d", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:08:36Z", "cyberarkpas.audit.issuer": "Administrator", @@ -1327,21 +1327,21 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615819476", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "1", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "PSM Disconnect", "cyberarkpas.audit.extra_details.application_type": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.user": "testark", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:49:12", "cyberarkpas.audit.extra_details.session_id": "27f74dce-f5d5-4c94-bf99-ca6aafe2c518", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.user": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T15:00:21Z", "cyberarkpas.audit.issuer": "Administrator", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/304_psm_upload_recording.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/304_psm_upload_recording.log-expected.json index 14603f0592b..67dae5fc704 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/304_psm_upload_recording.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/304_psm_upload_recording.log-expected.json @@ -5,12 +5,12 @@ "cyberarkpas.audit.desc": "PSM Upload Recording", "cyberarkpas.audit.extra_details.dst_host": "rhel7.cybr.com", "cyberarkpas.audit.extra_details.logon_account": "logon", + "cyberarkpas.audit.extra_details.other.user": "root", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_duration": "00:00:46", "cyberarkpas.audit.extra_details.session_id": "a4636750-50a2-492e-984c-e08743d8a883", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "root", "cyberarkpas.audit.file": "Root\\a4636750-50a2-492e-984c-e08743d8a883.SSH.txt", "cyberarkpas.audit.iso_timestamp": "2021-03-25T09:20:56Z", "cyberarkpas.audit.issuer": "PSMApp_COMP01", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/308_use_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/308_use_password.log-expected.json index 953a5211a77..e5f2e33591b 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/308_use_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/308_use_password.log-expected.json @@ -697,10 +697,10 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615814025", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "0", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "Use Password", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", @@ -786,10 +786,10 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615888216", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "4", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "Use Password", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/31_cpm_reconcile_password.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/31_cpm_reconcile_password.log-expected.json index 60aaf45b24e..6827026c78f 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/31_cpm_reconcile_password.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/31_cpm_reconcile_password.log-expected.json @@ -15,7 +15,7 @@ "cyberarkpas.audit.ca_properties.sequence_id": "1", "cyberarkpas.audit.ca_properties.user_name": "Administrator2", "cyberarkpas.audit.desc": "CPM Reconcile Password", - "cyberarkpas.audit.extra_details.address": "dbserver.cyberark.local", + "cyberarkpas.audit.extra_details.other.address": "dbserver.cyberark.local", "cyberarkpas.audit.extra_details.username": "Administrator2", "cyberarkpas.audit.file": "Root\\Operating System-WIN-SERVER-LOCAL-dbserver.cyberark.local-Administrator2", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/359_sql_command.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/359_sql_command.log-expected.json index aae4123d3cb..bf2a5247e8d 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/359_sql_command.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/359_sql_command.log-expected.json @@ -19,15 +19,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT USER FROM DUAL", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "69B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "4T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "69B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "4T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:44Z", "cyberarkpas.audit.issuer": "Administrator", @@ -104,15 +104,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "BEGIN DBMS_OUTPUT.DISABLE\\; END\\;", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "123B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "4T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "123B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "4T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:44Z", "cyberarkpas.audit.issuer": "Administrator", @@ -189,15 +189,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT ATTRIBUTE,SCOPE,NUMERIC_VALUE,CHAR_VALUE,DATE_VALUE FROM SYSTEM.PRODUCT_PRIVS WHERE (UPPER('SQL*Plus') LIKE UPPER(PRODUCT)) AND (UPPER(USER) LIKE USERID)", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "187B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "4T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "187B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "4T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:44Z", "cyberarkpas.audit.issuer": "Administrator", @@ -274,15 +274,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT CHAR_VALUE FROM SYSTEM.PRODUCT_PRIVS WHERE (UPPER('SQL*Plus') LIKE UPPER(PRODUCT)) AND ((UPPER(USER) LIKE USERID) OR (USERID \\= 'PUBLIC')) AND (UPPER(ATTRIBUTE) \\= 'ROLES')", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "380B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "4T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "380B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "4T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:44Z", "cyberarkpas.audit.issuer": "Administrator", @@ -359,15 +359,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "BEGIN DBMS_APPLICATION_INFO.SET_MODULE(:1,NULL)\\; END\\; (Parameters bound by position: 1\\=[SQL*Plus])", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "596B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "4T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "596B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "4T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:44Z", "cyberarkpas.audit.issuer": "Administrator", @@ -444,15 +444,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT DECODE('A','A','1','2') FROM DUAL", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "727B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "5T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "727B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "5T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:45Z", "cyberarkpas.audit.issuer": "Administrator", @@ -529,15 +529,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT INFO FROM SYSTEM.HELP WHERE UPPER(TOPIC) LIKE :1 ORDER BY TOPIC,SEQ (Parameters bound by position: 1\\=[HELP])", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "800B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "14T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "800B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "14T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:56:54Z", "cyberarkpas.audit.issuer": "Administrator", @@ -614,15 +614,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT * FROM DBA_USERS", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "1097B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "82T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "1097B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "82T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:58:02Z", "cyberarkpas.audit.issuer": "Administrator", @@ -699,15 +699,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "SELECT INFO FROM SYSTEM.HELP WHERE UPPER(TOPIC) LIKE :1 ORDER BY TOPIC,SEQ (Parameters bound by position: 1\\=[SHOW%])", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "948B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "25T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "948B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "25T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:57:05Z", "cyberarkpas.audit.issuer": "Administrator", @@ -784,15 +784,15 @@ "cyberarkpas.audit.desc": "SQL Command", "cyberarkpas.audit.extra_details.command": "select distinct owner from all_objects", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLPlus", - "cyberarkpas.audit.extra_details.data_base": "XE", "cyberarkpas.audit.extra_details.dst_host": "oracle.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "XE", + "cyberarkpas.audit.extra_details.other.sql_offset": "1153B", + "cyberarkpas.audit.extra_details.other.user": "HR", + "cyberarkpas.audit.extra_details.other.vid_offset": "124T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "0887c643-42f2-4a4f-806e-58c1689de0e6", - "cyberarkpas.audit.extra_details.sql_offset": "1153B", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.user": "HR", - "cyberarkpas.audit.extra_details.vid_offset": "124T", "cyberarkpas.audit.file": "Root\\Database-Oracle-oracle.cybr.com-HR", "cyberarkpas.audit.iso_timestamp": "2021-03-25T14:58:44Z", "cyberarkpas.audit.issuer": "Administrator", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/361_keystroke_logging.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/361_keystroke_logging.log-expected.json index 2824c5c7f3e..1d2c0d41d2e 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/361_keystroke_logging.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/361_keystroke_logging.log-expected.json @@ -14,13 +14,13 @@ "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "radiussrv.cyberark.local", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "3642B", + "cyberarkpas.audit.extra_details.other.user": "admin2", + "cyberarkpas.audit.extra_details.other.vid_offset": "125T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "499852f2-22b5-11eb-8bff-000c297aae88", "cyberarkpas.audit.extra_details.src_host": "10.2.0.6", - "cyberarkpas.audit.extra_details.ssh_offset": "3642B", - "cyberarkpas.audit.extra_details.user": "admin2", - "cyberarkpas.audit.extra_details.vid_offset": "125T", "cyberarkpas.audit.file": "Root\\Operating System-LINUX-SSH-radiussrv.cyberark.local-admin2", "cyberarkpas.audit.iso_timestamp": "2021-03-16T15:01:00Z", "cyberarkpas.audit.issuer": "Administrator", @@ -91,13 +91,13 @@ "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "1309B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "10T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "d284c268-2ba0-4366-af52-e33459b073a1", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "1309B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "10T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:49:49Z", "cyberarkpas.audit.issuer": "Administrator", @@ -185,13 +185,13 @@ "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "1312B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "6T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "29f340df-89e9-405a-beae-0216390cda42", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "1312B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "6T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:32:04Z", "cyberarkpas.audit.issuer": "Administrator", @@ -279,13 +279,13 @@ "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "1309B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "7T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "f1654cf8-8ce5-472a-8205-ba731b0fab46", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "1309B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "7T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:33:47Z", "cyberarkpas.audit.issuer": "Administrator", @@ -373,13 +373,13 @@ "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "1309B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "7T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "8b3d0b38-aef5-49d9-bdd7-d57706887d8b", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "1309B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "7T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:35:08Z", "cyberarkpas.audit.issuer": "Administrator", @@ -461,23 +461,23 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615814025", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "0", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "Keystroke logging", "cyberarkpas.audit.extra_details.command": "sudo su", "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "1309B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "8T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "27f74dce-f5d5-4c94-bf99-ca6aafe2c518", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "1309B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "8T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:11:18Z", "cyberarkpas.audit.issuer": "Administrator", @@ -559,23 +559,23 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615819476", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "1", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "Keystroke logging", "cyberarkpas.audit.extra_details.command": "(reverse-i-search)`grant': grant all privileges on *.* TO 'root'@'%' with grant option\\;", "cyberarkpas.audit.extra_details.connection_component_id": "PSMP-SSH", "cyberarkpas.audit.extra_details.dst_host": "34.123.103.115", "cyberarkpas.audit.extra_details.managed_account": "Yes", + "cyberarkpas.audit.extra_details.other.ssh_offset": "296291B", + "cyberarkpas.audit.extra_details.other.user": "testark", + "cyberarkpas.audit.extra_details.other.vid_offset": "2081T", "cyberarkpas.audit.extra_details.protocol": "SSH", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "27f74dce-f5d5-4c94-bf99-ca6aafe2c518", "cyberarkpas.audit.extra_details.src_host": "81.32.170.205", - "cyberarkpas.audit.extra_details.ssh_offset": "296291B", - "cyberarkpas.audit.extra_details.user": "testark", - "cyberarkpas.audit.extra_details.vid_offset": "2081T", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:45:51Z", "cyberarkpas.audit.issuer": "Administrator", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/38_cpm_verify_password_failed.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/38_cpm_verify_password_failed.log-expected.json index 6e9afaabf56..c33da1090c1 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/38_cpm_verify_password_failed.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/38_cpm_verify_password_failed.log-expected.json @@ -15,7 +15,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:19:58Z", @@ -99,7 +99,7 @@ "cyberarkpas.audit.ca_properties.user_dn": "ELASTIC.local", "cyberarkpas.audit.ca_properties.user_name": "bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", "cyberarkpas.audit.extra_details.username": "bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:25:32Z", @@ -182,7 +182,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC.local\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", "cyberarkpas.audit.extra_details.username": "ELASTIC.local\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-34.66.114.180-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:33:26Z", @@ -265,8 +265,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "1", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC.local\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "1", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "1", "cyberarkpas.audit.extra_details.username": "ELASTIC.local\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-34.66.114.180-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T15:04:11Z", @@ -349,8 +349,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "2", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC.local\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "2", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "2", "cyberarkpas.audit.extra_details.username": "ELASTIC.local\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-34.66.114.180-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T16:35:01Z", @@ -432,7 +432,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "10.0.1.20", + "cyberarkpas.audit.extra_details.other.address": "10.0.1.20", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T16:56:29Z", @@ -499,15 +499,15 @@ "cyberarkpas.audit.ca_properties.cpm_status": "failure", "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "mariadb", "cyberarkpas.audit.ca_properties.last_fail_date": "1615827554", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "mariadb", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "10.0.1.20", + "cyberarkpas.audit.extra_details.other.address": "10.0.1.20", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:01:07Z", @@ -574,15 +574,15 @@ "cyberarkpas.audit.ca_properties.cpm_status": "failure", "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "DRIVER={MariaDB ODBC 3.1 Driver};TCPIP=1;SERVER=localhost;UID=root;PWD=1234;DATABASE=test", "cyberarkpas.audit.ca_properties.last_fail_date": "1615827864", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "DRIVER={MariaDB ODBC 3.1 Driver};TCPIP=1;SERVER=localhost;UID=root;PWD=1234;DATABASE=test", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "10.0.1.20", + "cyberarkpas.audit.extra_details.other.address": "10.0.1.20", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:05:47Z", @@ -649,15 +649,15 @@ "cyberarkpas.audit.ca_properties.cpm_status": "failure", "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "DSN=mariadb;TCPIP=1;SERVER=localhost;UID=root;PWD=1234;DATABASE=test", "cyberarkpas.audit.ca_properties.last_fail_date": "1615828174", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "DSN=mariadb;TCPIP=1;SERVER=localhost;UID=root;PWD=1234;DATABASE=test", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "10.0.1.20", + "cyberarkpas.audit.extra_details.other.address": "10.0.1.20", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:10:25Z", @@ -733,7 +733,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "127.0.0.1", + "cyberarkpas.audit.extra_details.other.address": "127.0.0.1", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:28:07Z", @@ -802,16 +802,16 @@ "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.database": "test", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "mysql", "cyberarkpas.audit.ca_properties.last_fail_date": "1615829597", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "mysql", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.port": "3306", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "127.0.0.1", + "cyberarkpas.audit.extra_details.other.address": "127.0.0.1", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:33:17Z", @@ -880,16 +880,16 @@ "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.database": "test", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "Driver={MySQL ODBC 5.3 Unicode Driver};server=%ADDRESS%;user=%USER%;option=3;port=%PORT%;Password=%LOGONPASSWORD%", "cyberarkpas.audit.ca_properties.last_fail_date": "1615829907", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "Driver={MySQL ODBC 5.3 Unicode Driver};server=%ADDRESS%;user=%USER%;option=3;port=%PORT%;Password=%LOGONPASSWORD%", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.port": "3306", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "127.0.0.1", + "cyberarkpas.audit.extra_details.other.address": "127.0.0.1", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T17:38:27Z", @@ -958,16 +958,16 @@ "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.database": "test", "cyberarkpas.audit.ca_properties.device_type": "Database", - "cyberarkpas.audit.ca_properties.dsn": "mysql", "cyberarkpas.audit.ca_properties.last_fail_date": "1615831206", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.dsn": "mysql", "cyberarkpas.audit.ca_properties.policy_id": "MySQL", "cyberarkpas.audit.ca_properties.port": "3306", "cyberarkpas.audit.ca_properties.reset_immediately": "VerifyTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "root", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "Driver\\={MySQL ODBC 5.3 Unicode Driver}\\;server\\=127.0.0.1\\;user\\=root\\;option\\=3\\;port\\=3306\\;Password\\=1234", + "cyberarkpas.audit.extra_details.other.address": "Driver\\={MySQL ODBC 5.3 Unicode Driver}\\;server\\=127.0.0.1\\;user\\=root\\;option\\=3\\;port\\=3306\\;Password\\=1234", "cyberarkpas.audit.extra_details.username": "root", "cyberarkpas.audit.file": "Root\\Database-MySQL-10.0.1.20-root", "cyberarkpas.audit.iso_timestamp": "2021-03-15T18:00:07Z", @@ -1041,8 +1041,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "3", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC.local\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "3", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "3", "cyberarkpas.audit.extra_details.username": "ELASTIC.local\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-34.66.114.180-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T18:05:16Z", @@ -1125,8 +1125,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "4", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC.local\\bart", "cyberarkpas.audit.desc": "CPM Verify Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "4", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "4", "cyberarkpas.audit.extra_details.username": "ELASTIC.local\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-34.66.114.180-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-16T09:50:19Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/411_window_title.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/411_window_title.log-expected.json index 365c217d660..7b13d632c42 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/411_window_title.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/411_window_title.log-expected.json @@ -18,15 +18,15 @@ "cyberarkpas.audit.extra_details.command": "shutdown.exe, Shutdown Event Tracker", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-RDP", "cyberarkpas.audit.extra_details.dst_host": "dbserver.cyberark.local", + "cyberarkpas.audit.extra_details.other.rdp_offset": "218B", + "cyberarkpas.audit.extra_details.other.user": "Administrator2", + "cyberarkpas.audit.extra_details.other.vid_offset": "12T", "cyberarkpas.audit.extra_details.process_id": "4144", "cyberarkpas.audit.extra_details.process_name": "shutdown.exe", "cyberarkpas.audit.extra_details.protocol": "RDP", "cyberarkpas.audit.extra_details.psmid": "PSMServer_88f6598", - "cyberarkpas.audit.extra_details.rdp_offset": "218B", "cyberarkpas.audit.extra_details.session_id": "a1f46060-1de4-4f56-a8ba-71fdf3140ac1", "cyberarkpas.audit.extra_details.src_host": "10.2.0.6", - "cyberarkpas.audit.extra_details.user": "Administrator2", - "cyberarkpas.audit.extra_details.vid_offset": "12T", "cyberarkpas.audit.file": "Root\\Operating System-WIN-SERVER-LOCAL-dbserver.cyberark.local-Administrator2", "cyberarkpas.audit.iso_timestamp": "2021-03-16T17:11:42Z", "cyberarkpas.audit.issuer": "adm2", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/412_keystroke_logging.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/412_keystroke_logging.log-expected.json index 685a4a0586a..aef660f5e9b 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/412_keystroke_logging.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/412_keystroke_logging.log-expected.json @@ -18,15 +18,15 @@ "cyberarkpas.audit.desc": "Keystroke logging", "cyberarkpas.audit.extra_details.command": "SHOW DATABASES\\;", "cyberarkpas.audit.extra_details.connection_component_id": "PSM-SQLServerMgmtStudio", - "cyberarkpas.audit.extra_details.data_base": "master", "cyberarkpas.audit.extra_details.dst_host": "tgtsvr01.cybr.com", + "cyberarkpas.audit.extra_details.other.data_base": "master", + "cyberarkpas.audit.extra_details.other.txt_offset": "702B", + "cyberarkpas.audit.extra_details.other.user": "sa", + "cyberarkpas.audit.extra_details.other.vid_offset": "33T", "cyberarkpas.audit.extra_details.protocol": "SQLNet", "cyberarkpas.audit.extra_details.psmid": "PSMServer", "cyberarkpas.audit.extra_details.session_id": "975edc19-ad10-4b42-8098-f26afab40fac", "cyberarkpas.audit.extra_details.src_host": "127.0.0.1", - "cyberarkpas.audit.extra_details.txt_offset": "702B", - "cyberarkpas.audit.extra_details.user": "sa", - "cyberarkpas.audit.extra_details.vid_offset": "33T", "cyberarkpas.audit.file": "Root\\Database-MSSql-epmsvr01.cybr.com-sa", "cyberarkpas.audit.iso_timestamp": "2021-03-25T11:29:37Z", "cyberarkpas.audit.issuer": "Administrator", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/414_cpm_verify_ssh_key.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/414_cpm_verify_ssh_key.log-expected.json index fe2d5aedaf7..7fe96adaf04 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/414_cpm_verify_ssh_key.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/414_cpm_verify_ssh_key.log-expected.json @@ -6,12 +6,12 @@ "cyberarkpas.audit.ca_properties.cpm_status": "success", "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.device_type": "Operating System", - "cyberarkpas.audit.ca_properties.extra_pass3_folder": "Root", - "cyberarkpas.audit.ca_properties.extra_pass3_name": "Operating System-UnixSSH-rhel7.cybr.com-root", - "cyberarkpas.audit.ca_properties.extra_pass3_safe": "Linux Root", "cyberarkpas.audit.ca_properties.last_success_change": "1582315464", "cyberarkpas.audit.ca_properties.last_success_verification": "1616666646", "cyberarkpas.audit.ca_properties.last_task": "VerifyTask", + "cyberarkpas.audit.ca_properties.other.extra_pass3_folder": "Root", + "cyberarkpas.audit.ca_properties.other.extra_pass3_name": "Operating System-UnixSSH-rhel7.cybr.com-root", + "cyberarkpas.audit.ca_properties.other.extra_pass3_safe": "Linux Root", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSHKeys", "cyberarkpas.audit.ca_properties.privcloud": "privcloud", "cyberarkpas.audit.ca_properties.retries_count": "-1", @@ -19,7 +19,7 @@ "cyberarkpas.audit.ca_properties.tags": "SSH", "cyberarkpas.audit.ca_properties.user_name": "firecall1", "cyberarkpas.audit.desc": "CPM Verify SSH Key", - "cyberarkpas.audit.extra_details.address": "rhel7.cybr.com", + "cyberarkpas.audit.extra_details.other.address": "rhel7.cybr.com", "cyberarkpas.audit.extra_details.username": "firecall1", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSHKeys-rhel7.cybr.com-firecall1", "cyberarkpas.audit.iso_timestamp": "2021-03-25T10:04:06Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/459_general_audit.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/459_general_audit.log-expected.json index d607b784f41..2a33d77b170 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/459_general_audit.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/459_general_audit.log-expected.json @@ -17,8 +17,8 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountB", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "General Audit", - "cyberarkpas.audit.extra_details.dual_account_status": "Active", - "cyberarkpas.audit.extra_details.index": "2", + "cyberarkpas.audit.extra_details.other.dual_account_status": "Active", + "cyberarkpas.audit.extra_details.other.index": "2", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountB", "cyberarkpas.audit.iso_timestamp": "2021-03-08T10:19:42Z", "cyberarkpas.audit.issuer": "PasswordManager", @@ -75,8 +75,8 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountA", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "General Audit", - "cyberarkpas.audit.extra_details.dual_account_status": "Active", - "cyberarkpas.audit.extra_details.index": "1", + "cyberarkpas.audit.extra_details.other.dual_account_status": "Active", + "cyberarkpas.audit.extra_details.other.index": "1", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountA", "cyberarkpas.audit.iso_timestamp": "2021-03-10T14:38:57Z", "cyberarkpas.audit.issuer": "PasswordManager", @@ -133,8 +133,8 @@ "cyberarkpas.audit.ca_properties.user_name": "x_accountB", "cyberarkpas.audit.ca_properties.virtual_username": "virtual", "cyberarkpas.audit.desc": "General Audit", - "cyberarkpas.audit.extra_details.dual_account_status": "Active", - "cyberarkpas.audit.extra_details.index": "2", + "cyberarkpas.audit.extra_details.other.dual_account_status": "Active", + "cyberarkpas.audit.extra_details.other.index": "2", "cyberarkpas.audit.file": "Root\\Operating System-WindowsDesktopLocalAccountsRotationalPolicy-10.0.1.20-x_accountB", "cyberarkpas.audit.iso_timestamp": "2021-03-14T11:48:26Z", "cyberarkpas.audit.issuer": "PasswordManager", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/52_delete_file.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/52_delete_file.log-expected.json index 571cc11784d..c48c4268558 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/52_delete_file.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/52_delete_file.log-expected.json @@ -195,11 +195,11 @@ { "@timestamp": "2021-03-11T19:32:12.000Z", "cyberarkpas.audit.action": "Delete File", - "cyberarkpas.audit.ca_properties.__psm_live_sessions_1": "", - "cyberarkpas.audit.ca_properties.__psm_live_sessions_2": "", - "cyberarkpas.audit.ca_properties.__psm_live_sessions_3": "", - "cyberarkpas.audit.ca_properties.__psm_live_sessions_4": "", - "cyberarkpas.audit.ca_properties.__psm_live_sessions_5": "", + "cyberarkpas.audit.ca_properties.other.__psm_live_sessions_1": "", + "cyberarkpas.audit.ca_properties.other.__psm_live_sessions_2": "", + "cyberarkpas.audit.ca_properties.other.__psm_live_sessions_3": "", + "cyberarkpas.audit.ca_properties.other.__psm_live_sessions_4": "", + "cyberarkpas.audit.ca_properties.other.__psm_live_sessions_5": "", "cyberarkpas.audit.desc": "Delete File", "cyberarkpas.audit.file": "Root\\PSMPApp_VAGRANT.LiveSessions", "cyberarkpas.audit.iso_timestamp": "2021-03-11T19:32:12Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/57_cpm_change_password_failed.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/57_cpm_change_password_failed.log-expected.json index eaf206946a9..19bc08dc8aa 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/57_cpm_change_password_failed.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/57_cpm_change_password_failed.log-expected.json @@ -7,23 +7,23 @@ "cyberarkpas.audit.ca_properties.cpm_status": "failure", "cyberarkpas.audit.ca_properties.creation_method": "PVWA", "cyberarkpas.audit.ca_properties.device_type": "Operating System", - "cyberarkpas.audit.ca_properties.extra_pass3_folder": "Root", - "cyberarkpas.audit.ca_properties.extra_pass3_name": "Operating System-UnixSSH-rhel7.cybr.com-root", - "cyberarkpas.audit.ca_properties.extra_pass3_safe": "Linux Root", "cyberarkpas.audit.ca_properties.last_fail_date": "1616673608", "cyberarkpas.audit.ca_properties.last_success_change": "1616011989", "cyberarkpas.audit.ca_properties.last_success_reconciliation": "1576120341", "cyberarkpas.audit.ca_properties.last_success_verification": "1616580255", "cyberarkpas.audit.ca_properties.last_task": "ChangeTask", + "cyberarkpas.audit.ca_properties.other.extra_pass3_folder": "Root", + "cyberarkpas.audit.ca_properties.other.extra_pass3_name": "Operating System-UnixSSH-rhel7.cybr.com-root", + "cyberarkpas.audit.ca_properties.other.extra_pass3_safe": "Linux Root", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "No", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.privcloud": "privcloud", "cyberarkpas.audit.ca_properties.reset_immediately": "ChangeTask", "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.tags": "SSH", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "No", "cyberarkpas.audit.ca_properties.user_name": "firecall2", "cyberarkpas.audit.desc": "CPM Change Password Failed", - "cyberarkpas.audit.extra_details.address": "rhel7.cybr.com", + "cyberarkpas.audit.extra_details.other.address": "rhel7.cybr.com", "cyberarkpas.audit.extra_details.username": "firecall2", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-rhel7.cybr.com-firecall2", "cyberarkpas.audit.iso_timestamp": "2021-03-25T12:00:08Z", diff --git a/x-pack/filebeat/module/cyberarkpas/audit/test/60_cpm_reconcile_password_failed.log-expected.json b/x-pack/filebeat/module/cyberarkpas/audit/test/60_cpm_reconcile_password_failed.log-expected.json index 3b1ee72f9de..c473243de39 100644 --- a/x-pack/filebeat/module/cyberarkpas/audit/test/60_cpm_reconcile_password_failed.log-expected.json +++ b/x-pack/filebeat/module/cyberarkpas/audit/test/60_cpm_reconcile_password_failed.log-expected.json @@ -15,7 +15,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-11T21:12:22Z", @@ -99,8 +99,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "2", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "2", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "2", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:18:15Z", @@ -183,7 +183,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.123.103.115", + "cyberarkpas.audit.extra_details.other.address": "34.123.103.115", "cyberarkpas.audit.extra_details.username": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-14T13:46:13Z", @@ -265,8 +265,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "3", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "3", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "3", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-14T14:49:11Z", @@ -350,8 +350,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "4", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "4", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "4", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:12:18Z", @@ -434,8 +434,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "1", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.123.103.115", - "cyberarkpas.audit.extra_details.retriescount": "1", + "cyberarkpas.audit.extra_details.other.address": "34.123.103.115", + "cyberarkpas.audit.extra_details.other.retriescount": "1", "cyberarkpas.audit.extra_details.username": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T10:12:19Z", @@ -518,8 +518,8 @@ "cyberarkpas.audit.ca_properties.retries_count": "5", "cyberarkpas.audit.ca_properties.user_name": "ELASTIC\\bart", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.66.114.180", - "cyberarkpas.audit.extra_details.retriescount": "5", + "cyberarkpas.audit.extra_details.other.address": "34.66.114.180", + "cyberarkpas.audit.extra_details.other.retriescount": "5", "cyberarkpas.audit.extra_details.username": "ELASTIC\\bart", "cyberarkpas.audit.file": "Root\\Operating System-WinDomain-35.192.121.42-ELASTICbart", "cyberarkpas.audit.iso_timestamp": "2021-03-15T12:57:13Z", @@ -603,7 +603,7 @@ "cyberarkpas.audit.ca_properties.retries_count": "0", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.123.103.115", + "cyberarkpas.audit.extra_details.other.address": "34.123.103.115", "cyberarkpas.audit.extra_details.username": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T13:04:27Z", @@ -680,14 +680,14 @@ "cyberarkpas.audit.ca_properties.last_fail_date": "1615819476", "cyberarkpas.audit.ca_properties.last_success_verification": "1615803764", "cyberarkpas.audit.ca_properties.last_task": "ReconcileTask", + "cyberarkpas.audit.ca_properties.other.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.policy_id": "UnixSSH", "cyberarkpas.audit.ca_properties.reset_immediately": "ReconcileTask", "cyberarkpas.audit.ca_properties.retries_count": "1", - "cyberarkpas.audit.ca_properties.use_sudo_on_reconcile": "Yes", "cyberarkpas.audit.ca_properties.user_name": "testark", "cyberarkpas.audit.desc": "CPM Reconcile Password Failed", - "cyberarkpas.audit.extra_details.address": "34.123.103.115", - "cyberarkpas.audit.extra_details.retriescount": "1", + "cyberarkpas.audit.extra_details.other.address": "34.123.103.115", + "cyberarkpas.audit.extra_details.other.retriescount": "1", "cyberarkpas.audit.extra_details.username": "testark", "cyberarkpas.audit.file": "Root\\Operating System-UnixSSH-34.123.103.115-testark", "cyberarkpas.audit.iso_timestamp": "2021-03-15T14:44:37Z", diff --git a/x-pack/filebeat/module/cyberarkpas/fields.go b/x-pack/filebeat/module/cyberarkpas/fields.go index 2e48ca8da6d..f5c44e9a0f0 100644 --- a/x-pack/filebeat/module/cyberarkpas/fields.go +++ b/x-pack/filebeat/module/cyberarkpas/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCyberarkpas returns asset data. // This is the base64 encoded gzipped contents of module/cyberarkpas. func AssetCyberarkpas() string { - return "eJy0V0tv2zgQvudXzDEF6iDPLqJDASPZoFnEiBF72+5ejDE5kohQosChrOjfF9TDiWUpTeyUJ4uP+b55j0fwSGUAolySRfuYIR8AOOU0BXDlN8f2Eabj2QGAJBZWZU6ZNICvBwDw8hmEirTkowNofgXVjRGkmFAXwC9XZhRAZE2eNTuSQsy1W1TvAwhRMzVHrcjm81ku5lK59e6rMvpB60dbmrXrquENU6tWSlNEEsZCEDPMSORWuRLGnsML/dv10g5bvIVH2zhq2T1SWRgrO2cbHMcvP8GE4GKqTQGWhLHyqBdU4CKzJiPrFHEvdqjROUrpdXQhTJ46SMihRIdDYI4iY8tddZzHtJZRiYTD0Nj11siSRkcSvD7on/CnfiJe6u6GZodOCVihzglcjA6k4kxjyYDDXhBGEvfToSdncSHJodJ7OGGWkVChErU8aOT1xcIAj1Bp2sc3lUsaOIc2IleJ7AeL0FGB5aKy5kDcq+x3kLfTFrCgJWCWaSUqeZCgiFVKcDj9/mM8EAexYed/7aN0K+MzqBTyLCMfkNxRGoCeMMl8CZ38N7q6n0z/nf/90MtJsVk4lRA7TLrq18Qkui7jLVZrCRWt29k9zNsdCI1N0MHhw80VnJ2dXX4a5Hp6fHI2+jI6vZifHAfnfwUnl/8PcOac7D5W/O5LNORMForYQGGNo+ewPYJ5rBgUQ845al1WR+vbGVmvE8lqe539/T7XRgwH3BvZNsF914iqy9D6a7D8AKgoNZYWuDQrCuD8+PJLF8qIRVVYeLtTVQJSSU99R616CTFjtHNE/7aNMByyT3OV1mHkryH7IPM3r4lF3eQGMq6ht1BdFu9rAkYS3F6/o7Rl1shcuI8s+pYyS0yp44pEgzAAvyo+ssB3SruvcG8xgsViVwN87RwCPGABPyd3vprUaWdVpFLUrwwdft2nuoTGcFDElMLP2d286hMQI4OkZR5FKo2AUlxq2hLx51PIEvJAfXD01I2hreCsnwOljixJWD4Xq64qqbFJL8k1k1BcnJ+e91JZGqMJuzQ32PyIycVUu4dL1iZqa78wSaYVMRTKxfBwc+VhBttA2RkNW3qM4UfPCzMMB+YFphX52XofwFbGOwoHm9wKWnj3fZSqnV73ru6197QUWpP4vLO0CQcF8jOVI7gxtq4rTMy+l30G53twobSGZf3YEmoQWvlUbset22k/8dq/f8qOnn+sRPyqTv28Xh213joPbExbk8kErq/h27dgMglmsybnBrPrnzyF0wtoJ6xelitKpdnZcG/oXjVAv4lWZHmvf6ZvQK8QNhx7dPArAAD//75ppq4=" + return "eJzEWd9v2zYQfs9fcY8p0ARtknaoHwoEyYpmqJEgztpuL8KZPElEKFLjUXb83w+UZMeWJc+WvE1PsUR+993pfnxUzuCZFiMQiyk5dM858gmAV17TCG7CzWv3DA/XkxMASSycyr2yZgSfTwBgfRvEirTk8xOo/xqVK87AYEZNA+Hyi5xGkDhb5PUdSTEW2kfl/hHEqJnqR0vI+ucrLhZS+dXdnRjtRqtNW54tr5uaNzw4NVOaEpJwLQQxw4RE4ZRfwHXgsOb/8lqPwxZvEaxtPFqye6bF3DrZeLbB8Xr9J9gYfEpVKMCRsE6etxoVGOXO5uS8Im613YzMtmUhbGE8ZORRosfzxuI2nzf8ltIRN43/k/NrLuRZJBXjVNP2okNAyDnrIkkelR5Ghz36YgCEIwyxjTLyqR3gVMHeZuR6A4TXOcWNcjkQgGZKUBQW98coUEdYJdnQyJbJHIW/e0MoEypGDMlYZSS99N6tkX0Uo9KRRN/fjxKGi7JxRSJFkxwJK7QbI5RW2NLPemLOyKlYiSMgeuTn/gg2sSaSNkPVn0ZutRKLSPWv69w633+zUzOhbdHfvCMmH6ksI6nQk14MQPJOEUdlbfdGYfqrICNoSEg9Jv3ruWBykeyfEeX+QU1pppwPbTIgDQKyPt0xL2KN3pNpzNlXLeEpsa6ZDntLmKeUVhglJJzG1q1unTnS6ElCkCtlJ+A37aImoPbXUWHEKAEz1AWBT9GDVJxrXDBgt8gSVhK306EX77BDV+ypsSY5idAAKyyosdpkHh8uv5bzbEj9rKEMyj/Mc133+WGqQdgsQzNAO1ljqFTkkbBZbg0ZPyRAkn2UWu7f5arZU+ug3igZGkxIDsY5QsocJV9yZ70VVvcH4GyAD0zMIUVk4YZpkyXQEC5ODMuw/3V4xEo3DR80OMp5UfdDjy4hX0K2d+QEPc1xUZ4mus7camdDDibvHpYG5zRd71yQoUiVITh9+P7jumNIhVfVEu2DnF5ivAVloMhzCtOSqTkC6AWzXNMIxn+c3dyPH35/+vWxlZNiG3mVEXvMmu5XxFqOHFusVgglrbvJPTwt70BsXYYeTh+/3MDl5eWnN51cL969vzz7eHbx4en9u9HVL6P3n/7s4MzFVtYdFMXvWGhfZj/MUwtzZz29ztVzeEoVg2IouECtF+Wj1eqcXPCJZHl7JU3a37m2rUeYg9jWyf2thqo00upXpzYCUImxjiKc2hmN4Ordp49NU1ZEperh7a9kJUA4trY9Wg0XYsatY+TxPmExnHIoc2WqNArLkEOShZW3xKJSOx0VV9Pb7rGHKVQrCe5u99BeazNKFqLZlwcpUkd5OIUZzyWJ2kKH+dl8t/psb8xdCrShPUOH2ycIDud9A/B5a6Q84hx+jr+FblKVnVOJMqh3fPAM173RC6gDB/OUDPycfHsq5wSkyCBpWiSJMgmQKT8n/vcl5Ai5oz94emnm0FZyVtuBjCdHEqavzarpirEuayW5YhKLD1cXV61UptZqwibNDTY/UgpqoLTPC9Y2Wfb+IKe1Ioa58ik8frkJZjrHwKLxWXqldjA+tl6YYNyhF5hm5JQfdLJdYhzQONgWTlB5qD+Wq41Zd9D0GqyWYmezUHeONs3BHPmVyjl8sa7qK7Uq5rfgwwyeK61hWm12hBqEVqGUl3Lr7qGdePV+/604Bv6pEulOn9p57ZRa++qBDbU1Ho/h9ha+fh2Nx6PJpK65zur6rTBw8QGWCquV5YyMtL0Dt8f0qgy0h2hGjgf9V2wP66WFjRd7fvJ3AAAA//93QKi/" } From 87a8c857fcbfbcf3894e5e524349fec2c580fc5d Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 13:07:11 +0200 Subject: [PATCH 05/11] [Elastic Agent] Deprecate kibana-url param (#25364) The kibana-url param should not be used anymore and is replaced by --url. The flag should be full removed for 7.14. This PR marks it as deprecated and updates the description. --- x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index ebd06bfdd44..be02813e3d7 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -50,7 +50,7 @@ func newEnrollCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command func addEnrollFlags(cmd *cobra.Command) { cmd.Flags().StringP("url", "", "", "URL to enroll Agent into Fleet") - cmd.Flags().StringP("kibana-url", "k", "", "URL of Kibana to enroll Agent into Fleet") + cmd.Flags().StringP("kibana-url", "k", "", "URL of Fleet-Server to enroll Agent into Fleet-Server (deprecated)") cmd.Flags().StringP("enrollment-token", "t", "", "Enrollment token to use to enroll Agent into Fleet") cmd.Flags().StringP("fleet-server-es", "", "", "Start and run a Fleet Server along side this Elastic Agent connecting to the provided elasticsearch") cmd.Flags().StringP("fleet-server-es-ca", "", "", "Path to certificate authority to use with communicate with elasticsearch") From 1d998f423b681e3d93b1fe4dc34161980f168f49 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 13:52:31 +0200 Subject: [PATCH 06/11] [Elastic Agent] Update log message to contain fleet-server instead of Kibana (#25365) Quite a few log and error messages contained Kibana but instead it was referring to fleet-server now. This PR updates these log messages and modifies parts of the code to also refer to fleet-server. This should improve the user experience and reduce confusion when looking at logs. --- .../handlers/handler_action_policy_change.go | 6 +++--- x-pack/elastic-agent/pkg/agent/cmd/enroll.go | 2 +- x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go | 4 ++-- x-pack/elastic-agent/pkg/fleetapi/client/client.go | 14 +++++++------- .../pkg/fleetapi/client/client_test.go | 4 ++-- x-pack/elastic-agent/pkg/remote/client.go | 3 ++- 6 files changed, 17 insertions(+), 16 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go index 086742a816b..a82b39b6763 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/actions/handlers/handler_action_policy_change.go @@ -84,7 +84,7 @@ func (h *PolicyChange) Handle(ctx context.Context, a fleetapi.Action, acker stor } h.log.Debugf("handlerPolicyChange: emit configuration for action %+v", a) - err = h.handleKibanaHosts(ctx, c) + err = h.handleFleetServerHosts(ctx, c) if err != nil { return err } @@ -95,8 +95,8 @@ func (h *PolicyChange) Handle(ctx context.Context, a fleetapi.Action, acker stor return acker.Ack(ctx, action) } -func (h *PolicyChange) handleKibanaHosts(ctx context.Context, c *config.Config) (err error) { - // do not update kibana host from policy; no setters provided with local Fleet Server +func (h *PolicyChange) handleFleetServerHosts(ctx context.Context, c *config.Config) (err error) { + // do not update fleet-server host from policy; no setters provided with local Fleet Server if len(h.setters) == 0 { return nil } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index be02813e3d7..fc62d8deff8 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -63,7 +63,7 @@ func addEnrollFlags(cmd *cobra.Command) { cmd.Flags().BoolP("fleet-server-insecure-http", "", false, "Expose Fleet Server over HTTP (not recommended; insecure)") cmd.Flags().StringP("certificate-authorities", "a", "", "Comma separated list of root certificate for server verifications") cmd.Flags().StringP("ca-sha256", "p", "", "Comma separated list of certificate authorities hash pins used for certificate verifications") - cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to Kibana") + cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to fleet-server") cmd.Flags().StringP("staging", "", "", "Configures agent to download artifacts from a staging build") } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go index 9b51d4c3692..dc5e4753445 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll_cmd.go @@ -100,7 +100,7 @@ func (e *enrollCmdOption) remoteConfig() (remote.Config, error) { return remote.Config{}, err } if cfg.Protocol == remote.ProtocolHTTP && !e.Insecure { - return remote.Config{}, fmt.Errorf("connection to Kibana is insecure, strongly recommended to use a secure connection (override with --insecure)") + return remote.Config{}, fmt.Errorf("connection to fleet-server is insecure, strongly recommended to use a secure connection (override with --insecure)") } // Add any SSL options from the CLI. @@ -382,7 +382,7 @@ func (c *enrollCmd) enroll(ctx context.Context, persistentConfig map[string]inte resp, err := cmd.Execute(ctx, r) if err != nil { return errors.New(err, - "fail to execute request to Kibana", + "fail to execute request to fleet-server", errors.TypeNetwork) } diff --git a/x-pack/elastic-agent/pkg/fleetapi/client/client.go b/x-pack/elastic-agent/pkg/fleetapi/client/client.go index 86efb3d2d66..172600cfede 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client/client.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client/client.go @@ -59,10 +59,10 @@ func init() { } } -// NewAuthWithConfig returns a Kibana client that will: +// NewAuthWithConfig returns a fleet-server client that will: // // - Send the API Key on every HTTP request. -// - Ensure a minimun version of Kibana is required. +// - Ensure a minimun version of fleet-server is required. // - Send the Fleet User Agent on every HTTP request. func NewAuthWithConfig(log *logger.Logger, apiKey string, cfg remote.Config) (*remote.Client, error) { return remote.NewWithConfig(log, cfg, func(rt http.RoundTripper) (http.RoundTripper, error) { @@ -80,14 +80,14 @@ func NewAuthWithConfig(log *logger.Logger, apiKey string, cfg remote.Config) (*r }) } -// NewWithConfig takes a Kibana configuration and create a kibana.client with the appropriate tripper. +// NewWithConfig takes a fleet-server configuration and create a remote.client with the appropriate tripper. func NewWithConfig(log *logger.Logger, cfg remote.Config) (*remote.Client, error) { return remote.NewWithConfig(log, cfg, baseRoundTrippers) } -// ExtractError extracts error from a fleet response +// ExtractError extracts error from a fleet-server response func ExtractError(resp io.Reader) error { - // Lets try to extract a high level Kibana error. + // Lets try to extract a high level fleet-server error. e := &struct { StatusCode int `json:"statusCode"` Error string `json:"error"` @@ -104,10 +104,10 @@ func ExtractError(resp io.Reader) error { // System errors doesn't return a message, fleet code can return a Message key which has more // information. if len(e.Message) == 0 { - return fmt.Errorf("status code: %d, Kibana returned an error: %s", e.StatusCode, e.Error) + return fmt.Errorf("status code: %d, fleet-server returned an error: %s", e.StatusCode, e.Error) } return fmt.Errorf( - "status code: %d, Kibana returned an error: %s, message: %s", + "status code: %d, fleet-server returned an error: %s, message: %s", e.StatusCode, e.Error, e.Message, diff --git a/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go b/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go index db2242ce7a4..7fac802017f 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/client/client_test.go @@ -127,9 +127,9 @@ func TestHTTPClient(t *testing.T) { // case since we might deal with different format or error I make sense to test this method in // isolation. func TestExtract(t *testing.T) { - // The error before is returned when an exception or an internal occur in Kibana, they + // The error before is returned when an exception or an internal occur in fleet-server, they // are not only generated by the Fleet app. - t.Run("standard high level kibana errors", func(t *testing.T) { + t.Run("standard high level fleet-server errors", func(t *testing.T) { err := ExtractError(strings.NewReader(`{ "statusCode": 500, "Internal Server Error"}`)) assert.True(t, strings.Index(err.Error(), "500") > 0) assert.True(t, strings.Index(err.Error(), "Internal Server Error") > 0) diff --git a/x-pack/elastic-agent/pkg/remote/client.go b/x-pack/elastic-agent/pkg/remote/client.go index 2c7b36063e6..e12512d9ee0 100644 --- a/x-pack/elastic-agent/pkg/remote/client.go +++ b/x-pack/elastic-agent/pkg/remote/client.go @@ -137,7 +137,7 @@ func NewWithConfig(log *logger.Logger, cfg Config, wrapper wrapperFunc) (*Client url, err := common.MakeURL(string(cfg.Protocol), p, host, usedDefaultPort) if err != nil { - return nil, errors.Wrap(err, "invalid Kibana endpoint") + return nil, errors.Wrap(err, "invalid fleet-server endpoint") } clients[i] = &requestClient{ request: prefixRequestFactory(url), @@ -177,6 +177,7 @@ func (c *Client) Send( // Content-Type / Accepted type can be override from the called. req.Header.Set("Content-Type", "application/json") req.Header.Add("Accept", "application/json") + // TODO: Make this header specific to fleet-server or remove it req.Header.Set("kbn-xsrf", "1") // Without this Kibana will refuse to answer the request. // copy headers. From 0c5a38fc02f534359e688d84cf770840785749bf Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 14:24:35 +0200 Subject: [PATCH 07/11] [Elastic Agent] Update log message for capabilities (#25366) Before this change the log message was a bit cryptic when an input was blocked and I first had to look at the code to fully understand it: `input 'system/metrics' is left out due to capability restriction 'ID(*)'` ID in this context means input blocked. Not sure if this is obvious to users. I made the log message a bit more verbose to contain allow / deny in the log message. In addition, I moved the log message to the info level. If a user remove the capabilities for a certain input, I'm not sure if we should log it on the error level. On the other hand, Fleet should probably already make sure it is not sending down incompatible policies. --- x-pack/elastic-agent/pkg/capabilities/input.go | 10 +++++----- x-pack/elastic-agent/pkg/capabilities/output.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/elastic-agent/pkg/capabilities/input.go b/x-pack/elastic-agent/pkg/capabilities/input.go index 3459846ec14..57d5dc7768f 100644 --- a/x-pack/elastic-agent/pkg/capabilities/input.go +++ b/x-pack/elastic-agent/pkg/capabilities/input.go @@ -123,13 +123,13 @@ func (c *inputCapability) name() string { return c.Name } - t := "A" + t := "allow" if c.Type == denyKey { - t = "D" + t = "deny" } // e.g IA(*) or ID(system/*) - c.Name = fmt.Sprintf("I%s(%s)", t, c.Input) + c.Name = fmt.Sprintf("I %s(%s)", t, c.Input) return c.Name } @@ -164,8 +164,8 @@ func (c *inputCapability) renderInputs(inputs []map[string]interface{}) ([]map[s input[conditionKey] = isSupported if !isSupported { - msg := fmt.Sprintf("input '%s' is left out due to capability restriction '%s'", inputType, c.name()) - c.log.Errorf(msg) + msg := fmt.Sprintf("input '%s' is not run due to capability restriction '%s'", inputType, c.name()) + c.log.Infof(msg) c.reporter.Update(state.Degraded, msg, nil) } diff --git a/x-pack/elastic-agent/pkg/capabilities/output.go b/x-pack/elastic-agent/pkg/capabilities/output.go index 0b24c5838cf..dd7e0e46421 100644 --- a/x-pack/elastic-agent/pkg/capabilities/output.go +++ b/x-pack/elastic-agent/pkg/capabilities/output.go @@ -66,7 +66,7 @@ func (c *outputCapability) Apply(cfgMap map[string]interface{}) (map[string]inte if ok { renderedOutputs, err := c.renderOutputs(outputs) if err != nil { - c.log.Errorf("marking outputs failed for capability '%s': %v", c.name(), err) + c.log.Errorf("marking outputs as failed for the capability '%s': %v", c.name(), err) return cfgMap, err } @@ -89,13 +89,13 @@ func (c *outputCapability) name() string { return c.Name } - t := "A" + t := "allow" if c.Type == denyKey { - t = "D" + t = "deny" } // e.g OA(*) or OD(logstash) - c.Name = fmt.Sprintf("O%s(%s)", t, c.Output) + c.Name = fmt.Sprintf("Output %s(%s)", t, c.Output) return c.Name } From 9eca6883a8e4a4f7c06239c502319e24b71ee9df Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 14:28:50 +0200 Subject: [PATCH 08/11] Update docs around Elastic Agent and fleet-server environment variables (#24778) The goal of this update is to make it more descriptive which environment variables are used in which scenario. I also changed the order of the docs as I assume in most cases, the simple Elastic Agent enrollment case is needed, fleet-server is the more advanced and only in special cases the setup of fleet is needed (testing etc.). So I put these at the end. --- .../elastic-agent/pkg/agent/cmd/container.go | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index a370c4f3106..27fddb7c63c 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -65,49 +65,56 @@ environment variables to run inside of the container. The following actions are possible and grouped based on the actions. -* Preparing Kibana for Fleet - This prepares the Fleet plugin that exists inside of Kibana. This must either be enabled here or done externally - before Fleet Server will actually successfully start. +* Elastic Agent Fleet Enrollment + This enrolls the Elastic Agent into a Fleet Server. It is also possible to have this create a new enrollment token + for this specific Elastic Agent. - KIBANA_FLEET_SETUP - set to 1 enables this setup - KIBANA_FLEET_HOST - kibana host to enable Fleet on [$KIBANA_HOST] - KIBANA_FLEET_USERNAME - kibana username to enable Fleet [$KIBANA_USERNAME] - KIBANA_FLEET_PASSWORD - kibana password to enable Fleet [$KIBANA_PASSWORD] - KIBANA_FLEET_CA - path to certificate authority to use with communicate with Kibana [$KIBANA_CA] - KIBANA_REQUEST_RETRY_SLEEP - specifies sleep duration taken when agent performs a request to kibana [default 1s] - KIBANA_REQUEST_RETRY_COUNT - specifies number of retries agent performs when executing a request to kibana [default 30] + FLEET_ENROLL - set to 1 for enrollment into fleet-server. If not set, Elastic Agent is run in standalone mode. + FLEET_URL - URL of the Fleet Server to enroll into + FLEET_ENROLLMENT_TOKEN - token to use for enrollment. This is not needed in case FLEET_SERVER_ENABLED and FLEET_ENROLL is set. Then the token is fetched from Kibana. + FLEET_CA - path to certificate authority to use with communicate with Fleet Server [$KIBANA_CA] + FLEET_INSECURE - communicate with Fleet with either insecure HTTP or unverified HTTPS + + The following vars are need in the scenario that Elastic Agent should automatically fetch its own token. + + KIBANA_FLEET_HOST - kibana host to enable create enrollment token on [$KIBANA_HOST] + KIBANA_FLEET_USERNAME - kibana username to create enrollment token [$KIBANA_USERNAME] + KIBANA_FLEET_PASSWORD - kibana password to create enrollment token [$KIBANA_PASSWORD] + FLEET_TOKEN_NAME - token name to use for fetching token from Kibana. This requires Kibana configs to be set. + FLEET_TOKEN_POLICY_NAME - token policy name to use for fetching token from Kibana. This requires Kibana configs to be set. * Bootstrapping Fleet Server This bootstraps the Fleet Server to be run by this Elastic Agent. At least one Fleet Server is required in a Fleet - deployment for other Elastic Agent to bootstrap. + deployment for other Elastic Agent to bootstrap. In case the Elastic Agent is run without fleet-server. These variables + are not needed. - FLEET_SERVER_ENABLE - set to 1 enables bootstrapping of Fleet Server (forces FLEET_ENROLL enabled) + If FLEET_SERVER_ENABLE and FLEET_ENROLL is set but no FLEET_ENROLLMENT_TOKEN, the token is automatically fetched from Kibana. + + FLEET_SERVER_ENABLE - set to 1 enables bootstrapping of Fleet Server inside Elastic Agent (forces FLEET_ENROLL enabled) FLEET_SERVER_ELASTICSEARCH_HOST - elasticsearch host for Fleet Server to communicate with [$ELASTICSEARCH_HOST] FLEET_SERVER_ELASTICSEARCH_USERNAME - elasticsearch username for Fleet Server [$ELASTICSEARCH_USERNAME] FLEET_SERVER_ELASTICSEARCH_PASSWORD - elasticsearch password for Fleet Server [$ELASTICSEARCH_PASSWORD] FLEET_SERVER_ELASTICSEARCH_CA - path to certificate authority to use with communicate with elasticsearch [$ELASTICSEARCH_CA] FLEET_SERVER_SERVICE_TOKEN - service token to use for communication with elasticsearch FLEET_SERVER_POLICY_ID - policy ID for Fleet Server to use for itself ("Default Fleet Server policy" used when undefined) - FLEET_SERVER_HOST - binding host for Fleet Server HTTP (overrides the policy) + FLEET_SERVER_HOST - binding host for Fleet Server HTTP (overrides the policy). By default this is 0.0.0.0. FLEET_SERVER_PORT - binding port for Fleet Server HTTP (overrides the policy) FLEET_SERVER_CERT - path to certificate to use for HTTPS endpoint FLEET_SERVER_CERT_KEY - path to private key for certificate to use for HTTPS endpoint FLEET_SERVER_INSECURE_HTTP - expose Fleet Server over HTTP (not recommended; insecure) -* Elastic Agent Fleet Enrollment - This enrolls the Elastic Agent into a Fleet Server. It is also possible to have this create a new enrollment token - for this specific Elastic Agent. +* Preparing Kibana for Fleet + This prepares the Fleet plugin that exists inside of Kibana. This must either be enabled here or done externally + before Fleet Server will actually successfully start. All the Kibana variables are not needed in case Elastic Agent + should not setup Fleet. To manually trigger KIBANA_FLEET_SETUP navigate to Kibana -> Fleet -> Agents and enabled it. - FLEET_ENROLL - set to 1 for enrollment to occur - FLEET_URL - URL of the Fleet Server to enroll into - FLEET_ENROLLMENT_TOKEN - token to use for enrollment - FLEET_TOKEN_NAME - token name to use for fetching token from Kibana - FLEET_TOKEN_POLICY_NAME - token policy name to use for fetching token from Kibana - FLEET_CA - path to certificate authority to use with communicate with Fleet Server [$KIBANA_CA] - FLEET_INSECURE - communicate with Fleet with either insecure HTTP or un-verified HTTPS - KIBANA_FLEET_HOST - kibana host to enable create enrollment token on [$KIBANA_HOST] - KIBANA_FLEET_USERNAME - kibana username to create enrollment token [$KIBANA_USERNAME] - KIBANA_FLEET_PASSWORD - kibana password to create enrollment token [$KIBANA_PASSWORD] + KIBANA_FLEET_SETUP - set to 1 enables the setup of Fleet in Kibana by Elastic Agent. This was previously FLEET_SETUP. + KIBANA_FLEET_HOST - Kibana host accessible from fleet-server. [$KIBANA_HOST] + KIBANA_FLEET_USERNAME - kibana username to enable Fleet [$KIBANA_USERNAME] + KIBANA_FLEET_PASSWORD - kibana password to enable Fleet [$KIBANA_PASSWORD] + KIBANA_FLEET_CA - path to certificate authority to use with communicate with Kibana [$KIBANA_CA] + KIBANA_REQUEST_RETRY_SLEEP - specifies sleep duration taken when agent performs a request to kibana [default 1s] + KIBANA_REQUEST_RETRY_COUNT - specifies number of retries agent performs when executing a request to kibana [default 30] The following environment variables are provided as a convenience to prevent a large number of environment variable to be used when the same credentials will be used across all the possible actions above. @@ -121,6 +128,7 @@ be used when the same credentials will be used across all the possible actions a KIBANA_PASSWORD - kibana password [$ELASTICSEARCH_PASSWORD] KIBANA_CA - path to certificate authority to use with communicate with Kibana [$ELASTICSEARCH_CA] + By default when this command starts it will check for an existing fleet.yml. If that file already exists then all the above actions will be skipped, because the Elastic Agent has already been enrolled. To ensure that enrollment occurs on every start of the container set FLEET_FORCE to 1. From 448afd410eedbee72920527862919102f1c8f7bd Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Wed, 28 Apr 2021 16:47:25 +0200 Subject: [PATCH 09/11] Add structured logging to logs input (#25299) --- CHANGELOG.next.asciidoc | 1 + filebeat/input/log/harvester.go | 61 ++++++----- filebeat/input/log/harvester_test.go | 2 + filebeat/input/log/input.go | 135 ++++++++++++++---------- filebeat/input/log/input_other_test.go | 2 + filebeat/input/log/log.go | 21 ++-- filebeat/input/log/logger.go | 40 +++++++ filebeat/input/stdin/input.go | 1 + filebeat/tests/system/test_harvester.py | 2 +- filebeat/tests/system/test_input.py | 13 +-- filebeat/tests/system/test_stdin.py | 2 +- 11 files changed, 178 insertions(+), 102 deletions(-) create mode 100644 filebeat/input/log/logger.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 291b6379129..ed91b9a8f77 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -106,6 +106,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041] - Add User Agent Parser for Azure Sign In Logs Ingest Pipeline {pull}23201[23201] - Changes filebeat httpjson input's append transform to create a list even with only a single value{pull}25074[25074] +- Change logging in logs input to structure logging. Some log message formats have changed. {pull}25299[25299] - All url.* fields apart from url.original in the Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules are now url unescaped due to using the Elasticsearch uri_parts processor. {pull}24699[24699] *Heartbeat* diff --git a/filebeat/input/log/harvester.go b/filebeat/input/log/harvester.go index 0d4e6d6b539..34bf1e221d6 100644 --- a/filebeat/input/log/harvester.go +++ b/filebeat/input/log/harvester.go @@ -77,6 +77,8 @@ type OutletFactory func() channel.Outleter // Harvester contains all harvester related data type Harvester struct { + logger *logp.Logger + id uuid.UUID config config source harvester.Source // the source being watched @@ -120,6 +122,7 @@ type harvesterProgressMetrics struct { // NewHarvester creates a new harvester func NewHarvester( + logger *logp.Logger, config *common.Config, state file.State, states *file.States, @@ -132,7 +135,9 @@ func NewHarvester( return nil, err } + logger = logger.Named("harvester").With("harvester_id", id) h := &Harvester{ + logger: logger, config: defaultConfig(), state: state, states: states, @@ -204,7 +209,7 @@ func (h *Harvester) Setup() error { return err } - logp.Debug("harvester", "Harvester setup successful. Line terminator: %d", h.config.LineTerminator) + h.logger.Debugf("Harvester setup successful. Line terminator: %d", h.config.LineTerminator) return nil } @@ -234,6 +239,8 @@ func (h *Harvester) updateCurrentSize() error { // Run start the harvester and reads files line by line and sends events to the defined output func (h *Harvester) Run() error { + logger := h.logger + // Allow for some cleanup on termination if h.onTerminate != nil { defer h.onTerminate() @@ -287,7 +294,7 @@ func (h *Harvester) Run() error { select { // Applies when timeout is reached case <-closeTimeout: - logp.Info("Closing harvester because close_timeout was reached: %s", source) + logger.Infof("Closing harvester because close_timeout was reached: %s", source) // Required when reader loop returns and reader finished case <-h.done: } @@ -295,11 +302,11 @@ func (h *Harvester) Run() error { h.stop() err := h.reader.Close() if err != nil { - logp.Err("Failed to stop harvester for file %s: %v", h.state.Source, err) + logger.Errorf("Failed to stop harvester for file: %v", err) } }(h.state.Source) - logp.Info("Harvester started for file: %s", h.state.Source) + logger.Info("Harvester started for file.") h.doneWg.Add(1) go func() { @@ -318,21 +325,21 @@ func (h *Harvester) Run() error { if err != nil { switch err { case ErrFileTruncate: - logp.Info("File was truncated. Begin reading file from offset 0: %s", h.state.Source) + logger.Info("File was truncated. Begin reading file from offset 0.") h.state.Offset = 0 filesTruncated.Add(1) case ErrRemoved: - logp.Info("File was removed: %s. Closing because close_removed is enabled.", h.state.Source) + logger.Info("File was removed. Closing because close_removed is enabled.") case ErrRenamed: - logp.Info("File was renamed: %s. Closing because close_renamed is enabled.", h.state.Source) + logger.Info("File was renamed. Closing because close_renamed is enabled.") case ErrClosed: - logp.Info("Reader was closed: %s. Closing.", h.state.Source) + logger.Info("Reader was closed. Closing.") case io.EOF: - logp.Info("End of file reached: %s. Closing because close_eof is enabled.", h.state.Source) + logger.Info("End of file reached. Closing because close_eof is enabled.") case ErrInactive: - logp.Info("File is inactive: %s. Closing because close_inactive of %v reached.", h.state.Source, h.config.CloseInactive) + logger.Infof("File is inactive. Closing because close_inactive of %v reached.", h.config.CloseInactive) default: - logp.Err("Read line error: %v; File: %v", err, h.state.Source) + logger.Errorf("Read line error: %v", err) } return nil } @@ -370,7 +377,7 @@ func (h *Harvester) monitorFileSize() { case <-ticker.C: err := h.updateCurrentSize() if err != nil { - logp.Err("Error updating file size: %v; File: %v", err, h.state.Source) + h.logger.Errorf("Error updating file size: %v", err) } } } @@ -481,7 +488,7 @@ func (h *Harvester) SendStateUpdate() { h.publishState(h.state) - logp.Debug("harvester", "Update state: %s, offset: %v", h.state.Source, h.state.Offset) + h.logger.Debugf("Update state (offset: %v).", h.state.Offset) h.states.Update(h.state) } @@ -491,14 +498,14 @@ func (h *Harvester) shouldExportLine(line string) bool { if len(h.config.IncludeLines) > 0 { if !harvester.MatchAny(h.config.IncludeLines, line) { // drop line - logp.Debug("harvester", "Drop line as it does not match any of the include patterns %s", line) + h.logger.Debugf("Drop line as it does not match any of the include patterns %s", line) return false } } if len(h.config.ExcludeLines) > 0 { if harvester.MatchAny(h.config.ExcludeLines, line) { // drop line - logp.Debug("harvester", "Drop line as it does match one of the exclude patterns%s", line) + h.logger.Debugf("Drop line as it does match one of the exclude patterns%s", line) return false } } @@ -539,6 +546,8 @@ func (h *Harvester) openFile() error { } func (h *Harvester) validateFile(f *os.File) error { + logger := h.logger + info, err := f.Stat() if err != nil { return fmt.Errorf("Failed getting stats for file %s: %s", h.state.Source, err) @@ -557,9 +566,9 @@ func (h *Harvester) validateFile(f *os.File) error { if err != nil { if err == transform.ErrShortSrc { - logp.Info("Initialising encoding for '%v' failed due to file being too short", f) + logger.Infof("Initialising encoding for '%v' failed due to file being too short", f) } else { - logp.Err("Initialising encoding for '%v' failed: %v", f, err) + logger.Errorf("Initialising encoding for '%v' failed: %v", f, err) } return err } @@ -570,7 +579,7 @@ func (h *Harvester) validateFile(f *os.File) error { return err } - logp.Debug("harvester", "Setting offset for file: %s. Offset: %d ", h.state.Source, offset) + logger.Debugf("Setting offset: %d ", offset) h.state.Offset = offset return nil @@ -579,12 +588,12 @@ func (h *Harvester) validateFile(f *os.File) error { func (h *Harvester) initFileOffset(file *os.File) (int64, error) { // continue from last known offset if h.state.Offset > 0 { - logp.Debug("harvester", "Set previous offset for file: %s. Offset: %d ", h.state.Source, h.state.Offset) + h.logger.Debugf("Set previous offset: %d ", h.state.Offset) return file.Seek(h.state.Offset, os.SEEK_SET) } // get offset from file in case of encoding factory was required to read some data. - logp.Debug("harvester", "Setting offset for file based on seek: %s", h.state.Source) + h.logger.Debug("Setting offset to: 0") return file.Seek(0, os.SEEK_CUR) } @@ -605,8 +614,8 @@ func (h *Harvester) cleanup() { // Mark harvester as finished h.state.Finished = true - logp.Debug("harvester", "Stopping harvester for file: %s", h.state.Source) - defer logp.Debug("harvester", "harvester cleanup finished for file: %s", h.state.Source) + h.logger.Debugf("Stopping harvester.") + defer h.logger.Debugf("harvester cleanup finished.") // Make sure file is closed as soon as harvester exits // If file was never opened, it can't be closed @@ -615,14 +624,14 @@ func (h *Harvester) cleanup() { // close file handler h.source.Close() - logp.Debug("harvester", "Closing file: %s", h.state.Source) + h.logger.Debugf("Closing file") harvesterOpenFiles.Add(-1) // On completion, push offset so we can continue where we left off if we relaunch on the same file // Only send offset if file object was created successfully h.SendStateUpdate() } else { - logp.Warn("Stopping harvester, NOT closing file as file info not available: %s", h.state.Source) + h.logger.Warn("Stopping harvester, NOT closing file as file info not available.") } harvesterClosed.Add(1) @@ -645,12 +654,12 @@ func (h *Harvester) newLogFileReader() (reader.Reader, error) { var r reader.Reader var err error - logp.Debug("harvester", "newLogFileReader with config.MaxBytes: %d", h.config.MaxBytes) + h.logger.Debugf("newLogFileReader with config.MaxBytes: %d", h.config.MaxBytes) // TODO: NewLineReader uses additional buffering to deal with encoding and testing // for new lines in input stream. Simple 8-bit based encodings, or plain // don't require 'complicated' logic. - h.log, err = NewLog(h.source, h.config.LogConfig) + h.log, err = NewLog(h.logger, h.source, h.config.LogConfig) if err != nil { return nil, err } diff --git a/filebeat/input/log/harvester_test.go b/filebeat/input/log/harvester_test.go index 537851979ca..dba61b5d0ea 100644 --- a/filebeat/input/log/harvester_test.go +++ b/filebeat/input/log/harvester_test.go @@ -30,6 +30,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/reader" "github.com/elastic/beats/v7/libbeat/reader/readfile" "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" @@ -75,6 +76,7 @@ func TestReadLine(t *testing.T) { source := File{File: readFile} h := Harvester{ + logger: logp.NewLogger("harvester"), config: config{ LogConfig: LogConfig{ CloseInactive: 500 * time.Millisecond, diff --git a/filebeat/input/log/input.go b/filebeat/input/log/input.go index 1b203adcf5e..b5a5cc7543f 100644 --- a/filebeat/input/log/input.go +++ b/filebeat/input/log/input.go @@ -27,6 +27,8 @@ import ( "sync" "time" + "github.com/gofrs/uuid" + "github.com/elastic/beats/v7/filebeat/channel" "github.com/elastic/beats/v7/filebeat/harvester" "github.com/elastic/beats/v7/filebeat/input" @@ -61,6 +63,7 @@ func init() { // Input contains the input and its config type Input struct { cfg *common.Config + logger *logp.Logger config config states *file.States harvesters *harvester.Registry @@ -130,7 +133,11 @@ func NewInput( meta = nil } + uuid, _ := uuid.NewV4() + logger := logp.NewLogger("input").With("input_id", uuid) + p := &Input{ + logger: logger, config: inputConfig, cfg: cfg, harvesters: harvester.NewRegistry(), @@ -144,7 +151,7 @@ func NewInput( // Create empty harvester to check if configs are fine // TODO: Do config validation instead - _, err = p.createHarvester(file.State{}, nil) + _, err = p.createHarvester(logger, file.State{}, nil) if err != nil { return nil, err } @@ -154,7 +161,7 @@ func NewInput( return nil, err } - logp.Info("Configured paths: %v", p.config.Paths) + logger.Infof("Configured paths: %v", p.config.Paths) cleanupNeeded = false go p.stopWhenDone() @@ -166,7 +173,9 @@ func NewInput( // It goes through all states coming from the registry. Only the states which match the glob patterns of // the input will be loaded and updated. All other states will not be touched. func (p *Input) loadStates(states []file.State) error { - logp.Debug("input", "exclude_files: %s. Number of states: %d", p.config.ExcludeFiles, len(states)) + logger := p.logger + + logger.Debugf("exclude_files: %s. Number of states: %d", p.config.ExcludeFiles, len(states)) for _, state := range states { // Check if state source belongs to this input. If yes, update the state. @@ -190,19 +199,20 @@ func (p *Input) loadStates(states []file.State) error { // Update input states and send new states to registry err := p.updateState(state) if err != nil { - logp.Err("Problem putting initial state: %+v", err) + logger.Errorf("Problem putting initial state: %+v", err) return err } } } - logp.Debug("input", "input with previous states loaded: %v", p.states.Count()) + logger.Debugf("input with previous states loaded: %v", p.states.Count()) return nil } // Run runs the input func (p *Input) Run() { - logp.Debug("input", "Start next scan") + logger := p.logger + logger.Debug("Start next scan") // TailFiles is like ignore_older = 1ns and only on startup if p.config.TailFiles { @@ -223,61 +233,64 @@ func (p *Input) Run() { if p.config.CleanInactive > 0 || p.config.CleanRemoved { beforeCount := p.states.Count() cleanedStates, pendingClean := p.states.Cleanup() - logp.Debug("input", "input states cleaned up. Before: %d, After: %d, Pending: %d", + logger.Debugf("input states cleaned up. Before: %d, After: %d, Pending: %d", beforeCount, beforeCount-cleanedStates, pendingClean) } // Marking removed files to be cleaned up. Cleanup happens after next scan to make sure all states are updated first if p.config.CleanRemoved { for _, state := range p.states.GetStates() { + stateLogger := loggerWithState(logger, state) + // os.Stat will return an error in case the file does not exist stat, err := os.Stat(state.Source) if err != nil { if os.IsNotExist(err) { - p.removeState(state) - logp.Debug("input", "Remove state for file as file removed: %s", state.Source) + p.removeState(stateLogger, state) + stateLogger.Debugf("Remove state for file as file removed: %s", state.Source) } else { - logp.Err("input state for %s was not removed: %s", state.Source, err) + stateLogger.Errorf("input state for %s was not removed: %s", state.Source, err) } } else { // Check if existing source on disk and state are the same. Remove if not the case. newState := file.NewState(stat, state.Source, p.config.Type, p.meta, p.fileStateIdentifier) if state.IdentifierName != newState.IdentifierName { - logp.Debug("input", "file_identity configuration for file has changed from %s to %s, generating new id", state.IdentifierName, newState.IdentifierName) + stateLogger.Debugf("file_identity configuration for file has changed from %s to %s, generating new id", state.IdentifierName, newState.IdentifierName) state.Id, state.IdentifierName = p.fileStateIdentifier.GenerateID(state) } if !state.IsEqual(&newState) { - p.removeState(state) - logp.Debug("input", "Remove state of file as its identity has changed: %s", state.Source) + p.removeState(stateLogger, state) + stateLogger.Debugf("Remove state of file as its identity has changed: %s", state.Source) } } } } } -func (p *Input) removeState(state file.State) { +func (p *Input) removeState(logger *logp.Logger, state file.State) { // Only clean up files where state is Finished if !state.Finished { - logp.Debug("input", "State for file not removed because harvester not finished: %s", state.Source) + logger.Debugf("State for file not removed because harvester not finished: %s", state.Source) return } state.TTL = 0 err := p.updateState(state) if err != nil { - logp.Err("File cleanup state update error: %s", err) + logger.Errorf("File cleanup state update error: %s", err) } } // getFiles returns all files which have to be harvested // All globs are expanded and then directory and excluded files are removed func (p *Input) getFiles() map[string]os.FileInfo { + logger := p.logger paths := map[string]os.FileInfo{} for _, path := range p.config.Paths { matches, err := filepath.Glob(path) if err != nil { - logp.Err("glob(%s) failed: %v", path, err) + logger.Errorf("glob(%s) failed: %v", path, err) continue } @@ -287,32 +300,32 @@ func (p *Input) getFiles() map[string]os.FileInfo { // check if the file is in the exclude_files list if p.isFileExcluded(file) { - logp.Debug("input", "Exclude file: %s", file) + logger.Debugf("Exclude file: %s", file) continue } // Fetch Lstat File info to detected also symlinks fileInfo, err := os.Lstat(file) if err != nil { - logp.Debug("input", "lstat(%s) failed: %s", file, err) + logger.Debugf("lstat(%s) failed: %s", file, err) continue } if fileInfo.IsDir() { - logp.Debug("input", "Skipping directory: %s", file) + logger.Debugf("Skipping directory: %s", file) continue } isSymlink := fileInfo.Mode()&os.ModeSymlink > 0 if isSymlink && !p.config.Symlinks { - logp.Debug("input", "File %s skipped as it is a symlink.", file) + logger.Debugf("File %s skipped as it is a symlink.", file) continue } // Fetch Stat file info which fetches the inode. In case of a symlink, the original inode is fetched fileInfo, err = os.Stat(file) if err != nil { - logp.Debug("input", "stat(%s) failed: %s", file, err) + logger.Debugf("stat(%s) failed: %s", file, err) continue } @@ -321,7 +334,7 @@ func (p *Input) getFiles() map[string]os.FileInfo { if p.config.Symlinks { for _, finfo := range paths { if os.SameFile(finfo, fileInfo) { - logp.Info("Same file found as symlink and original. Skipping file: %s (as it same as %s)", file, finfo.Name()) + logger.Infof("Same file found as symlink and original. Skipping file: %s (as it same as %s)", file, finfo.Name()) continue OUTER } } @@ -347,7 +360,7 @@ func (p *Input) matchesFile(filePath string) bool { // Evaluate if glob matches filePath match, err := filepath.Match(glob, filePath) if err != nil { - logp.Debug("input", "Error matching glob: %s", err) + p.logger.Debugf("Error matching glob: %s", err) continue } @@ -436,7 +449,7 @@ func getFileState(path string, info os.FileInfo, p *Input) (file.State, error) { if err != nil { return file.State{}, fmt.Errorf("could not fetch abs path for file %s: %s", absolutePath, err) } - logp.Debug("input", "Check file for harvesting: %s", absolutePath) + p.logger.Debugf("Check file for harvesting: %s", absolutePath) // Create new state for comparison newState := file.NewState(info, absolutePath, p.config.Type, p.meta, p.fileStateIdentifier) return newState, nil @@ -452,6 +465,8 @@ func getKeys(paths map[string]os.FileInfo) []string { // Scan starts a scanGlob for each provided path/glob func (p *Input) scan() { + logger := p.logger + var sortInfos []FileSortInfo var files []string @@ -462,7 +477,7 @@ func (p *Input) scan() { if p.config.ScanSort != "" { sortInfos, err = getSortedFiles(p.config.ScanOrder, p.config.ScanSort, getSortInfos(paths)) if err != nil { - logp.Err("Failed to sort files during scan due to error %s", err) + logger.Errorf("Failed to sort files during scan due to error %s", err) } } @@ -471,6 +486,7 @@ func (p *Input) scan() { } for i := 0; i < len(paths); i++ { + logger = p.logger // reset logger on each loop var path string var info os.FileInfo @@ -485,49 +501,53 @@ func (p *Input) scan() { select { case <-p.done: - logp.Info("Scan aborted because input stopped.") + logger.Info("Scan aborted because input stopped.") return default: } newState, err := getFileState(path, info, p) if err != nil { - logp.Err("Skipping file %s due to error %s", path, err) + logger.Errorf("Skipping file %s due to error %s", path, err) } + logger = loggerWithState(logger, newState) + // Load last state isNewState := p.states.IsNew(newState) // Ignores all files which fall under ignore_older if p.isIgnoreOlder(newState) { - err := p.handleIgnoreOlder(isNewState, newState) + err := p.handleIgnoreOlder(logger, isNewState, newState) if err != nil { - logp.Err("Updating ignore_older state error: %s", err) + logger.Errorf("Updating ignore_older state error: %s", err) } continue } // Decides if previous state exists if isNewState { - logp.Debug("input", "Start harvester for new file: %s", newState.Source) - err := p.startHarvester(newState, 0) + logger.Debugf("Start harvester for new file: %s", newState.Source) + err := p.startHarvester(logger, newState, 0) if err == errHarvesterLimit { - logp.Debug("input", harvesterErrMsg, newState.Source, err) + logger.Debugf(harvesterErrMsg, newState.Source, err) continue } if err != nil { - logp.Err(harvesterErrMsg, newState.Source, err) + logger.Errorf(harvesterErrMsg, newState.Source, err) } } else { lastState := p.states.FindPrevious(newState) - p.harvestExistingFile(newState, lastState) + p.harvestExistingFile(logger, newState, lastState) } } } // harvestExistingFile continues harvesting a file with a known state if needed -func (p *Input) harvestExistingFile(newState file.State, oldState file.State) { - logp.Debug("input", "Update existing file for harvesting: %s, offset: %v", newState.Source, oldState.Offset) +func (p *Input) harvestExistingFile(logger *logp.Logger, newState file.State, oldState file.State) { + logger = loggerWithOldState(logger, oldState) + + logger.Debugf("Update existing file for harvesting: %s, offset: %v", newState.Source, oldState.Offset) // No harvester is running for the file, start a new harvester // It is important here that only the size is checked and not modification time, as modification time could be incorrect on windows @@ -536,20 +556,20 @@ func (p *Input) harvestExistingFile(newState file.State, oldState file.State) { // Resume harvesting of an old file we've stopped harvesting from // This could also be an issue with force_close_older that a new harvester is started after each scan but not needed? // One problem with comparing modTime is that it is in seconds, and scans can happen more then once a second - logp.Debug("input", "Resuming harvesting of file: %s, offset: %d, new size: %d", newState.Source, oldState.Offset, newState.Fileinfo.Size()) - err := p.startHarvester(newState, oldState.Offset) + logger.Debugf("Resuming harvesting of file: %s, offset: %d, new size: %d", newState.Source, oldState.Offset, newState.Fileinfo.Size()) + err := p.startHarvester(logger, newState, oldState.Offset) if err != nil { - logp.Err("Harvester could not be started on existing file: %s, Err: %s", newState.Source, err) + logger.Errorf("Harvester could not be started on existing file: %s, Err: %s", newState.Source, err) } return } // File size was reduced -> truncated file if oldState.Finished && newState.Fileinfo.Size() < oldState.Offset { - logp.Debug("input", "Old file was truncated. Starting from the beginning: %s, offset: %d, new size: %d ", newState.Source, newState.Offset, newState.Fileinfo.Size()) - err := p.startHarvester(newState, 0) + logger.Debugf("Old file was truncated. Starting from the beginning: %s, offset: %d, new size: %d ", newState.Source, newState.Offset, newState.Fileinfo.Size()) + err := p.startHarvester(logger, newState, 0) if err != nil { - logp.Err("Harvester could not be started on truncated file: %s, Err: %s", newState.Source, err) + logger.Errorf("Harvester could not be started on truncated file: %s, Err: %s", newState.Source, err) } filesTruncated.Add(1) @@ -560,41 +580,41 @@ func (p *Input) harvestExistingFile(newState file.State, oldState file.State) { if oldState.Source != "" && oldState.Source != newState.Source { // This does not start a new harvester as it is assume that the older harvester is still running // or no new lines were detected. It sends only an event status update to make sure the new name is persisted. - logp.Debug("input", "File rename was detected: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset) + logger.Debugf("File rename was detected: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset) if oldState.Finished { - logp.Debug("input", "Updating state for renamed file: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset) + logger.Debugf("Updating state for renamed file: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset) // Update state because of file rotation oldState.Source = newState.Source oldState.TTL = newState.TTL err := p.updateState(oldState) if err != nil { - logp.Err("File rotation state update error: %s", err) + logger.Errorf("File rotation state update error: %s", err) } filesRenamed.Add(1) } else { - logp.Debug("input", "File rename detected but harvester not finished yet.") + logger.Debugf("File rename detected but harvester not finished yet.") } } if !oldState.Finished { // Nothing to do. Harvester is still running and file was not renamed - logp.Debug("input", "Harvester for file is still running: %s", newState.Source) + logger.Debugf("Harvester for file is still running: %s", newState.Source) } else { - logp.Debug("input", "File didn't change: %s", newState.Source) + logger.Debugf("File didn't change: %s", newState.Source) } } // handleIgnoreOlder handles states which fall under ignore older // Based on the state information it is decided if the state information has to be updated or not -func (p *Input) handleIgnoreOlder(isNewState bool, newState file.State) error { - logp.Debug("input", "Ignore file because ignore_older reached: %s", newState.Source) +func (p *Input) handleIgnoreOlder(logger *logp.Logger, isNewState bool, newState file.State) error { + logger.Debugf("Ignore file because ignore_older reached: %s", newState.Source) if !isNewState { lastState := p.states.FindPrevious(newState) if !lastState.Finished { - logp.Info("File is falling under ignore_older before harvesting is finished. Adjust your close_* settings: %s", newState.Source) + logger.Infof("File is falling under ignore_older before harvesting is finished. Adjust your close_* settings: %s", newState.Source) } // Old state exist, no need to update it return nil @@ -602,7 +622,7 @@ func (p *Input) handleIgnoreOlder(isNewState bool, newState file.State) error { // Make sure file is not falling under clean_inactive yet if p.isCleanInactive(newState) { - logp.Debug("input", "Do not write state for ignore_older because clean_inactive reached") + logger.Debugf("Do not write state for ignore_older because clean_inactive reached") return nil } @@ -669,9 +689,10 @@ func subOutletWrap(outlet channel.Outleter) func() channel.Outleter { } // createHarvester creates a new harvester instance from the given state -func (p *Input) createHarvester(state file.State, onTerminate func()) (*Harvester, error) { +func (p *Input) createHarvester(logger *logp.Logger, state file.State, onTerminate func()) (*Harvester, error) { // Each wraps the outlet, for closing the outlet individually h, err := NewHarvester( + logger, p.cfg, state, p.states, @@ -688,7 +709,7 @@ func (p *Input) createHarvester(state file.State, onTerminate func()) (*Harveste // startHarvester starts a new harvester with the given offset // In case the HarvesterLimit is reached, an error is returned -func (p *Input) startHarvester(state file.State, offset int64) error { +func (p *Input) startHarvester(logger *logp.Logger, state file.State, offset int64) error { if p.numHarvesters.Inc() > p.config.HarvesterLimit && p.config.HarvesterLimit > 0 { p.numHarvesters.Dec() harvesterSkipped.Add(1) @@ -699,7 +720,7 @@ func (p *Input) startHarvester(state file.State, offset int64) error { state.Offset = offset // Create harvester with state - h, err := p.createHarvester(state, func() { p.numHarvesters.Dec() }) + h, err := p.createHarvester(logger, state, func() { p.numHarvesters.Dec() }) if err != nil { p.numHarvesters.Dec() return err @@ -758,7 +779,7 @@ func (p *Input) doUpdate(state file.State) error { Private: state, }) if !ok { - logp.Info("input outlet closed") + p.logger.Info("input outlet closed") return errors.New("input outlet closed") } return nil diff --git a/filebeat/input/log/input_other_test.go b/filebeat/input/log/input_other_test.go index 9324e7f2d04..552979af1b9 100644 --- a/filebeat/input/log/input_other_test.go +++ b/filebeat/input/log/input_other_test.go @@ -27,6 +27,7 @@ import ( "github.com/elastic/beats/v7/filebeat/input/file" "github.com/elastic/beats/v7/filebeat/input/inputtest" "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" ) var matchTests = []struct { @@ -145,6 +146,7 @@ var initStateTests = []struct { func TestInit(t *testing.T) { for _, test := range initStateTests { p := Input{ + logger: logp.NewLogger("harvester"), config: config{ Paths: test.paths, }, diff --git a/filebeat/input/log/log.go b/filebeat/input/log/log.go index 60728143764..19f50d7f256 100644 --- a/filebeat/input/log/log.go +++ b/filebeat/input/log/log.go @@ -29,6 +29,8 @@ import ( // Log contains all log related data type Log struct { + logger *logp.Logger + fs harvester.Source offset int64 config LogConfig @@ -39,6 +41,7 @@ type Log struct { // NewLog creates a new log instance to read log sources func NewLog( + logger *logp.Logger, fs harvester.Source, config LogConfig, ) (*Log, error) { @@ -52,6 +55,7 @@ func NewLog( } return &Log{ + logger: logger, fs: fs, offset: offset, config: config, @@ -104,7 +108,7 @@ func (f *Log) Read(buf []byte) (int, error) { return totalN, err } - logp.Debug("harvester", "End of file reached: %s; Backoff now.", f.fs.Name()) + f.logger.Debugf("End of file reached: %s; Backoff now.", f.fs.Name()) f.wait() } } @@ -113,7 +117,7 @@ func (f *Log) Read(buf []byte) (int, error) { // based on the config options. func (f *Log) errorChecks(err error) error { if err != io.EOF { - logp.Err("Unexpected state reading from %s; error: %s", f.fs.Name(), err) + f.logger.Errorf("Unexpected state reading from %s; error: %s", f.fs.Name(), err) return err } @@ -121,7 +125,7 @@ func (f *Log) errorChecks(err error) error { // Stdin is not continuable if !f.fs.Continuable() { - logp.Debug("harvester", "Source is not continuable: %s", f.fs.Name()) + f.logger.Debugf("Source is not continuable: %s", f.fs.Name()) return err } @@ -134,14 +138,13 @@ func (f *Log) errorChecks(err error) error { // calling the stat function info, statErr := f.fs.Stat() if statErr != nil { - logp.Err("Unexpected error reading from %s; error: %s", f.fs.Name(), statErr) + f.logger.Errorf("Unexpected error reading from %s; error: %s", f.fs.Name(), statErr) return statErr } // check if file was truncated if info.Size() < f.offset { - logp.Debug("harvester", - "File was truncated as offset (%d) > size (%d): %s", f.offset, info.Size(), f.fs.Name()) + f.logger.Debugf("File was truncated as offset (%d) > size (%d): %s", f.offset, info.Size(), f.fs.Name()) return ErrFileTruncate } @@ -167,14 +170,14 @@ func (f *Log) checkFileDisappearedErrors() error { // calling the stat function info, statErr := f.fs.Stat() if statErr != nil { - logp.Err("Unexpected error reading from %s; error: %s", f.fs.Name(), statErr) + f.logger.Errorf("Unexpected error reading from %s; error: %s", f.fs.Name(), statErr) return statErr } if f.config.CloseRenamed { // Check if the file can still be found under the same path if !file.IsSameFile(f.fs.Name(), info) { - logp.Debug("harvester", "close_renamed is enabled and file %s has been renamed", f.fs.Name()) + f.logger.Debugf("close_renamed is enabled and file %s has been renamed", f.fs.Name()) return ErrRenamed } } @@ -182,7 +185,7 @@ func (f *Log) checkFileDisappearedErrors() error { if f.config.CloseRemoved { // Check if the file name exists. See https://github.com/elastic/filebeat/issues/93 if f.fs.Removed() { - logp.Debug("harvester", "close_removed is enabled and file %s has been removed", f.fs.Name()) + f.logger.Debugf("close_removed is enabled and file %s has been removed", f.fs.Name()) return ErrRemoved } } diff --git a/filebeat/input/log/logger.go b/filebeat/input/log/logger.go new file mode 100644 index 00000000000..f48cae94ce6 --- /dev/null +++ b/filebeat/input/log/logger.go @@ -0,0 +1,40 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package log + +import ( + "github.com/elastic/beats/v7/filebeat/input/file" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func loggerWithState(logger *logp.Logger, state file.State) *logp.Logger { + return logger.With( + "source", state.Source, + "state_id", state.Id, + "finished", state.Finished, + "os_id", state.FileStateOS, + ) +} + +func loggerWithOldState(logger *logp.Logger, oldState file.State) *logp.Logger { + return logger.With( + "old_source", oldState.Source, + "old_finished", oldState.Finished, + "old_os_id", oldState.FileStateOS, + ) +} diff --git a/filebeat/input/stdin/input.go b/filebeat/input/stdin/input.go index 0e8fbd0fc10..04dfe12ba39 100644 --- a/filebeat/input/stdin/input.go +++ b/filebeat/input/stdin/input.go @@ -88,6 +88,7 @@ func (p *Input) Run() { func (p *Input) createHarvester(state file.State) (*log.Harvester, error) { // Each harvester gets its own copy of the outlet h, err := log.NewHarvester( + logp.NewLogger("stdin"), p.cfg, state, nil, nil, func() channel.Outleter { diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index 659253c4ec6..f56a51d0fe0 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -370,7 +370,7 @@ def test_truncated_file_closed(self): # Wait until harvester is closed self.wait_until( lambda: self.log_contains( - "Stopping harvester for file"), + "Stopping harvester."), max_timeout=15) # Write 1 line -> truncation diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index 7192fd956d5..f9dbd138a2e 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -178,8 +178,7 @@ def test_rotating_close_inactive_low_write_rate(self): # wait for file to be closed due to close_inactive self.wait_until( - lambda: self.log_contains( - "Closing file: {}\n".format(os.path.abspath(testfile))), + lambda: self.log_contains("Closing file"), max_timeout=10) # wait a bit longer (on 1.0.1 this would cause the harvester @@ -305,8 +304,7 @@ def test_close_inactive(self): # wait for file to be closed due to close_inactive self.wait_until( - lambda: self.log_contains( - "Closing file: {}\n".format(os.path.abspath(testfile))), + lambda: self.log_contains("Closing file"), max_timeout=10) # write second line @@ -359,8 +357,7 @@ def test_close_inactive_file_removal(self): # wait for file to be closed due to close_inactive self.wait_until( - lambda: self.log_contains( - "Closing file: {}\n".format(os.path.abspath(testfile))), + lambda: self.log_contains("Closing file"), max_timeout=10) filebeat.check_kill_and_wait() @@ -407,7 +404,7 @@ def test_close_inactive_file_rotation_and_removal(self): self.wait_until( lambda: self.log_contains( # Still checking for old file name as filename does not change in harvester - "Closing file: "), + "Closing file"), max_timeout=10) filebeat.check_kill_and_wait() @@ -467,7 +464,7 @@ def test_close_inactive_file_rotation_and_removal2(self): self.wait_until( lambda: self.log_contains_count( # Checking if two files were closed - "Closing file: ") == 2, + "Closing file") == 2, max_timeout=10) filebeat.check_kill_and_wait() diff --git a/filebeat/tests/system/test_stdin.py b/filebeat/tests/system/test_stdin.py index 6d701fdfbf5..a8283925ab8 100644 --- a/filebeat/tests/system/test_stdin.py +++ b/filebeat/tests/system/test_stdin.py @@ -24,7 +24,7 @@ def test_stdin(self): self.wait_until( lambda: self.log_contains( - "Harvester started for file: -"), + "Harvester started for file."), max_timeout=10) iterations1 = 5 From 89d2b36765e57deaae816ff927b50117dec985eb Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Wed, 28 Apr 2021 16:58:30 +0200 Subject: [PATCH 10/11] [elastic-agent] Improve log messages (#25372) This PR improves the log messages to make them easier to consume by users. * Change unroll to unenroll, that is what we use in the UI too * Trigger unenroll only when the maxCounter is exceeded (nit) * Move Artifact has been built with security disable to info. This only happens for internal builds * Move log message around binary already exists to debug level as this is an expected behaviour * Be more specific about binary downloads * For state changes, move the message first. That is the most interesting part for a user --- .../agent/application/gateway/fleet/fleet_gateway.go | 10 ++++++---- x-pack/elastic-agent/pkg/agent/cmd/run.go | 2 +- .../pkg/agent/operation/operation_fetch.go | 4 ++-- x-pack/elastic-agent/pkg/reporter/log/format.go | 4 ++-- x-pack/elastic-agent/pkg/reporter/log/reporter.go | 2 +- x-pack/elastic-agent/pkg/reporter/log/reporter_test.go | 2 +- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go index 6018e2bc5f9..8b208257828 100644 --- a/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/gateway/fleet/fleet_gateway.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/scheduler" ) +// Max number of times an invalid API Key is checked const maxUnauthCounter int = 6 // Default Configuration for the Fleet Gateway. @@ -244,8 +245,8 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, if isUnauth(err) { f.unauthCounter++ - if f.shouldUnroll() { - f.log.Warnf("retrieved unauthorized for '%d' times. Unrolling.", f.unauthCounter) + if f.shouldUnenroll() { + f.log.Warnf("retrieved an invalid api key error '%d' times. Starting to unenroll the elastic agent.", f.unauthCounter) return &fleetapi.CheckinResponse{ Actions: []fleetapi.Action{&fleetapi.ActionUnenroll{ActionID: "", ActionType: "UNENROLL", IsDetected: true}}, }, nil @@ -273,8 +274,9 @@ func (f *fleetGateway) execute(ctx context.Context) (*fleetapi.CheckinResponse, return resp, nil } -func (f *fleetGateway) shouldUnroll() bool { - return f.unauthCounter >= maxUnauthCounter +// shouldUnenroll checks if the max number of trying an invalid key is reached +func (f *fleetGateway) shouldUnenroll() bool { + return f.unauthCounter > maxUnauthCounter } func isUnauth(err error) bool { diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 2c0bb76cdb0..e49d60b5937 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -130,7 +130,7 @@ func run(streams *cli.IOStreams, override cfgOverrider) error { // Windows: Mark } if allowEmptyPgp, _ := release.PGP(); allowEmptyPgp { - logger.Warn("Artifact has been build with security disabled. Elastic Agent will not verify signatures of used artifacts.") + logger.Info("Artifact has been built with security disabled. Elastic Agent will not verify signatures of the artifacts.") } execPath, err := reexecPath() diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go b/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go index 0f66842abce..00450a641c9 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_fetch.go @@ -58,7 +58,7 @@ func (o *operationFetch) Check(_ context.Context, _ Application) (bool, error) { return true, nil } - o.logger.Infof("%s.%s already exists in %s. Skipping operation %s", o.program.BinaryName(), o.program.Version(), fullPath, o.Name()) + o.logger.Debugf("binary '%s.%s' already exists in %s. Skipping operation %s", o.program.BinaryName(), o.program.Version(), fullPath, o.Name()) return false, err } @@ -72,7 +72,7 @@ func (o *operationFetch) Run(ctx context.Context, application Application) (err fullPath, err := o.downloader.Download(ctx, o.program.Spec(), o.program.Version()) if err == nil { - o.logger.Infof("operation '%s' downloaded %s.%s into %s", o.Name(), o.program.BinaryName(), o.program.Version(), fullPath) + o.logger.Infof("downloaded binary '%s.%s' into '%s' as part of operation '%s'", o.program.BinaryName(), o.program.Version(), fullPath, o.Name()) } return err diff --git a/x-pack/elastic-agent/pkg/reporter/log/format.go b/x-pack/elastic-agent/pkg/reporter/log/format.go index dcb08f5b1c4..3cac93aa0cf 100644 --- a/x-pack/elastic-agent/pkg/reporter/log/format.go +++ b/x-pack/elastic-agent/pkg/reporter/log/format.go @@ -20,8 +20,8 @@ const ( ) const ( - // e.g "2006-01-02T15:04:05: type: 'STATE': event type: 'STARTING' message: Application 'filebeat' is starting." - defaultLogFormat = "%s: type: '%s': sub_type: '%s' message: %s" + // e.g "2006-01-02T15:04:05 - message: Application 'filebeat' is starting. - type: 'STATE' - event type: 'STARTING'" + defaultLogFormat = "%s - message: %s - type: '%s' - sub_type: '%s'" timeFormat = time.RFC3339 ) diff --git a/x-pack/elastic-agent/pkg/reporter/log/reporter.go b/x-pack/elastic-agent/pkg/reporter/log/reporter.go index 8651d7b800b..a202bd7cbda 100644 --- a/x-pack/elastic-agent/pkg/reporter/log/reporter.go +++ b/x-pack/elastic-agent/pkg/reporter/log/reporter.go @@ -47,9 +47,9 @@ func (r *Reporter) Close() error { return nil } func defaultFormatFunc(e reporter.Event) string { return fmt.Sprintf(defaultLogFormat, e.Time().Format(timeFormat), + e.Message(), e.Type(), e.SubType(), - e.Message(), ) } diff --git a/x-pack/elastic-agent/pkg/reporter/log/reporter_test.go b/x-pack/elastic-agent/pkg/reporter/log/reporter_test.go index f58518ca70e..7aa7362f1fa 100644 --- a/x-pack/elastic-agent/pkg/reporter/log/reporter_test.go +++ b/x-pack/elastic-agent/pkg/reporter/log/reporter_test.go @@ -94,5 +94,5 @@ func (testEvent) Payload() map[string]interface{} { return map[string]interface{ func DefaultString(event testEvent) string { timestamp := event.timestamp.Format(timeFormat) - return fmt.Sprintf("%s: type: '%s': sub_type: '%s' message: message", timestamp, event.Type(), event.SubType()) + return fmt.Sprintf("%s - message: message - type: '%s' - sub_type: '%s'", timestamp, event.Type(), event.SubType()) } From d09dfb01d8d97ae6fcfa025f8babcdc2dbb7123d Mon Sep 17 00:00:00 2001 From: Alex Resnick Date: Wed, 28 Apr 2021 10:58:24 -0500 Subject: [PATCH 11/11] Add ZooKeeper Module to Filebeat (#25128) Adds a new module for ZooKeeper audit and service logs. --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 87 +++ filebeat/docs/modules/zookeeper.asciidoc | 93 +++ filebeat/docs/modules_list.asciidoc | 2 + x-pack/filebeat/filebeat.reference.yml | 17 + x-pack/filebeat/include/list.go | 1 + .../module/zookeeper/_meta/config.yml | 15 + .../module/zookeeper/_meta/docs.asciidoc | 80 +++ .../module/zookeeper/_meta/fields.yml | 10 + .../module/zookeeper/audit/_meta/fields.yml | 30 + .../module/zookeeper/audit/config/audit.yml | 12 + .../zookeeper/audit/ingest/pipeline.yml | 101 +++ .../module/zookeeper/audit/manifest.yml | 17 + .../module/zookeeper/audit/test/audit.log | 23 + .../audit/test/audit.log-expected.json | 677 ++++++++++++++++++ x-pack/filebeat/module/zookeeper/fields.go | 23 + .../module/zookeeper/log/_meta/fields.yml | 6 + .../module/zookeeper/log/config/log.yml | 12 + .../module/zookeeper/log/ingest/pipeline.yml | 68 ++ .../module/zookeeper/log/manifest.yml | 13 + .../module/zookeeper/log/test/zookeeper.log | 34 + .../log/test/zookeeper.log-expected.json | 614 ++++++++++++++++ x-pack/filebeat/module/zookeeper/module.yml | 0 .../filebeat/modules.d/zookeeper.yml.disabled | 18 + 24 files changed, 1954 insertions(+) create mode 100644 filebeat/docs/modules/zookeeper.asciidoc create mode 100644 x-pack/filebeat/module/zookeeper/_meta/config.yml create mode 100644 x-pack/filebeat/module/zookeeper/_meta/docs.asciidoc create mode 100644 x-pack/filebeat/module/zookeeper/_meta/fields.yml create mode 100644 x-pack/filebeat/module/zookeeper/audit/_meta/fields.yml create mode 100644 x-pack/filebeat/module/zookeeper/audit/config/audit.yml create mode 100644 x-pack/filebeat/module/zookeeper/audit/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/zookeeper/audit/manifest.yml create mode 100644 x-pack/filebeat/module/zookeeper/audit/test/audit.log create mode 100644 x-pack/filebeat/module/zookeeper/audit/test/audit.log-expected.json create mode 100644 x-pack/filebeat/module/zookeeper/fields.go create mode 100644 x-pack/filebeat/module/zookeeper/log/_meta/fields.yml create mode 100644 x-pack/filebeat/module/zookeeper/log/config/log.yml create mode 100644 x-pack/filebeat/module/zookeeper/log/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/zookeeper/log/manifest.yml create mode 100644 x-pack/filebeat/module/zookeeper/log/test/zookeeper.log create mode 100644 x-pack/filebeat/module/zookeeper/log/test/zookeeper.log-expected.json create mode 100644 x-pack/filebeat/module/zookeeper/module.yml create mode 100644 x-pack/filebeat/modules.d/zookeeper.yml.disabled diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ed91b9a8f77..ad25a484fba 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -828,6 +828,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `awsfargate` module to collect container logs from Amazon ECS on Fargate. {pull}25041[25041] - New module `cyberarkpas` for CyberArk Privileged Access Security audit logs. {pull}24803[24803] - Add `uri_parts` processor to Apache, Nginx, IIS, Traefik, S3Access, Cisco, F5, Fortinet, Google Workspace, Imperva, Microsoft, Netscout, O365, Sophos, Squid, Suricata, Zeek, Zia, Zoom, and ZScaler modules ingest pipelines. {issue}19088[19088] {pull}24699[24699] +- New module `zookeeper` for Zookeeper service and audit logs {issue}25061[25061] {pull}25128[25128] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 31d8b4491d1..3d793f98ac9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -91,6 +91,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> @@ -164031,6 +164032,92 @@ type: boolean -- +[[exported-fields-zookeeper]] +== ZooKeeper fields + +ZooKeeper Module + + + +[float] +=== zookeeper + + + + +[float] +=== audit + +ZooKeeper Audit logs. + + + +*`zookeeper.audit.session`*:: ++ +-- +Client session id + + +type: keyword + +-- + +*`zookeeper.audit.znode`*:: ++ +-- +Path of the znode + + +type: keyword + +-- + +*`zookeeper.audit.znode_type`*:: ++ +-- +Type of znode in case of creation operation + + +type: keyword + +-- + +*`zookeeper.audit.acl`*:: ++ +-- +String representation of znode ACL like cdrwa(create, delete,read, write, admin). This is logged only for setAcl operation + + +type: keyword + +-- + +*`zookeeper.audit.result`*:: ++ +-- +Result of the operation. Possible values are (success/failure/invoked). Result "invoked" is used for serverStop operation because stop is logged before ensuring that server actually stopped. + + +type: keyword + +-- + +*`zookeeper.audit.user`*:: ++ +-- +Comma separated list of users who are associate with a client session + + +type: keyword + +-- + +[float] +=== log + +ZooKeeper logs. + + [[exported-fields-zoom]] == Zoom fields diff --git a/filebeat/docs/modules/zookeeper.asciidoc b/filebeat/docs/modules/zookeeper.asciidoc new file mode 100644 index 00000000000..ea6a3df0a24 --- /dev/null +++ b/filebeat/docs/modules/zookeeper.asciidoc @@ -0,0 +1,93 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-zookeeper]] +:modulename: zookeeper +:has-dashboards: false + +== ZooKeeper module + +The +{modulename}+ module collects and parses the logs created by https://zookeeper.apache.org/[Apache ZooKeeper] + +include::../include/what-happens.asciidoc[] + +include::../include/gs-link.asciidoc[] + +[float] +=== Compatibility + +The +{modulename}+ module was tested with logs from versions 3.7.0. + +include::../include/configuring-intro.asciidoc[] + +The following example shows how to set paths in the +modules.d/{modulename}.yml+ +file to override the default paths for logs: + +[source,yaml] +----- +- module: zookeeper + audit: + enabled: true + var.paths: + - "/path/to/logs/zookeeper_audit.log*" + log: + enabled: true + var.paths: + - "/path/to/logs/zookeeper.log*" +----- + + +To specify the same settings at the command line, you use: + +[source,yaml] +----- +-M "zookeeper.audit.var.paths=[/path/to/logs/zookeeper_audit.log*]" -M "zookeeper.log.var.paths=[/path/to/logs/zookeeper.log*]" +----- + +[float] +=== Audit logging + +Audit logging is available since Zookeeper 3.6.0, but it is disabled by default. To enable it, you can add the following setting to the configuration file: +["source","sh"] +---------------------- +audit.enable=true +---------------------- + +//set the fileset name used in the included example +:fileset_ex: audit + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `audit` fileset settings + +include::../include/var-paths.asciidoc[] + +include::../include/timezone-support.asciidoc[] + +:fileset_ex!: + +//set the fileset name used in the included example +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +include::../include/var-paths.asciidoc[] + +include::../include/timezone-support.asciidoc[] + +:fileset_ex!: + +:modulename!: + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 3e554cc6407..bb588001ee1 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -70,6 +70,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> @@ -144,5 +145,6 @@ include::modules/threatintel.asciidoc[] include::modules/tomcat.asciidoc[] include::modules/traefik.asciidoc[] include::modules/zeek.asciidoc[] +include::modules/zookeeper.asciidoc[] include::modules/zoom.asciidoc[] include::modules/zscaler.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 75a10511b47..0ee874e5c17 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2365,6 +2365,23 @@ filebeat.modules: # Filebeat will choose the paths depending on your OS. #var.paths: +#------------------------------ ZooKeeper Module ------------------------------ +- module: zookeeper + # All logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #--------------------------------- Zoom Module --------------------------------- - module: zoom webhook: diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 94340a8c11c..995cc2a7a0e 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -60,6 +60,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/filebeat/module/threatintel" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/tomcat" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zeek" + _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zookeeper" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zoom" _ "github.com/elastic/beats/v7/x-pack/filebeat/module/zscaler" _ "github.com/elastic/beats/v7/x-pack/filebeat/processors/add_nomad_metadata" diff --git a/x-pack/filebeat/module/zookeeper/_meta/config.yml b/x-pack/filebeat/module/zookeeper/_meta/config.yml new file mode 100644 index 00000000000..a31d217a5ec --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/_meta/config.yml @@ -0,0 +1,15 @@ +- module: zookeeper + # All logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/zookeeper/_meta/docs.asciidoc b/x-pack/filebeat/module/zookeeper/_meta/docs.asciidoc new file mode 100644 index 00000000000..2e680ce7b2d --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/_meta/docs.asciidoc @@ -0,0 +1,80 @@ +:modulename: zookeeper +:has-dashboards: false + +== ZooKeeper module + +The +{modulename}+ module collects and parses the logs created by https://zookeeper.apache.org/[Apache ZooKeeper] + +include::../include/what-happens.asciidoc[] + +include::../include/gs-link.asciidoc[] + +[float] +=== Compatibility + +The +{modulename}+ module was tested with logs from versions 3.7.0. + +include::../include/configuring-intro.asciidoc[] + +The following example shows how to set paths in the +modules.d/{modulename}.yml+ +file to override the default paths for logs: + +[source,yaml] +----- +- module: zookeeper + audit: + enabled: true + var.paths: + - "/path/to/logs/zookeeper_audit.log*" + log: + enabled: true + var.paths: + - "/path/to/logs/zookeeper.log*" +----- + + +To specify the same settings at the command line, you use: + +[source,yaml] +----- +-M "zookeeper.audit.var.paths=[/path/to/logs/zookeeper_audit.log*]" -M "zookeeper.log.var.paths=[/path/to/logs/zookeeper.log*]" +----- + +[float] +=== Audit logging + +Audit logging is available since Zookeeper 3.6.0, but it is disabled by default. To enable it, you can add the following setting to the configuration file: +["source","sh"] +---------------------- +audit.enable=true +---------------------- + +//set the fileset name used in the included example +:fileset_ex: audit + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `audit` fileset settings + +include::../include/var-paths.asciidoc[] + +include::../include/timezone-support.asciidoc[] + +:fileset_ex!: + +//set the fileset name used in the included example +:fileset_ex: log + +include::../include/config-option-intro.asciidoc[] + +[float] +==== `log` fileset settings + +include::../include/var-paths.asciidoc[] + +include::../include/timezone-support.asciidoc[] + +:fileset_ex!: + +:modulename!: diff --git a/x-pack/filebeat/module/zookeeper/_meta/fields.yml b/x-pack/filebeat/module/zookeeper/_meta/fields.yml new file mode 100644 index 00000000000..e9ba99f8d39 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: zookeeper + title: "ZooKeeper" + release: beta + description: > + ZooKeeper Module + fields: + - name: zookeeper + type: group + description: > + fields: diff --git a/x-pack/filebeat/module/zookeeper/audit/_meta/fields.yml b/x-pack/filebeat/module/zookeeper/audit/_meta/fields.yml new file mode 100644 index 00000000000..1bc5d6b79e5 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/_meta/fields.yml @@ -0,0 +1,30 @@ +- name: audit + type: group + description: > + ZooKeeper Audit logs. + release: beta + fields: + - name: session + type: keyword + description: > + Client session id + - name: znode + type: keyword + description: > + Path of the znode + - name: znode_type + type: keyword + description: > + Type of znode in case of creation operation + - name: acl + type: keyword + description: > + String representation of znode ACL like cdrwa(create, delete,read, write, admin). This is logged only for setAcl operation + - name: result + type: keyword + description: > + Result of the operation. Possible values are (success/failure/invoked). Result "invoked" is used for serverStop operation because stop is logged before ensuring that server actually stopped. + - name: user + type: keyword + description: > + Comma separated list of users who are associate with a client session diff --git a/x-pack/filebeat/module/zookeeper/audit/config/audit.yml b/x-pack/filebeat/module/zookeeper/audit/config/audit.yml new file mode 100644 index 00000000000..759bd4ed456 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/config/audit.yml @@ -0,0 +1,12 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +processors: + - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.9.0 diff --git a/x-pack/filebeat/module/zookeeper/audit/ingest/pipeline.yml b/x-pack/filebeat/module/zookeeper/audit/ingest/pipeline.yml new file mode 100644 index 00000000000..83ee590664d --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/ingest/pipeline.yml @@ -0,0 +1,101 @@ +description: Pipeline for parsing ZooKeeper audit messages +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: message + target_field: log.original + ignore_missing: true +- grok: + field: log.original + patterns: + - '%{TIMESTAMP_ISO8601:zookeeper.audit.timestamp}%{SPACE}%{LOGLEVEL:log.level}%{SPACE}%{CALLER_CLASS:log.logger}:%{SPACE}%{GREEDYDATA:message}' + pattern_definitions: + CALLER_CLASS: (%{JAVACLASS}|%{NOTSPACE}) +- rename: + field: '@timestamp' + target_field: event.created + if: ctx?.zookeeper?.audit?.timestamp != null +- date: + if: ctx?.zookeeper?.audit?.timestamp != null && ctx.event.timezone == null + field: zookeeper.audit.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx?.zookeeper?.audit?.timestamp != null && ctx.event.timezone != null + field: zookeeper.audit.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- kv: + field: message + field_split: "\\s+" + value_split: "=" + target_field: zookeeper.audit +- remove: + field: + - message + - zookeeper.audit.timestamp + ignore_missing: true +- rename: + field: zookeeper.audit.operation + target_field: event.action + ignore_missing: true +- set: + field: event.outcome + value: "{{zookeeper.audit.result}}" + ignore_empty_value: true + if: '["success","failure"].contains(ctx.zookeeper?.audit?.result)' +- rename: + field: zookeeper.audit.ip + target_field: client.address + ignore_missing: true +- convert: + field: client.address + target_field: client.ip + type: ip + ignore_missing: true +- geoip: + field: client.ip + target_field: client.geo + if: ctx?.client?.ip != null +- geoip: + database_file: GeoLite2-ASN.mmdb + field: client.ip + target_field: client.as + properties: + - asn + - organization_name + ignore_missing: true + if: ctx?.client?.ip != null +- set: + field: user.id + value: "{{zookeeper.audit.user}}" + ignore_empty_value: true +- split: + field: zookeeper.audit.user + separator: "," + ignore_missing: true +- append: + field: related.user + value: '{{user.id}}' + if: ctx?.user?.id != null +- append: + field: related.ip + value: '{{client.ip}}' + if: ctx?.client?.ip != null +on_failure: +- set: + field: error.log + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zookeeper/audit/manifest.yml b/x-pack/filebeat/module/zookeeper/audit/manifest.yml new file mode 100644 index 00000000000..1b86fede37d --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/manifest.yml @@ -0,0 +1,17 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/zookeeper/zookeeper_audit.log* + os.darwin: + - /var/log/zookeeper/zookeeper_audit.log* + os.windows: + - c:/programdata/zookeeper/zookeeper_audit.log* + +ingest_pipeline: ingest/pipeline.yml +input: config/audit.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/zookeeper/audit/test/audit.log b/x-pack/filebeat/module/zookeeper/audit/test/audit.log new file mode 100644 index 00000000000..81c5023693f --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/test/audit.log @@ -0,0 +1,23 @@ +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: user=zookeeper/192.168.1.3 operation=serverStart result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/a znode_type=persistent result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/a znode_type=persistent result=failure +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/a result=failure +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/a result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setAcl znode=/a acl=world:anyone:cdrwa result=failure +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setAcl znode=/a acl=world:anyone:cdrwa result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/b znode_type=persistent result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/b result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/b result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=multiOperation result=failure +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/a result=failure +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/a result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730001 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/ephemral znode_type=ephemral result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730001 user=zookeeper/192.168.1.3 operation=ephemeralZNodeDeletionOnSessionCloseOrExpire znode=/ephemral result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=reconfig znode=/zookeeper/config result=success +2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: user=zookeeper/192.168.1.3 operation=serverStop result=invoked +2021-04-27 11:51:32,074 INFO audit.Log4jAuditLogger: user=jaime operation=serverStart result=success +2021-04-27 11:52:05,689 INFO audit.Log4jAuditLogger: session=0x1004380147c0000 user=127.0.0.1 ip=127.0.0.1 operation=create znode=/a znode_type=persistent result=success +2021-04-27 11:53:56,256 INFO audit.Log4jAuditLogger: session=0x1004380147c0000 user=127.0.0.1 ip=127.0.0.1 operation=setData znode=/a result=success +2021-04-27 11:53:58,941 INFO audit.Log4jAuditLogger: session=0x1004380147c0000 user=127.0.0.1 ip=127.0.0.1 operation=setData znode=/a result=success +2021-04-27 11:55:01,453 INFO audit.Log4jAuditLogger: session=0x1004380147c0000 user=127.0.0.1 ip=127.0.0.1 operation=delete znode=/a result=success +2021-04-27 11:55:36,466 INFO audit.Log4jAuditLogger: user=jaime operation=serverStop result=invoked diff --git a/x-pack/filebeat/module/zookeeper/audit/test/audit.log-expected.json b/x-pack/filebeat/module/zookeeper/audit/test/audit.log-expected.json new file mode 100644 index 00000000000..2aa1db05271 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/audit/test/audit.log-expected.json @@ -0,0 +1,677 @@ +[ + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "event.action": "serverStart", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 0, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: user=zookeeper/192.168.1.3 operation=serverStart result=success", + "related.user": [ + "zookeeper/192.168.1.3" + ], + "service.type": "zookeeper", + "user.id": "zookeeper/192.168.1.3", + "zookeeper.audit.result": "success", + "zookeeper.audit.user": [ + "zookeeper/192.168.1.3" + ] + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "create", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 119, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/a znode_type=persistent result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a", + "zookeeper.audit.znode_type": "persistent" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "create", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 319, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/a znode_type=persistent result=failure", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "failure", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a", + "zookeeper.audit.znode_type": "persistent" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "setData", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 519, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/a result=failure", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "failure", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "setData", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 696, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/a result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "setAcl", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 873, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setAcl znode=/a acl=world:anyone:cdrwa result=failure", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.acl": "world:anyone:cdrwa", + "zookeeper.audit.result": "failure", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "setAcl", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 1074, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setAcl znode=/a acl=world:anyone:cdrwa result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.acl": "world:anyone:cdrwa", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "create", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 1275, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/b znode_type=persistent result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/b", + "zookeeper.audit.znode_type": "persistent" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "setData", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 1475, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=setData znode=/b result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/b" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "delete", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 1652, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/b result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/b" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "multiOperation", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 1829, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=multiOperation result=failure", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "failure", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ] + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "delete", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "failure", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2002, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/a result=failure", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "failure", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "delete", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2179, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=delete znode=/a result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "create", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2356, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730001 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=create znode=/ephemral znode_type=ephemral result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730001", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/ephemral", + "zookeeper.audit.znode_type": "ephemral" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "event.action": "ephemeralZNodeDeletionOnSessionCloseOrExpire", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2556, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730001 user=zookeeper/192.168.1.3 operation=ephemeralZNodeDeletionOnSessionCloseOrExpire znode=/ephemral result=success", + "related.user": [ + "zookeeper/192.168.1.3" + ], + "service.type": "zookeeper", + "user.id": "zookeeper/192.168.1.3", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730001", + "zookeeper.audit.user": [ + "zookeeper/192.168.1.3" + ], + "zookeeper.audit.znode": "/ephemral" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "client.address": "192.168.1.2", + "client.ip": "192.168.1.2", + "event.action": "reconfig", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2749, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: session=0x19344730000 user=192.168.1.2,zkcli@HADOOP.COM ip=192.168.1.2 operation=reconfig znode=/zookeeper/config result=success", + "related.ip": [ + "192.168.1.2" + ], + "related.user": [ + "192.168.1.2,zkcli@HADOOP.COM" + ], + "service.type": "zookeeper", + "user.id": "192.168.1.2,zkcli@HADOOP.COM", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x19344730000", + "zookeeper.audit.user": [ + "192.168.1.2", + "zkcli@HADOOP.COM" + ], + "zookeeper.audit.znode": "/zookeeper/config" + }, + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "event.action": "serverStop", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 2938, + "log.original": "2021-04-19 01:58:20,588 INFO audit.Log4jAuditLogger: user=zookeeper/192.168.1.3 operation=serverStop result=invoked", + "related.user": [ + "zookeeper/192.168.1.3" + ], + "service.type": "zookeeper", + "user.id": "zookeeper/192.168.1.3", + "zookeeper.audit.result": "invoked", + "zookeeper.audit.user": [ + "zookeeper/192.168.1.3" + ] + }, + { + "@timestamp": "2021-04-27T11:51:32.074-02:00", + "event.action": "serverStart", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3057, + "log.original": "2021-04-27 11:51:32,074 INFO audit.Log4jAuditLogger: user=jaime\toperation=serverStart\tresult=success", + "related.user": [ + "jaime" + ], + "service.type": "zookeeper", + "user.id": "jaime", + "zookeeper.audit.result": "success", + "zookeeper.audit.user": [ + "jaime" + ] + }, + { + "@timestamp": "2021-04-27T11:52:05.689-02:00", + "client.address": "127.0.0.1", + "client.ip": "127.0.0.1", + "event.action": "create", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3158, + "log.original": "2021-04-27 11:52:05,689 INFO audit.Log4jAuditLogger: session=0x1004380147c0000\tuser=127.0.0.1\tip=127.0.0.1\toperation=create\tznode=/a\tznode_type=persistent\tresult=success", + "related.ip": [ + "127.0.0.1" + ], + "related.user": [ + "127.0.0.1" + ], + "service.type": "zookeeper", + "user.id": "127.0.0.1", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x1004380147c0000", + "zookeeper.audit.user": [ + "127.0.0.1" + ], + "zookeeper.audit.znode": "/a", + "zookeeper.audit.znode_type": "persistent" + }, + { + "@timestamp": "2021-04-27T11:53:56.256-02:00", + "client.address": "127.0.0.1", + "client.ip": "127.0.0.1", + "event.action": "setData", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3328, + "log.original": "2021-04-27 11:53:56,256 INFO audit.Log4jAuditLogger: session=0x1004380147c0000\tuser=127.0.0.1\tip=127.0.0.1\toperation=setData\tznode=/a\tresult=success", + "related.ip": [ + "127.0.0.1" + ], + "related.user": [ + "127.0.0.1" + ], + "service.type": "zookeeper", + "user.id": "127.0.0.1", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x1004380147c0000", + "zookeeper.audit.user": [ + "127.0.0.1" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-27T11:53:58.941-02:00", + "client.address": "127.0.0.1", + "client.ip": "127.0.0.1", + "event.action": "setData", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3477, + "log.original": "2021-04-27 11:53:58,941 INFO audit.Log4jAuditLogger: session=0x1004380147c0000\tuser=127.0.0.1\tip=127.0.0.1\toperation=setData\tznode=/a\tresult=success", + "related.ip": [ + "127.0.0.1" + ], + "related.user": [ + "127.0.0.1" + ], + "service.type": "zookeeper", + "user.id": "127.0.0.1", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x1004380147c0000", + "zookeeper.audit.user": [ + "127.0.0.1" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-27T11:55:01.453-02:00", + "client.address": "127.0.0.1", + "client.ip": "127.0.0.1", + "event.action": "delete", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.outcome": "success", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3626, + "log.original": "2021-04-27 11:55:01,453 INFO audit.Log4jAuditLogger: session=0x1004380147c0000\tuser=127.0.0.1\tip=127.0.0.1\toperation=delete\tznode=/a\tresult=success", + "related.ip": [ + "127.0.0.1" + ], + "related.user": [ + "127.0.0.1" + ], + "service.type": "zookeeper", + "user.id": "127.0.0.1", + "zookeeper.audit.result": "success", + "zookeeper.audit.session": "0x1004380147c0000", + "zookeeper.audit.user": [ + "127.0.0.1" + ], + "zookeeper.audit.znode": "/a" + }, + { + "@timestamp": "2021-04-27T11:55:36.466-02:00", + "event.action": "serverStop", + "event.dataset": "zookeeper.audit", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "fileset.name": "audit", + "input.type": "log", + "log.level": "INFO", + "log.logger": "audit.Log4jAuditLogger", + "log.offset": 3774, + "log.original": "2021-04-27 11:55:36,466 INFO audit.Log4jAuditLogger: user=jaime\toperation=serverStop\tresult=invoked", + "related.user": [ + "jaime" + ], + "service.type": "zookeeper", + "user.id": "jaime", + "zookeeper.audit.result": "invoked", + "zookeeper.audit.user": [ + "jaime" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zookeeper/fields.go b/x-pack/filebeat/module/zookeeper/fields.go new file mode 100644 index 00000000000..478dd3ed932 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package zookeeper + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "zookeeper", asset.ModuleFieldsPri, AssetZookeeper); err != nil { + panic(err) + } +} + +// AssetZookeeper returns asset data. +// This is the base64 encoded gzipped contents of module/zookeeper. +func AssetZookeeper() string { + return "eJysVE2L2zAUvOdXDDntwm727kMh5NgWlu6eeikv0iQWUSSjJyV4f32xG+erbkvB72TmSTPzxvZ7xo5thY8Yd2TDNAOyy54V5t9j/Nxj8xmQ6CnKCmtmmQGWapJrsouhwqcZAJzP42u0xXMGbBy91apvPyPInrdSXeW2YYVtiqU5ISPct1TXdFKsy2d0jO6PlL/qYnvZUcHHrS6uTtxPPtS9oWtTSlUXw01vsLZje4zJ3vX+YrCrlXcMeeCFs6OyHyFaTif6KrlG3CDXHKG+Ef3RyUyn/N427JR7brgAI9oDJlG6S4gNU/806kmMn87MW04ubJHYJCpDPhkY3C1XX+DdjjA2HeWhd8gnWHpmPiWKfcIxuQ4Tu3fhcYH32imcdp/alhYx+BabmKDMS+P/MVuiFp+nG+9bzze857P2Aq9R1a09cRBfqJBEPGgxhqovG3G+JL64cIg72sfFwDM/IfNuwKK0p8nSgektx+aigDWNFCW0gy9xrLmJiWDQ0gefa8knAojJRbxv+zsN7WI0oqLn7TJBQKu43wuUjSTJtPBO+7g6FcWxjn0yohqNk0wcXa4hMDe/7G97y8ftNFvrf/fVzwAAAP//jcqrBQ==" +} diff --git a/x-pack/filebeat/module/zookeeper/log/_meta/fields.yml b/x-pack/filebeat/module/zookeeper/log/_meta/fields.yml new file mode 100644 index 00000000000..820d0edfc77 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/_meta/fields.yml @@ -0,0 +1,6 @@ +- name: log + type: group + description: > + ZooKeeper logs. + release: beta + fields: diff --git a/x-pack/filebeat/module/zookeeper/log/config/log.yml b/x-pack/filebeat/module/zookeeper/log/config/log.yml new file mode 100644 index 00000000000..759bd4ed456 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/config/log.yml @@ -0,0 +1,12 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] +processors: + - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.9.0 diff --git a/x-pack/filebeat/module/zookeeper/log/ingest/pipeline.yml b/x-pack/filebeat/module/zookeeper/log/ingest/pipeline.yml new file mode 100644 index 00000000000..8a42ca81f75 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/ingest/pipeline.yml @@ -0,0 +1,68 @@ +description: Pipeline for parsing ZooKeeper log messages +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: message + target_field: log.original +- grok: + field: log.original + trace_match: true + patterns: + - '%{TIMESTAMP_ISO8601:zookeeper.log.timestamp}%{SPACE}\[myid:(%{POSINT})?\]%{SPACE}-%{SPACE}%{LOGLEVEL:log.level}%{SPACE}\[%{DATA:zookeeper.log.process}\]%{SPACE}-%{SPACE}%{GREEDYDATA:message}' +- grok: + field: zookeeper.log.process + ignore_missing: true + ignore_failure: true + patterns: + - '%{GREEDYDATA:process.thread.name}:%{CALLER_CLASS:log.logger}@%{POSINT:log.origin.file.line}' + pattern_definitions: + CALLER_CLASS: (%{JAVACLASS}|%{NOTSPACE}) +- remove: + field: zookeeper.log.process + ignore_missing: true +- rename: + field: '@timestamp' + target_field: event.created +- date: + if: ctx.event.timezone == null + field: zookeeper.log.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss,SSS + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- date: + if: ctx.event.timezone != null + field: zookeeper.log.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd HH:mm:ss,SSS + timezone: '{{ event.timezone }}' + on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' +- remove: + field: zookeeper.log.timestamp +- set: + field: event.kind + value: event +- script: + lang: painless + source: >- + def errorLevels = ["ERROR", "FATAL"]; + if (ctx?.log?.level != null) { + if (errorLevels.contains(ctx.log.level)) { + ctx.event.type = "error"; + } else { + ctx.event.type = "info"; + } + } +on_failure: +- set: + field: error.log + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/zookeeper/log/manifest.yml b/x-pack/filebeat/module/zookeeper/log/manifest.yml new file mode 100644 index 00000000000..bef2a214aff --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/manifest.yml @@ -0,0 +1,13 @@ +module_version: 1.0 + +var: + - name: paths + default: + - /var/log/zookeeper/zookeeper.log* + os.darwin: + - /var/log/zookeeper/zookeeper.log* + os.windows: + - c:/programdata/zookeeper/zookeeper.log* + +ingest_pipeline: ingest/pipeline.yml +input: config/log.yml diff --git a/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log b/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log new file mode 100644 index 00000000000..429265121a6 --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log @@ -0,0 +1,34 @@ +2021-04-19 01:58:20,588 [myid:] - INFO [main:QuorumPeerConfig@117] - Reading configuration from: /app/zookeeper/bin/../conf/zoo.cfg +2021-04-19 01:58:20,608 [myid:] - INFO [main:QuorumPeerConfig@327] - clientPortAddress is 0.0.0.0/0.0.0.0:2181 +2021-04-19 01:58:20,609 [myid:] - INFO [main:QuorumPeerConfig@331] - secureClientPort is not set +2021-04-19 01:58:20,620 [myid:] - WARN [main:QuorumPeerConfig@590] - No server failure will be tolerated. You need at least 3 servers. +2021-04-19 01:58:20,645 [myid:1] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3 +2021-04-19 01:58:20,645 [myid:1] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 1 +2021-04-19 01:58:20,650 [myid:1] - INFO [main:ManagedUtil@46] - Log4j found with jmx enabled. +2021-04-19 01:58:20,653 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@138] - Purge task started. +2021-04-19 01:58:20,686 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@144] - Purge task completed. +2021-04-19 01:58:20,729 [myid:1] - INFO [main:QuorumPeerMain@138] - Starting quorum peer +2021-04-19 01:58:21,411 [myid:1] - INFO [main:JettyAdminServer@112] - Started AdminServer on address 0.0.0.0, port 18081 and command URL /commands +2021-04-19 01:58:21,453 [myid:1] - INFO [QuorumPeerListener:QuorumCnxManager$Listener@636] - My election bind port: 0.0.0.0/0.0.0.0:3888 +2021-04-19 01:58:21,482 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):QuorumPeer@1055] - LOOKING +2021-04-19 01:58:21,484 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):FastLeaderElection@894] - New election. My id = 1, proposed zxid=0x0 +2021-04-19 01:58:21,488 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@688] - Notification: 2 (message format version), 1 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEPoch), LOOKING (my state)0 (n.config version) +2021-04-19 01:58:21,691 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):MBeanRegistry@128] - Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=LeaderElection] +2021-04-19 01:58:21,691 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):QuorumPeer@1143] - LEADING +2021-04-19 01:58:21,698 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@63] - TCP NoDelay set to: true +2021-04-19 01:58:21,698 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@83] - zookeeper.leader.maxConcurrentSnapshots = 10 +2021-04-19 01:58:21,701 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@85] - zookeeper.leader.maxConcurrentSnapshotTimeout = 5 +2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:zookeeper.version=3.5.3-beta-8ce24f9e675cbefffb8f21a47e06b42864475a60, built on 04/03/2017 16:19 GMT +2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:host.name=ip-192-168-40-10.ec2.internal +2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:java.version=1.8.0_144 +2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:java.vendor=Oracle Corporation +2021-04-19 01:58:31,148 [myid:1] - INFO [NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:2181:NIOServerCnxnFactory$AcceptThread@296] - Accepted socket connection from /172.17.0.2:55730 +2021-04-19 01:58:31,149 [myid:1] - INFO [NIOWorkerThread-2:ZooKeeperServer@1013] - Client attempting to establish new session at /172.17.0.2:55730 +2021-04-19 01:58:31,152 [myid:1] - INFO [CommitProcWorkThread-4:ZooKeeperServer@727] - Established session 0x10000318e5f0003 with negotiated timeout 15000 for client /172.17.0.2:55730 +2021-04-19 01:58:31,157 [myid:1] - INFO [NIOWorkerThread-4:ZooKeeperServer@1050] - got auth packet /172.17.0.2:55730 +2021-04-19 01:58:31,157 [myid:1] - INFO [NIOWorkerThread-4:ZooKeeperServer@1068] - auth success /172.17.0.2:55730 +2021-04-19 02:10:20,448 [myid:1] - INFO [ProcessThread(sid:1 cport:-1)::PrepRequestProcessor@613] - Processed session termination for sessionid: 0x10000318e5f0020 +2021-04-19 03:05:03,964 [myid:1] - WARN [NIOWorkerThread-7:NIOServerCnxn@366] - Unable to read additional data from client sessionid 0x10000318e5f0022, likely client has closed socket +2021-04-19 03:05:03,965 [myid:1] - INFO [NIOWorkerThread-7:MBeanRegistry@128] - Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=Leader,name3=Connections,name4=172.17.0.2,name5=0x10000318e5f0022] +2021-04-19 03:05:03,965 [myid:1] - INFO [NIOWorkerThread-7:NIOServerCnxn@627] - Closed socket connection for client /172.17.0.2:58658 which had sessionid 0x10000318e5f0022 +2021-04-19 03:05:14,058 [myid:1] - INFO [SessionTracker:ZooKeeperServer@391] - Expiring session 0x10000318e5f0022, timeout of 10000ms exceeded diff --git a/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log-expected.json b/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log-expected.json new file mode 100644 index 00000000000..3d83f7f56cd --- /dev/null +++ b/x-pack/filebeat/module/zookeeper/log/test/zookeeper.log-expected.json @@ -0,0 +1,614 @@ +[ + { + "@timestamp": "2021-04-19T01:58:20.588-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeerConfig", + "log.offset": 0, + "log.origin.file.line": "117", + "log.original": "2021-04-19 01:58:20,588 [myid:] - INFO [main:QuorumPeerConfig@117] - Reading configuration from: /app/zookeeper/bin/../conf/zoo.cfg", + "message": "Reading configuration from: /app/zookeeper/bin/../conf/zoo.cfg", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.608-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeerConfig", + "log.offset": 133, + "log.origin.file.line": "327", + "log.original": "2021-04-19 01:58:20,608 [myid:] - INFO [main:QuorumPeerConfig@327] - clientPortAddress is 0.0.0.0/0.0.0.0:2181", + "message": "clientPortAddress is 0.0.0.0/0.0.0.0:2181", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.609-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeerConfig", + "log.offset": 245, + "log.origin.file.line": "331", + "log.original": "2021-04-19 01:58:20,609 [myid:] - INFO [main:QuorumPeerConfig@331] - secureClientPort is not set", + "message": "secureClientPort is not set", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.620-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "WARN", + "log.logger": "QuorumPeerConfig", + "log.offset": 343, + "log.origin.file.line": "590", + "log.original": "2021-04-19 01:58:20,620 [myid:] - WARN [main:QuorumPeerConfig@590] - No server failure will be tolerated. You need at least 3 servers.", + "message": "No server failure will be tolerated. You need at least 3 servers.", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.645-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "DatadirCleanupManager", + "log.offset": 479, + "log.origin.file.line": "78", + "log.original": "2021-04-19 01:58:20,645 [myid:1] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3", + "message": "autopurge.snapRetainCount set to 3", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.645-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "DatadirCleanupManager", + "log.offset": 589, + "log.origin.file.line": "79", + "log.original": "2021-04-19 01:58:20,645 [myid:1] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 1", + "message": "autopurge.purgeInterval set to 1", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.650-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ManagedUtil", + "log.offset": 697, + "log.origin.file.line": "46", + "log.original": "2021-04-19 01:58:20,650 [myid:1] - INFO [main:ManagedUtil@46] - Log4j found with jmx enabled.", + "message": "Log4j found with jmx enabled.", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.653-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "DatadirCleanupManager$PurgeTask", + "log.offset": 792, + "log.origin.file.line": "138", + "log.original": "2021-04-19 01:58:20,653 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@138] - Purge task started.", + "message": "Purge task started.", + "process.thread.name": "PurgeTask", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.686-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "DatadirCleanupManager$PurgeTask", + "log.offset": 903, + "log.origin.file.line": "144", + "log.original": "2021-04-19 01:58:20,686 [myid:1] - INFO [PurgeTask:DatadirCleanupManager$PurgeTask@144] - Purge task completed.", + "message": "Purge task completed.", + "process.thread.name": "PurgeTask", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:20.729-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeerMain", + "log.offset": 1016, + "log.origin.file.line": "138", + "log.original": "2021-04-19 01:58:20,729 [myid:1] - INFO [main:QuorumPeerMain@138] - Starting quorum peer", + "message": "Starting quorum peer", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.411-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "JettyAdminServer", + "log.offset": 1106, + "log.origin.file.line": "112", + "log.original": "2021-04-19 01:58:21,411 [myid:1] - INFO [main:JettyAdminServer@112] - Started AdminServer on address 0.0.0.0, port 18081 and command URL /commands", + "message": "Started AdminServer on address 0.0.0.0, port 18081 and command URL /commands", + "process.thread.name": "main", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.453-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumCnxManager$Listener", + "log.offset": 1254, + "log.origin.file.line": "636", + "log.original": "2021-04-19 01:58:21,453 [myid:1] - INFO [QuorumPeerListener:QuorumCnxManager$Listener@636] - My election bind port: 0.0.0.0/0.0.0.0:3888", + "message": "My election bind port: 0.0.0.0/0.0.0.0:3888", + "process.thread.name": "QuorumPeerListener", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.482-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeer", + "log.offset": 1392, + "log.origin.file.line": "1055", + "log.original": "2021-04-19 01:58:21,482 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):QuorumPeer@1055] - LOOKING", + "message": "LOOKING", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.484-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "FastLeaderElection", + "log.offset": 1518, + "log.origin.file.line": "894", + "log.original": "2021-04-19 01:58:21,484 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):FastLeaderElection@894] - New election. My id = 1, proposed zxid=0x0", + "message": "New election. My id = 1, proposed zxid=0x0", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.488-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "FastLeaderElection", + "log.offset": 1687, + "log.origin.file.line": "688", + "log.original": "2021-04-19 01:58:21,488 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@688] - Notification: 2 (message format version), 1 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEPoch), LOOKING (my state)0 (n.config version)", + "message": "Notification: 2 (message format version), 1 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEPoch), LOOKING (my state)0 (n.config version)", + "process.thread.name": "WorkerReceiver[myid=1]", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.691-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "MBeanRegistry", + "log.offset": 1951, + "log.origin.file.line": "128", + "log.original": "2021-04-19 01:58:21,691 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):MBeanRegistry@128] - Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=LeaderElection]", + "message": "Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=LeaderElection]", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.691-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "QuorumPeer", + "log.offset": 2182, + "log.origin.file.line": "1143", + "log.original": "2021-04-19 01:58:21,691 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):QuorumPeer@1143] - LEADING", + "message": "LEADING", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.698-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Leader", + "log.offset": 2308, + "log.origin.file.line": "63", + "log.original": "2021-04-19 01:58:21,698 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@63] - TCP NoDelay set to: true", + "message": "TCP NoDelay set to: true", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.698-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Leader", + "log.offset": 2445, + "log.origin.file.line": "83", + "log.original": "2021-04-19 01:58:21,698 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@83] - zookeeper.leader.maxConcurrentSnapshots = 10", + "message": "zookeeper.leader.maxConcurrentSnapshots = 10", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.701-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Leader", + "log.offset": 2602, + "log.origin.file.line": "85", + "log.original": "2021-04-19 01:58:21,701 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Leader@85] - zookeeper.leader.maxConcurrentSnapshotTimeout = 5", + "message": "zookeeper.leader.maxConcurrentSnapshotTimeout = 5", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.717-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Environment", + "log.offset": 2764, + "log.origin.file.line": "109", + "log.original": "2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:zookeeper.version=3.5.3-beta-8ce24f9e675cbefffb8f21a47e06b42864475a60, built on 04/03/2017 16:19 GMT", + "message": "Server environment:zookeeper.version=3.5.3-beta-8ce24f9e675cbefffb8f21a47e06b42864475a60, built on 04/03/2017 16:19 GMT", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.717-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Environment", + "log.offset": 3002, + "log.origin.file.line": "109", + "log.original": "2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:host.name=ip-192-168-40-10.ec2.internal", + "message": "Server environment:host.name=ip-192-168-40-10.ec2.internal", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.717-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Environment", + "log.offset": 3179, + "log.origin.file.line": "109", + "log.original": "2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:java.version=1.8.0_144", + "message": "Server environment:java.version=1.8.0_144", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:21.717-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "Environment", + "log.offset": 3339, + "log.origin.file.line": "109", + "log.original": "2021-04-19 01:58:21,717 [myid:1] - INFO [QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled):Environment@109] - Server environment:java.vendor=Oracle Corporation", + "message": "Server environment:java.vendor=Oracle Corporation", + "process.thread.name": "QuorumPeer[myid=1](plain=/0.0.0.0:2181)(secure=disabled)", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:31.148-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "NIOServerCnxnFactory$AcceptThread", + "log.offset": 3507, + "log.origin.file.line": "296", + "log.original": "2021-04-19 01:58:31,148 [myid:1] - INFO [NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:2181:NIOServerCnxnFactory$AcceptThread@296] - Accepted socket connection from /172.17.0.2:55730", + "message": "Accepted socket connection from /172.17.0.2:55730", + "process.thread.name": "NIOServerCxnFactory.AcceptThread:0.0.0.0/0.0.0.0:2181", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:31.149-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ZooKeeperServer", + "log.offset": 3694, + "log.origin.file.line": "1013", + "log.original": "2021-04-19 01:58:31,149 [myid:1] - INFO [NIOWorkerThread-2:ZooKeeperServer@1013] - Client attempting to establish new session at /172.17.0.2:55730", + "message": "Client attempting to establish new session at /172.17.0.2:55730", + "process.thread.name": "NIOWorkerThread-2", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:31.152-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ZooKeeperServer", + "log.offset": 3842, + "log.origin.file.line": "727", + "log.original": "2021-04-19 01:58:31,152 [myid:1] - INFO [CommitProcWorkThread-4:ZooKeeperServer@727] - Established session 0x10000318e5f0003 with negotiated timeout 15000 for client /172.17.0.2:55730", + "message": "Established session 0x10000318e5f0003 with negotiated timeout 15000 for client /172.17.0.2:55730", + "process.thread.name": "CommitProcWorkThread-4", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:31.157-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ZooKeeperServer", + "log.offset": 4027, + "log.origin.file.line": "1050", + "log.original": "2021-04-19 01:58:31,157 [myid:1] - INFO [NIOWorkerThread-4:ZooKeeperServer@1050] - got auth packet /172.17.0.2:55730", + "message": "got auth packet /172.17.0.2:55730", + "process.thread.name": "NIOWorkerThread-4", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T01:58:31.157-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ZooKeeperServer", + "log.offset": 4145, + "log.origin.file.line": "1068", + "log.original": "2021-04-19 01:58:31,157 [myid:1] - INFO [NIOWorkerThread-4:ZooKeeperServer@1068] - auth success /172.17.0.2:55730", + "message": "auth success /172.17.0.2:55730", + "process.thread.name": "NIOWorkerThread-4", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T02:10:20.448-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "PrepRequestProcessor", + "log.offset": 4260, + "log.origin.file.line": "613", + "log.original": "2021-04-19 02:10:20,448 [myid:1] - INFO [ProcessThread(sid:1 cport:-1)::PrepRequestProcessor@613] - Processed session termination for sessionid: 0x10000318e5f0020", + "message": "Processed session termination for sessionid: 0x10000318e5f0020", + "process.thread.name": "ProcessThread(sid:1 cport:-1):", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T03:05:03.964-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "WARN", + "log.logger": "NIOServerCnxn", + "log.offset": 4424, + "log.origin.file.line": "366", + "log.original": "2021-04-19 03:05:03,964 [myid:1] - WARN [NIOWorkerThread-7:NIOServerCnxn@366] - Unable to read additional data from client sessionid 0x10000318e5f0022, likely client has closed socket", + "message": "Unable to read additional data from client sessionid 0x10000318e5f0022, likely client has closed socket", + "process.thread.name": "NIOWorkerThread-7", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T03:05:03.965-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "MBeanRegistry", + "log.offset": 4609, + "log.origin.file.line": "128", + "log.original": "2021-04-19 03:05:03,965 [myid:1] - INFO [NIOWorkerThread-7:MBeanRegistry@128] - Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=Leader,name3=Connections,name4=172.17.0.2,name5=0x10000318e5f0022]", + "message": "Unregister MBean [org.apache.ZooKeeperService:name0=ReplicatedServer_id1,name1=replica.1,name2=Leader,name3=Connections,name4=172.17.0.2,name5=0x10000318e5f0022]", + "process.thread.name": "NIOWorkerThread-7", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T03:05:03.965-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "NIOServerCnxn", + "log.offset": 4852, + "log.origin.file.line": "627", + "log.original": "2021-04-19 03:05:03,965 [myid:1] - INFO [NIOWorkerThread-7:NIOServerCnxn@627] - Closed socket connection for client /172.17.0.2:58658 which had sessionid 0x10000318e5f0022", + "message": "Closed socket connection for client /172.17.0.2:58658 which had sessionid 0x10000318e5f0022", + "process.thread.name": "NIOWorkerThread-7", + "service.type": "zookeeper" + }, + { + "@timestamp": "2021-04-19T03:05:14.058-02:00", + "event.dataset": "zookeeper.log", + "event.kind": "event", + "event.module": "zookeeper", + "event.timezone": "-02:00", + "event.type": "info", + "fileset.name": "log", + "input.type": "log", + "log.level": "INFO", + "log.logger": "ZooKeeperServer", + "log.offset": 5025, + "log.origin.file.line": "391", + "log.original": "2021-04-19 03:05:14,058 [myid:1] - INFO [SessionTracker:ZooKeeperServer@391] - Expiring session 0x10000318e5f0022, timeout of 10000ms exceeded", + "message": "Expiring session 0x10000318e5f0022, timeout of 10000ms exceeded", + "process.thread.name": "SessionTracker", + "service.type": "zookeeper" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/zookeeper/module.yml b/x-pack/filebeat/module/zookeeper/module.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/filebeat/modules.d/zookeeper.yml.disabled b/x-pack/filebeat/modules.d/zookeeper.yml.disabled new file mode 100644 index 00000000000..34273eacff4 --- /dev/null +++ b/x-pack/filebeat/modules.d/zookeeper.yml.disabled @@ -0,0 +1,18 @@ +# Module: zookeeper +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-zookeeper.html + +- module: zookeeper + # All logs + audit: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + # All logs + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: