From c7bc53023288c7d34afd631c0c3e3552f6219419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chema=20Mart=C3=ADnez?= Date: Sat, 27 Apr 2024 08:57:55 +0200 Subject: [PATCH] [zscaler_zia] Fix mapping of source.ip and source.nat.ip (#9727) * Fix mapping of source.ip and source.nat.ip * Update changelog * updated web datastream pipeline tests --------- Co-authored-by: Shourie Ganguly --- .../docker/sample_logs/web-http_endpoint.log | 2 +- packages/zscaler_zia/changelog.yml | 5 ++ .../test-web-http-endpoint.log-expected.json | 16 ++-- .../test/pipeline/test-web.log-expected.json | 88 +++++++------------ .../elasticsearch/ingest_pipeline/default.yml | 8 +- .../data_stream/web/sample_event.json | 6 +- packages/zscaler_zia/docs/README.md | 6 +- packages/zscaler_zia/manifest.yml | 2 +- 8 files changed, 58 insertions(+), 75 deletions(-) diff --git a/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log b/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log index 074c9a963b8..2e5fefd09d8 100644 --- a/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log +++ b/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log @@ -1 +1 @@ -{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"test@example.com","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"1.128.3.4","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} +{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"test@example.com","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"192.168.1.35","cintip":"203.0.113.5","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}} diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml index 639a2331ac1..90607aefdcc 100644 --- a/packages/zscaler_zia/changelog.yml +++ b/packages/zscaler_zia/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.19.1" + changes: + - description: Fix mapping of source.ip and source.nat.ip + type: bugfix + link: https://github.com/elastic/integrations/pull/9727 - version: "2.19.0" changes: - description: Set sensitive values as secret. diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json index 475a6756a69..3710a7395d0 100644 --- a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json +++ b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json @@ -43,8 +43,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.145" + "81.2.69.145", + "81.2.69.193" ], "user": [ "test", @@ -57,9 +57,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -169,8 +167,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.145" + "81.2.69.145", + "81.2.69.193" ], "user": [ "test", @@ -183,9 +181,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json index 25bbff8a1a5..dadc3e9b71e 100644 --- a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json +++ b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json @@ -43,8 +43,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.145" + "81.2.69.145", + "81.2.69.193" ], "user": [ "test", @@ -57,9 +57,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -171,8 +169,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "89.160.20.156" + "89.160.20.156", + "81.2.69.193" ], "user": [ "test", @@ -185,9 +183,7 @@ "ruleset": "SSLPol" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -294,8 +290,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "89.160.20.112" + "89.160.20.112", + "81.2.69.193" ], "user": [ "test", @@ -308,9 +304,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -422,8 +416,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.144" + "81.2.69.144", + "81.2.69.193" ], "user": [ "test", @@ -436,9 +430,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -550,8 +542,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test", @@ -564,9 +556,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -674,8 +664,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "89.160.20.112" + "89.160.20.112", + "81.2.69.193" ], "user": [ "test", @@ -688,9 +678,7 @@ "ruleset": "None" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -802,8 +790,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test", @@ -816,9 +804,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -930,8 +916,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test" @@ -942,9 +928,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -1050,8 +1034,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test", @@ -1063,9 +1047,7 @@ "ruleset": "FwFilter" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -1173,8 +1155,8 @@ "TestMachine35" ], "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test", @@ -1187,9 +1169,7 @@ "ruleset": "None" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" @@ -1289,8 +1269,8 @@ }, "related": { "ip": [ - "81.2.69.193", - "81.2.69.143" + "81.2.69.143", + "81.2.69.193" ], "user": [ "test", @@ -1302,9 +1282,7 @@ "ruleset": "None" }, "source": { - "nat": { - "ip": "81.2.69.193" - } + "ip": "81.2.69.193" }, "tags": [ "preserve_original_event" diff --git a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml index d0df2ee03b9..1614baadd69 100644 --- a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml +++ b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml @@ -78,14 +78,14 @@ processors: field: event.type value: info - convert: - field: json.cip + field: json.cintip target_field: source.nat.ip if: ctx.json?.cip != ctx.json?.cintip type: ip ignore_missing: true on_failure: - remove: - field: json.cip + field: json.cintip - append: field: error.message value: '{{{_ingest.on_failure_message}}}' @@ -364,13 +364,13 @@ processors: target_field: zscaler_zia.web.bandwidth_throttle ignore_missing: true - convert: - field: json.cintip + field: json.cip target_field: source.ip type: ip ignore_missing: true on_failure: - remove: - field: json.cintip + field: json.cip - append: field: error.message value: '{{{_ingest.on_failure_message}}}' diff --git a/packages/zscaler_zia/data_stream/web/sample_event.json b/packages/zscaler_zia/data_stream/web/sample_event.json index 93e83150c5e..51a3b40b75e 100644 --- a/packages/zscaler_zia/data_stream/web/sample_event.json +++ b/packages/zscaler_zia/data_stream/web/sample_event.json @@ -63,6 +63,7 @@ "TestMachine35" ], "ip": [ + "203.0.113.5", "1.128.3.4" ], "user": [ @@ -76,8 +77,9 @@ }, "source": { "nat": { - "ip": "1.128.3.4" - } + "ip": "203.0.113.5" + }, + "ip": "192.168.1.35" }, "tags": [ "forwarded", diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md index c0917efb2f9..caf67bad596 100644 --- a/packages/zscaler_zia/docs/README.md +++ b/packages/zscaler_zia/docs/README.md @@ -767,6 +767,7 @@ An example event for `web` looks as following: "TestMachine35" ], "ip": [ + "203.0.113.5", "1.128.3.4" ], "user": [ @@ -780,8 +781,9 @@ An example event for `web` looks as following: }, "source": { "nat": { - "ip": "1.128.3.4" - } + "ip": "203.0.113.5" + }, + "ip": "192.168.1.35" }, "tags": [ "forwarded", diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml index 9ff2e157ec1..abba8599039 100644 --- a/packages/zscaler_zia/manifest.yml +++ b/packages/zscaler_zia/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: zscaler_zia title: Zscaler Internet Access -version: "2.19.0" +version: "2.19.1" description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent. type: integration categories: