[zscaler_zia] Fix mapping of source.ip and source.nat.ip (elastic#9727)

* Fix mapping of source.ip and source.nat.ip * Update changelog * updated web datastream pipeline tests --------- Co-authored-by: Shourie Ganguly <[email protected]>
milan-elastic · Apr 27, 2024 · c7bc530 · c7bc530
1 parent 4750ea8
commit c7bc530
Show file tree

Hide file tree

Showing 8 changed files with 58 additions and 75 deletions.
diff --git a/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log b/packages/zscaler_zia/_dev/deploy/docker/sample_logs/web-http_endpoint.log
@@ -1 +1 @@
-{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"[email protected]","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"1.128.3.4","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}
+{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"[email protected]","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"192.168.1.35","cintip":"203.0.113.5","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}
diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.19.1"
+  changes:
+    - description: Fix mapping of source.ip and source.nat.ip
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/9727
 - version: "2.19.0"
   changes:
     - description: Set sensitive values as secret.

diff --git a/...s/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json b/...s/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json
@@ -43,8 +43,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.145"
+                    "81.2.69.145",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -57,9 +57,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -169,8 +167,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.145"
+                    "81.2.69.145",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -183,9 +181,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"

diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json
@@ -43,8 +43,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.145"
+                    "81.2.69.145",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -57,9 +57,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -171,8 +169,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "89.160.20.156"
+                    "89.160.20.156",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -185,9 +183,7 @@
                 "ruleset": "SSLPol"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -294,8 +290,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "89.160.20.112"
+                    "89.160.20.112",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -308,9 +304,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -422,8 +416,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.144"
+                    "81.2.69.144",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -436,9 +430,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -550,8 +542,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -564,9 +556,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -674,8 +664,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "89.160.20.112"
+                    "89.160.20.112",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -688,9 +678,7 @@
                 "ruleset": "None"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -802,8 +790,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -816,9 +804,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -930,8 +916,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test"
@@ -942,9 +928,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -1050,8 +1034,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -1063,9 +1047,7 @@
                 "ruleset": "FwFilter"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -1173,8 +1155,8 @@
                     "TestMachine35"
                 ],
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -1187,9 +1169,7 @@
                 "ruleset": "None"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"
@@ -1289,8 +1269,8 @@
             },
             "related": {
                 "ip": [
-                    "81.2.69.193",
-                    "81.2.69.143"
+                    "81.2.69.143",
+                    "81.2.69.193"
                 ],
                 "user": [
                     "test",
@@ -1302,9 +1282,7 @@
                 "ruleset": "None"
             },
             "source": {
-                "nat": {
-                    "ip": "81.2.69.193"
-                }
+                "ip": "81.2.69.193"
             },
             "tags": [
                 "preserve_original_event"

diff --git a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml
@@ -78,14 +78,14 @@ processors:
       field: event.type
       value: info
   - convert:
-      field: json.cip
+      field: json.cintip
       target_field: source.nat.ip
       if: ctx.json?.cip != ctx.json?.cintip
       type: ip
       ignore_missing: true
       on_failure:
         - remove:
-            field: json.cip
+            field: json.cintip
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'
@@ -364,13 +364,13 @@ processors:
       target_field: zscaler_zia.web.bandwidth_throttle
       ignore_missing: true
   - convert:
-      field: json.cintip
+      field: json.cip
       target_field: source.ip
       type: ip
       ignore_missing: true
       on_failure:
         - remove:
-            field: json.cintip
+            field: json.cip
         - append:
             field: error.message
             value: '{{{_ingest.on_failure_message}}}'

diff --git a/packages/zscaler_zia/data_stream/web/sample_event.json b/packages/zscaler_zia/data_stream/web/sample_event.json
@@ -63,6 +63,7 @@
             "TestMachine35"
         ],
         "ip": [
+            "203.0.113.5",
             "1.128.3.4"
         ],
         "user": [
@@ -76,8 +77,9 @@
     },
     "source": {
         "nat": {
-            "ip": "1.128.3.4"
-        }
+            "ip": "203.0.113.5"
+        },
+        "ip": "192.168.1.35"
     },
     "tags": [
         "forwarded",

diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md
@@ -767,6 +767,7 @@ An example event for `web` looks as following:
             "TestMachine35"
         ],
         "ip": [
+            "203.0.113.5",
             "1.128.3.4"
         ],
         "user": [
@@ -780,8 +781,9 @@ An example event for `web` looks as following:
     },
     "source": {
         "nat": {
-            "ip": "1.128.3.4"
-        }
+            "ip": "203.0.113.5"
+        },
+        "ip": "192.168.1.35"
     },
     "tags": [
         "forwarded",

diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: zscaler_zia
 title: Zscaler Internet Access
-version: "2.19.0"
+version: "2.19.1"
 description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent.
 type: integration
 categories:
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"[email protected]","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"1.128.3.4","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}
		{ "sourcetype" : "zscalernss-web", "event" :{"time":"2021-12-31 08:08:08","login":"[email protected]","proto":"HTTP_PROXY","eurl":"www.example.com","action":"Blocked","appname":"General Browsing","appclass":"General Browsing","reqsize":"600","respsize":"65","stime":"0","ctime":"0","urlclass":"Business Use","urlsupercat":"Information Technology","urlcat":"Web Search","malwarecat":"None","threatname":"None","riskscore":"0","dlpeng":"None","dlpdict":"None","location":"Test DB","dept":"Unknown","cip":"192.168.1.35","cintip":"203.0.113.5","sip":"1.128.3.4","reqmethod":"CONNECT","respcode":"200","eua":"Windows Microsoft Windows 10 Pro ZTunnel/1.0","ereferer":"None","ruletype":"FwFilter","rulelabel":"Zscaler Proxy Traffic","contenttype":"Other","unscannabletype":"None","deviceowner":"administrator1","devicehostname":"TestMachine35"}}