cisco_duo: fix handling of IP addresses with port numbers

packages/cisco_duo/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.1.5"
+  changes:
+    - description: Fix handling of IP addresses with port numbers.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/xxxx
 - version: "1.1.4"
   changes:
     - description: Fix dashboard issues.

packages/cisco_duo/data_stream/auth/_dev/test/pipeline/test-auth.log

@@ -2,4 +2,5 @@
 {"access_device":{"browser":"Chrome","browser_version":"92.0.4515.107","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"89.160.20.156","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"[email protected]","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-07-23T07:21:51.271776+00:00","ood_software":null,"reason":"user_approved","result":"success","timestamp":1627024911,"txid":"fa59a691-9139-43e9-9854-f9e1dbf72af5","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}
 {"access_device":{"browser":"Chrome","browser_version":"92.0.4515.131","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"89.160.20.156","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"[email protected]","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-08-12T09:14:23.060168+00:00","ood_software":null,"reason":"user_approved","result":"success","timestamp":1628759663,"txid":"861a81e7-1f60-4865-95eb-57d9c43ce073","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}
 {"access_device":{"browser":"Chrome","browser_version":"92.0.4515.107","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"89.160.20.156","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-07-23T07:20:54.700050+00:00","ood_software":null,"reason":"user_marked_fraud","result":"fraud","timestamp":1627024854,"txid":"78e1a910-350b-4226-828b-edb0ac2f2e3c","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}
-{"access_device":{"browser":"Chrome","browser_version":"92.0.4515.107","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"89.160.20.156","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-07-23T07:19:34.702203+00:00","ood_software":null,"reason":"user_mistake","result":"denied","timestamp":1627024774,"txid":"e22120cd-7388-424f-aa0a-b60cad42d8f3","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}
+{"access_device":{"browser":"Chrome","browser_version":"92.0.4515.107","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"89.160.20.156","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-07-23T07:19:34.702203+00:00","ood_software":null,"reason":"user_mistake","result":"denied","timestamp":1627024774,"txid":"e22120cd-7388-424f-aa0a-b60cad42d8f3","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}
+{"access_device":{"browser":"Chrome","browser_version":"92.0.4515.107","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.112:1234","is_encryption_enabled":"unknown","is_firewall_enabled":"unknown","is_password_set":"unknown","java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Windows","os_version":"10"},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Duo Access Gateway Launcher"},"auth_device":{"ip":"192.168.225.254:4321","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"+91 12345 12345"},"email":"","event_type":"authentication","factor":"duo_push","isotimestamp":"2021-07-23T07:19:34.702203+00:00","ood_software":null,"reason":"user_mistake","result":"denied","timestamp":1627024774,"txid":"e22120cd-7388-424f-aa0a-b60cad42d8f3","user":{"groups":["AD Sync"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway"}}

packages/cisco_duo/data_stream/auth/_dev/test/pipeline/test-auth.log-expected.json

@@ -583,6 +583,110 @@
                 },
                 "version": "92.0.4515.107"
             }
+        },
+        {
+            "@timestamp": "2021-07-23T07:19:34.000Z",
+            "cisco_duo": {
+                "auth": {
+                    "access_device": {
+                        "flash_version": "uninstalled",
+                        "ip": "89.160.20.112",
+                        "is_encryption_enabled": "unknown",
+                        "is_firewall_enabled": "unknown",
+                        "is_password_set": "unknown",
+                        "java_version": "uninstalled",
+                        "location": {
+                            "city": "Ann Arbor",
+                            "country": "United States",
+                            "state": "Michigan"
+                        },
+                        "port": 1234
+                    },
+                    "application": {
+                        "key": "DIY231J8BR23QK4UKBY8",
+                        "name": "Duo Access Gateway Launcher"
+                    },
+                    "auth_device": {
+                        "ip": "192.168.225.254",
+                        "location": {
+                            "city": "Ann Arbor",
+                            "country": "United States",
+                            "state": "Michigan"
+                        },
+                        "name": "+91 12345 12345",
+                        "port": 4321
+                    },
+                    "event_type": "authentication",
+                    "factor": "duo_push",
+                    "reason": "user_mistake",
+                    "result": "denied",
+                    "txid": "e22120cd-7388-424f-aa0a-b60cad42d8f3"
+                }
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "event": {
+                "category": "authentication",
+                "kind": "event",
+                "original": "{\"access_device\":{\"browser\":\"Chrome\",\"browser_version\":\"92.0.4515.107\",\"flash_version\":\"uninstalled\",\"hostname\":null,\"ip\":\"89.160.20.112:1234\",\"is_encryption_enabled\":\"unknown\",\"is_firewall_enabled\":\"unknown\",\"is_password_set\":\"unknown\",\"java_version\":\"uninstalled\",\"location\":{\"city\":\"Ann Arbor\",\"country\":\"United States\",\"state\":\"Michigan\"},\"os\":\"Windows\",\"os_version\":\"10\"},\"alias\":\"\",\"application\":{\"key\":\"DIY231J8BR23QK4UKBY8\",\"name\":\"Duo Access Gateway Launcher\"},\"auth_device\":{\"ip\":\"192.168.225.254:4321\",\"location\":{\"city\":\"Ann Arbor\",\"country\":\"United States\",\"state\":\"Michigan\"},\"name\":\"+91 12345 12345\"},\"email\":\"\",\"event_type\":\"authentication\",\"factor\":\"duo_push\",\"isotimestamp\":\"2021-07-23T07:19:34.702203+00:00\",\"ood_software\":null,\"reason\":\"user_mistake\",\"result\":\"denied\",\"timestamp\":1627024774,\"txid\":\"e22120cd-7388-424f-aa0a-b60cad42d8f3\",\"user\":{\"groups\":[\"AD Sync\"],\"key\":\"DU3KC77WJ06Y5HIV7XKQ\",\"name\":\"narroway\"}}",
+                "outcome": "failed",
+                "reason": "user_mistake",
+                "type": "info"
+            },
+            "related": {
+                "ip": [
+                    "89.160.20.112",
+                    "192.168.225.254"
+                ]
+            },
+            "source": {
+                "address": "89.160.20.112:1234",
+                "as": {
+                    "number": 29518,
+                    "organization": {
+                        "name": "Bredband2 AB"
+                    }
+                },
+                "geo": {
+                    "city_name": "Linköping",
+                    "continent_name": "Europe",
+                    "country_iso_code": "SE",
+                    "country_name": "Sweden",
+                    "location": {
+                        "lat": 58.4167,
+                        "lon": 15.6167
+                    },
+                    "region_iso_code": "SE-E",
+                    "region_name": "Östergötland County"
+                },
+                "ip": "89.160.20.112",
+                "port": 1234,
+                "user": {
+                    "group": {
+                        "name": [
+                            "AD Sync"
+                        ]
+                    },
+                    "id": "DU3KC77WJ06Y5HIV7XKQ",
+                    "name": "narroway"
+                }
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "id": "DU3KC77WJ06Y5HIV7XKQ",
+                "name": "narroway"
+            },
+            "user_agent": {
+                "name": "Chrome",
+                "os": {
+                    "name": "Windows",
+                    "version": "10"
+                },
+                "version": "92.0.4515.107"
+            }
         }
     ]
 }

packages/cisco_duo/data_stream/auth/_dev/test/pipeline/test-common-config.yml

@@ -1,5 +1,3 @@
-dynamic_fields:
-  event.ingested: ".*"
 fields:
   tags:
     - preserve_original_event

packages/cisco_duo/data_stream/auth/elasticsearch/ingest_pipeline/default.yml

@@ -45,13 +45,45 @@ processors:
       copy_from: json.reason
       ignore_failure: true
   - set: 
-      field: source.ip
+      field: source.address
       copy_from: json.access_device.ip
+  - dissect:
+      field: json.access_device.ip
+      pattern: "%{json.access_device.ip}:%{json.access_device.port}"
+      ignore_missing: true
       ignore_failure: true
+      if: "ctx?.json?.access_device?.ip.contains(':')"
+  - convert:
+      field: json.access_device.ip
+      type: ip
+      ignore_missing: true
+  - convert:
+      field: json.access_device.port
+      type: long
+      ignore_missing: true
   - set: 
-      field: source.address
+      field: source.ip
       copy_from: json.access_device.ip
       ignore_failure: true
+  - set:
+      field: source.port
+      copy_from: json.access_device.port
+      ignore_failure: true
+
+  - dissect:
+      field: json.auth_device.ip
+      pattern: "%{json.auth_device.ip}:%{json.auth_device.port}"
+      ignore_missing: true
+      ignore_failure: true
+      if: "ctx?.json?.auth_device?.ip.contains(':')"
+  - convert:
+      field: json.auth_device.ip
+      type: ip
+      ignore_missing: true
+  - convert:
+      field: json.auth_device.port
+      type: long
+      ignore_missing: true
   - set: 
       field: source.address
       copy_from: json.access_device.hostname
@@ -184,6 +216,10 @@ processors:
       field: json.access_device.ip
       target_field: cisco_duo.auth.access_device.ip
       ignore_missing: true
+  - rename:
+      field: json.access_device.port
+      target_field: cisco_duo.auth.access_device.port
+      ignore_missing: true
   - rename:
       field: json.access_device.is_encryption_enabled
       target_field: cisco_duo.auth.access_device.is_encryption_enabled
@@ -240,6 +276,10 @@ processors:
       field: json.auth_device.ip
       target_field: cisco_duo.auth.auth_device.ip
       ignore_missing: true
+  - rename:
+      field: json.auth_device.port
+      target_field: cisco_duo.auth.auth_device.port
+      ignore_missing: true
   - rename:
       field: json.auth_device.location.city
       target_field: cisco_duo.auth.auth_device.location.city

packages/cisco_duo/data_stream/auth/fields/ecs.yml

@@ -20,6 +20,8 @@
   name: related.ip
 - external: ecs
   name: source.ip
+- external: ecs
+  name: source.port
 - external: ecs
   name: source.address
 - external: ecs

packages/cisco_duo/data_stream/auth/fields/fields.yml

@@ -52,6 +52,10 @@
           type: ip
           description: |
             The access device's IP address.
+        - name: port
+          type: long
+          description: |
+            The access device's port number.
         - name: is_encryption_enabled
           type: keyword
           description: |
@@ -105,6 +109,10 @@
           type: ip
           description: |
             The IP address of the authentication device.
+        - name: port
+          type: long
+          description: |
+            The network port of the authentication device.
         - name: location
           type: group
           fields:

packages/cisco_duo/docs/README.md

@@ -314,6 +314,7 @@ An example event for `auth` looks as following:
 | cisco_duo.auth.access_device.location.city | The city name of the access device using geoip location. | keyword |
 | cisco_duo.auth.access_device.location.country | The country of the access device using geoip location. | keyword |
 | cisco_duo.auth.access_device.location.state | The state name of the access device using geoip location. | keyword |
+| cisco_duo.auth.access_device.port | The access device's port number. | long |
 | cisco_duo.auth.access_device.security_agents | Reports the security agents present on the endpoint as detected by the Duo Device Health app. | keyword |
 | cisco_duo.auth.alias | The username alias used to log in. | keyword |
 | cisco_duo.auth.application.key | The application's integration_key. | keyword |
@@ -332,6 +333,7 @@ An example event for `auth` looks as following:
 | cisco_duo.auth.auth_device.location.country | The country of the authentication device using geoip location. | keyword |
 | cisco_duo.auth.auth_device.location.state | The state name of the authentication device using geoip location. | keyword |
 | cisco_duo.auth.auth_device.name | The name of the authentication device. | keyword |
+| cisco_duo.auth.auth_device.port | The network port of the authentication device. | long |
 | cisco_duo.auth.email | The email address of the user, if known to Duo, otherwise none. | keyword |
 | cisco_duo.auth.event_type | The type of activity logged. | keyword |
 | cisco_duo.auth.factor | The authentication factor. | keyword |
@@ -399,6 +401,7 @@ An example event for `auth` looks as following:
 | source.geo.region_iso_code | Region ISO code. | keyword |
 | source.geo.region_name | Region name. | keyword |
 | source.ip | IP address of the source (IPv4 or IPv6). | ip |
+| source.port | Port of the source. | long |
 | source.user.email | User email address. | keyword |
 | source.user.group.name | Name of the group. | keyword |
 | source.user.id | Unique identifier of the user. | keyword |

packages/cisco_duo/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: cisco_duo
 title: Cisco Duo
-version: 1.1.4
+version: 1.1.5
 license: basic
 description: Collect logs from Cisco Duo with Elastic Agent.
 type: integration