[Filebeat][Okta] Ingest Pipeline for Okta Module drops debug_context fields

[BenB196]

The following needs to be added to the Filebeat mapping:

{
  "_doc": {
    "dynamic_templates": [],
    "properties": {
      "okta": {
        "type": "object",
        "properties": {
          "debug_context": {
            "type": "object",
            "properties": {
              "debug_data": {
                "type": "object",
                "properties": {
                  "suspicious_activity_event_type": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_state": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_longitude": {
                    "type": "float"
                  },
                  "suspicious_activity_event_ip": {
                    "type": "ip"
                  },
                  "suspicious_activity_event_latitude": {
                    "type": "float"
                  },
                  "suspicious_activity_event_city": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_browser": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_transaction_id": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_id": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_os": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_event_country": {
                    "ignore_above": 1024,
                    "type": "keyword"
                  },
                  "suspicious_activity_timestamp": {
                    "type": "date"
                  }
                }
              }
            }
          }
        }
      }
    }
  }
}

The following processors need to be added to the ingest pipeline prior to json being dropped:

  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityBrowser",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_browser",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventCity",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_city",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventCountry",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_country",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventId",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_id",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventIp",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_ip",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventLatitude",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_latitude",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventLongitude",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_longitude",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventState",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_state",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventTransactionId",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_transaction_id",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityEventType",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_event_type",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityOs",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_os",
      "ignore_missing": true
    }
  },
  {
    "rename": {
      "ignore_failure": true,
      "field": "json.debugContext.debugData.suspiciousActivityTimestamp",
      "target_field": "okta.debug_context.debug_data.suspicious_activity_timestamp",
      "ignore_missing": true
    }
  },

For confirmed bugs, please report:

• Version: 7.12.1
• Operating System: Docker
• Discuss Forum URL: https://discuss.elastic.co/t/okta-module-drops-a-lot-of-debugcontext-information-from-events/272588
• Steps to Reproduce:

[elasticmachine]

Pinging @elastic/security-external-integrations (Team:Security-External Integrations)

[jamiehynds]

Thanks for taking the time to adjust the pipeline @BenB196. Could you submit a PR and we can work through your additions there?

[legoguy1000]

@jamiehynds I can do this. @BenB196 can you provide some sample logs from Okta that contain these "new" fields. The sample data that is currently there doesn't have those fields to test.

[BenB196]

@legoguy1000 sorry just getting around to this now. Thanks for opening this PR. Do you still need an example, of the event?

[legoguy1000]

@legoguy1000 sorry just getting around to this now. Thanks for opening this PR. Do you still need an example, of the event?

No problem. I created 1 event by just using the fields u provided so if u have some real ones that would be good just to make sure I didn't mess up.

[BenB196]

@legoguy1000 here is an example event, I obfuscated the data with the same info that was in the example you had where applicable to keep things consistent:

{
    "actor": {
        "alternateId": "[email protected]",
        "detailEntry": null,
        "displayName": "xxxxxx",
        "id": "00u1abvz4pYqdM8ms4x6",
        "type": "User"
    },
    "authenticationContext": {
        "authenticationProvider": null,
        "authenticationStep": 0,
        "credentialProvider": null,
        "credentialType": null,
        "externalSessionId": "102bZDNFfWaQSyEZQuDgWt-uQ",
        "interface": null,
        "issuer": null
    },
    "client": {
        "device": "Computer",
        "geographicalContext": {
            "city": "Dublin",
            "country": "United States",
            "geolocation": {
                "lat": 37.7201,
                "lon": -121.919
            },
            "postalCode": "94568",
            "state": "California"
        },
        "id": null,
        "ipAddress": "108.255.197.247",
        "userAgent": {
            "browser": "FIREFOX",
            "os": "Mac OS X",
            "rawUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0"
        },
        "zone": "null"
    },
    "debugContext": {
        "debugData": {
            "requestId": "<random_id_string>",
            "requestUri": "<uri_endpoint>",
            "suspiciousActivityBrowser": "browser",
            "suspiciousActivityEventCity": "New York City",
            "suspiciousActivityEventCountry": "United States",
            "suspiciousActivityEventId": "1234567",
            "suspiciousActivityEventIp": "10.50.14.5",
            "suspiciousActivityEventLatitude": "40.744960",
            "suspiciousActivityEventLongitude": "-73.988590",
            "suspiciousActivityEventState": "New York",
            "suspiciousActivityEventTransactionId": "12345678900",
            "suspiciousActivityEventType": "system.email.new_device_notification.sent_message",
            "suspiciousActivityOs": "Windows 10",
            "suspiciousActivityTimestamp": "2021-05-08T21:50:16.594Z",
            "url": "<url>"
        }
    },
    "device": null,
    "displayMessage": "User report suspicious activity",
    "eventType": "user.account.report_suspicious_activity_by_enduser",
    "legacyEventType": "core.user.account.report_suspicious_activity_by_enduser",
    "outcome": {
        "reason": null,
        "result": "SUCCESS"
    },
    "published": "2020-02-14T20:18:57.762Z",
    "request": {
        "ipChain": [{
                "geographicalContext": {
                    "city": "Dublin",
                    "country": "United States",
                    "geolocation": {
                        "lat": 37.7201,
                        "lon": -121.919
                    },
                    "postalCode": "94568",
                    "state": "California"
                },
                "ip": "108.255.197.247",
                "source": null,
                "version": "V4"
            }
        ]
    },
    "securityContext": {
        "asNumber": 7018,
        "asOrg": "AT&T Services, Inc.",
        "domain": "att.com",
        "isProxy": false,
        "isp": "AT&T Corp."
    },
    "severity": "WARN",
    "target": [{
            "alternateId": "[email protected]",
            "detailEntry": null,
            "displayName": "xxxxxx",
            "id": "00u1abvz4pYqdM8ms4x6",
            "type": "User"
        }
    ],
    "transaction": {
        "detail": {},
        "id": "XkcAsWb8WjwDP76xh@1v8wAABp0",
        "type": "WEB"
    },
    "uuid": "36a3b6b3-fcc0-47a0-96bd-95330cfdb658",
    "version": "0"
}

[legoguy1000]

I updated the sample data and all good. I also added a uri_parts processor to the pipeline to parse the okta.debug_context.debug_data.url field in the url.* field.