From b2210e7b3b1f7ce5377b70d64015283be9d32861 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 5 Mar 2020 22:35:31 +0100 Subject: [PATCH] CEF module: Support Check Point devices This adds a new ingest pipeline and fields to populate from Check Point CEF logs. Closes #16041 --- filebeat/docs/fields.asciidoc | 554 ++++++++++++++++++ x-pack/filebeat/module/cef/fields.go | 2 +- .../filebeat/module/cef/log/_meta/fields.yml | 250 ++++++++ .../module/cef/log/ingest/cp-pipeline.yml | 306 ++++++++++ .../module/cef/log/ingest/fp-pipeline.yml | 2 +- .../module/cef/log/ingest/pipeline.yml | 3 + x-pack/filebeat/module/cef/log/manifest.yml | 1 + .../module/cef/log/test/checkpoint.log | 3 + .../cef/log/test/checkpoint.log-expected.json | 182 ++++++ 9 files changed, 1301 insertions(+), 2 deletions(-) create mode 100644 x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d3425983b30..e37e30f96eb 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4754,6 +4754,560 @@ type: keyword -- +[float] +=== checkpoint + +Fields for Check Point custom string mappings. + + + +*`checkpoint.app_risk`*:: ++ +-- +Application risk. + +type: keyword + +-- + +*`checkpoint.app_severity`*:: ++ +-- +Application threat severity. + +type: keyword + +-- + +*`checkpoint.app_sig_id`*:: ++ +-- +The signature ID which the application was detected by. + +type: keyword + +-- + +*`checkpoint.auth_method`*:: ++ +-- +Password authentication protocol used. + +type: keyword + +-- + +*`checkpoint.category`*:: ++ +-- +Category. + +type: keyword + +-- + +*`checkpoint.confidence_level`*:: ++ +-- +Confidence level determined. + +type: keyword + +-- + +*`checkpoint.connectivity_state`*:: ++ +-- +Connectivity state. + +type: keyword + +-- + +*`checkpoint.cookie`*:: ++ +-- +IKE cookie. + +type: keyword + +-- + +*`checkpoint.dst_phone_number`*:: ++ +-- +Destination IP-Phone. + +type: keyword + +-- + +*`checkpoint.email_control`*:: ++ +-- +Engine name. + +type: keyword + +-- + +*`checkpoint.email_id`*:: ++ +-- +Internal email ID. + +type: keyword + +-- + +*`checkpoint.email_recipients_num`*:: ++ +-- +Number of recipients. + +type: long + +-- + +*`checkpoint.email_session_id`*:: ++ +-- +Internal email session ID. + +type: keyword + +-- + +*`checkpoint.email_spool_id`*:: ++ +-- +Internal email spool ID. + +type: keyword + +-- + +*`checkpoint.email_subject`*:: ++ +-- +Email subject. + +type: keyword + +-- + +*`checkpoint.event_count`*:: ++ +-- +Number of events associated with the log. + +type: long + +-- + +*`checkpoint.file_hash`*:: ++ +-- +File hash (SHA1 or MD5). + +type: keyword + +-- + +*`checkpoint.frequency`*:: ++ +-- +Scan frequency. + +type: keyword + +-- + +*`checkpoint.icmp_type`*:: ++ +-- +ICMP type. + +type: long + +-- + +*`checkpoint.icmp_code`*:: ++ +-- +ICMP code. + +type: long + +-- + +*`checkpoint.identity_type`*:: ++ +-- +Identity type. + +type: keyword + +-- + +*`checkpoint.incident_extension`*:: ++ +-- +Format of original data. + +type: keyword + +-- + +*`checkpoint.integrity_av_invoke_type`*:: ++ +-- +Scan invoke type. + +type: keyword + +-- + +*`checkpoint.peer_gateway`*:: ++ +-- +Main IP of the peer Security Gateway. + +type: ip + +-- + +*`checkpoint.performance_impact`*:: ++ +-- +Protection performance impact. + +type: keyword + +-- + +*`checkpoint.protection_id`*:: ++ +-- +Protection malware ID. + +type: keyword + +-- + +*`checkpoint.protection_name`*:: ++ +-- +Specific signature name of the attack. + +type: keyword + +-- + +*`checkpoint.protection_type`*:: ++ +-- +Type of protection used to detect the attack. + +type: keyword + +-- + +*`checkpoint.scan_result`*:: ++ +-- +Scan result. + +type: keyword + +-- + +*`checkpoint.sensor_mode`*:: ++ +-- +Sensor mode. + +type: keyword + +-- + +*`checkpoint.severity`*:: ++ +-- +Threat severity. + +type: keyword + +-- + +*`checkpoint.malware_status`*:: ++ +-- +Malware status. + +type: keyword + +-- + +*`checkpoint.subscription_expiration`*:: ++ +-- +The expiration date of the subscription. + +type: date + +-- + +*`checkpoint.tcp_flags`*:: ++ +-- +TCP packet flags. + +type: keyword + +-- + +*`checkpoint.termination_reason`*:: ++ +-- +Termination reason. + +type: keyword + +-- + +*`checkpoint.update_status`*:: ++ +-- +Update status. + +type: keyword + +-- + +*`checkpoint.user_status`*:: ++ +-- +User response. + +type: keyword + +-- + +*`checkpoint.uuid`*:: ++ +-- +External ID. + +type: keyword + +-- + +*`checkpoint.virus_name`*:: ++ +-- +Virus name. + +type: keyword + +-- + +*`checkpoint.malware_name`*:: ++ +-- +Malware name. + +type: keyword + +-- + +*`checkpoint.malware_family`*:: ++ +-- +Malware family. + +type: keyword + +-- + +*`checkpoint.voip_log_type`*:: ++ +-- +VoIP log types. + +type: keyword + +-- + +[float] +=== cef.extensions + +Extra vendor-specific extensions. + + + +*`cef.extensions.cp_app_risk`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.cp_severity`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.ifname`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.inzone`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.layer_uuid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.layer_name`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.logid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.loguid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.match_id`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.nat_addtnl_rulenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.nat_rulenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.origin`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.originsicname`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.outzone`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.parent_rule`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.product`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.rule_action`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.rule_uid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.sequencenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.service_id`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.version`*:: ++ +-- +type: keyword + +-- + +*`observer.ingress.zone`*:: ++ +-- +-- + +*`observer.egress.zone`*:: ++ +-- +-- + +*`observer.interface.name`*:: ++ +-- +-- + [[exported-fields-cisco]] == Cisco fields diff --git a/x-pack/filebeat/module/cef/fields.go b/x-pack/filebeat/module/cef/fields.go index ce9bfda5de2..f4d5d5f2d46 100644 --- a/x-pack/filebeat/module/cef/fields.go +++ b/x-pack/filebeat/module/cef/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCef returns asset data. // This is the base64 encoded gzipped contents of module/cef. func AssetCef() string { - return "eJx8kMFq8zAQhO9+inmB5AF0+C/5a+ihp5RejZFWzhJZK7Syi9++yImDk0L3JHZGsx9zwJUWA0v+MIqbAjVA4RLI4PTWNoAjtZlTYYkG/xoA+FiN8JKRyRLPHIfqRpBBITNlnBcNMhzxeSHcctE7p5gpOslriiay7NnCMwWn4FgtXA+hCMqFNqU+HVlx1FnySFksqd5jUpaZHemxwd1v1n2dA2I/kqmklpJwLA8JKEsigyHLlHZbR76fQunWKAPfB6Un+Vcb27Q32NpK+7iH06RFRpxLriWNfUocB919fGXec8+cJ+3YPYkb+ZWWb8mv2h+Adb5qIt7/Nz8BAAD//0k3k/4=" + return "eJy8Wc9v6zYMvvevILDLdmiAd3iXHAY8pO0WDN0L0O5dBVWmHS62qEm02+yvHyTnV9u0Vu1iOQWR9H0fKVIilUvY4HYOBsvLhou2xgsAIalxDovrmwuAAoPx5ITYzuHXCwCA2zQRSvbg0SB1ZKs4G2quAnCHHu62oeZqBvdrhB4XdFEE6NAW7BNKcGioJAMlYV0EIBunUCQCYZA17kfi1wINF6gMluA8GwxhB+M8d1RgmF3Abv48/R4/l2B1g/Oo1KBjsnIYApCtwzlUnlt38muBpW5rUQlqDqWuAz4bfuWN/eemFxu9cnPgg0UbhBu4Ex+d1GjnyFbhZOFLzae6O/JtUFQ8G9wr3+D2kf3LsXcExs+PiAjLq8PA6y97crNGs/kfnbaIfLBKXjO918Jzr80y3aadU57CZqzbvjlXk9EpEiPO7LWTXtIF7NCTbD+DUtYetcAeMoedqglREnM0UGW1tB5heQWPazLrlHX6RNajDlCgoBEs4GFQVitr1aCsebSulQ4hzklYaGUvxHkWNlxDG7AYkGG0YMV+9L4sduuHaNiWVKA1qGrssB5Nd8CBhJMc7huyw4aytWiEOpKtCqIFJ2g4IEFCGqTmDY2mW/5xvUMYoCmCKLdmi8q2zQP6sYRXGIRsH0rL1eUqQg5QY6OpVoateB69t9e2IosJMYtufEIvraC3uu6BYHmVxefRkCO0EqKDz3LXbKv3iP9MGwNcwhEsiztgCMT282zeAebaHhzzJ3o8wWVztw9/o5HRYdUz9iBDhB1aUYZbe54uf38TUgAdAhvS8UZ4JOnvjFj3vS+jpBrVWof1WJtvqEaIAPDz3e/fvgB7uL36+ssQrcd/WrRm9GVwZ7Q9ogywkWmcitCjPL1c3K7S1ByWWBmPZ4mrh1iKeP3K9m17cnJkB5JlljWJU+GToI2ZPDpU2DdaYsiyp4pijhZa9CC/YBWLL6U7RbbjDU6yPUVOj5NjvkP0qtKCj/p8sJJ7j+1WU7zaotExISMa3KFpo0HwWw87qMCX0XOxoqHG6fHn08pzrBlT4XYEhR50SMVh7YSz+URAo+tHnWrcfOL4w+ht37e4x/I6wu03RotoM9RdnEiZEoH3W5d4j3CpfI6ddl/U5ysKRlvlMbT16KBI+dBDDJGhDexV89YRl0OWIKAZPuemdnH3H+rcdtGYqvU2jOW83cV0jzJkYPtwWKrwyZFPdfBZ7uJ1D/GqbTxCpOn7wD6lGVAkxqmy1tVo++8XK3DabFAg4QzxpXYqSVYedRh/s9wfkaBHGqBuXfTRxO3+K4Hk7XYb0E+lC+hjojq2YSh72nb8IX39tCugB4/m/k1syqncv4Fl9GH7BJ1Ctk/PD9CVuqF69Bm0J+xRhpzJ5FTN1aSr5QcvV7HqTyvOxeThoQDL2aGgOw3Jz35VvH4Sr3dvzZeHd+Yj9XsPiWf9ZJz60KPiWyAfumDOV6ZldjS+Udn+y3bC+lpv0avsVH8HY5odNVeTJHA1yYRGi1nnFqVnEawWpYtCbK18W+NbTy7ZWJNB+v5o6vpAZtrGcivTItRpHztH3/+XNRbEc9FmNj1nASK90ubN6iofZFKchv6xYlpkBPQdxUZwgpAOfXYbf/jyE9x/v/p+KHqE4XpxB19mX0HbAjw23J1cqocIeoiC0c/IVh5DmD2Lp1ezMGcSWUFfaoOzFN3/BQAA//9c6pxr" } diff --git a/x-pack/filebeat/module/cef/log/_meta/fields.yml b/x-pack/filebeat/module/cef/log/_meta/fields.yml index 0d24bf8458f..16e54d71cf1 100644 --- a/x-pack/filebeat/module/cef/log/_meta/fields.yml +++ b/x-pack/filebeat/module/cef/log/_meta/fields.yml @@ -8,3 +8,253 @@ type: keyword description: > Virus ID + + +- name: checkpoint + type: group + default_field: false + description: > + Fields for Check Point custom string mappings. + fields: + - name: app_risk + type: keyword + description: Application risk. + + - name: app_severity + type: keyword + description: Application threat severity. + + - name: app_sig_id + type: keyword + description: The signature ID which the application was detected by. + + - name: auth_method + type: keyword + description: Password authentication protocol used. + + - name: category + type: keyword + description: Category. + + - name: confidence_level + type: keyword + description: Confidence level determined. + + - name: connectivity_state + type: keyword + description: Connectivity state. + + - name: cookie + type: keyword + description: IKE cookie. + + - name: dst_phone_number + type: keyword + description: Destination IP-Phone. + + - name: email_control + type: keyword + description: Engine name. + + - name: email_id + type: keyword + description: Internal email ID. + + - name: email_recipients_num + type: long + description: Number of recipients. + + - name: email_session_id + type: keyword + description: Internal email session ID. + + - name: email_spool_id + type: keyword + description: Internal email spool ID. + + - name: email_subject + type: keyword + description: Email subject. + + - name: event_count + type: long + description: Number of events associated with the log. + + - name: file_hash + type: keyword + description: File hash (SHA1 or MD5). + + - name: frequency + type: keyword + description: Scan frequency. + + - name: icmp_type + type: long + description: ICMP type. + + - name: icmp_code + type: long + description: ICMP code. + + - name: identity_type + type: keyword + description: Identity type. + + - name: incident_extension + type: keyword + description: Format of original data. + + - name: integrity_av_invoke_type + type: keyword + description: Scan invoke type. + + - name: peer_gateway + type: ip + description: Main IP of the peer Security Gateway. + + - name: performance_impact + type: keyword + description: Protection performance impact. + + - name: protection_id + type: keyword + description: Protection malware ID. + + - name: protection_name + type: keyword + description: Specific signature name of the attack. + + - name: protection_type + type: keyword + description: Type of protection used to detect the attack. + + - name: scan_result + type: keyword + description: Scan result. + + - name: sensor_mode + type: keyword + description: Sensor mode. + + - name: severity + type: keyword + description: Threat severity. + + - name: malware_status + type: keyword + description: Malware status. + + - name: subscription_expiration + type: date + description: The expiration date of the subscription. + + - name: tcp_flags + type: keyword + description: TCP packet flags. + + - name: termination_reason + type: keyword + description: Termination reason. + + - name: update_status + type: keyword + description: Update status. + + - name: user_status + type: keyword + description: User response. + + - name: uuid + type: keyword + description: External ID. + + - name: virus_name + type: keyword + description: Virus name. + + - name: malware_name + type: keyword + description: Malware name. + + - name: malware_family + type: keyword + description: Malware family. + + - name: voip_log_type + type: keyword + description: VoIP log types. + +- name: cef.extensions + type: group + default_field: false + description: > + Extra vendor-specific extensions. + fields: + + - name: cp_app_risk + type: keyword + + - name: cp_severity + type: keyword + + - name: ifname + type: keyword + + - name: inzone + type: keyword + + - name: layer_uuid + type: keyword + + - name: layer_name + type: keyword + + - name: logid + type: keyword + + - name: loguid + type: keyword + + - name: match_id + type: keyword + + - name: nat_addtnl_rulenum + type: keyword + + - name: nat_rulenum + type: keyword + + - name: origin + type: keyword + + - name: originsicname + type: keyword + + - name: outzone + type: keyword + + - name: parent_rule + type: keyword + + - name: product + type: keyword + + - name: rule_action + type: keyword + + - name: rule_uid + type: keyword + + - name: sequencenum + type: keyword + + - name: service_id + type: keyword + + - name: version + type: keyword + +# TODO: Update to ECS 1.5 and remove. +- name: observer.ingress.zone +- name: observer.egress.zone +- name: observer.interface.name diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml new file mode 100644 index 00000000000..1cf4399dbb5 --- /dev/null +++ b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml @@ -0,0 +1,306 @@ +--- +description: Pipeline for Check Point CEF + +processors: + # This script is mapping CEF extensions to ECS when possible. Otherwise + # it maps them to fields under the `checkpoint` group using Check Point log + # field names. + # + # [1] Description of Check Point CEF extensions: + # https://community.checkpoint.com/t5/Logging-and-Reporting/Log-Exporter-CEF-Field-Mappings/td-p/41060 + # [2] Description of Check Point log field names (sk144192): + # https://supportcenter.checkpoint.com/supportcenter/portal?eventSubmit_doGoviewsolutiondetails=&solutionid=sk144192 + # + # Note that in some cases the CEF extension name doesn't accurately describe + # its contents. For example sntdom/sourceNtDomain, which is used to store + # Check Point's domain_name, documented as "Domain name sent to DNS request". + # + # This script processes the `params.extensions` list below. This list consists + # of two different kinds of mappings, the simpler has a source ext `name` + # and a `to` field. It copies the given extension field to the target `to`. + # + # When the `labels` dict is defined, the target field depends on the value of + # the accompanying label field. For example, the field deviceCustomIPv6Address2 + # is mapped to `source.ip` only when the extension deviceCustomIPv6Address2Label + # exists and its value is "Source IPv6 Address". + # + # Also it can convert the destination value by simple mapping when the + # convert key exists. Values without an entry in the convert dict are not + # copied and the target field remains unset. + # + # The output of this processor is a single field, `_tmp_copy`, that contains + # a list of actions `{"to": "target_field", "value":"field value"}` that is + # later executed using a foreach processor. This is done to avoid complex + # de-dotting and other gotchas of setting arbitrary fields in Painless. + - script: + lang: painless + params: + extensions: + - name: cp_app_risk + to: checkpoint.app_risk + + - name: cp_app_risk + to: event.risk_score + # This mapping is a mix of [1] and [2] above. + convert: + unknown: 0 + informational: 0 + very-low: 1 + low: 2 + medium: 3 + high: 4 + very-high: 5 + critical: 5 + + - name: cp_severity + to: checkpoint.severity + + - name: cp_severity + to: event.severity + convert: + # This mapping is a mix of [1] and [2] above. + unknown: 0 + informational: 0 + very-low: 1 + low: 1 + medium: 2 + high: 3 + very-high: 4 + critical: 4 + + # Number of events associated with the log + - name: baseEventCount + to: checkpoint.event_count + + # Log type + - name: deviceExternalId + to: observer.type + + # Product Family + - name: deviceFacility + to: observer.type + convert: + '0': Network + '1': Endpoint + '2': Access + '3': Threat + '4': Mobile + + # Gateway interface, where the connection is received from in case of an outbound connection + - name: deviceInboundInterface + to: observer.ingress.interface.name + + # Gateway interface, where the connection is sent from, in case of an inbound connection + - name: deviceOutboundInterface + to: observer.egress.interface.name + + - name: externalId + to: checkpoint.uuid + + - name: fileHash + to: checkpoint.file_hash + + - name: reason + to: checkpoint.termination_reason + + # Possibly an IKE cookie + - name: checkrequestCookies + to: checkpoint.cookie + + # Domain name sent to DNS request + - name: sourceNtDomain + to: dns.question.name + + # CVE registry entry + - name: Signature + to: vulnerability.id + + - name: Recipient + to: destination.user.email + + - name: Sender + to: source.user.email + + - name: deviceCustomFloatingPoint1 + labels: + update version: observer.version + + - name: deviceCustomIPv6Address2 + labels: + source ipv6 address: source.ip + + - name: deviceCustomIPv6Address3 + labels: + destination ipv6 address: destination.ip + + - name: deviceCustomNumber1 + labels: + payload: network.bytes + elapsed time in seconds: host.uptime + email recipients number: checkpoint.email_recipients_num + + - name: deviceCustomNumber2 + labels: + duration in seconds: event.duration + icmp type: checkpoint.icmp_type + + - name: deviceCustomNumber3 + labels: + icmp code: checkpoint.icmp_code + + - name: deviceCustomString1 + labels: + application rule name: rule.name + dlp rule name: rule.name + threat prevention rule name: rule.name + connectivity state: checkpoint.connectivity_state + email id: checkpoint.email_id + voip log type: checkpoint.voip_log_type + + - name: deviceCustomString2 + labels: + # Protection malware id + protection id: checkpoint.protection_id + update status: checkpoint.update_status + email subject: checkpoint.email_subject + sensor mode: checkpoint.sensor_mode + scan invoke type: checkpoint.integrity_av_invoke_type + category: checkpoint.category + # Matched categories + categories: rule.category + peer gateway: checkpoint.peer_gateway + + - name: deviceCustomString6 + labels: + application name: process.name + virus name: checkpoint.virus_name + malware name: checkpoint.malware_name + malware family: checkpoint.malware_family + + - name: deviceCustomString3 + labels: + user group: group.name + # Format of original data. + incident extension: checkpoint.incident_extension + identity type: checkpoint.identity_type + email spool id: checkpoint.email_spool_id + # Type of protection used to detect the attack + protection type: checkpoint.protection_type + + - name: deviceCustomString4 + labels: + malware status: checkpoint.spyware_status + destination os: os.name + scan result: checkpoint.scan_result + frequency: checkpoint.frequency + protection name: checkpoint.protection_name + user response: checkpoint.user_status + email control: checkpoint.email_control + tcp flags: checkpoint.tcp_flags + threat prevention rule id: rule.id + + - name: deviceCustomString5 + labels: + matched category: rule.category + authentication method: checkpoint.auth_method + email session id: checkpoint.email_session_id + vlan id: network.vlan.id + + - name: deviceCustomDate2 + labels: + subscription expiration: checkpoint.subscription_expiration + + - name: deviceFlexNumber1 + labels: + confidence: checkpoint.confidence_level + + - name: deviceFlexNumber2 + labels: + destination phone number: checkpoint.dst_phone_number + performance impact: checkpoint.performance_impact + + - name: flexString1 + labels: + application signature id: checkpoint.app_sig_id + + - name: flexString2 + labels: + malware action: event.action + attack information: event.action + + - name: rule_uid + to: rule.uuid + + - name: ifname + to: observer.interface.name + + - name: inzone + to: observer.ingress.zone + + - name: outzone + to: observer.egress.zone + + - name: product + to: observer.product + + source: | + def actions = new ArrayList(); + def exts = ctx.cef?.extensions; + if (exts == null) return; + for (entry in params.extensions) { + def value = exts[entry.name]; + if (value == null || + (entry.convert != null && + (value=entry.convert[value.toLowerCase()]) == null)) + continue; + if (entry.to != null) { + actions.add([ + "value": value, + "to": entry.to + ]); + continue; + } + def label = exts[entry.name + "Label"]; + if (label == null) continue; + def dest = entry.labels[label.toLowerCase()]; + if (dest == null) continue; + actions.add([ + "value": value, + "to": dest + ]); + } + ctx["_tmp_copy"] = actions; + + - foreach: + field: _tmp_copy + processor: + set: + field: "{{_ingest._value.to}}" + value: "{{_ingest._value.value}}" + + - remove: + field: _tmp_copy + + # event.duration is a string and contains seconds. Convert to long nanos. + - script: + params: + second_to_nanos: 1000000000 + lang: painless + source: | + def duration = ctx.event?.duration; + if (duration == null) return; + ctx.event.duration = Long.parseLong(duration) * params.second_to_nanos; + on_failure: + - remove: + field: event.duration + ignore_missing: true + + # checkpoint.file_hash can be either MD5 or SHA1. + - set: + field: 'file.hash.md5' + value: '{{checkpoint.file_hash}}' + if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==32' + - set: + field: 'file.hash.sha1' + value: '{{checkpoint.file_hash}}' + if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==40' diff --git a/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml index 3fe032c00fb..1459d521cce 100644 --- a/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml @@ -24,4 +24,4 @@ processors: - set: field: cef.forcepoint.virus_id value: "{{cef.extensions.deviceCustomString4}}" - if: "ctx.cef?.extensions?.deviceCustomString4 != null" + if: "ctx.cef?.extensions?.deviceCustomString4 != null" diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 2d14926e7f8..75a86ea2758 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -80,6 +80,9 @@ processors: - pipeline: name: '{< IngestPipeline "fp-pipeline" >}' if: "ctx.cef?.device?.vendor == 'FORCEPOINT'" + - pipeline: + name: '{< IngestPipeline "cp-pipeline" >}' + if: "ctx.cef?.device?.vendor == 'Check Point'" on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/cef/log/manifest.yml b/x-pack/filebeat/module/cef/log/manifest.yml index 670a3188a4e..60115d99b40 100644 --- a/x-pack/filebeat/module/cef/log/manifest.yml +++ b/x-pack/filebeat/module/cef/log/manifest.yml @@ -16,6 +16,7 @@ var: ingest_pipeline: - ingest/pipeline.yml - ingest/fp-pipeline.yml + - ingest/cp-pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log b/x-pack/filebeat/module/cef/log/test/checkpoint.log index e69de29bb2d..8951c3edade 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log @@ -0,0 +1,3 @@ +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\=R80,O\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json index e69de29bb2d..e9753166af1 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json @@ -0,0 +1,182 @@ +[ + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.destinationAddress": "52.173.84.157", + "cef.extensions.destinationPort": 443, + "cef.extensions.destinationTranslatedAddress": "0.0.0.0", + "cef.extensions.destinationTranslatedPort": 0, + "cef.extensions.deviceAction": "Accept", + "cef.extensions.deviceCustomDate2": "2017-10-16T10:42:13.713Z", + "cef.extensions.deviceCustomDate2Label": "This field is made up", + "cef.extensions.deviceCustomString2Label": "Rule Name", + "cef.extensions.deviceCustomString5": "Business / Economy", + "cef.extensions.deviceCustomString5Label": "Matched Category", + "cef.extensions.deviceDirection": 0, + "cef.extensions.deviceReceiptTime": "2018-11-26T22:17:32.000Z", + "cef.extensions.ifname": "eth0", + "cef.extensions.inzone": "Internal", + "cef.extensions.layer_name": "Network", + "cef.extensions.layer_uuid": "b406b732-2437-4848-9741-6eae1f5bf112", + "cef.extensions.logid": "0", + "cef.extensions.loguid": "{0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001}", + "cef.extensions.match_id": "4", + "cef.extensions.nat_addtnl_rulenum": "1", + "cef.extensions.nat_rulenum": "4", + "cef.extensions.origin": "192.168.101.254", + "cef.extensions.originsicname": "CN=R80,O=R80_M..6u6bdo", + "cef.extensions.outzone": "External", + "cef.extensions.parent_rule": "0", + "cef.extensions.product": "VPN-1 & FireWall-1", + "cef.extensions.rule_action": "Accept", + "cef.extensions.rule_uid": "9e5e6e74-aa9a-4693-b9fe-53712dd27bea", + "cef.extensions.sequencenum": "1", + "cef.extensions.service_id": "https", + "cef.extensions.sourceAddress": "192.168.101.100", + "cef.extensions.sourcePort": 49363, + "cef.extensions.sourceTranslatedAddress": "192.168.103.254", + "cef.extensions.sourceTranslatedPort": 35398, + "cef.extensions.transportProtocol": "6", + "cef.extensions.version": "5", + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "destination.as.number": 8075, + "destination.as.organization.name": "Microsoft Corporation", + "destination.geo.city_name": "Des Moines", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 41.6006, + "destination.geo.location.lon": -93.6112, + "destination.geo.region_iso_code": "US-IA", + "destination.geo.region_name": "Iowa", + "destination.ip": "52.173.84.157", + "destination.nat.ip": "0.0.0.0", + "destination.nat.port": 0, + "destination.port": 443, + "event.action": "Accept", + "event.code": "Log", + "event.dataset": "cef.log", + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", + "fileset.name": "log", + "input.type": "log", + "log.offset": 0, + "message": "https", + "network.community_id": "1:yRLApDaheTmJZHL4UUDMjcHWAik=", + "network.direction": "inbound", + "network.transport": "6", + "observer.egress.zone": "External", + "observer.ingress.zone": "Internal", + "observer.interface.name": "eth0", + "observer.product": "VPN-1 & FireWall-1", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "rule.category": "Business / Economy", + "rule.uuid": "9e5e6e74-aa9a-4693-b9fe-53712dd27bea", + "service.type": "cef", + "source.ip": "192.168.101.100", + "source.nat.ip": "192.168.103.254", + "source.nat.port": 35398, + "source.port": 49363, + "tags": [ + "cef" + ] + }, + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.destinationPort": 25, + "cef.extensions.deviceAction": "Bypass", + "cef.extensions.deviceCustomDate2": "2020-04-11T10:42:13.000Z", + "cef.extensions.deviceCustomDate2Label": "Subscription expiration", + "cef.extensions.deviceCustomNumber1Label": "Email Recipients Number", + "cef.extensions.deviceCustomString1Label": "Email ID", + "cef.extensions.deviceCustomString4": "SMTP Policy Restrictions", + "cef.extensions.deviceCustomString4Label": "Email Control", + "cef.extensions.deviceCustomString5Label": "Email Session ID", + "cef.extensions.deviceDirection": 0, + "cef.extensions.deviceReceiptTime": "2018-12-19T09:22:10.000Z", + "cef.extensions.fileHash": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "cef.extensions.message": "Encrypted session", + "cef.extensions.sourcePort": 4001, + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "checkpoint.email_control": "SMTP Policy Restrictions", + "checkpoint.file_hash": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "checkpoint.subscription_expiration": "2020-04-11T10:42:13.000Z", + "destination.port": 25, + "event.action": "Bypass", + "event.code": "Log", + "event.dataset": "cef.log", + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", + "file.hash.sha1": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "fileset.name": "log", + "input.type": "log", + "log.offset": 875, + "message": "Encrypted session", + "network.direction": "inbound", + "observer.product": "VPN-1 & FireWall-1", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "service.type": "cef", + "source.port": 4001, + "tags": [ + "cef" + ] + }, + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.baseEventCount": "12", + "cef.extensions.cp_app_risk": "High", + "cef.extensions.cp_severity": "Very-High", + "cef.extensions.deviceAction": "Drop", + "cef.extensions.deviceCustomIPv6Address2": "fd00::555", + "cef.extensions.deviceCustomIPv6Address2Label": "Source IPv6 Address", + "cef.extensions.deviceCustomIPv6Address3": "::1", + "cef.extensions.deviceCustomIPv6Address3Label": "Destination IPv6 Address", + "cef.extensions.deviceCustomNumber2": 5, + "cef.extensions.deviceCustomNumber2Label": "Duration in Seconds", + "cef.extensions.deviceFacility": "4", + "cef.extensions.fileHash": "580a783c1cb2b20613323f715d231a69", + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "checkpoint.app_risk": "High", + "checkpoint.event_count": "12", + "checkpoint.file_hash": "580a783c1cb2b20613323f715d231a69", + "checkpoint.severity": "Very-High", + "destination.ip": "::1", + "event.action": "Drop", + "event.code": "Log", + "event.dataset": "cef.log", + "event.duration": 5000000000, + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", + "event.risk_score": "4", + "event.severity": "4", + "file.hash.md5": "580a783c1cb2b20613323f715d231a69", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1291, + "message": "https", + "observer.product": "VPN-1 & FireWall-1", + "observer.type": "Mobile", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "service.type": "cef", + "source.ip": "fd00::555", + "tags": [ + "cef" + ] + } +] \ No newline at end of file