From f52e4524ef0d8f7f1a6df2a6af3d0e642bbf96fb Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Thu, 10 Dec 2020 15:26:02 -0500 Subject: [PATCH] [Filebeat] Allow cef and checkpoint modules to override network directionality based off of zones (#23066) * [Filebeat] Allow cef and checkpoint modules to override network directionality based off of zones * Remove _temp_ * Add changelog entry * run mage update and add variable reference * Don't override categorization if no zone set * Update cef pipeline --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/filebeat.reference.yml | 16 +++++ x-pack/filebeat/module/cef/_meta/config.yml | 8 +++ .../filebeat/module/cef/log/config/input.yml | 14 +++++ .../module/cef/log/ingest/cp-pipeline.yml | 61 +++++++++++++++++++ .../module/cef/log/ingest/pipeline.yml | 4 ++ x-pack/filebeat/module/cef/log/manifest.yml | 2 + .../module/checkpoint/_meta/config.yml | 8 +++ .../checkpoint/firewall/config/firewall.yml | 13 ++++ .../checkpoint/firewall/ingest/pipeline.yml | 60 ++++++++++++++++++ .../module/checkpoint/firewall/manifest.yml | 2 + x-pack/filebeat/modules.d/cef.yml.disabled | 8 +++ .../modules.d/checkpoint.yml.disabled | 8 +++ 13 files changed, 205 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5b9f4943e64..54d4fa40f17 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -759,6 +759,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add top_level_domain enrichment for suricata/eve fileset. {pull}23046[23046] - Add top_level_domain enrichment for zeek/dns fileset. {pull}23046[23046] - Add `network.direction` to netflow/log fileset. {pull}23052[23052] +- Allow cef and checkpoint modules to override network directionality based off of zones {pull}23066[23066] *Heartbeat* diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 0abd5c15fed..e75e0a72467 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -461,6 +461,14 @@ filebeat.modules: syslog_host: localhost syslog_port: 9003 + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + #------------------------------ Checkpoint Module ------------------------------ - module: checkpoint firewall: @@ -476,6 +484,14 @@ filebeat.modules: # The UDP port to listen for syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] + #-------------------------------- Cisco Module -------------------------------- - module: cisco asa: diff --git a/x-pack/filebeat/module/cef/_meta/config.yml b/x-pack/filebeat/module/cef/_meta/config.yml index 6ea927cc972..1b9ff319441 100644 --- a/x-pack/filebeat/module/cef/_meta/config.yml +++ b/x-pack/filebeat/module/cef/_meta/config.yml @@ -4,3 +4,11 @@ var: syslog_host: localhost syslog_port: 9003 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/x-pack/filebeat/module/cef/log/config/input.yml b/x-pack/filebeat/module/cef/log/config/input.yml index e6f277c2ca0..4568f659c3a 100644 --- a/x-pack/filebeat/module/cef/log/config/input.yml +++ b/x-pack/filebeat/module/cef/log/config/input.yml @@ -29,3 +29,17 @@ processors: target: '' fields: ecs.version: 1.7.0 + +{{ if .external_zones }} + - add_fields: + target: _temp_ + fields: + external_zones: {{ .external_zones | tojson }} +{{ end }} + +{{ if .internal_zones }} + - add_fields: + target: _temp_ + fields: + internal_zones: {{ .internal_zones | tojson }} +{{ end }} diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml index eea2f8fd592..89a1eca22f3 100644 --- a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml @@ -337,3 +337,64 @@ processors: field: event.category value: intrusion_detection if: 'ctx.event?.category != "malware" && (ctx.checkpoint?.protection_type != null || ctx.cef.extensions?.flexString2Label == "Attack Information")' + + # Handle zone-based network directionality + - set: + field: network.direction + value: inbound + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) + - set: + field: network.direction + value: outbound + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) + - set: + field: network.direction + value: internal + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) + - set: + field: network.direction + value: external + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) + - set: + field: network.direction + value: unknown + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ( + ( + !ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + !ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) + ) || + ( + !ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) && + !ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) + ) + ) + diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 7dab1ca3382..676f66a943a 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -87,6 +87,10 @@ processors: - pipeline: name: '{< IngestPipeline "cp-pipeline" >}' if: "ctx.cef?.device?.vendor == 'Check Point'" + - remove: + field: + - _temp_ + ignore_missing: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/cef/log/manifest.yml b/x-pack/filebeat/module/cef/log/manifest.yml index 777ac5010be..d1314088d69 100644 --- a/x-pack/filebeat/module/cef/log/manifest.yml +++ b/x-pack/filebeat/module/cef/log/manifest.yml @@ -12,6 +12,8 @@ var: default: 9003 - name: input default: syslog + - name: internal_zones + - name: external_zones ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/checkpoint/_meta/config.yml b/x-pack/filebeat/module/checkpoint/_meta/config.yml index 4408a7ba5f2..8ed0c7d11c2 100644 --- a/x-pack/filebeat/module/checkpoint/_meta/config.yml +++ b/x-pack/filebeat/module/checkpoint/_meta/config.yml @@ -11,3 +11,11 @@ # The UDP port to listen for syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index bcbe32b9ae8..e0fa537fc88 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -29,3 +29,16 @@ processors: target: '' fields: ecs.version: 1.7.0 +{{ if .external_zones }} + - add_fields: + target: _temp_ + fields: + external_zones: {{ .external_zones | tojson }} +{{ end }} + +{{ if .internal_zones }} + - add_fields: + target: _temp_ + fields: + internal_zones: {{ .internal_zones | tojson }} +{{ end }} diff --git a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml index d22d9a65eaf..975a0e76104 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/ingest/pipeline.yml @@ -781,6 +781,65 @@ processors: field: destination.as.organization_name target_field: destination.as.organization.name ignore_missing: true +# Handle zone-based network directionality +- set: + field: network.direction + value: inbound + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) +- set: + field: network.direction + value: outbound + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) +- set: + field: network.direction + value: internal + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) +- set: + field: network.direction + value: external + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) +- set: + field: network.direction + value: unknown + if: > + ctx?._temp_?.external_zones != null && + ctx?._temp_?.internal_zones != null && + ctx?.observer?.ingress?.zone != null && + ctx?.observer?.egress?.zone != null && + ( + ( + !ctx._temp_.external_zones.contains(ctx.observer.egress.zone) && + !ctx._temp_.internal_zones.contains(ctx.observer.egress.zone) + ) || + ( + !ctx._temp_.external_zones.contains(ctx.observer.ingress.zone) && + !ctx._temp_.internal_zones.contains(ctx.observer.ingress.zone) + ) + ) - remove: field: - checkpoint.client_outbound_packets @@ -801,6 +860,7 @@ processors: - checkpoint.uid - checkpoint.time - syslog5424_ts + - _temp_ ignore_missing: true on_failure: - set: diff --git a/x-pack/filebeat/module/checkpoint/firewall/manifest.yml b/x-pack/filebeat/module/checkpoint/firewall/manifest.yml index 69301541669..5d5d3dd4dcb 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/manifest.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/manifest.yml @@ -10,6 +10,8 @@ var: - name: input default: syslog - name: ssl + - name: internal_zones + - name: external_zones ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/modules.d/cef.yml.disabled b/x-pack/filebeat/modules.d/cef.yml.disabled index 3da653da87f..bb8eca97d6b 100644 --- a/x-pack/filebeat/modules.d/cef.yml.disabled +++ b/x-pack/filebeat/modules.d/cef.yml.disabled @@ -7,3 +7,11 @@ var: syslog_host: localhost syslog_port: 9003 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ] diff --git a/x-pack/filebeat/modules.d/checkpoint.yml.disabled b/x-pack/filebeat/modules.d/checkpoint.yml.disabled index 6963ecbef99..03db911f192 100644 --- a/x-pack/filebeat/modules.d/checkpoint.yml.disabled +++ b/x-pack/filebeat/modules.d/checkpoint.yml.disabled @@ -14,3 +14,11 @@ # The UDP port to listen for syslog traffic. Defaults to 9001. #var.syslog_port: 9001 + + # Set internal security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.internal_zones: [ "Internal" ] + + # Set external security zones. used to override parsed network.direction + # based on zone egress and ingress + #var.external_zones: [ "External" ]