From 471f0c83bd020e3cef63bff23c9cfa58211e741c Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 3 Nov 2023 10:17:12 +0100 Subject: [PATCH] Add network processor in addition to interface based direction resolution. --- .../fortinet/firewall/config/firewall.yml | 7 ++++ .../fortinet/firewall/ingest/pipeline.yml | 40 +++++++++++++------ .../module/fortinet/firewall/manifest.yml | 2 + .../firewall/test/event-nul.log-expected.json | 1 + .../firewall/test/event.log-expected.json | 3 ++ .../firewall/test/traffic.log-expected.json | 13 ++++++ .../firewall/test/utm.log-expected.json | 15 +++++++ 7 files changed, 68 insertions(+), 13 deletions(-) diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml index debef17bb199..c6d86333f558 100644 --- a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml @@ -44,3 +44,10 @@ processors: fields: internal_interfaces: {{ .internal_interfaces | tojson }} {{ end }} + +{{ if .internal_networks }} + - add_fields: + target: _temp + fields: + internal_networks: {{ .internal_networks | tojson }} +{{ end }} diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml index 244b5e1726ed..1b06c97e209b 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml @@ -169,6 +169,33 @@ processors: field: fortinet.firewall.level target_field: log.level ignore_missing: true +- remove: + field: + - message + ignore_missing: true +- pipeline: + name: '{< IngestPipeline "event" >}' + if: "ctx.fortinet?.firewall?.type == 'event'" +- pipeline: + name: '{< IngestPipeline "traffic" >}' + if: "ctx.fortinet?.firewall?.type == 'traffic'" +- pipeline: + name: '{< IngestPipeline "utm" >}' + if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'" +- rename: + field: fortinet.firewall.dir + target_field: network.direction + ignore_missing: true + if: ctx.network?.direction == null +- rename: + field: fortinet.firewall.direction + target_field: network.direction + ignore_missing: true + if: ctx.network?.direction == null +- network_direction: + internal_networks_field: _temp.internal_networks + ignore_missing: true + if: ctx.network?.direction == null # Handle interface-based network directionality - set: field: network.direction @@ -228,19 +255,6 @@ processors: !ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name) ) ) -- remove: - field: - - message - ignore_missing: true -- pipeline: - name: '{< IngestPipeline "event" >}' - if: "ctx.fortinet?.firewall?.type == 'event'" -- pipeline: - name: '{< IngestPipeline "traffic" >}' - if: "ctx.fortinet?.firewall?.type == 'traffic'" -- pipeline: - name: '{< IngestPipeline "utm" >}' - if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'" - rename: field: fortinet.firewall.reason target_field: event.reason diff --git a/x-pack/filebeat/module/fortinet/firewall/manifest.yml b/x-pack/filebeat/module/fortinet/firewall/manifest.yml index f76743d96d82..9ea97eaf88cc 100644 --- a/x-pack/filebeat/module/fortinet/firewall/manifest.yml +++ b/x-pack/filebeat/module/fortinet/firewall/manifest.yml @@ -11,6 +11,8 @@ var: default: udp - name: internal_interfaces - name: external_interfaces + - name: internal_networks + default: [private] ingest_pipeline: - ingest/pipeline.yml diff --git a/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json index 89c0254c9149..6027b583bd6d 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json @@ -79,6 +79,7 @@ "log.level": "error", "log.offset": 361, "message": "IPsec phase 1 error", + "network.direction": "external", "network.type": "ipv4", "observer.name": "testswitch3", "observer.product": "Fortigate", diff --git a/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json index debdaf06adb5..fd25700f576b 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json @@ -79,6 +79,7 @@ "log.level": "error", "log.offset": 413, "message": "IPsec phase 1 error", + "network.direction": "external", "network.type": "ipv4", "observer.name": "testswitch3", "observer.product": "Fortigate", @@ -690,6 +691,7 @@ "log.level": "information", "log.offset": 6204, "message": "Administrator admin logged in successfully from ssh(172.16.200.254)", + "network.direction": "internal", "network.type": "ipv4", "observer.product": "Fortigate", "observer.type": "firewall", @@ -791,6 +793,7 @@ "log.level": "notice", "log.offset": 7146, "message": "User bob succeeded in authentication", + "network.direction": "internal", "network.type": "ipv4", "observer.product": "Fortigate", "observer.type": "firewall", diff --git a/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json index b36a6ac66724..c15eb7519961 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json @@ -48,6 +48,7 @@ "log.offset": 0, "network.bytes": 0, "network.community_id": "1:8bNES6YDdszaPi28xM3VcSwJdbg=", + "network.direction": "outbound", "network.iana_number": "17", "network.protocol": "snmp", "network.transport": "udp", @@ -121,6 +122,7 @@ "log.level": "warning", "log.offset": 576, "network.community_id": "1:XfJvzLBAUivFLHoCO1CY6XoPK/8=", + "network.direction": "outbound", "network.iana_number": "17", "network.protocol": "dns", "network.transport": "udp", @@ -192,6 +194,7 @@ "log.offset": 1173, "network.bytes": 504096, "network.community_id": "1:0Eqo4bxxbVP3bSKsiVJ4ynR2uB8=", + "network.direction": "external", "network.iana_number": "17", "network.packets": 1769018, "network.protocol": "portname", @@ -281,6 +284,7 @@ "network.application": "icmp6/25/0", "network.bytes": 3034, "network.community_id": "1:88ozpFHdjx44KUIm/9vfDoO2jsk=", + "network.direction": "external", "network.iana_number": "58", "network.packets": 4, "network.protocol": "icmp6/1/0", @@ -364,6 +368,7 @@ "network.application": "PING", "network.bytes": 10, "network.community_id": "1:egBfbLgtrijMKr5zptNNyYdllaE=", + "network.direction": "external", "network.iana_number": "1", "network.packets": 40, "network.protocol": "ping", @@ -437,6 +442,7 @@ "log.level": "warning", "log.offset": 2990, "network.community_id": "1:8S1phidNTgIiEGM89KsStyENoH8=", + "network.direction": "internal", "network.iana_number": "17", "network.protocol": "udp/12302", "network.transport": "udp", @@ -535,6 +541,7 @@ "network.application": "Skype.Portals", "network.bytes": 78577, "network.community_id": "1:hTeZu8dnUyDg40++rBqS1lZF7AQ=", + "network.direction": "external", "network.iana_number": "6", "network.packets": 183, "network.protocol": "https", @@ -619,6 +626,7 @@ "log.offset": 4660, "network.bytes": 5940, "network.community_id": "1:2HKGEYlW4AJ/Af+zmajWDRu3kog=", + "network.direction": "outbound", "network.iana_number": "17", "network.packets": 11, "network.protocol": "udp/7878", @@ -691,6 +699,7 @@ "log.offset": 5177, "network.bytes": 0, "network.community_id": "1:FUNVChD2K8kyVo9eGl6FqaTLwPM=", + "network.direction": "external", "network.iana_number": "6", "network.packets": 0, "network.protocol": "https", @@ -784,6 +793,7 @@ "network.application": "HTTP.BROWSER", "network.bytes": 2698, "network.community_id": "1:mS2/WPDX46+WauGLEZvCIQ/IKK0=", + "network.direction": "internal", "network.iana_number": "6", "network.packets": 12, "network.protocol": "http", @@ -863,6 +873,7 @@ "log.offset": 6777, "network.bytes": 3874, "network.community_id": "1:6Q3s77giRtaDlbjtG7Qfum6LzEk=", + "network.direction": "outbound", "network.iana_number": "6", "network.packets": 19, "network.protocol": "http", @@ -943,6 +954,7 @@ "log.offset": 7629, "network.bytes": 9677, "network.community_id": "1:h1lO9dsjUlBQibNPDwk2LSH5uV4=", + "network.direction": "internal", "network.iana_number": "6", "network.packets": 17, "network.protocol": "http", @@ -1028,6 +1040,7 @@ "log.offset": 8377, "network.bytes": 7580, "network.community_id": "1:J2etn+6EN21BXHPPJZQeRpj+C3k=", + "network.direction": "outbound", "network.iana_number": "6", "network.packets": 45, "network.protocol": "https", diff --git a/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json index 109d58f54149..d9863533fe52 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json @@ -375,6 +375,7 @@ "log.offset": 2826, "message": "Domain is monitored", "network.community_id": "1:CEgDbLCgjPBvzcMawzocROJhmOU=", + "network.direction": "outbound", "network.iana_number": "17", "network.transport": "udp", "network.type": "ipv4", @@ -454,6 +455,7 @@ "log.offset": 3386, "message": "Domain is monitored", "network.community_id": "1:CEgDbLCgjPBvzcMawzocROJhmOU=", + "network.direction": "outbound", "network.iana_number": "17", "network.transport": "udp", "network.type": "ipv4", @@ -613,6 +615,7 @@ "log.offset": 4640, "message": "Domain is monitored", "network.community_id": "1:kD75mqzsTS7oiRllEEF3faHh2ic=", + "network.direction": "outbound", "network.iana_number": "17", "network.transport": "udp", "network.type": "ipv4", @@ -682,6 +685,7 @@ "log.level": "information", "log.offset": 5194, "network.community_id": "1:r3fHI3hh+39DMVJua+CV3LAi34M=", + "network.direction": "outbound", "network.iana_number": "17", "network.transport": "udp", "network.type": "ipv4", @@ -742,6 +746,7 @@ "log.offset": 5658, "message": "Server certificate passed", "network.community_id": "1:DPYPEQ6CL+DsivLJV6otkkVV6S8=", + "network.direction": "outbound", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1089,6 +1094,7 @@ "log.offset": 9040, "message": "anomaly: icmp_flood, 51 > threshold 50", "network.community_id": "1:/EwPCnPnhunCBJc8C73Iy8WlrhM=", + "network.direction": "internal", "network.iana_number": "1", "network.protocol": "ping", "network.transport": "icmp", @@ -1267,6 +1273,7 @@ "log.offset": 10820, "message": "Server certificate blocked", "network.community_id": "1:3JAdUt0lSMifcZEPoVJn1SC8tdE=", + "network.direction": "outbound", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1320,6 +1327,7 @@ "log.offset": 11247, "message": "Server certificate blocked", "network.community_id": "1:+CuXSKFw5mhoSjpYrUOYxAYOzaU=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1373,6 +1381,7 @@ "log.offset": 11675, "message": "Server certificate blocked", "network.community_id": "1:xeLbgVy2CNJ3q/bxUWxBBt6cGKM=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1426,6 +1435,7 @@ "log.offset": 12097, "message": "Connection is blocked due to unsupported SSL traffic", "network.community_id": "1:PohXhOT4cmeI1agRXluSxRuXkvM=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "smtps", "network.transport": "tcp", @@ -1480,6 +1490,7 @@ "log.offset": 12581, "message": "Server certificate blocked", "network.community_id": "1:gg6I8tZchtWCopsLdNDN7E84ZbU=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1534,6 +1545,7 @@ "log.offset": 13012, "message": "Certificate blacklisted", "network.community_id": "1:/tDtPynm8PUjA7+AXhG5maLXczU=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1587,6 +1599,7 @@ "log.offset": 13474, "message": "SSL connection exempted", "network.community_id": "1:o4PokgFFuw7PzgWghlu55zAVFAQ=", + "network.direction": "outbound", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1640,6 +1653,7 @@ "log.offset": 13890, "message": "SSL connection exempted", "network.community_id": "1:q6lEK+V8YAiHWchN6gVt5i1lbm8=", + "network.direction": "internal", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp", @@ -1693,6 +1707,7 @@ "log.offset": 14301, "message": "SSL connection exempted", "network.community_id": "1:fc1FAipY32n2Km+Fczx/L3cxBPE=", + "network.direction": "outbound", "network.iana_number": "6", "network.protocol": "https", "network.transport": "tcp",