Add network processor in addition to interface based direction resolu…

…tion. (elastic#37023)
Scholar-Li · Feb 5, 2024 · a74e95d · a74e95d
1 parent 8c7918f
commit a74e95d
Show file tree

Hide file tree

Showing 11 changed files with 87 additions and 13 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -261,6 +261,7 @@ is collected by it.
 - Avoid unwanted publication of Okta entity records. {pull}36770[36770]
 - Add support for Digest Authentication to CEL input. {issue}35514[35514] {pull}36932[36932]
 - Use filestream input with file_identity.fingerprint as default for hints autodiscover. {issue}35984[35984] {pull}36950[36950]
+- Add network processor in addition to interface based direction resolution. {pull}37023[37023]
 
 *Auditbeat*
 

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -1031,6 +1031,12 @@ filebeat.modules:
     # set to leverage this functionality.
     #var.external_interfaces: [ "WAN" ]
 
+    # List of internal networks. Supports IPv4 and IPv6 addresses and ranges in CIDR notation.
+    # Also supports the named ranges listed
+    # [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/network-direction-processor.html#supported-named-network-ranges).
+    # This can't be used in combination with internal_interfaces and external_interfaces.
+    #var.internal_networks: [ "private" ]
+
   clientendpoint:
     enabled: false
 

diff --git a/x-pack/filebeat/module/fortinet/_meta/config.yml b/x-pack/filebeat/module/fortinet/_meta/config.yml
@@ -22,6 +22,12 @@
     # set to leverage this functionality.
     #var.external_interfaces: [ "WAN" ]
 
+    # List of internal networks. Supports IPv4 and IPv6 addresses and ranges in CIDR notation.
+    # Also supports the named ranges listed
+    # [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/network-direction-processor.html#supported-named-network-ranges).
+    # This can't be used in combination with internal_interfaces and external_interfaces.
+    #var.internal_networks: [ "private" ]
+
   clientendpoint:
     enabled: false
 

diff --git a/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml b/x-pack/filebeat/module/fortinet/firewall/config/firewall.yml
@@ -44,3 +44,10 @@ processors:
       fields:
         internal_interfaces: {{ .internal_interfaces | tojson }}
 {{ end }}
+
+{{ if .internal_networks }}
+  - add_fields:
+      target: _temp
+      fields:
+        internal_networks: {{ .internal_networks | tojson }}
+{{ end }}
diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/pipeline.yml
@@ -169,6 +169,33 @@ processors:
     field: fortinet.firewall.level
     target_field: log.level
     ignore_missing: true
+- remove:
+    field:
+      - message
+    ignore_missing: true
+- pipeline:
+    name: '{< IngestPipeline "event" >}'
+    if: "ctx.fortinet?.firewall?.type == 'event'"
+- pipeline:
+    name: '{< IngestPipeline "traffic" >}'
+    if: "ctx.fortinet?.firewall?.type == 'traffic'"
+- pipeline:
+    name: '{< IngestPipeline "utm" >}'
+    if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
+- rename:
+    field: fortinet.firewall.dir
+    target_field: network.direction
+    ignore_missing: true
+    if: ctx.network?.direction == null
+- rename:
+    field: fortinet.firewall.direction
+    target_field: network.direction
+    ignore_missing: true
+    if: ctx.network?.direction == null
+- network_direction:
+    internal_networks_field: _temp.internal_networks
+    ignore_missing: true
+    if: ctx.network?.direction == null
 # Handle interface-based network directionality
 - set:
     field: network.direction
@@ -228,19 +255,6 @@ processors:
           !ctx._temp.internal_interfaces.contains(ctx.observer.ingress.interface.name)
         )
       )
-- remove:
-    field:
-      - message
-    ignore_missing: true
-- pipeline:
-    name: '{< IngestPipeline "event" >}'
-    if: "ctx.fortinet?.firewall?.type == 'event'"
-- pipeline:
-    name: '{< IngestPipeline "traffic" >}'
-    if: "ctx.fortinet?.firewall?.type == 'traffic'"
-- pipeline:
-    name: '{< IngestPipeline "utm" >}'
-    if: "ctx.fortinet?.firewall?.type == 'utm' || ctx.fortinet?.firewall?.type == 'dns'"
 - rename:
     field: fortinet.firewall.reason
     target_field: event.reason

diff --git a/x-pack/filebeat/module/fortinet/firewall/manifest.yml b/x-pack/filebeat/module/fortinet/firewall/manifest.yml
@@ -11,6 +11,8 @@ var:
     default: udp
   - name: internal_interfaces
   - name: external_interfaces
+  - name: internal_networks
+    default: [private]
 
 ingest_pipeline:
   - ingest/pipeline.yml

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/event-nul.log-expected.json
@@ -79,6 +79,7 @@
         "log.level": "error",
         "log.offset": 361,
         "message": "IPsec phase 1 error",
+        "network.direction": "external",
         "network.type": "ipv4",
         "observer.name": "testswitch3",
         "observer.product": "Fortigate",

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/event.log-expected.json
@@ -79,6 +79,7 @@
         "log.level": "error",
         "log.offset": 413,
         "message": "IPsec phase 1 error",
+        "network.direction": "external",
         "network.type": "ipv4",
         "observer.name": "testswitch3",
         "observer.product": "Fortigate",
@@ -690,6 +691,7 @@
         "log.level": "information",
         "log.offset": 6204,
         "message": "Administrator admin logged in successfully from ssh(172.16.200.254)",
+        "network.direction": "internal",
         "network.type": "ipv4",
         "observer.product": "Fortigate",
         "observer.type": "firewall",
@@ -791,6 +793,7 @@
         "log.level": "notice",
         "log.offset": 7146,
         "message": "User bob succeeded in authentication",
+        "network.direction": "internal",
         "network.type": "ipv4",
         "observer.product": "Fortigate",
         "observer.type": "firewall",

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/traffic.log-expected.json
@@ -48,6 +48,7 @@
         "log.offset": 0,
         "network.bytes": 0,
         "network.community_id": "1:8bNES6YDdszaPi28xM3VcSwJdbg=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.protocol": "snmp",
         "network.transport": "udp",
@@ -121,6 +122,7 @@
         "log.level": "warning",
         "log.offset": 576,
         "network.community_id": "1:XfJvzLBAUivFLHoCO1CY6XoPK/8=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.protocol": "dns",
         "network.transport": "udp",
@@ -192,6 +194,7 @@
         "log.offset": 1173,
         "network.bytes": 504096,
         "network.community_id": "1:0Eqo4bxxbVP3bSKsiVJ4ynR2uB8=",
+        "network.direction": "external",
         "network.iana_number": "17",
         "network.packets": 1769018,
         "network.protocol": "portname",
@@ -281,6 +284,7 @@
         "network.application": "icmp6/25/0",
         "network.bytes": 3034,
         "network.community_id": "1:88ozpFHdjx44KUIm/9vfDoO2jsk=",
+        "network.direction": "external",
         "network.iana_number": "58",
         "network.packets": 4,
         "network.protocol": "icmp6/1/0",
@@ -364,6 +368,7 @@
         "network.application": "PING",
         "network.bytes": 10,
         "network.community_id": "1:egBfbLgtrijMKr5zptNNyYdllaE=",
+        "network.direction": "external",
         "network.iana_number": "1",
         "network.packets": 40,
         "network.protocol": "ping",
@@ -437,6 +442,7 @@
         "log.level": "warning",
         "log.offset": 2990,
         "network.community_id": "1:8S1phidNTgIiEGM89KsStyENoH8=",
+        "network.direction": "internal",
         "network.iana_number": "17",
         "network.protocol": "udp/12302",
         "network.transport": "udp",
@@ -535,6 +541,7 @@
         "network.application": "Skype.Portals",
         "network.bytes": 78577,
         "network.community_id": "1:hTeZu8dnUyDg40++rBqS1lZF7AQ=",
+        "network.direction": "external",
         "network.iana_number": "6",
         "network.packets": 183,
         "network.protocol": "https",
@@ -619,6 +626,7 @@
         "log.offset": 4660,
         "network.bytes": 5940,
         "network.community_id": "1:2HKGEYlW4AJ/Af+zmajWDRu3kog=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.packets": 11,
         "network.protocol": "udp/7878",
@@ -691,6 +699,7 @@
         "log.offset": 5177,
         "network.bytes": 0,
         "network.community_id": "1:FUNVChD2K8kyVo9eGl6FqaTLwPM=",
+        "network.direction": "external",
         "network.iana_number": "6",
         "network.packets": 0,
         "network.protocol": "https",
@@ -784,6 +793,7 @@
         "network.application": "HTTP.BROWSER",
         "network.bytes": 2698,
         "network.community_id": "1:mS2/WPDX46+WauGLEZvCIQ/IKK0=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.packets": 12,
         "network.protocol": "http",
@@ -863,6 +873,7 @@
         "log.offset": 6777,
         "network.bytes": 3874,
         "network.community_id": "1:6Q3s77giRtaDlbjtG7Qfum6LzEk=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.packets": 19,
         "network.protocol": "http",
@@ -943,6 +954,7 @@
         "log.offset": 7629,
         "network.bytes": 9677,
         "network.community_id": "1:h1lO9dsjUlBQibNPDwk2LSH5uV4=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.packets": 17,
         "network.protocol": "http",
@@ -1028,6 +1040,7 @@
         "log.offset": 8377,
         "network.bytes": 7580,
         "network.community_id": "1:J2etn+6EN21BXHPPJZQeRpj+C3k=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.packets": 45,
         "network.protocol": "https",

diff --git a/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/utm.log-expected.json
@@ -375,6 +375,7 @@
         "log.offset": 2826,
         "message": "Domain is monitored",
         "network.community_id": "1:CEgDbLCgjPBvzcMawzocROJhmOU=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.transport": "udp",
         "network.type": "ipv4",
@@ -454,6 +455,7 @@
         "log.offset": 3386,
         "message": "Domain is monitored",
         "network.community_id": "1:CEgDbLCgjPBvzcMawzocROJhmOU=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.transport": "udp",
         "network.type": "ipv4",
@@ -613,6 +615,7 @@
         "log.offset": 4640,
         "message": "Domain is monitored",
         "network.community_id": "1:kD75mqzsTS7oiRllEEF3faHh2ic=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.transport": "udp",
         "network.type": "ipv4",
@@ -682,6 +685,7 @@
         "log.level": "information",
         "log.offset": 5194,
         "network.community_id": "1:r3fHI3hh+39DMVJua+CV3LAi34M=",
+        "network.direction": "outbound",
         "network.iana_number": "17",
         "network.transport": "udp",
         "network.type": "ipv4",
@@ -742,6 +746,7 @@
         "log.offset": 5658,
         "message": "Server certificate passed",
         "network.community_id": "1:DPYPEQ6CL+DsivLJV6otkkVV6S8=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1089,6 +1094,7 @@
         "log.offset": 9040,
         "message": "anomaly: icmp_flood, 51 > threshold 50",
         "network.community_id": "1:/EwPCnPnhunCBJc8C73Iy8WlrhM=",
+        "network.direction": "internal",
         "network.iana_number": "1",
         "network.protocol": "ping",
         "network.transport": "icmp",
@@ -1267,6 +1273,7 @@
         "log.offset": 10820,
         "message": "Server certificate blocked",
         "network.community_id": "1:3JAdUt0lSMifcZEPoVJn1SC8tdE=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1320,6 +1327,7 @@
         "log.offset": 11247,
         "message": "Server certificate blocked",
         "network.community_id": "1:+CuXSKFw5mhoSjpYrUOYxAYOzaU=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1373,6 +1381,7 @@
         "log.offset": 11675,
         "message": "Server certificate blocked",
         "network.community_id": "1:xeLbgVy2CNJ3q/bxUWxBBt6cGKM=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1426,6 +1435,7 @@
         "log.offset": 12097,
         "message": "Connection is blocked due to unsupported SSL traffic",
         "network.community_id": "1:PohXhOT4cmeI1agRXluSxRuXkvM=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "smtps",
         "network.transport": "tcp",
@@ -1480,6 +1490,7 @@
         "log.offset": 12581,
         "message": "Server certificate blocked",
         "network.community_id": "1:gg6I8tZchtWCopsLdNDN7E84ZbU=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1534,6 +1545,7 @@
         "log.offset": 13012,
         "message": "Certificate blacklisted",
         "network.community_id": "1:/tDtPynm8PUjA7+AXhG5maLXczU=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1587,6 +1599,7 @@
         "log.offset": 13474,
         "message": "SSL connection exempted",
         "network.community_id": "1:o4PokgFFuw7PzgWghlu55zAVFAQ=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1640,6 +1653,7 @@
         "log.offset": 13890,
         "message": "SSL connection exempted",
         "network.community_id": "1:q6lEK+V8YAiHWchN6gVt5i1lbm8=",
+        "network.direction": "internal",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",
@@ -1693,6 +1707,7 @@
         "log.offset": 14301,
         "message": "SSL connection exempted",
         "network.community_id": "1:fc1FAipY32n2Km+Fczx/L3cxBPE=",
+        "network.direction": "outbound",
         "network.iana_number": "6",
         "network.protocol": "https",
         "network.transport": "tcp",

diff --git a/x-pack/filebeat/modules.d/fortinet.yml.disabled b/x-pack/filebeat/modules.d/fortinet.yml.disabled
@@ -25,6 +25,12 @@
     # set to leverage this functionality.
     #var.external_interfaces: [ "WAN" ]
 
+    # List of internal networks. Supports IPv4 and IPv6 addresses and ranges in CIDR notation.
+    # Also supports the named ranges listed
+    # [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/network-direction-processor.html#supported-named-network-ranges).
+    # This can't be used in combination with internal_interfaces and external_interfaces.
+    #var.internal_networks: [ "private" ]
+
   clientendpoint:
     enabled: false