From 306a5a887bf0eafdd942eb329ebb7c3c00f1028e Mon Sep 17 00:00:00 2001 From: Marius Iversen Date: Wed, 27 May 2020 17:40:58 +0200 Subject: [PATCH] [Filebeat][Fortinet]Converting certain fields to type long (#18736) * certain fields was set to string instead of long, causing issues with painless and did not conform with the ECS standards --- .../module/fortinet/firewall/ingest/event.yml | 45 ++++++-- .../fortinet/firewall/ingest/traffic.yml | 43 +++++-- .../module/fortinet/firewall/ingest/utm.yml | 58 +++++++--- .../firewall/test/fortinet.log-expected.json | 106 +++++++++--------- 4 files changed, 168 insertions(+), 84 deletions(-) diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml index 7365c802a83b..30b32592d253 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/event.yml @@ -61,19 +61,25 @@ processors: field: fortinet.firewall.remip target_field: destination.ip ignore_missing: true - if: "ctx.destination?.ip == null" -- rename: + if: "ctx.destination?.ip == null" +- convert: field: fortinet.firewall.dstport target_field: destination.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.remport target_field: destination.port + type: long + ignore_failure: true ignore_missing: true if: "ctx.destination?.port == null" -- rename: +- convert: field: fortinet.firewall.rcvdbyte target_field: destination.bytes + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.daddr @@ -93,9 +99,11 @@ processors: field: fortinet.firewall.group target_field: source.user.group.name ignore_missing: true -- rename: +- convert: field: fortinet.firewall.sentbyte target_field: source.bytes + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.srcip @@ -115,13 +123,17 @@ processors: target_field: source.mac ignore_missing: true if: "ctx.source?.mac == null" -- rename: +- convert: field: fortinet.firewall.srcport target_field: source.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.locport target_field: source.port + type: long + ignore_failure: true ignore_missing: true if: "ctx.source?.port == null" - rename: @@ -140,9 +152,11 @@ processors: field: fortinet.firewall.file target_field: file.name ignore_missing: true -- rename: +- convert: field: fortinet.firewall.filesize target_field: file.size + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.level @@ -197,9 +211,11 @@ processors: field: fortinet.firewall.url target_field: url.path ignore_missing: true -- rename: +- convert: field: fortinet.firewall.sess_duration + type: long target_field: event.duration + ignore_failure: true ignore_missing: true if: "ctx.event?.duration == null" - geoip: @@ -289,6 +305,17 @@ processors: field: related.user value: "{{source.user.name}}" if: "ctx.source?.user?.name != null" +- remove: + field: + - fortinet.firewall.dstport + - fortinet.firewall.remport + - fortinet.firewall.rcvdbyte + - fortinet.firewall.sentbyte + - fortinet.firewall.srcport + - fortinet.firewall.locport + - fortinet.firewall.filesize + - fortinet.firewall.sess_duration + ignore_missing: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml index 35aa4f68153d..2269a0ac286f 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/traffic.yml @@ -45,21 +45,29 @@ processors: field: fortinet.firewall.tranip target_field: destination.nat.ip ignore_missing: true -- rename: +- convert: field: fortinet.firewall.dstport target_field: destination.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.tranport target_field: destination.nat.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.rcvdbyte target_field: destination.bytes + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.rcvdpkt target_field: destination.packets + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.dstcollectedemail @@ -77,9 +85,11 @@ processors: field: fortinet.firewall.group target_field: source.user.group.name ignore_missing: true -- rename: +- convert: field: fortinet.firewall.sentbyte target_field: source.bytes + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.srcdomain @@ -93,9 +103,11 @@ processors: field: fortinet.firewall.srcmac target_field: source.mac ignore_missing: true -- rename: +- convert: field: fortinet.firewall.srcport target_field: source.port + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.unauthuser @@ -110,17 +122,21 @@ processors: field: fortinet.firewall.collectedemail target_field: source.user.email ignore_missing: true -- rename: +- convert: field: fortinet.firewall.sentpkt target_field: source.packets + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.transip target_field: source.nat.ip ignore_missing: true -- rename: +- convert: field: fortinet.firewall.transport target_field: source.nat.port + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.app @@ -280,6 +296,17 @@ processors: field: related.user value: "{{destination.user.name}}" if: "ctx.destination?.user?.name != null" +- remove: + field: + - fortinet.firewall.dstport + - fortinet.firewall.tranport + - fortinet.firewall.rcvdbyte + - fortinet.firewall.rcvdpkt + - fortinet.firewall.sentbyte + - fortinet.firewall.srcport + - fortinet.firewall.sentpkt + - fortinet.firewall.transport + ignore_missing: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml index 3fe35bbacf86..a85f09c332ab 100644 --- a/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml +++ b/x-pack/filebeat/module/fortinet/firewall/ingest/utm.yml @@ -30,24 +30,32 @@ processors: field: fortinet.firewall.remip target_field: destination.ip ignore_missing: true - if: "ctx.destination?.ip == null" -- rename: + if: "ctx.destination?.ip == null" +- convert: field: fortinet.firewall.dst_port target_field: destination.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.remport target_field: destination.port + type: long + ignore_failure: true ignore_missing: true if: "ctx.destination?.port == null" -- rename: +- convert: field: fortinet.firewall.dstport target_field: destination.port + type: long + ignore_failure: true ignore_missing: true if: "ctx.destination?.port == null" -- rename: +- convert: field: fortinet.firewall.rcvdbyte target_field: destination.bytes + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.recipient @@ -61,18 +69,31 @@ processors: field: fortinet.firewall.locip target_field: source.ip ignore_missing: true -- rename: +- convert: field: fortinet.firewall.locport target_field: source.port + type: long + ignore_failure: true ignore_missing: true -- rename: +- convert: field: fortinet.firewall.src_port target_field: source.port + type: long + ignore_failure: true ignore_missing: true if: "ctx.source?.port == null" -- rename: +- convert: + field: fortinet.firewall.srcport + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.source?.port == null" +- convert: field: fortinet.firewall.sentbyte target_field: source.bytes + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.srcdomain @@ -87,11 +108,6 @@ processors: field: fortinet.firewall.srcmac target_field: source.mac ignore_missing: true -- rename: - field: fortinet.firewall.srcport - target_field: source.port - ignore_missing: true - if: "ctx.source?.port == null" - rename: field: fortinet.firewall.unauthuser target_field: source.user.name @@ -171,9 +187,11 @@ processors: field: fortinet.firewall.filename target_field: file.name ignore_missing: true -- rename: +- convert: field: fortinet.firewall.filesize target_field: file.size + type: long + ignore_failure: true ignore_missing: true - rename: field: fortinet.firewall.filetype @@ -390,6 +408,18 @@ processors: field: related.hash value: "{{fortinet.file.hash.crc32}}" if: "ctx.fortinet?.file?.hash?.crc32 != null" +- remove: + field: + - fortinet.firewall.dst_port + - fortinet.firewall.remport + - fortinet.firewall.dstport + - fortinet.firewall.rcvdbyte + - fortinet.firewall.locport + - fortinet.firewall.src_port + - fortinet.firewall.srcport + - fortinet.firewall.sentbyte + - fortinet.firewall.filesize + ignore_missing: true on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json index 667eb25b9b50..815e73b5dae8 100644 --- a/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json +++ b/x-pack/filebeat/module/fortinet/firewall/test/fortinet.log-expected.json @@ -3,13 +3,13 @@ "@timestamp": "2020-04-23T12:17:48.000-05:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", - "destination.bytes": "1130", + "destination.bytes": 1130, "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "443", + "destination.port": 443, "event.action": "ftgd_blk", "event.category": [ "network" @@ -40,7 +40,7 @@ "log.level": "warning", "log.offset": 0, "message": "URL belongs to a denied category in policy", - "network.bytes": "11521130", + "network.bytes": 2282, "network.direction": "outgoing", "network.iana_number": "6", "network.protocol": "https", @@ -62,9 +62,9 @@ "rule.id": "100602", "rule.ruleset": "elasticruleset", "service.type": "fortinet", - "source.bytes": "1152", + "source.bytes": 1152, "source.ip": "192.168.2.1", - "source.port": "61930", + "source.port": 61930, "source.user.group.name": "elasticgroup", "source.user.name": "elasticuser", "tags": [ @@ -77,13 +77,13 @@ "@timestamp": "2020-04-23T12:17:45.000-05:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", - "destination.bytes": "6812", + "destination.bytes": 6812, "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "443", + "destination.port": 443, "event.action": "ftgd_allow", "event.category": [ "network" @@ -114,7 +114,7 @@ "log.level": "notice", "log.offset": 707, "message": "URL belongs to an allowed category in policy", - "network.bytes": "35456812", + "network.bytes": 10357, "network.direction": "outgoing", "network.iana_number": "6", "network.protocol": "https", @@ -136,9 +136,9 @@ "rule.id": "38", "rule.ruleset": "elasticruleset", "service.type": "fortinet", - "source.bytes": "3545", + "source.bytes": 3545, "source.ip": "192.168.2.1", - "source.port": "65236", + "source.port": 65236, "source.user.group.name": "elasticgroup", "source.user.name": "elasticuser", "tags": [ @@ -156,7 +156,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "443", + "destination.port": 443, "event.action": "signature", "event.category": [ "network" @@ -210,7 +210,7 @@ "rule.ruleset": "elasticruleset", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "59790", + "source.port": 59790, "source.user.group.name": "elasticgroup", "source.user.name": "elasticuser", "tags": [ @@ -229,7 +229,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "53", + "destination.port": 53, "dns.id": "2234", "dns.question.class": "IN", "dns.question.name": "elastic.example.com", @@ -281,7 +281,7 @@ "rule.ruleset": "test", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "53430", + "source.port": 53430, "tags": [ "fortinet-firewall" ] @@ -295,7 +295,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "443", + "destination.port": 443, "event.action": "signature", "event.category": [ "network" @@ -349,7 +349,7 @@ "rule.ruleset": "elasticruleset", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "63012", + "source.port": 63012, "source.user.group.name": "elasticgroup", "source.user.name": "elasticuser", "tags": [ @@ -367,7 +367,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "53", + "destination.port": 53, "dns.id": "2352", "dns.question.class": "IN", "dns.question.name": "elastic.co", @@ -419,7 +419,7 @@ "rule.ruleset": "elastictest", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "54438", + "source.port": 54438, "tags": [ "fortinet-firewall" ] @@ -433,7 +433,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "53", + "destination.port": 53, "dns.id": "235", "dns.question.class": "IN", "dns.question.name": "elastic.co", @@ -478,7 +478,7 @@ "rule.ruleset": "elastictest", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "54788", + "source.port": 54788, "tags": [ "fortinet-firewall" ] @@ -492,7 +492,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.4.4", - "destination.port": "443", + "destination.port": 443, "event.action": "ssl-anomalies", "event.category": [ "network" @@ -540,7 +540,7 @@ "rule.ruleset": "somecerts", "service.type": "fortinet", "source.ip": "192.168.2.1", - "source.port": "59726", + "source.port": 59726, "source.user.group.name": "elasticgroup2", "source.user.name": "elasticuser2", "tags": [ @@ -601,7 +601,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.4.4", - "destination.port": "500", + "destination.port": 500, "event.category": [ "network" ], @@ -650,7 +650,7 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "8.8.8.8", - "source.port": "500", + "source.port": 500, "tags": [ "fortinet-firewall" ] @@ -664,7 +664,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.4.5.4", - "destination.port": "500", + "destination.port": 500, "event.category": [ "network" ], @@ -717,7 +717,7 @@ "source.geo.location.lat": 48.8582, "source.geo.location.lon": 2.3387, "source.ip": "9.9.9.9", - "source.port": "500", + "source.port": 500, "tags": [ "fortinet-firewall" ] @@ -821,7 +821,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.5.4", - "destination.port": "500", + "destination.port": 500, "event.category": [ "network" ], @@ -872,7 +872,7 @@ "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "7.6.3.4", - "source.port": "500", + "source.port": 500, "tags": [ "fortinet-firewall" ] @@ -1160,7 +1160,7 @@ "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.port": "53", + "destination.port": 53, "event.action": "dns", "event.category": [ "network" @@ -1213,7 +1213,7 @@ "rule.uuid": "2345de-b143-52134d8-6654f-4654sdfg16f431", "service.type": "fortinet", "source.ip": "192.168.1.6", - "source.port": "53438", + "source.port": 53438, "tags": [ "fortinet-firewall" ] @@ -1222,14 +1222,14 @@ "@timestamp": "2020-04-23T12:11:51.000-05:00", "destination.as.number": 40386, "destination.as.organization.name": "Bloomip Inc.", - "destination.bytes": "65446", + "destination.bytes": 65446, "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.6.4.7", - "destination.packets": "1045601", - "destination.port": "6000", + "destination.packets": 1045601, + "destination.port": 6000, "event.action": "accept", "event.category": [ "network" @@ -1265,9 +1265,9 @@ "input.type": "log", "log.level": "notice", "log.offset": 11142, - "network.bytes": "43865065446", + "network.bytes": 504096, "network.iana_number": "17", - "network.packets": "7234171045601", + "network.packets": 1769018, "network.protocol": "portname", "observer.egress.interface.name": "wan1", "observer.ingress.interface.name": "port1", @@ -1288,7 +1288,7 @@ "service.type": "fortinet", "source.as.number": 4808, "source.as.organization.name": "China Unicom Beijing Province Network", - "source.bytes": "438650", + "source.bytes": 438650, "source.geo.city_name": "Beijing", "source.geo.continent_name": "Asia", "source.geo.country_iso_code": "CN", @@ -1298,9 +1298,9 @@ "source.geo.region_name": "Beijing", "source.ip": "192.168.10.10", "source.nat.ip": "123.123.123.123", - "source.nat.port": "60964", - "source.packets": "723417", - "source.port": "6000", + "source.nat.port": 60964, + "source.packets": 723417, + "source.port": 6000, "tags": [ "fortinet-firewall" ] @@ -1309,13 +1309,13 @@ "@timestamp": "2020-04-23T12:11:48.000-05:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", - "destination.bytes": "20", + "destination.bytes": 20, "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "2001:4860:4860::8888", - "destination.packets": "0", + "destination.packets": 0, "event.action": "accept", "event.category": [ "network" @@ -1348,9 +1348,9 @@ "log.level": "notice", "log.offset": 11876, "network.application": "icmp6/25/0", - "network.bytes": "301420", + "network.bytes": 3034, "network.iana_number": "58", - "network.packets": "40", + "network.packets": 4, "network.protocol": "icmp6/1/0", "observer.egress.interface.name": "unknown0", "observer.ingress.interface.name": "port1", @@ -1369,13 +1369,13 @@ "service.type": "fortinet", "source.as.number": 15169, "source.as.organization.name": "Google LLC", - "source.bytes": "3014", + "source.bytes": 3014, "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "2001:4860:4860::8888", - "source.packets": "4", + "source.packets": 4, "tags": [ "fortinet-firewall" ] @@ -1384,13 +1384,13 @@ "@timestamp": "2020-04-23T13:10:57.000-04:00", "destination.as.number": 15169, "destination.as.organization.name": "Google LLC", - "destination.bytes": "10", + "destination.bytes": 10, "destination.geo.continent_name": "North America", "destination.geo.country_iso_code": "US", "destination.geo.location.lat": 37.751, "destination.geo.location.lon": -97.822, "destination.ip": "8.8.8.8", - "destination.packets": "40", + "destination.packets": 40, "event.action": "accept", "event.category": [ "network" @@ -1425,9 +1425,9 @@ "log.level": "notice", "log.offset": 12426, "network.application": "PING", - "network.bytes": "010", + "network.bytes": 10, "network.iana_number": "1", - "network.packets": "040", + "network.packets": 40, "network.protocol": "ping", "observer.egress.interface.name": "unknown0", "observer.ingress.interface.name": "wan1", @@ -1444,13 +1444,13 @@ "rule.id": "0", "rule.ruleset": "rulepolicy", "service.type": "fortinet", - "source.bytes": "0", + "source.bytes": 0, "source.geo.continent_name": "North America", "source.geo.country_iso_code": "US", "source.geo.location.lat": 37.751, "source.geo.location.lon": -97.822, "source.ip": "9.7.7.7", - "source.packets": "0", + "source.packets": 0, "tags": [ "fortinet-firewall" ] @@ -1458,7 +1458,7 @@ { "@timestamp": "2020-04-23T12:14:39.000-05:00", "destination.ip": "192.168.100.100", - "destination.port": "1235", + "destination.port": 1235, "event.action": "ip-conn", "event.category": [ "network" @@ -1515,7 +1515,7 @@ "rule.uuid": "654cc-b6542-53467u8-e45234-1566casd35f7836", "service.type": "fortinet", "source.ip": "192.168.1.1", - "source.port": "62493", + "source.port": 62493, "source.user.name": "elasticsuper", "tags": [ "fortinet-firewall"