From cc4cf910194ded16187c076b652979a863d659ea Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 11 Mar 2019 19:48:30 +0100 Subject: [PATCH 1/5] NetFlow: Add ACL ID datatype for CISCO ASA Cisco ASA NetFlow implementation requires a new field datatype, ACL ID, with 12-byte length, that encodes the following information: - First four bytes are the ACL name ID. - Next four bytes are the ACL entry ID / Object-Group ID. - Final four bytes are the Extended ACL Entry ID. Following with Logstash tradition, these fields are converted to string featuring the 3 ID's in encoded in hex and separated by a hypen. --- .../input/netflow/decoder/fields/types.go | 30 ++++++++++++++++ .../netflow/decoder/fields/types_test.go | 35 +++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/x-pack/filebeat/input/netflow/decoder/fields/types.go b/x-pack/filebeat/input/netflow/decoder/fields/types.go index 383bc5d23f0..20ab1107df8 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/types.go +++ b/x-pack/filebeat/input/netflow/decoder/fields/types.go @@ -6,6 +6,7 @@ package fields import ( "encoding/binary" + "encoding/hex" "errors" "fmt" "io" @@ -322,6 +323,34 @@ func (u UnsupportedDecoder) Decode(data []byte) (interface{}, error) { var _ Decoder = (*UnsupportedDecoder)(nil) +type ACLIDDecoder struct{} + +const aclIDLength = 12 + +func (u ACLIDDecoder) MinLength() uint16 { + return aclIDLength +} + +func (u ACLIDDecoder) MaxLength() uint16 { + return aclIDLength +} + +func (u ACLIDDecoder) Decode(data []byte) (interface{}, error) { + if len(data) != aclIDLength { + return nil, ErrOutOfBounds + } + // Encode a [12]byte to a hex string in the form: + // "11223344-55667788-99aabbcc" + var result [aclIDLength*2 + 2]byte + hex.Encode(result[:8], data[:4]) + hex.Encode(result[9:17], data[4:8]) + hex.Encode(result[18:], data[8:]) + result[8], result[17] = '-', '-' + return string(result[:]), nil +} + +var _ Decoder = (*OctetArrayDecoder)(nil) + // RFC5610 fields var ( OctetArray = OctetArrayDecoder{} @@ -347,4 +376,5 @@ var ( BasicList = UnsupportedDecoder{} SubTemplateList = UnsupportedDecoder{} SubTemplateMultiList = UnsupportedDecoder{} + ACLID = ACLIDDecoder{} ) diff --git a/x-pack/filebeat/input/netflow/decoder/fields/types_test.go b/x-pack/filebeat/input/netflow/decoder/fields/types_test.go index 765e69dd398..ffe56c831dd 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/types_test.go +++ b/x-pack/filebeat/input/netflow/decoder/fields/types_test.go @@ -1092,3 +1092,38 @@ func TestUnsupported(t *testing.T) { }, }) } + +func TestACLID(t *testing.T) { + doTest(t, ACLID, 12, 12, []testCase{ + { + title: "Empty", + bytes: []byte{}, + err: true, + }, + { + title: "Sample", + bytes: []byte{ + 0x10, 0x21, 0x32, 0x43, + 0x54, 0x65, 0x76, 0x87, + 0x98, 0xA9, 0xBA, 0xCD}, + value: "10213243-54657687-98a9bacd", + }, + { + title: "Short", + bytes: []byte{ + 0x10, 0x21, 0x32, 0x43, + 0x54, 0x65, 0x76, 0x87, + 0x98, 0xA9, 0xBA}, + err: true, + }, + { + title: "Long", + bytes: []byte{ + 0x10, 0x21, 0x32, 0x43, + 0x54, 0x65, 0x76, 0x87, + 0x98, 0xA9, 0xBA, 0xCD, + 0xDF}, + err: true, + }, + }) +} From b3ef4784ceb8e724893af512f8e8a51021b4bb4a Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 11 Mar 2019 23:57:46 +0100 Subject: [PATCH 2/5] NetFlow: Add Cisco ASA fields Cisco ASA uses some custom fields for NetFlow V9. From "Cisco ASA NetFlow Implementation Guide": - 33000 NF_F_INGRESS_ACL_ID, renamed to ingress_acl_id - 33001 NF_F_EGRESS_ACL_ID, renamed to egress_acl_id - 33002 NF_F_FW_EXT_EVENT, renamed to fw_ext_event - 40000 NF_F_USERNAME, renamed to username Some devices also use the following fields, from "Information Elements for Stealthwatch v7.0": - 40001 ASAXlateSourceAddressIPV4, renamed to xlate_source_address_ipv4 - 40002 ASAXlateDestinationAddressIPV4, renamed to xlate_destination_address_ipv4 - 40003 ASAXlateSourcePort, renamed to xlate_source_port - 40004 ASAXlateDestinationPort, renamed to xlate_destination_port - 40005 ASAFirewallEvent, renamed to firewall_event --- x-pack/filebeat/input/netflow/decoder/fields/cisco.csv | 9 +++++++++ x-pack/filebeat/input/netflow/decoder/fields/gen.go | 3 ++- x-pack/filebeat/input/netflow/decoder/fields/types.go | 4 +++- .../input/netflow/decoder/fields/zfields_cisco.go | 9 +++++++++ x-pack/filebeat/input/netflow/definitions.go | 2 +- 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv b/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv index 8939dc97fc5..653a275d06f 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv +++ b/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv @@ -267,3 +267,12 @@ netscalerUnknown432,5951,432,unsigned8 netscalerUnknown433,5951,433,unsigned8 netscalerUnknown453,5951,453,unsigned64 netscalerUnknown465,5951,465,unsigned32 +ingressAclID,0,33000,aclid +egressAclID,0,33001,aclid +fwExtEvent,0,33002,unsigned16 +username,0,40000,string +XlateSourceAddressIPV4,0,40001,ipv4Address +XlateDestinationAddressIPV4,0,40002,ipv4Address +XlateSourcePort,0,40003,unsigned16 +XlateDestinationPort,0,40004,unsigned16 +FirewallEvent,0,40005,unsigned8 diff --git a/x-pack/filebeat/input/netflow/decoder/fields/gen.go b/x-pack/filebeat/input/netflow/decoder/fields/gen.go index 27e28148f94..743c1a062cb 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/gen.go +++ b/x-pack/filebeat/input/netflow/decoder/fields/gen.go @@ -61,6 +61,7 @@ var TypeNames = []string{ "BasicList", "SubTemplateList", "SubTemplateMultiList", + "ACLID", } func write(w io.Writer, msg string) { @@ -176,7 +177,7 @@ func main() { write(outHandle, fmt.Sprintf(`} func init() { - if err := RegisterFields(%s); err != nil { + if err := RegisterGlobalFields(%s); err != nil { panic(err) } } diff --git a/x-pack/filebeat/input/netflow/decoder/fields/types.go b/x-pack/filebeat/input/netflow/decoder/fields/types.go index 20ab1107df8..2832a9acc58 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/types.go +++ b/x-pack/filebeat/input/netflow/decoder/fields/types.go @@ -376,5 +376,7 @@ var ( BasicList = UnsupportedDecoder{} SubTemplateList = UnsupportedDecoder{} SubTemplateMultiList = UnsupportedDecoder{} - ACLID = ACLIDDecoder{} ) + +// ACLID field added for Cisco ASA devices +var ACLID = ACLIDDecoder{} diff --git a/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go b/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go index 05dce245848..ae37275d528 100644 --- a/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go +++ b/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go @@ -277,6 +277,15 @@ var CiscoFields = FieldDict{ Key{EnterpriseID: 5951, FieldID: 433}: {Name: "netscalerUnknown433", Decoder: Unsigned8}, Key{EnterpriseID: 5951, FieldID: 453}: {Name: "netscalerUnknown453", Decoder: Unsigned64}, Key{EnterpriseID: 5951, FieldID: 465}: {Name: "netscalerUnknown465", Decoder: Unsigned32}, + Key{EnterpriseID: 0, FieldID: 33000}: {Name: "ingressAclID", Decoder: ACLID}, + Key{EnterpriseID: 0, FieldID: 33001}: {Name: "egressAclID", Decoder: ACLID}, + Key{EnterpriseID: 0, FieldID: 33002}: {Name: "fwExtEvent", Decoder: Unsigned16}, + Key{EnterpriseID: 0, FieldID: 40000}: {Name: "username", Decoder: String}, + Key{EnterpriseID: 0, FieldID: 40001}: {Name: "XlateSourceAddressIPV4", Decoder: Ipv4Address}, + Key{EnterpriseID: 0, FieldID: 40002}: {Name: "XlateDestinationAddressIPV4", Decoder: Ipv4Address}, + Key{EnterpriseID: 0, FieldID: 40003}: {Name: "XlateSourcePort", Decoder: Unsigned16}, + Key{EnterpriseID: 0, FieldID: 40004}: {Name: "XlateDestinationPort", Decoder: Unsigned16}, + Key{EnterpriseID: 0, FieldID: 40005}: {Name: "FirewallEvent", Decoder: Unsigned8}, } func init() { diff --git a/x-pack/filebeat/input/netflow/definitions.go b/x-pack/filebeat/input/netflow/definitions.go index 3d1832d4dec..4227334d96a 100644 --- a/x-pack/filebeat/input/netflow/definitions.go +++ b/x-pack/filebeat/input/netflow/definitions.go @@ -41,7 +41,7 @@ var logstashName2Decoder = map[string]fields.Decoder{ "uint64": fields.Unsigned64, "octet_array": fields.OctetArray, "octetarray": fields.OctetArray, - "acl_id_asa": fields.UnsupportedDecoder{}, + "acl_id_asa": fields.ACLID, "mpls_label_stack_octets": fields.UnsupportedDecoder{}, "application_id": fields.UnsupportedDecoder{}, "forwarding_status": fields.UnsupportedDecoder{}, From e67b705ab125d796699e2f43d8d691fbea3cf1a4 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 12 Mar 2019 01:09:11 +0100 Subject: [PATCH 3/5] Update test data --- .../golden/Netflow-9-Cisco-ASA-2.golden.json | 8 ++++++ .../golden/Netflow-9-Cisco-ASA.golden.json | 28 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json index 409a4e910b1..aff611b8fd4 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json @@ -485,6 +485,7 @@ "netflow": { "destination_ipv4_address": "192.168.0.18", "destination_transport_port": 80, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 4, "exporter": { "address": "192.0.2.1:4444", @@ -499,6 +500,7 @@ "fw_ext_event": 0, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "3edcde49-0aa62ac3-a8a2a76b", "ingress_interface": 3, "observation_time_milliseconds": "2016-07-21T13:50:33.385Z", "post_napt_destination_transport_port": 80, @@ -682,6 +684,7 @@ "netflow": { "destination_ipv4_address": "192.168.0.17", "destination_transport_port": 80, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 4, "exporter": { "address": "192.0.2.1:4444", @@ -696,6 +699,7 @@ "fw_ext_event": 0, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "3edcde49-0aa62ac3-56e8512e", "ingress_interface": 3, "observation_time_milliseconds": "2016-07-21T13:50:35.035Z", "post_napt_destination_transport_port": 80, @@ -879,6 +883,7 @@ "netflow": { "destination_ipv4_address": "192.168.0.17", "destination_transport_port": 80, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 4, "exporter": { "address": "192.0.2.1:4444", @@ -893,6 +898,7 @@ "fw_ext_event": 0, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "3edcde49-0aa62ac3-56e8512e", "ingress_interface": 3, "observation_time_milliseconds": "2016-07-21T13:50:35.785Z", "post_napt_destination_transport_port": 80, @@ -1076,6 +1082,7 @@ "netflow": { "destination_ipv4_address": "192.168.0.18", "destination_transport_port": 80, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 4, "exporter": { "address": "192.0.2.1:4444", @@ -1090,6 +1097,7 @@ "fw_ext_event": 0, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "3edcde49-0aa62ac3-a8a2a76b", "ingress_interface": 3, "observation_time_milliseconds": "2016-07-21T13:50:36.395Z", "post_napt_destination_transport_port": 80, diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json index cf291331445..3657cc1cbd7 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json @@ -23,6 +23,7 @@ "netflow": { "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 17549, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -37,6 +38,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:49.599Z", "octet_total_count": 56, @@ -90,6 +92,7 @@ "netflow": { "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -104,6 +107,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:50.179Z", "octet_total_count": 56, @@ -157,6 +161,7 @@ "netflow": { "destination_ipv4_address": "192.168.23.22", "destination_transport_port": 17549, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -171,6 +176,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:50.219Z", "octet_total_count": 56, @@ -224,6 +230,7 @@ "netflow": { "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -238,6 +245,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:50.419Z", "octet_total_count": 56, @@ -291,6 +299,7 @@ "netflow": { "destination_ipv4_address": "192.168.23.20", "destination_transport_port": 17805, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -305,6 +314,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:50.429Z", "octet_total_count": 56, @@ -358,6 +368,7 @@ "netflow": { "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -372,6 +383,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:50.619Z", "octet_total_count": 56, @@ -425,6 +437,7 @@ "netflow": { "destination_ipv4_address": "192.168.14.11", "destination_transport_port": 17805, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -439,6 +452,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:50.639Z", "octet_total_count": 56, @@ -492,6 +506,7 @@ "netflow": { "destination_ipv4_address": "192.168.14.1", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -506,6 +521,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:50.639Z", "octet_total_count": 56, @@ -559,6 +575,7 @@ "netflow": { "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 17805, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -573,6 +590,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:50.639Z", "octet_total_count": 56, @@ -626,6 +644,7 @@ "netflow": { "destination_ipv4_address": "192.168.23.1", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -640,6 +659,7 @@ "fw_ext_event": 2016, "icmp_code_ipv4": 3, "icmp_type_ipv4": 3, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:51.269Z", "octet_total_count": 160, @@ -693,6 +713,7 @@ "netflow": { "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -707,6 +728,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:51.269Z", "octet_total_count": 56, @@ -760,6 +782,7 @@ "netflow": { "destination_ipv4_address": "192.168.23.22", "destination_transport_port": 18061, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -774,6 +797,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:51.289Z", "octet_total_count": 56, @@ -827,6 +851,7 @@ "netflow": { "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 3, "exporter": { "address": "192.0.2.1:4444", @@ -841,6 +866,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 2, "observation_time_milliseconds": "2015-10-09T09:47:51.489Z", "octet_total_count": 56, @@ -894,6 +920,7 @@ "netflow": { "destination_ipv4_address": "192.168.23.20", "destination_transport_port": 18061, + "egress_acl_id": "00000000-00000000-00000000", "egress_interface": 2, "exporter": { "address": "192.0.2.1:4444", @@ -908,6 +935,7 @@ "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, + "ingress_acl_id": "0f8e7ff3-fc1a030f-00000000", "ingress_interface": 3, "observation_time_milliseconds": "2015-10-09T09:47:51.509Z", "octet_total_count": 56, From 10a1b9d4b5919cf86c5f905a6e98169f7274b7c8 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 14 Mar 2019 09:46:49 +0100 Subject: [PATCH 4/5] Rename Cisco ASA fields in test --- .../input/netflow/testdata/dat_tests.yaml | 1 + .../fields/netflow9_cisco_asa_custom.yaml | 13 +- .../golden/Netflow-9-Cisco-ASA-2.golden.json | 16 +- .../golden/Netflow-9-Cisco-ASA.golden.json | 196 +++++++++--------- 4 files changed, 114 insertions(+), 112 deletions(-) diff --git a/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml b/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml index cd4b4048261..f8d69ffc70c 100644 --- a/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml +++ b/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml @@ -92,6 +92,7 @@ tests: - netflow9_test_cisco_asa_2_tpl_27x.dat - netflow9_test_cisco_asa_2_data.dat custom_fields: + # This renames some fields to test the loading fields from file feature. - netflow9_cisco_asa_custom.yaml Netflow 9 ipt_netflow reduced size encoding: files: diff --git a/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml b/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml index bcf40b96760..57c6aba2383 100644 --- a/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml +++ b/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml @@ -1,3 +1,4 @@ +# This renames some fields to test the loading fields from file feature. 33000: - :acl_id_asa - :ingress_acl_id @@ -9,19 +10,19 @@ - :fw_ext_event 40000: - :string -- :username +- :asa_username 40001: - :ip4_addr -- :xlate_src_addr_ipv4 +- :asa_xlate_src_addr_ipv4 40002: - :ip4_addr -- :xlate_dst_addr_ipv4 +- :asa_xlate_dst_addr_ipv4 40003: - :uint16 -- :xlate_src_port +- :asa_xlate_src_port 40004: - :uint16 -- :xlate_dst_port +- :asa_xlate_dst_port 40005: - :uint8 -- :fw_event +- :asa_fw_event diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json index aff611b8fd4..3cd8a553a61 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json @@ -483,6 +483,7 @@ "locality": "private" }, "netflow": { + "asa_username": "", "destination_ipv4_address": "192.168.0.18", "destination_transport_port": 80, "egress_acl_id": "00000000-00000000-00000000", @@ -510,8 +511,7 @@ "protocol_identifier": 6, "source_ipv4_address": "192.168.0.1", "source_transport_port": 56649, - "type": "netflow_flow", - "username": "" + "type": "netflow_flow" }, "network": { "community_id": "1:IZ8RrSqt8oeb2F2Rp9296zm54bc=", @@ -682,6 +682,7 @@ "locality": "private" }, "netflow": { + "asa_username": "", "destination_ipv4_address": "192.168.0.17", "destination_transport_port": 80, "egress_acl_id": "00000000-00000000-00000000", @@ -709,8 +710,7 @@ "protocol_identifier": 6, "source_ipv4_address": "192.168.0.2", "source_transport_port": 61777, - "type": "netflow_flow", - "username": "" + "type": "netflow_flow" }, "network": { "community_id": "1:E1vNamQGw5X+X+vT1g7ui6Nc3O0=", @@ -881,6 +881,7 @@ "locality": "private" }, "netflow": { + "asa_username": "", "destination_ipv4_address": "192.168.0.17", "destination_transport_port": 80, "egress_acl_id": "00000000-00000000-00000000", @@ -908,8 +909,7 @@ "protocol_identifier": 6, "source_ipv4_address": "192.168.0.1", "source_transport_port": 56650, - "type": "netflow_flow", - "username": "" + "type": "netflow_flow" }, "network": { "community_id": "1:pkwcoe/zjCLerUgj+HGAwwt4wV8=", @@ -1080,6 +1080,7 @@ "locality": "private" }, "netflow": { + "asa_username": "", "destination_ipv4_address": "192.168.0.18", "destination_transport_port": 80, "egress_acl_id": "00000000-00000000-00000000", @@ -1107,8 +1108,7 @@ "protocol_identifier": 6, "source_ipv4_address": "192.168.0.1", "source_transport_port": 56651, - "type": "netflow_flow", - "username": "" + "type": "netflow_flow" }, "network": { "community_id": "1:35/w0D/WO1QvBp8O+Vd95Nb+tt4=", diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json index 3657cc1cbd7..63dcdbf0cf2 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA.golden.json @@ -21,6 +21,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "2.2.2.11", + "asa_xlate_dst_port": 17549, + "asa_xlate_src_addr_ipv4": "192.168.14.1", + "asa_xlate_src_port": 0, "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 17549, "egress_acl_id": "00000000-00000000-00000000", @@ -34,7 +40,6 @@ }, "flow_id": 8500, "flow_start_milliseconds": "2015-10-09T09:47:47.569Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -45,12 +50,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.14.1", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "2.2.2.11", - "xlate_dst_port": 17549, - "xlate_src_addr_ipv4": "192.168.14.1", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -90,6 +90,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "164.164.37.11", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "192.168.23.22", + "asa_xlate_src_port": 17549, "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -103,7 +109,6 @@ }, "flow_id": 8501, "flow_start_milliseconds": "2015-10-09T09:47:48.169Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -114,12 +119,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.23.22", "source_transport_port": 17549, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "164.164.37.11", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "192.168.23.22", - "xlate_src_port": 17549 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -159,6 +159,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.23.22", + "asa_xlate_dst_port": 17549, + "asa_xlate_src_addr_ipv4": "164.164.37.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.23.22", "destination_transport_port": 17549, "egress_acl_id": "00000000-00000000-00000000", @@ -172,7 +178,6 @@ }, "flow_id": 8502, "flow_start_milliseconds": "2015-10-09T09:47:48.179Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -183,12 +188,7 @@ "protocol_identifier": 1, "source_ipv4_address": "164.164.37.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.23.22", - "xlate_dst_port": 17549, - "xlate_src_addr_ipv4": "164.164.37.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -228,6 +228,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "164.164.37.11", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "192.168.23.20", + "asa_xlate_src_port": 17805, "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -241,7 +247,6 @@ }, "flow_id": 8503, "flow_start_milliseconds": "2015-10-09T09:47:48.399Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -252,12 +257,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.23.20", "source_transport_port": 17805, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "164.164.37.11", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "192.168.23.20", - "xlate_src_port": 17805 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -297,6 +297,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.23.20", + "asa_xlate_dst_port": 17805, + "asa_xlate_src_addr_ipv4": "164.164.37.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.23.20", "destination_transport_port": 17805, "egress_acl_id": "00000000-00000000-00000000", @@ -310,7 +316,6 @@ }, "flow_id": 8504, "flow_start_milliseconds": "2015-10-09T09:47:48.409Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -321,12 +326,7 @@ "protocol_identifier": 1, "source_ipv4_address": "164.164.37.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.23.20", - "xlate_dst_port": 17805, - "xlate_src_addr_ipv4": "164.164.37.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -366,6 +366,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "2.2.2.11", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "192.168.14.11", + "asa_xlate_src_port": 17805, "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -379,7 +385,6 @@ }, "flow_id": 8505, "flow_start_milliseconds": "2015-10-09T09:47:48.589Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -390,12 +395,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.14.11", "source_transport_port": 17805, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "2.2.2.11", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "192.168.14.11", - "xlate_src_port": 17805 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -435,6 +435,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.14.11", + "asa_xlate_dst_port": 17805, + "asa_xlate_src_addr_ipv4": "2.2.2.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.14.11", "destination_transport_port": 17805, "egress_acl_id": "00000000-00000000-00000000", @@ -448,7 +454,6 @@ }, "flow_id": 8506, "flow_start_milliseconds": "2015-10-09T09:47:48.599Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -459,12 +464,7 @@ "protocol_identifier": 1, "source_ipv4_address": "2.2.2.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.14.11", - "xlate_dst_port": 17805, - "xlate_src_addr_ipv4": "2.2.2.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -504,6 +504,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.14.1", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "2.2.2.11", + "asa_xlate_src_port": 17805, "destination_ipv4_address": "192.168.14.1", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -517,7 +523,6 @@ }, "flow_id": 8507, "flow_start_milliseconds": "2015-10-09T09:47:48.609Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -528,12 +533,7 @@ "protocol_identifier": 1, "source_ipv4_address": "2.2.2.11", "source_transport_port": 17805, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.14.1", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "2.2.2.11", - "xlate_src_port": 17805 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -573,6 +573,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "2.2.2.11", + "asa_xlate_dst_port": 17805, + "asa_xlate_src_addr_ipv4": "192.168.14.1", + "asa_xlate_src_port": 0, "destination_ipv4_address": "2.2.2.11", "destination_transport_port": 17805, "egress_acl_id": "00000000-00000000-00000000", @@ -586,7 +592,6 @@ }, "flow_id": 8508, "flow_start_milliseconds": "2015-10-09T09:47:48.619Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -597,12 +602,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.14.1", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "2.2.2.11", - "xlate_dst_port": 17805, - "xlate_src_addr_ipv4": "192.168.14.1", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -642,6 +642,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.23.1", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "164.164.37.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.23.1", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -655,7 +661,6 @@ }, "flow_id": 8525, "flow_start_milliseconds": "2015-10-09T09:47:51.269Z", - "fw_event": 2, "fw_ext_event": 2016, "icmp_code_ipv4": 3, "icmp_type_ipv4": 3, @@ -666,12 +671,7 @@ "protocol_identifier": 1, "source_ipv4_address": "164.164.37.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.23.1", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "164.164.37.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 160, @@ -711,6 +711,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "164.164.37.11", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "192.168.23.22", + "asa_xlate_src_port": 18061, "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -724,7 +730,6 @@ }, "flow_id": 8509, "flow_start_milliseconds": "2015-10-09T09:47:49.249Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -735,12 +740,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.23.22", "source_transport_port": 18061, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "164.164.37.11", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "192.168.23.22", - "xlate_src_port": 18061 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -780,6 +780,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.23.22", + "asa_xlate_dst_port": 18061, + "asa_xlate_src_addr_ipv4": "164.164.37.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.23.22", "destination_transport_port": 18061, "egress_acl_id": "00000000-00000000-00000000", @@ -793,7 +799,6 @@ }, "flow_id": 8510, "flow_start_milliseconds": "2015-10-09T09:47:49.259Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -804,12 +809,7 @@ "protocol_identifier": 1, "source_ipv4_address": "164.164.37.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.23.22", - "xlate_dst_port": 18061, - "xlate_src_addr_ipv4": "164.164.37.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -849,6 +849,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "164.164.37.11", + "asa_xlate_dst_port": 0, + "asa_xlate_src_addr_ipv4": "192.168.23.20", + "asa_xlate_src_port": 18061, "destination_ipv4_address": "164.164.37.11", "destination_transport_port": 0, "egress_acl_id": "00000000-00000000-00000000", @@ -862,7 +868,6 @@ }, "flow_id": 8511, "flow_start_milliseconds": "2015-10-09T09:47:49.469Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 8, @@ -873,12 +878,7 @@ "protocol_identifier": 1, "source_ipv4_address": "192.168.23.20", "source_transport_port": 18061, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "164.164.37.11", - "xlate_dst_port": 0, - "xlate_src_addr_ipv4": "192.168.23.20", - "xlate_src_port": 18061 + "type": "netflow_flow" }, "network": { "bytes": 56, @@ -918,6 +918,12 @@ "locality": "public" }, "netflow": { + "asa_fw_event": 2, + "asa_username": "", + "asa_xlate_dst_addr_ipv4": "192.168.23.20", + "asa_xlate_dst_port": 18061, + "asa_xlate_src_addr_ipv4": "164.164.37.11", + "asa_xlate_src_port": 0, "destination_ipv4_address": "192.168.23.20", "destination_transport_port": 18061, "egress_acl_id": "00000000-00000000-00000000", @@ -931,7 +937,6 @@ }, "flow_id": 8512, "flow_start_milliseconds": "2015-10-09T09:47:49.479Z", - "fw_event": 2, "fw_ext_event": 2025, "icmp_code_ipv4": 0, "icmp_type_ipv4": 0, @@ -942,12 +947,7 @@ "protocol_identifier": 1, "source_ipv4_address": "164.164.37.11", "source_transport_port": 0, - "type": "netflow_flow", - "username": "", - "xlate_dst_addr_ipv4": "192.168.23.20", - "xlate_dst_port": 18061, - "xlate_src_addr_ipv4": "164.164.37.11", - "xlate_src_port": 0 + "type": "netflow_flow" }, "network": { "bytes": 56, From ebea41be0473b539ca5bab4c3e059a3bb4860e9c Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 28 Mar 2019 14:53:57 +0100 Subject: [PATCH 5/5] CHANGELOG updated --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 793c94615b1..2e0f1acc47b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -317,6 +317,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for MySQL 8.0 slow logs and tests also for Percona 8.0 and MariaDB 10.3. {pull}11417[11417] - New Filebeat coredns module to ingest coredns logs. It supports both native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200] - New module for Cisco ASA logs. {issue}9200[9200] {pull}11171[11171] +- Added support for Cisco ASA fields to the netflow input. {pull}11201[11201] *Heartbeat*