[Filebeat] NetFlow: Add Cisco ASA fields (#11201)

Cisco ASA uses some custom fields for NetFlow V9. From "Cisco ASA NetFlow Implementation Guide": - 33000 NF_F_INGRESS_ACL_ID, renamed to ingress_acl_id - 33001 NF_F_EGRESS_ACL_ID, renamed to egress_acl_id - 33002 NF_F_FW_EXT_EVENT, renamed to fw_ext_event - 40000 NF_F_USERNAME, renamed to username Some devices also use the following fields, from "Information Elements for Stealthwatch v7.0": - 40001 ASAXlateSourceAddressIPV4, renamed to xlate_source_address_ipv4 - 40002 ASAXlateDestinationAddressIPV4, renamed to xlate_destination_address_ipv4 - 40003 ASAXlateSourcePort, renamed to xlate_source_port - 40004 ASAXlateDestinationPort, renamed to xlate_destination_port - 40005 ASAFirewallEvent, renamed to firewall_event Cisco ASA NetFlow implementation requires a new field datatype, ACL ID, with 12-byte length, that encodes the following information: - First four bytes are the ACL name ID. - Next four bytes are the ACL entry ID / Object-Group ID. - Final four bytes are the Extended ACL Entry ID. Following with Logstash tradition, these fields are converted to string featuring the 3 ID's in encoded in hex and separated by a hypen.
elastic · Apr 1, 2019 · 5c04770 · 5c04770
1 parent f81ac68
commit 5c04770
Show file tree

Hide file tree

Showing 11 changed files with 239 additions and 114 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -319,6 +319,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for MySQL 8.0 slow logs and tests also for Percona 8.0 and MariaDB 10.3. {pull}11417[11417]
 - New Filebeat coredns module to ingest coredns logs. It supports both native coredns deployment and coredns deployment in kubernetes. {pull}11200[11200]
 - New module for Cisco ASA logs. {issue}9200[9200] {pull}11171[11171]
+- Added support for Cisco ASA fields to the netflow input. {pull}11201[11201]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv b/x-pack/filebeat/input/netflow/decoder/fields/cisco.csv
@@ -267,3 +267,12 @@ netscalerUnknown432,5951,432,unsigned8
 netscalerUnknown433,5951,433,unsigned8
 netscalerUnknown453,5951,453,unsigned64
 netscalerUnknown465,5951,465,unsigned32
+ingressAclID,0,33000,aclid
+egressAclID,0,33001,aclid
+fwExtEvent,0,33002,unsigned16
+username,0,40000,string
+XlateSourceAddressIPV4,0,40001,ipv4Address
+XlateDestinationAddressIPV4,0,40002,ipv4Address
+XlateSourcePort,0,40003,unsigned16
+XlateDestinationPort,0,40004,unsigned16
+FirewallEvent,0,40005,unsigned8
diff --git a/x-pack/filebeat/input/netflow/decoder/fields/gen.go b/x-pack/filebeat/input/netflow/decoder/fields/gen.go
@@ -61,6 +61,7 @@ var TypeNames = []string{
 	"BasicList",
 	"SubTemplateList",
 	"SubTemplateMultiList",
+	"ACLID",
 }
 
 func write(w io.Writer, msg string) {
@@ -176,7 +177,7 @@ func main() {
 	write(outHandle, fmt.Sprintf(`}
 
 func init() {
-  if err := RegisterFields(%s); err != nil {
+  if err := RegisterGlobalFields(%s); err != nil {
     panic(err)
   }
 }

diff --git a/x-pack/filebeat/input/netflow/decoder/fields/types.go b/x-pack/filebeat/input/netflow/decoder/fields/types.go
@@ -6,6 +6,7 @@ package fields
 
 import (
 	"encoding/binary"
+	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
@@ -322,6 +323,34 @@ func (u UnsupportedDecoder) Decode(data []byte) (interface{}, error) {
 
 var _ Decoder = (*UnsupportedDecoder)(nil)
 
+type ACLIDDecoder struct{}
+
+const aclIDLength = 12
+
+func (u ACLIDDecoder) MinLength() uint16 {
+	return aclIDLength
+}
+
+func (u ACLIDDecoder) MaxLength() uint16 {
+	return aclIDLength
+}
+
+func (u ACLIDDecoder) Decode(data []byte) (interface{}, error) {
+	if len(data) != aclIDLength {
+		return nil, ErrOutOfBounds
+	}
+	// Encode a [12]byte to a hex string in the form:
+	// "11223344-55667788-99aabbcc"
+	var result [aclIDLength*2 + 2]byte
+	hex.Encode(result[:8], data[:4])
+	hex.Encode(result[9:17], data[4:8])
+	hex.Encode(result[18:], data[8:])
+	result[8], result[17] = '-', '-'
+	return string(result[:]), nil
+}
+
+var _ Decoder = (*OctetArrayDecoder)(nil)
+
 // RFC5610 fields
 var (
 	OctetArray           = OctetArrayDecoder{}
@@ -348,3 +377,6 @@ var (
 	SubTemplateList      = UnsupportedDecoder{}
 	SubTemplateMultiList = UnsupportedDecoder{}
 )
+
+// ACLID field added for Cisco ASA devices
+var ACLID = ACLIDDecoder{}
diff --git a/x-pack/filebeat/input/netflow/decoder/fields/types_test.go b/x-pack/filebeat/input/netflow/decoder/fields/types_test.go
@@ -1092,3 +1092,38 @@ func TestUnsupported(t *testing.T) {
 		},
 	})
 }
+
+func TestACLID(t *testing.T) {
+	doTest(t, ACLID, 12, 12, []testCase{
+		{
+			title: "Empty",
+			bytes: []byte{},
+			err:   true,
+		},
+		{
+			title: "Sample",
+			bytes: []byte{
+				0x10, 0x21, 0x32, 0x43,
+				0x54, 0x65, 0x76, 0x87,
+				0x98, 0xA9, 0xBA, 0xCD},
+			value: "10213243-54657687-98a9bacd",
+		},
+		{
+			title: "Short",
+			bytes: []byte{
+				0x10, 0x21, 0x32, 0x43,
+				0x54, 0x65, 0x76, 0x87,
+				0x98, 0xA9, 0xBA},
+			err: true,
+		},
+		{
+			title: "Long",
+			bytes: []byte{
+				0x10, 0x21, 0x32, 0x43,
+				0x54, 0x65, 0x76, 0x87,
+				0x98, 0xA9, 0xBA, 0xCD,
+				0xDF},
+			err: true,
+		},
+	})
+}
diff --git a/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go b/x-pack/filebeat/input/netflow/decoder/fields/zfields_cisco.go
@@ -277,6 +277,15 @@ var CiscoFields = FieldDict{
 	Key{EnterpriseID: 5951, FieldID: 433}: {Name: "netscalerUnknown433", Decoder: Unsigned8},
 	Key{EnterpriseID: 5951, FieldID: 453}: {Name: "netscalerUnknown453", Decoder: Unsigned64},
 	Key{EnterpriseID: 5951, FieldID: 465}: {Name: "netscalerUnknown465", Decoder: Unsigned32},
+	Key{EnterpriseID: 0, FieldID: 33000}:  {Name: "ingressAclID", Decoder: ACLID},
+	Key{EnterpriseID: 0, FieldID: 33001}:  {Name: "egressAclID", Decoder: ACLID},
+	Key{EnterpriseID: 0, FieldID: 33002}:  {Name: "fwExtEvent", Decoder: Unsigned16},
+	Key{EnterpriseID: 0, FieldID: 40000}:  {Name: "username", Decoder: String},
+	Key{EnterpriseID: 0, FieldID: 40001}:  {Name: "XlateSourceAddressIPV4", Decoder: Ipv4Address},
+	Key{EnterpriseID: 0, FieldID: 40002}:  {Name: "XlateDestinationAddressIPV4", Decoder: Ipv4Address},
+	Key{EnterpriseID: 0, FieldID: 40003}:  {Name: "XlateSourcePort", Decoder: Unsigned16},
+	Key{EnterpriseID: 0, FieldID: 40004}:  {Name: "XlateDestinationPort", Decoder: Unsigned16},
+	Key{EnterpriseID: 0, FieldID: 40005}:  {Name: "FirewallEvent", Decoder: Unsigned8},
 }
 
 func init() {

diff --git a/x-pack/filebeat/input/netflow/definitions.go b/x-pack/filebeat/input/netflow/definitions.go
@@ -41,7 +41,7 @@ var logstashName2Decoder = map[string]fields.Decoder{
 	"uint64":                  fields.Unsigned64,
 	"octet_array":             fields.OctetArray,
 	"octetarray":              fields.OctetArray,
-	"acl_id_asa":              fields.UnsupportedDecoder{},
+	"acl_id_asa":              fields.ACLID,
 	"mpls_label_stack_octets": fields.UnsupportedDecoder{},
 	"application_id":          fields.UnsupportedDecoder{},
 	"forwarding_status":       fields.UnsupportedDecoder{},

diff --git a/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml b/x-pack/filebeat/input/netflow/testdata/dat_tests.yaml
@@ -92,6 +92,7 @@ tests:
       - netflow9_test_cisco_asa_2_tpl_27x.dat
       - netflow9_test_cisco_asa_2_data.dat
     custom_fields:
+      # This renames some fields to test the loading fields from file feature.
       - netflow9_cisco_asa_custom.yaml
   Netflow 9 ipt_netflow reduced size encoding:
     files:

diff --git a/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml b/x-pack/filebeat/input/netflow/testdata/fields/netflow9_cisco_asa_custom.yaml
@@ -1,3 +1,4 @@
+# This renames some fields to test the loading fields from file feature.
 33000:
 - :acl_id_asa
 - :ingress_acl_id
@@ -9,19 +10,19 @@
 - :fw_ext_event
 40000:
 - :string
-- :username
+- :asa_username
 40001:
 - :ip4_addr
-- :xlate_src_addr_ipv4
+- :asa_xlate_src_addr_ipv4
 40002:
 - :ip4_addr
-- :xlate_dst_addr_ipv4
+- :asa_xlate_dst_addr_ipv4
 40003:
 - :uint16
-- :xlate_src_port
+- :asa_xlate_src_port
 40004:
 - :uint16
-- :xlate_dst_port
+- :asa_xlate_dst_port
 40005:
 - :uint8
-- :fw_event
+- :asa_fw_event
diff --git a/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/Netflow-9-Cisco-ASA-2.golden.json
@@ -483,8 +483,10 @@
           "locality": "private"
         },
         "netflow": {
+          "asa_username": "",
           "destination_ipv4_address": "192.168.0.18",
           "destination_transport_port": 80,
+          "egress_acl_id": "00000000-00000000-00000000",
           "egress_interface": 4,
           "exporter": {
             "address": "192.0.2.1:4444",
@@ -499,6 +501,7 @@
           "fw_ext_event": 0,
           "icmp_code_ipv4": 0,
           "icmp_type_ipv4": 0,
+          "ingress_acl_id": "3edcde49-0aa62ac3-a8a2a76b",
           "ingress_interface": 3,
           "observation_time_milliseconds": "2016-07-21T13:50:33.385Z",
           "post_napt_destination_transport_port": 80,
@@ -508,8 +511,7 @@
           "protocol_identifier": 6,
           "source_ipv4_address": "192.168.0.1",
           "source_transport_port": 56649,
-          "type": "netflow_flow",
-          "username": ""
+          "type": "netflow_flow"
         },
         "network": {
           "community_id": "1:IZ8RrSqt8oeb2F2Rp9296zm54bc=",
@@ -680,8 +682,10 @@
           "locality": "private"
         },
         "netflow": {
+          "asa_username": "",
           "destination_ipv4_address": "192.168.0.17",
           "destination_transport_port": 80,
+          "egress_acl_id": "00000000-00000000-00000000",
           "egress_interface": 4,
           "exporter": {
             "address": "192.0.2.1:4444",
@@ -696,6 +700,7 @@
           "fw_ext_event": 0,
           "icmp_code_ipv4": 0,
           "icmp_type_ipv4": 0,
+          "ingress_acl_id": "3edcde49-0aa62ac3-56e8512e",
           "ingress_interface": 3,
           "observation_time_milliseconds": "2016-07-21T13:50:35.035Z",
           "post_napt_destination_transport_port": 80,
@@ -705,8 +710,7 @@
           "protocol_identifier": 6,
           "source_ipv4_address": "192.168.0.2",
           "source_transport_port": 61777,
-          "type": "netflow_flow",
-          "username": ""
+          "type": "netflow_flow"
         },
         "network": {
           "community_id": "1:E1vNamQGw5X+X+vT1g7ui6Nc3O0=",
@@ -877,8 +881,10 @@
           "locality": "private"
         },
         "netflow": {
+          "asa_username": "",
           "destination_ipv4_address": "192.168.0.17",
           "destination_transport_port": 80,
+          "egress_acl_id": "00000000-00000000-00000000",
           "egress_interface": 4,
           "exporter": {
             "address": "192.0.2.1:4444",
@@ -893,6 +899,7 @@
           "fw_ext_event": 0,
           "icmp_code_ipv4": 0,
           "icmp_type_ipv4": 0,
+          "ingress_acl_id": "3edcde49-0aa62ac3-56e8512e",
           "ingress_interface": 3,
           "observation_time_milliseconds": "2016-07-21T13:50:35.785Z",
           "post_napt_destination_transport_port": 80,
@@ -902,8 +909,7 @@
           "protocol_identifier": 6,
           "source_ipv4_address": "192.168.0.1",
           "source_transport_port": 56650,
-          "type": "netflow_flow",
-          "username": ""
+          "type": "netflow_flow"
         },
         "network": {
           "community_id": "1:pkwcoe/zjCLerUgj+HGAwwt4wV8=",
@@ -1074,8 +1080,10 @@
           "locality": "private"
         },
         "netflow": {
+          "asa_username": "",
           "destination_ipv4_address": "192.168.0.18",
           "destination_transport_port": 80,
+          "egress_acl_id": "00000000-00000000-00000000",
           "egress_interface": 4,
           "exporter": {
             "address": "192.0.2.1:4444",
@@ -1090,6 +1098,7 @@
           "fw_ext_event": 0,
           "icmp_code_ipv4": 0,
           "icmp_type_ipv4": 0,
+          "ingress_acl_id": "3edcde49-0aa62ac3-a8a2a76b",
           "ingress_interface": 3,
           "observation_time_milliseconds": "2016-07-21T13:50:36.395Z",
           "post_napt_destination_transport_port": 80,
@@ -1099,8 +1108,7 @@
           "protocol_identifier": 6,
           "source_ipv4_address": "192.168.0.1",
           "source_transport_port": 56651,
-          "type": "netflow_flow",
-          "username": ""
+          "type": "netflow_flow"
         },
         "network": {
           "community_id": "1:35/w0D/WO1QvBp8O+Vd95Nb+tt4=",