From 6e084ebd4d9eaf7c5682603210bd8d99bd016aa7 Mon Sep 17 00:00:00 2001 From: Dan Kortschak <90160302+efd6@users.noreply.github.com> Date: Wed, 20 Apr 2022 06:39:38 +0930 Subject: [PATCH] x-pack/filebeat/input/netflow: replace invalid field value (#31295) * x-pack/filebeat/input/netflow: reduce lint and silence linter * x-pack/filebeat/input/netflow: replace invalid field value --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/netflow/convert.go | 60 +++++--- .../golden/ipfix_cisco.pcap.golden.json | 145 ++++++++++++++---- 3 files changed, 152 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9934014983c0..1ad3ad811fa2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -52,6 +52,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif - Do not emit error log when filestream reader reaches EOF and `close.reader.on_eof` is enabled. {pull}31109[31109] - m365_defender: Fix processing when alerts.entities is an empty list. {issue}31223[31223] {pull}31227[31227] - Prevent filestream from rereading whole files if they are rotated using rename. {pull}31268[31268] +- Netflow: replace invalid field value. {pull}31295[31295] *Heartbeat* diff --git a/x-pack/filebeat/input/netflow/convert.go b/x-pack/filebeat/input/netflow/convert.go index bb445968569c..9f2de13c6524 100644 --- a/x-pack/filebeat/input/netflow/convert.go +++ b/x-pack/filebeat/input/netflow/convert.go @@ -34,7 +34,13 @@ func toBeatEvent(flow record.Record, internalNetworks []string) (event beat.Even } } -func toBeatEventCommon(flow record.Record) (event beat.Event) { +func toBeatEventCommon(flow record.Record) beat.Event { + const ( + flowType = "netflow_flow" + optionsType = "netflow_options" + unknownType = "netflow_unknown" + ) + // replace net.HardwareAddress with its String() representation fixMacAddresses(flow.Fields) // Nest Exporter into netflow fields @@ -43,11 +49,11 @@ func toBeatEventCommon(flow record.Record) (event beat.Event) { // Nest Type into netflow fields switch flow.Type { case record.Flow: - flow.Fields["type"] = "netflow_flow" + flow.Fields["type"] = flowType case record.Options: - flow.Fields["type"] = "netflow_options" + flow.Fields["type"] = optionsType default: - flow.Fields["type"] = "netflow_unknown" + flow.Fields["type"] = unknownType } // ECS Fields -- event @@ -57,7 +63,7 @@ func toBeatEventCommon(flow record.Record) (event beat.Event) { "category": []string{"network_traffic", "network"}, "action": flow.Fields["type"], } - if ecsEvent["action"] == "netflow_flow" { + if ecsEvent["action"] == flowType { ecsEvent["type"] = []string{"connection"} } // ECS Fields -- device @@ -66,13 +72,14 @@ func toBeatEventCommon(flow record.Record) (event beat.Event) { ecsDevice["ip"] = extractIPFromIPPort(exporter) } - event.Timestamp = flow.Timestamp - event.Fields = common.MapStr{ - "netflow": fieldNameConverter.ToSnakeCase(flow.Fields), - "event": ecsEvent, - "observer": ecsDevice, + return beat.Event{ + Timestamp: flow.Timestamp, + Fields: common.MapStr{ + "netflow": fieldNameConverter.ToSnakeCase(flow.Fields), + "event": ecsEvent, + "observer": ecsDevice, + }, } - return } func extractIPFromIPPort(address string) string { @@ -101,8 +108,8 @@ func optionsToBeatEvent(flow record.Record) beat.Event { return toBeatEventCommon(flow) } -func flowToBeatEvent(flow record.Record, internalNetworks []string) (event beat.Event) { - event = toBeatEventCommon(flow) +func flowToBeatEvent(flow record.Record, internalNetworks []string) beat.Event { + event := toBeatEventCommon(flow) ecsEvent, ok := event.Fields["event"].(common.MapStr) if !ok { @@ -122,10 +129,10 @@ func flowToBeatEvent(flow record.Record, internalNetworks []string) (event beat. hasStartUptime = hasStartUptime && startUptime <= sysUptime hasEndUptime = hasEndUptime && endUptime <= sysUptime if hasStartUptime { - ecsEvent["start"] = flow.Timestamp.Add((time.Duration(startUptime) - time.Duration(sysUptime)) * time.Millisecond) + ecsEvent["start"] = flow.Timestamp.Add(time.Duration(startUptime-sysUptime) * time.Millisecond) } if hasEndUptime { - ecsEvent["end"] = flow.Timestamp.Add((time.Duration(endUptime) - time.Duration(sysUptime)) * time.Millisecond) + ecsEvent["end"] = flow.Timestamp.Add(time.Duration(endUptime-sysUptime) * time.Millisecond) } if hasStartUptime && hasEndUptime { ecsEvent["duration"] = ecsEvent["end"].(time.Time).Sub(ecsEvent["start"].(time.Time)).Nanoseconds() @@ -280,7 +287,7 @@ func flowToBeatEvent(flow record.Record, internalNetworks []string) (event beat. if biflowDir == 2 { ecsDest, ecsSource = ecsSource, ecsDest } - ecsEvent["category"] = "network_session" + ecsEvent["category"] = []string{"network", "session"} // Assume source is the client in biflows. event.Fields["client"] = ecsSource @@ -318,7 +325,7 @@ func flowToBeatEvent(flow record.Record, internalNetworks []string) (event beat. if len(relatedIP) > 0 { event.Fields["related"] = common.MapStr{"ip": uniqueIPs(relatedIP)} } - return + return event } // unique returns ips lexically sorted and with repeated elements @@ -344,10 +351,10 @@ func uniqueIPs(ips []net.IP) []net.IP { func getKeyUint64(dict record.Map, key string) (value uint64, found bool) { iface, found := dict[key] if !found { - return + return value, found } value, found = iface.(uint64) - return + return value, found } func getKeyUint64Alternatives(dict record.Map, keys ...string) (value uint64, found bool) { @@ -355,29 +362,29 @@ func getKeyUint64Alternatives(dict record.Map, keys ...string) (value uint64, fo for _, key := range keys { if iface, found = dict[key]; found { if value, found = iface.(uint64); found { - return + return value, found } } } - return + return value, found } func getKeyString(dict record.Map, key string) (value string, found bool) { iface, found := dict[key] if !found { - return + return value, found } value, found = iface.(string) - return + return value, found } func getKeyIP(dict record.Map, key string) (value net.IP, found bool) { iface, found := dict[key] if !found { - return + return value, found } value, found = iface.(net.IP) - return + return value, found } // Replaces each net.HardwareAddr in the dictionary with its string representation @@ -437,6 +444,7 @@ func getIPLocality(internalNetworks []string, ips ...net.IP) Locality { return LocalityInternal } +//nolint:godox // Bad linter! // TODO: create table from https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml // They have a CSV file available for conversion. @@ -467,6 +475,7 @@ func (p IPProtocol) String() string { func flowID(srcIP, dstIP net.IP, srcPort, dstPort uint16, proto uint8) string { h := xxhash.New() // Both flows will have the same ID. + //nolint:errcheck // Hash writes never fail. if srcPort >= dstPort { h.Write(srcIP) binary.Write(h, binary.BigEndian, srcPort) @@ -478,6 +487,7 @@ func flowID(srcIP, dstIP net.IP, srcPort, dstPort uint16, proto uint8) string { h.Write(srcIP) binary.Write(h, binary.BigEndian, srcPort) } + //nolint:errcheck // Hash writes never fail. binary.Write(h, binary.BigEndian, proto) return base64.RawURLEncoding.EncodeToString(h.Sum(nil)) diff --git a/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json b/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json index 149355177e94..6c19f8c9c457 100644 --- a/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json +++ b/x-pack/filebeat/input/netflow/testdata/golden/ipfix_cisco.pcap.golden.json @@ -15,7 +15,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -102,7 +105,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -189,7 +195,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -276,7 +285,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -363,7 +375,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -450,7 +465,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -537,7 +555,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -624,7 +645,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -711,7 +735,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -798,7 +825,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -885,7 +915,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -972,7 +1005,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1059,7 +1095,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1146,7 +1185,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1233,7 +1275,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1320,7 +1365,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1407,7 +1455,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1494,7 +1545,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1581,7 +1635,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1668,7 +1725,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1755,7 +1815,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1842,7 +1905,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -1929,7 +1995,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2016,7 +2085,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2103,7 +2175,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2190,7 +2265,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2277,7 +2355,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2364,7 +2445,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection" @@ -2451,7 +2535,10 @@ }, "event": { "action": "netflow_flow", - "category": "network_session", + "category": [ + "network", + "session" + ], "kind": "event", "type": [ "connection"