From ef554d361280dd8004d1672464486650852f06ad Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Wed, 9 Feb 2022 17:50:57 +0100 Subject: [PATCH 1/8] Resolving trap OIDs to readable names --- pkg/logs/config/config.go | 4 +- pkg/logs/internal/launchers/traps/launcher.go | 7 +- pkg/logs/internal/tailers/traps/tailer.go | 16 +- .../internal/tailers/traps/tailer_test.go | 19 +- pkg/snmp/traps/format.go | 182 ------------- pkg/snmp/traps/format_test.go | 184 ------------- pkg/snmp/traps/formatter.go | 247 +++++++++++++++++ pkg/snmp/traps/formatter_test.go | 250 ++++++++++++++++++ pkg/snmp/traps/oid_resolver.go | 185 +++++++++++++ pkg/snmp/traps/oid_resolver_test.go | 219 +++++++++++++++ pkg/snmp/traps/server_test.go | 53 +++- pkg/snmp/traps/testing.go | 20 -- pkg/snmp/traps/traps_db.go | 33 +++ ...-Traps-OIDs-to-names-70de58eecc4892aa.yaml | 11 + 14 files changed, 1016 insertions(+), 414 deletions(-) delete mode 100644 pkg/snmp/traps/format.go delete mode 100644 pkg/snmp/traps/format_test.go create mode 100644 pkg/snmp/traps/formatter.go create mode 100644 pkg/snmp/traps/formatter_test.go create mode 100644 pkg/snmp/traps/oid_resolver.go create mode 100644 pkg/snmp/traps/oid_resolver_test.go create mode 100644 pkg/snmp/traps/traps_db.go create mode 100644 releasenotes/notes/Resolve-SNMP-Traps-OIDs-to-names-70de58eecc4892aa.yaml diff --git a/pkg/logs/config/config.go b/pkg/logs/config/config.go index 4e0c3a10c4b810..b59eb2a10dcc70 100644 --- a/pkg/logs/config/config.go +++ b/pkg/logs/config/config.go @@ -78,8 +78,8 @@ func SNMPTrapsSource() *LogSource { // source to forward SNMP traps as logs. return NewLogSource(SnmpTraps, &LogsConfig{ Type: SnmpTrapsType, - Service: "snmp", - Source: "snmp", + Service: "snmp-traps", + Source: "snmp-traps", }) } return nil diff --git a/pkg/logs/internal/launchers/traps/launcher.go b/pkg/logs/internal/launchers/traps/launcher.go index f46987a7ec0ee4..33f3f8e1d3f40b 100644 --- a/pkg/logs/internal/launchers/traps/launcher.go +++ b/pkg/logs/internal/launchers/traps/launcher.go @@ -9,6 +9,7 @@ import ( "github.com/DataDog/datadog-agent/pkg/logs/config" tailer "github.com/DataDog/datadog-agent/pkg/logs/internal/tailers/traps" "github.com/DataDog/datadog-agent/pkg/logs/pipeline" + "github.com/DataDog/datadog-agent/pkg/security/log" "github.com/DataDog/datadog-agent/pkg/snmp/traps" ) @@ -36,7 +37,11 @@ func (l *Launcher) Start() { func (l *Launcher) startNewTailer(source *config.LogSource, inputChan chan *traps.SnmpPacket) { outputChan := l.pipelineProvider.NextPipelineChan() - l.tailer = tailer.NewTailer(source, inputChan, outputChan) + oidResolver, err := traps.NewMultiFilesOIDResolver() + if err != nil { + log.Errorf("unable to load traps database: %w", err) + } + l.tailer = tailer.NewTailer(oidResolver, source, inputChan, outputChan) l.tailer.Start() } diff --git a/pkg/logs/internal/tailers/traps/tailer.go b/pkg/logs/internal/tailers/traps/tailer.go index aaba34bee9c249..021a00e6134e6d 100644 --- a/pkg/logs/internal/tailers/traps/tailer.go +++ b/pkg/logs/internal/tailers/traps/tailer.go @@ -6,7 +6,6 @@ package traps import ( - "encoding/json" "time" "github.com/DataDog/datadog-agent/pkg/logs/config" @@ -18,17 +17,19 @@ import ( // Tailer consumes and processes a stream of trap packets, and sends them to a stream of log messages. type Tailer struct { source *config.LogSource + formatter traps.Formatter inputChan traps.PacketsChannel outputChan chan *message.Message done chan interface{} } // NewTailer returns a new Tailer -func NewTailer(source *config.LogSource, inputChan traps.PacketsChannel, outputChan chan *message.Message) *Tailer { +func NewTailer(oidResolver traps.OIDResolver, source *config.LogSource, inputChan traps.PacketsChannel, outputChan chan *message.Message) *Tailer { return &Tailer{ source: source, inputChan: inputChan, outputChan: outputChan, + formatter: traps.NewJSONFormatter(oidResolver), done: make(chan interface{}, 1), } } @@ -50,19 +51,14 @@ func (t *Tailer) run() { // Loop terminates when the channel is closed. for packet := range t.inputChan { - data, err := traps.FormatPacketToJSON(packet) + data, err := t.formatter.FormatPacket(packet) if err != nil { log.Errorf("failed to format packet: %s", err) continue } t.source.BytesRead.Add(int64(len(data))) - content, err := json.Marshal(data) - if err != nil { - log.Errorf("failed to serialize packet data to JSON: %s", err) - continue - } origin := message.NewOrigin(t.source) - origin.SetTags(traps.GetTags(packet)) - t.outputChan <- message.NewMessage(content, origin, message.StatusInfo, time.Now().UnixNano()) + origin.SetTags(t.formatter.GetTags(packet)) + t.outputChan <- message.NewMessage(data, origin, message.StatusInfo, time.Now().UnixNano()) } } diff --git a/pkg/logs/internal/tailers/traps/tailer_test.go b/pkg/logs/internal/tailers/traps/tailer_test.go index 690e5ebe4171d4..ca8f78f856c88b 100644 --- a/pkg/logs/internal/tailers/traps/tailer_test.go +++ b/pkg/logs/internal/tailers/traps/tailer_test.go @@ -6,7 +6,6 @@ package traps import ( - "encoding/json" "net" "testing" "time" @@ -23,7 +22,7 @@ import ( func TestTrapsShouldReceiveMessages(t *testing.T) { inputChan := make(traps.PacketsChannel, 1) outputChan := make(chan *message.Message) - tailer := NewTailer(config.NewLogSource("test", &config.LogsConfig{}), inputChan, outputChan) + tailer := NewTailer(&traps.NoOpOIDResolver{}, config.NewLogSource("test", &config.LogsConfig{}), inputChan, outputChan) tailer.Start() p := &traps.SnmpPacket{ @@ -46,18 +45,22 @@ func TestTrapsShouldReceiveMessages(t *testing.T) { return } + formattedPacket := format(t, p) assert.Equal(t, message.StatusInfo, msg.GetStatus()) - assert.Equal(t, format(t, p), msg.Content) - assert.Equal(t, traps.GetTags(p), msg.Origin.Tags()) + assert.Equal(t, formattedPacket, msg.Content) + assert.Equal(t, []string{ + "snmp_version:2", + "device_namespace:default", + "snmp_device:127.0.0.1", + }, msg.Origin.Tags()) close(inputChan) tailer.WaitFlush() } func format(t *testing.T, p *traps.SnmpPacket) []byte { - data, err := traps.FormatPacketToJSON(p) + formatter := traps.NewJSONFormatter(nil) + formattedPacket, err := formatter.FormatPacket(p) assert.NoError(t, err) - content, err := json.Marshal(data) - assert.NoError(t, err) - return content + return formattedPacket } diff --git a/pkg/snmp/traps/format.go b/pkg/snmp/traps/format.go deleted file mode 100644 index 732bf5d52807b5..00000000000000 --- a/pkg/snmp/traps/format.go +++ /dev/null @@ -1,182 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016-present Datadog, Inc. - -package traps - -import ( - "fmt" - "strings" - - "github.com/gosnmp/gosnmp" -) - -const ( - sysUpTimeInstanceOID = "1.3.6.1.2.1.1.3.0" - snmpTrapOID = "1.3.6.1.6.3.1.1.4.1.0" -) - -// FormatPacketToJSON converts an SNMP trap packet to a JSON-serializable object. -func FormatPacketToJSON(packet *SnmpPacket) (map[string]interface{}, error) { - if packet.Content.Version == gosnmp.Version1 { - return formatV1Trap(packet), nil - } - return formatTrap(packet) -} - -// GetTags returns a list of tags associated to an SNMP trap packet. -func GetTags(packet *SnmpPacket) []string { - return []string{ - fmt.Sprintf("snmp_version:%s", formatVersion(packet)), - fmt.Sprintf("device_namespace:%s", GetNamespace()), - fmt.Sprintf("snmp_device:%s", packet.Addr.IP.String()), - } -} - -func formatVersion(packet *SnmpPacket) string { - switch packet.Content.Version { - case gosnmp.Version3: - return "3" - case gosnmp.Version2c: - return "2" - case gosnmp.Version1: - return "1" - default: - return "unknown" - } -} - -func formatV1Trap(packet *SnmpPacket) map[string]interface{} { - data := make(map[string]interface{}) - data["uptime"] = uint32(packet.Content.Timestamp) - enterpriseOid := normalizeOID(packet.Content.Enterprise) - genericTrap := packet.Content.GenericTrap - specificTrap := packet.Content.SpecificTrap - var trapOID string - if genericTrap == 6 { - // Vendor-specific trap - trapOID = fmt.Sprintf("%s.0.%d", enterpriseOid, specificTrap) - } else { - // Generic trap - trapOID = fmt.Sprintf("%s.%d", genericTrapOid, genericTrap+1) - } - data["oid"] = trapOID - data["enterprise_oid"] = enterpriseOid - data["generic_trap"] = genericTrap - data["specific_trap"] = specificTrap - data["variables"] = parseVariables(packet.Content.Variables) - - return data -} - -func formatTrap(packet *SnmpPacket) (map[string]interface{}, error) { - /* - An SNMP v2 or v3 trap packet consists in the following variables (PDUs): - {sysUpTime.0, snmpTrapOID.0, additionalDataVariables...} - See: https://tools.ietf.org/html/rfc3416#section-4.2.6 - */ - variables := packet.Content.Variables - if len(variables) < 2 { - return nil, fmt.Errorf("expected at least 2 variables, got %d", len(variables)) - } - - data := make(map[string]interface{}) - - uptime, err := parseSysUpTime(variables[0]) - if err != nil { - return nil, err - } - data["uptime"] = uptime - - trapOID, err := parseSnmpTrapOID(variables[1]) - if err != nil { - return nil, err - } - data["oid"] = trapOID - - data["variables"] = parseVariables(variables[2:]) - - return data, nil -} - -func normalizeOID(value string) string { - // OIDs can be formatted as ".1.2.3..." ("absolute form") or "1.2.3..." ("relative form"). - // Convert everything to relative form, like we do in the Python check. - return strings.TrimLeft(value, ".") -} - -func parseSysUpTime(variable gosnmp.SnmpPDU) (uint32, error) { - name := normalizeOID(variable.Name) - if name != sysUpTimeInstanceOID { - return 0, fmt.Errorf("expected OID %s, got %s", sysUpTimeInstanceOID, name) - } - - value, ok := variable.Value.(uint32) - if !ok { - return 0, fmt.Errorf("expected uptime to be uint32 (got %v of type %T)", variable.Value, variable.Value) - } - - return value, nil -} - -func parseSnmpTrapOID(variable gosnmp.SnmpPDU) (string, error) { - name := normalizeOID(variable.Name) - if name != snmpTrapOID { - return "", fmt.Errorf("expected OID %s, got %s", snmpTrapOID, name) - } - - value := "" - switch variable.Value.(type) { - case string: - value = variable.Value.(string) - case []byte: - value = string(variable.Value.([]byte)) - default: - return "", fmt.Errorf("expected snmpTrapOID to be a string (got %v of type %T)", variable.Value, variable.Value) - } - - return normalizeOID(value), nil -} - -func parseVariables(variables []gosnmp.SnmpPDU) []map[string]interface{} { - var parsedVariables []map[string]interface{} - - for _, variable := range variables { - parsedVariable := make(map[string]interface{}) - parsedVariable["oid"] = normalizeOID(variable.Name) - parsedVariable["type"] = formatType(variable) - parsedVariable["value"] = formatValue(variable) - parsedVariables = append(parsedVariables, parsedVariable) - } - - return parsedVariables -} - -func formatType(variable gosnmp.SnmpPDU) string { - switch variable.Type { - case gosnmp.Integer, gosnmp.Uinteger32: - return "integer" - case gosnmp.OctetString: - return "string" - case gosnmp.ObjectIdentifier: - return "oid" - case gosnmp.Counter32: - return "counter32" - case gosnmp.Counter64: - return "counter64" - case gosnmp.Gauge32: - return "gauge32" - default: - return "other" - } -} - -func formatValue(variable gosnmp.SnmpPDU) interface{} { - switch variable.Value.(type) { - case []byte: - return string(variable.Value.([]byte)) - default: - return variable.Value - } -} diff --git a/pkg/snmp/traps/format_test.go b/pkg/snmp/traps/format_test.go deleted file mode 100644 index ac68c0da65317d..00000000000000 --- a/pkg/snmp/traps/format_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2020-present Datadog, Inc. - -package traps - -import ( - "net" - "testing" - - "github.com/gosnmp/gosnmp" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func createTestV1GenericPacket() *SnmpPacket { - examplePacket := &gosnmp.SnmpPacket{Version: gosnmp.Version1, SnmpTrap: LinkDownv1GenericTrap} - examplePacket.Variables = examplePacket.SnmpTrap.Variables - return &SnmpPacket{ - Content: examplePacket, - Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, - } -} - -func createTestV1SpecificPacket() *SnmpPacket { - examplePacket := &gosnmp.SnmpPacket{Version: gosnmp.Version1, SnmpTrap: AlarmActiveStatev1SpecificTrap} - examplePacket.Variables = examplePacket.SnmpTrap.Variables - return &SnmpPacket{ - Content: examplePacket, - Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, - } -} - -func createTestPacket() *SnmpPacket { - examplePacket := &gosnmp.SnmpPacket{ - Version: gosnmp.Version2c, - Community: "public", - Variables: NetSNMPExampleHeartbeatNotification.Variables, - } - return &SnmpPacket{ - Content: examplePacket, - Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, - } -} - -func TestFormatPacketV1Generic(t *testing.T) { - packet := createTestV1GenericPacket() - data, err := FormatPacketToJSON(packet) - require.NoError(t, err) - - assert.Equal(t, "1.3.6.1.6.3.1.1.5.3", data["oid"]) - assert.NotNil(t, data["uptime"]) - assert.NotNil(t, data["enterprise_oid"]) - assert.NotNil(t, data["generic_trap"]) - assert.NotNil(t, data["specific_trap"]) - - variables, ok := data["variables"].([]map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, len(variables), 3) - - assert.Equal(t, "1.3.6.1.6.3.1.1.5", data["enterprise_oid"]) - assert.Equal(t, 2, data["generic_trap"]) - assert.Equal(t, 0, data["specific_trap"]) - - ifIndex := variables[0] - assert.Equal(t, ifIndex["oid"], "1.3.6.1.2.1.2.2.1.1") - assert.Equal(t, ifIndex["type"], "integer") - assert.Equal(t, ifIndex["value"], 2) - - ifAdminStatus := variables[1] - assert.Equal(t, ifAdminStatus["oid"], "1.3.6.1.2.1.2.2.1.7") - assert.Equal(t, ifAdminStatus["type"], "integer") - assert.Equal(t, ifAdminStatus["value"], 1) - - ifOperStatus := variables[2] - assert.Equal(t, ifOperStatus["oid"], "1.3.6.1.2.1.2.2.1.8") - assert.Equal(t, ifOperStatus["type"], "integer") - assert.Equal(t, ifOperStatus["value"], 2) -} - -func TestFormatPacketV1Specific(t *testing.T) { - packet := createTestV1SpecificPacket() - data, err := FormatPacketToJSON(packet) - require.NoError(t, err) - - assert.Equal(t, "1.3.6.1.2.1.118.0.2", data["oid"]) - assert.NotNil(t, data["uptime"]) - assert.NotNil(t, data["enterprise_oid"]) - assert.NotNil(t, data["generic_trap"]) - assert.NotNil(t, data["specific_trap"]) - - variables, ok := data["variables"].([]map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, len(variables), 2) - - assert.Equal(t, "1.3.6.1.2.1.118", data["enterprise_oid"]) - assert.Equal(t, 6, data["generic_trap"]) - assert.Equal(t, 2, data["specific_trap"]) - - alarmActiveModelPointer := variables[0] - assert.Equal(t, alarmActiveModelPointer["oid"], "1.3.6.1.2.1.118.1.2.2.1.13") - assert.Equal(t, alarmActiveModelPointer["type"], "string") - assert.Equal(t, alarmActiveModelPointer["value"], "foo") - - alarmActiveResourceID := variables[1] - assert.Equal(t, alarmActiveResourceID["oid"], "1.3.6.1.2.1.118.1.2.2.1.10") - assert.Equal(t, alarmActiveResourceID["type"], "string") - assert.Equal(t, alarmActiveResourceID["value"], "bar") - -} - -func TestFormatPacketToJSON(t *testing.T) { - packet := createTestPacket() - - data, err := FormatPacketToJSON(packet) - require.NoError(t, err) - - assert.Equal(t, "1.3.6.1.4.1.8072.2.3.0.1", data["oid"]) - assert.NotNil(t, data["uptime"]) - - variables, ok := data["variables"].([]map[string]interface{}) - assert.True(t, ok) - assert.Equal(t, len(variables), 2) - - heartBeatRate := variables[0] - assert.Equal(t, heartBeatRate["oid"], "1.3.6.1.4.1.8072.2.3.2.1") - assert.Equal(t, heartBeatRate["type"], "integer") - assert.Equal(t, heartBeatRate["value"], 1024) - - heartBeatName := variables[1] - assert.Equal(t, heartBeatName["oid"], "1.3.6.1.4.1.8072.2.3.2.2") - assert.Equal(t, heartBeatName["type"], "string") - assert.Equal(t, heartBeatName["value"], "test") -} - -func TestFormatPacketToJSONShouldFailIfNotEnoughVariables(t *testing.T) { - packet := createTestPacket() - - packet.Content.Variables = []gosnmp.SnmpPDU{ - // No variables at all. - } - _, err := FormatPacketToJSON(packet) - require.Error(t, err) - - packet.Content.Variables = []gosnmp.SnmpPDU{ - // sysUpTimeInstance and data, but no snmpTrapOID - {Name: "1.3.6.1.2.1.1.3.0", Type: gosnmp.TimeTicks, Value: uint32(1000)}, - {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, - {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, - } - _, err = FormatPacketToJSON(packet) - require.Error(t, err) - - packet.Content.Variables = []gosnmp.SnmpPDU{ - // snmpTrapOID and data, but no sysUpTimeInstance - {Name: "1.3.6.1.6.3.1.1.4.1.0", Type: gosnmp.OctetString, Value: "1.3.6.1.4.1.8072.2.3.0.1"}, - {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, - {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, - } - _, err = FormatPacketToJSON(packet) - require.Error(t, err) -} - -func TestGetTags(t *testing.T) { - packet := createTestPacket() - tags := GetTags(packet) - assert.Equal(t, tags, []string{ - "snmp_version:2", - "device_namespace:default", - "snmp_device:127.0.0.1", - }) -} - -func TestGetTagsForUnsupportedVersionShouldStillSucceed(t *testing.T) { - packet := createTestPacket() - packet.Content.Version = 12 - tags := GetTags(packet) - assert.Equal(t, tags, []string{ - "snmp_version:unknown", - "device_namespace:default", - "snmp_device:127.0.0.1", - }) -} diff --git a/pkg/snmp/traps/formatter.go b/pkg/snmp/traps/formatter.go new file mode 100644 index 00000000000000..7e5d3f1db47161 --- /dev/null +++ b/pkg/snmp/traps/formatter.go @@ -0,0 +1,247 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package traps + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/DataDog/datadog-agent/pkg/util/log" + "github.com/gosnmp/gosnmp" +) + +// Formatter is an interface to extract and format raw SNMP Traps +type Formatter interface { + FormatPacket(packet *SnmpPacket) ([]byte, error) + GetTags(packet *SnmpPacket) []string +} + +// JSONFormatter is a Formatter implementation that transforms Traps into JSON +type JSONFormatter struct { + oidResolver OIDResolver + namespace string +} + +type trapVariable struct { + OID string `json:"oid"` + VarType string `json:"type"` + Value interface{} `json:"value"` +} + +const ( + sysUpTimeInstanceOID = "1.3.6.1.2.1.1.3.0" + snmpTrapOID = "1.3.6.1.6.3.1.1.4.1.0" +) + +// NewJSONFormatter creates a new JSONFormatter instance with an optional OIDResolver variable. +func NewJSONFormatter(oidResolver OIDResolver) JSONFormatter { + namespace := GetNamespace() + if oidResolver == nil { + return JSONFormatter{NoOpOIDResolver{}, namespace} + } + return JSONFormatter{oidResolver, namespace} +} + +// FormatPacket converts a raw SNMP trap packet to a FormattedSnmpPacket containing the JSON data and the tags to attach +func (f JSONFormatter) FormatPacket(packet *SnmpPacket) ([]byte, error) { + var formattedData map[string]interface{} + var err error + if packet.Content.Version == gosnmp.Version1 { + formattedData = f.formatV1Trap(packet.Content) + } else { + formattedData, err = f.formatTrap(packet.Content) + if err != nil { + return nil, err + } + } + return json.Marshal(formattedData) +} + +// GetTags returns a list of tags associated to an SNMP trap packet. +func (f JSONFormatter) GetTags(packet *SnmpPacket) []string { + return []string{ + fmt.Sprintf("snmp_version:%s", formatVersion(packet.Content)), + fmt.Sprintf("device_namespace:%s", f.namespace), + fmt.Sprintf("snmp_device:%s", packet.Addr.IP.String()), + } +} + +func (f JSONFormatter) formatV1Trap(packet *gosnmp.SnmpPacket) map[string]interface{} { + data := make(map[string]interface{}) + data["uptime"] = uint32(packet.Timestamp) + enterpriseOid := NormalizeOID(packet.Enterprise) + genericTrap := packet.GenericTrap + specificTrap := packet.SpecificTrap + var trapOID string + if genericTrap == 6 { + // Vendor-specific trap + trapOID = fmt.Sprintf("%s.0.%d", enterpriseOid, specificTrap) + } else { + // Generic trap + trapOID = fmt.Sprintf("%s.%d", genericTrapOid, genericTrap+1) + } + data["trap_oid"] = trapOID + trapMetadata, err := f.oidResolver.GetTrapMetadata(trapOID) + if err != nil { + log.Debugf("unable to resolve OID: %s", err) + } else { + data["trap_name"] = trapMetadata.Name + } + data["enterprise_oid"] = enterpriseOid + data["generic_trap"] = genericTrap + data["specific_trap"] = specificTrap + variables := parseVariables(packet.Variables) + data["variables_raw"] = variables + for _, variable := range variables { + varMetadata, err := f.oidResolver.GetVariableMetadata(trapOID, variable.OID) + if err != nil { + log.Debugf("unable to enrich variable: %s", err) + continue + } + data[varMetadata.Name] = variable.Value + } + return data +} + +func (f JSONFormatter) formatTrap(packet *gosnmp.SnmpPacket) (map[string]interface{}, error) { + /* + An SNMP v2 or v3 trap packet consists in the following variables (PDUs): + {sysUpTime.0, snmpTrapOID.0, additionalDataVariables...} + See: https://tools.ietf.org/html/rfc3416#section-4.2.6 + */ + variables := packet.Variables + if len(variables) < 2 { + return nil, fmt.Errorf("expected at least 2 variables, got %d", len(variables)) + } + + data := make(map[string]interface{}) + + uptime, err := parseSysUpTime(variables[0]) + if err != nil { + return nil, err + } + data["uptime"] = uptime + + trapOID, err := parseSnmpTrapOID(variables[1]) + if err != nil { + return nil, err + } + data["trap_oid"] = trapOID + + trapMetadata, err := f.oidResolver.GetTrapMetadata(trapOID) + if err != nil { + log.Debugf("unable to resolve OID: %s", err) + } else { + data["trap_name"] = trapMetadata.Name + } + + parsedVariables := parseVariables(variables[2:]) + data["variables_raw"] = parsedVariables + for _, variable := range parsedVariables { + varMetadata, err := f.oidResolver.GetVariableMetadata(trapOID, variable.OID) + if err != nil { + log.Debugf("unable to enrich variable: %s", err) + continue + } + data[varMetadata.Name] = variable.Value + } + return data, nil +} + +// NormalizeOID convert an OID from the absolute form ".1.2.3..." to a relative form "1.2.3..." +func NormalizeOID(value string) string { + // OIDs can be formatted as ".1.2.3..." ("absolute form") or "1.2.3..." ("relative form"). + // Convert everything to relative form, like we do in the Python check. + return strings.TrimLeft(value, ".") +} + +func parseSysUpTime(variable gosnmp.SnmpPDU) (uint32, error) { + name := NormalizeOID(variable.Name) + if name != sysUpTimeInstanceOID { + return 0, fmt.Errorf("expected OID %s, got %s", sysUpTimeInstanceOID, name) + } + + value, ok := variable.Value.(uint32) + if !ok { + return 0, fmt.Errorf("expected uptime to be uint32 (got %v of type %T)", variable.Value, variable.Value) + } + + return value, nil +} + +func parseSnmpTrapOID(variable gosnmp.SnmpPDU) (string, error) { + name := NormalizeOID(variable.Name) + if name != snmpTrapOID { + return "", fmt.Errorf("expected OID %s, got %s", snmpTrapOID, name) + } + + value := "" + switch variable.Value.(type) { + case string: + value = variable.Value.(string) + case []byte: + value = string(variable.Value.([]byte)) + default: + return "", fmt.Errorf("expected snmpTrapOID to be a string (got %v of type %T)", variable.Value, variable.Value) + } + + return NormalizeOID(value), nil +} + +func parseVariables(variables []gosnmp.SnmpPDU) []trapVariable { + var parsedVariables []trapVariable + + for _, variable := range variables { + varOID := NormalizeOID(variable.Name) + varType := formatType(variable) + varValue := formatValue(variable) + parsedVariables = append(parsedVariables, trapVariable{OID: varOID, VarType: varType, Value: varValue}) + } + + return parsedVariables +} + +func formatType(variable gosnmp.SnmpPDU) string { + switch variable.Type { + case gosnmp.Integer, gosnmp.Uinteger32: + return "integer" + case gosnmp.OctetString: + return "string" + case gosnmp.ObjectIdentifier: + return "oid" + case gosnmp.Counter32: + return "counter32" + case gosnmp.Counter64: + return "counter64" + case gosnmp.Gauge32: + return "gauge32" + default: + return "other" + } +} + +func formatValue(variable gosnmp.SnmpPDU) interface{} { + switch variable.Value.(type) { + case []byte: + return string(variable.Value.([]byte)) + default: + return variable.Value + } +} + +func formatVersion(packet *gosnmp.SnmpPacket) string { + switch packet.Version { + case gosnmp.Version3: + return "3" + case gosnmp.Version2c: + return "2" + case gosnmp.Version1: + return "1" + default: + return "unknown" + } +} diff --git a/pkg/snmp/traps/formatter_test.go b/pkg/snmp/traps/formatter_test.go new file mode 100644 index 00000000000000..731022e5535a4a --- /dev/null +++ b/pkg/snmp/traps/formatter_test.go @@ -0,0 +1,250 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2020-present Datadog, Inc. + +package traps + +import ( + "encoding/json" + "net" + "testing" + + "github.com/gosnmp/gosnmp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var defaultFormatter = NewJSONFormatter(NoOpOIDResolver{}) + +func createTestV1GenericPacket() *SnmpPacket { + examplePacket := &gosnmp.SnmpPacket{Version: gosnmp.Version1, SnmpTrap: LinkDownv1GenericTrap} + examplePacket.Variables = examplePacket.SnmpTrap.Variables + return &SnmpPacket{ + Content: examplePacket, + Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, + } +} + +func createTestV1SpecificPacket() *SnmpPacket { + examplePacket := &gosnmp.SnmpPacket{Version: gosnmp.Version1, SnmpTrap: AlarmActiveStatev1SpecificTrap} + examplePacket.Variables = examplePacket.SnmpTrap.Variables + return &SnmpPacket{ + Content: examplePacket, + Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, + } +} + +func createTestPacket() *SnmpPacket { + examplePacket := &gosnmp.SnmpPacket{ + Version: gosnmp.Version2c, + Community: "public", + Variables: NetSNMPExampleHeartbeatNotification.Variables, + } + return &SnmpPacket{ + Content: examplePacket, + Addr: &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 13156}, + } +} + +func TestFormatPacketV1Generic(t *testing.T) { + packet := createTestV1GenericPacket() + formattedPacket, err := defaultFormatter.FormatPacket(packet) + require.NoError(t, err) + data := make(map[string]interface{}) + err = json.Unmarshal(formattedPacket, &data) + require.NoError(t, err) + + assert.Equal(t, "1.3.6.1.6.3.1.1.5.3", data["trap_oid"]) + assert.NotNil(t, data["uptime"]) + assert.NotNil(t, data["enterprise_oid"]) + assert.NotNil(t, data["generic_trap"]) + assert.NotNil(t, data["specific_trap"]) + + variables := make([]map[string]interface{}, 3) + for i := 0; i < 3; i++ { + variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + } + + assert.Equal(t, "1.3.6.1.6.3.1.1.5", data["enterprise_oid"]) + assert.EqualValues(t, 2, data["generic_trap"]) + assert.EqualValues(t, 0, data["specific_trap"]) + + ifIndex := variables[0] + assert.EqualValues(t, ifIndex["oid"], "1.3.6.1.2.1.2.2.1.1") + assert.EqualValues(t, ifIndex["type"], "integer") + assert.EqualValues(t, ifIndex["value"], 2) + + ifAdminStatus := variables[1] + assert.EqualValues(t, ifAdminStatus["oid"], "1.3.6.1.2.1.2.2.1.7") + assert.EqualValues(t, ifAdminStatus["type"], "integer") + assert.EqualValues(t, ifAdminStatus["value"], 1) + + ifOperStatus := variables[2] + assert.EqualValues(t, ifOperStatus["oid"], "1.3.6.1.2.1.2.2.1.8") + assert.EqualValues(t, ifOperStatus["type"], "integer") + assert.EqualValues(t, ifOperStatus["value"], 2) +} + +func TestFormatPacketV1Specific(t *testing.T) { + packet := createTestV1SpecificPacket() + formattedPacket, err := defaultFormatter.FormatPacket(packet) + require.NoError(t, err) + data := make(map[string]interface{}) + err = json.Unmarshal(formattedPacket, &data) + require.NoError(t, err) + + assert.Equal(t, "1.3.6.1.2.1.118.0.2", data["trap_oid"]) + assert.NotNil(t, data["uptime"]) + assert.NotNil(t, data["enterprise_oid"]) + assert.NotNil(t, data["generic_trap"]) + assert.NotNil(t, data["specific_trap"]) + + variables := make([]map[string]interface{}, 2) + for i := 0; i < 2; i++ { + variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + } + + assert.Equal(t, "1.3.6.1.2.1.118", data["enterprise_oid"]) + assert.EqualValues(t, 6, data["generic_trap"]) + assert.EqualValues(t, 2, data["specific_trap"]) + + alarmActiveModelPointer := variables[0] + assert.Equal(t, alarmActiveModelPointer["oid"], "1.3.6.1.2.1.118.1.2.2.1.13") + assert.EqualValues(t, alarmActiveModelPointer["type"], "string") + assert.EqualValues(t, alarmActiveModelPointer["value"], "foo") + + alarmActiveResourceID := variables[1] + assert.Equal(t, alarmActiveResourceID["oid"], "1.3.6.1.2.1.118.1.2.2.1.10") + assert.EqualValues(t, alarmActiveResourceID["type"], "string") + assert.EqualValues(t, alarmActiveResourceID["value"], "bar") + +} + +func TestFormatPacketToJSON(t *testing.T) { + packet := createTestPacket() + + formattedPacket, err := defaultFormatter.FormatPacket(packet) + require.NoError(t, err) + data := make(map[string]interface{}) + err = json.Unmarshal(formattedPacket, &data) + require.NoError(t, err) + + assert.Equal(t, "1.3.6.1.4.1.8072.2.3.0.1", data["trap_oid"]) + assert.NotNil(t, data["uptime"]) + + variables := make([]map[string]interface{}, 2) + for i := 0; i < 2; i++ { + variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + } + + heartBeatRate := variables[0] + assert.Equal(t, heartBeatRate["oid"], "1.3.6.1.4.1.8072.2.3.2.1") + assert.EqualValues(t, heartBeatRate["type"], "integer") + assert.EqualValues(t, heartBeatRate["value"], 1024) + + heartBeatName := variables[1] + assert.Equal(t, heartBeatName["oid"], "1.3.6.1.4.1.8072.2.3.2.2") + assert.EqualValues(t, heartBeatName["type"], "string") + assert.EqualValues(t, heartBeatName["value"], "test") +} + +func TestFormatPacketToJSONShouldFailIfNotEnoughVariables(t *testing.T) { + packet := createTestPacket() + + packet.Content.Variables = []gosnmp.SnmpPDU{ + // No variables at all. + } + _, err := defaultFormatter.FormatPacket(packet) + require.Error(t, err) + + packet.Content.Variables = []gosnmp.SnmpPDU{ + // sysUpTimeInstance and data, but no snmpTrapOID + {Name: "1.3.6.1.2.1.1.3.0", Type: gosnmp.TimeTicks, Value: uint32(1000)}, + {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, + {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, + } + _, err = defaultFormatter.FormatPacket(packet) + require.Error(t, err) + + packet.Content.Variables = []gosnmp.SnmpPDU{ + // snmpTrapOID and data, but no sysUpTimeInstance + {Name: "1.3.6.1.6.3.1.1.4.1.0", Type: gosnmp.OctetString, Value: "1.3.6.1.4.1.8072.2.3.0.1"}, + {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, + {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, + } + _, err = defaultFormatter.FormatPacket(packet) + require.Error(t, err) +} + +func TestGetTags(t *testing.T) { + packet := createTestPacket() + assert.Equal(t, defaultFormatter.GetTags(packet), []string{ + "snmp_version:2", + "device_namespace:default", + "snmp_device:127.0.0.1", + }) +} + +func TestGetTagsForUnsupportedVersionShouldStillSucceed(t *testing.T) { + packet := createTestPacket() + packet.Content.Version = 12 + assert.Equal(t, defaultFormatter.GetTags(packet), []string{ + "snmp_version:unknown", + "device_namespace:default", + "snmp_device:127.0.0.1", + }) +} + +func TestNewJSONFormatterWithNilStillWorks(t *testing.T) { + var formatter Formatter = NewJSONFormatter(nil) + packet := createTestPacket() + _, err := formatter.FormatPacket(packet) + require.NoError(t, err) + tags := formatter.GetTags(packet) + assert.Equal(t, tags, []string{ + "snmp_version:2", + "device_namespace:default", + "snmp_device:127.0.0.1", + }) +} + +func TestFormatterWithResolverAndTrapV2(t *testing.T) { + formatter := NewJSONFormatter(resolverWithData) + packet := createTestPacket() + data, err := formatter.FormatPacket(packet) + require.NoError(t, err) + content := make(map[string]interface{}) + json.Unmarshal(data, &content) + + assert.EqualValues(t, "netSnmpExampleHeartbeatNotification", content["trap_name"]) + assert.EqualValues(t, 1024, content["netSnmpExampleHeartbeatRate"]) + + tags := formatter.GetTags(packet) + assert.Equal(t, tags, []string{ + "snmp_version:2", + "device_namespace:default", + "snmp_device:127.0.0.1", + }) +} + +func TestFormatterWithResolverAndTrapV1Generic(t *testing.T) { + formatter := NewJSONFormatter(resolverWithData) + packet := createTestV1GenericPacket() + data, err := formatter.FormatPacket(packet) + require.NoError(t, err) + content := make(map[string]interface{}) + json.Unmarshal(data, &content) + + assert.EqualValues(t, "ifDown", content["trap_name"]) + assert.EqualValues(t, 2, content["ifIndex"]) + assert.EqualValues(t, 1, content["ifAdminStatus"]) + assert.EqualValues(t, 2, content["ifOperStatus"]) + + tags := formatter.GetTags(packet) + assert.Equal(t, tags, []string{ + "snmp_version:1", + "device_namespace:default", + "snmp_device:127.0.0.1", + }) +} diff --git a/pkg/snmp/traps/oid_resolver.go b/pkg/snmp/traps/oid_resolver.go new file mode 100644 index 00000000000000..0d5d3a2e8a8a3d --- /dev/null +++ b/pkg/snmp/traps/oid_resolver.go @@ -0,0 +1,185 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2022-present Datadog, Inc.package traps + +package traps + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/DataDog/datadog-agent/pkg/config" + "github.com/DataDog/datadog-agent/pkg/util/log" + "gopkg.in/yaml.v2" +) + +const ddTrapDBFileNamePrefix string = "dd_traps_db" + +type unmarshaller func(data []byte, v interface{}) error + +// OIDResolver is a interface to get Trap and Variable metadata from OIDs +type OIDResolver interface { + GetTrapMetadata(trapOID string) (TrapMetadata, error) + GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) +} + +// NoOpOIDResolver is a dummy OIDResolver implementation that is unable to get any Trap or Variable metadata. +type NoOpOIDResolver struct{} + +// GetTrapMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetTrapMetadata(trapOID string) (TrapMetadata, error) { + return TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + +// GetVariableMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) { + return VariableMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + +// MultiFilesOIDResolver is an OIDResolver implementation that can be configured with multiple input files. +// Trap OIDs conflicts are resolved using the name of the source file in alphabetical order and by giving +// the less priority to Datadog's own database shipped with the agent. +// Variable OIDs conflicts are fully resolved by also looking at the trap OID. A given trap OID only +// exist in a single file (after the previous conflict resolution), meaning that we get the variable +// metadata from that same file. +type MultiFilesOIDResolver struct { + traps TrapSpec +} + +// NewMultiFilesOIDResolver creates a new MultiFilesOIDResolver instance by loading json or yaml files +// (optionnally gzipped) located in the directory snmp.d/traps_db/ +func NewMultiFilesOIDResolver() (*MultiFilesOIDResolver, error) { + oidResolver := &MultiFilesOIDResolver{traps: make(TrapSpec)} + confdPath := config.Datadog.GetString("confd_path") + trapsDBRoot := filepath.Join(confdPath, "snmp.d", "traps_db") + files, err := os.ReadDir(trapsDBRoot) + if err != nil { + return nil, fmt.Errorf("failed to read dir `%s`: %v", trapsDBRoot, err) + } + fileNames := getSortedFileNames(files) + for _, fileName := range fileNames { + err := oidResolver.updateFromFile(filepath.Join(trapsDBRoot, fileName)) + if err != nil { + log.Warnf("unable to load trap db file %s: %s", fileName, err) + } + } + return oidResolver, nil +} + +// GetTrapMetadata returns TrapMetadata for a given trapOID +func (or *MultiFilesOIDResolver) GetTrapMetadata(trapOID string) (TrapMetadata, error) { + trapOID = strings.TrimSuffix(NormalizeOID(trapOID), ".0") + trapData, ok := or.traps[trapOID] + if !ok { + return TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) + } + return trapData, nil +} + +// GetVariableMetadata returns VariableMetadata for a given variableOID and trapOID. +// The trapOID should not be needed in theory but the Datadog Agent allows to define multiple variable names for the +// same OID as long as they are defined in different file. The trapOID is used to differentiate between these files. +func (or *MultiFilesOIDResolver) GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) { + trapOID = strings.TrimSuffix(NormalizeOID(trapOID), ".0") + varOID = strings.TrimSuffix(NormalizeOID(varOID), ".0") + trapData, ok := or.traps[trapOID] + if !ok { + return VariableMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) + } + varData, ok := trapData.variableSpecPtr[varOID] + if !ok { + return VariableMetadata{}, fmt.Errorf("variable OID %s is not defined", varOID) + } + return varData, nil +} + +func getSortedFileNames(files []fs.DirEntry) []string { + fileNames := make([]string, 0, len(files)) + for _, file := range files { + if file.IsDir() { + log.Debugf("not loading traps data from path %s: file is directory", file.Name()) + continue + } + fileNames = append(fileNames, file.Name()) + } + + // Sort files alphabetically but put first the one shipped with the agent + sort.Slice(fileNames, func(i, j int) bool { + fileNameI := strings.ToLower(fileNames[i]) + fileNameJ := strings.ToLower(fileNames[j]) + if strings.HasPrefix(fileNameI, ddTrapDBFileNamePrefix) { + return true + } else if strings.HasPrefix(fileNameJ, ddTrapDBFileNamePrefix) { + return false + } + return fileNameI < fileNameJ + }) + return fileNames +} + +func (or *MultiFilesOIDResolver) updateFromFile(filePath string) error { + var fileReader io.ReadCloser + fileReader, err := os.Open(filePath) + if err != nil { + return err + } + defer fileReader.Close() + if strings.HasSuffix(filePath, ".gz") { + filePath = strings.TrimSuffix(filePath, ".gz") + uncompressor, err := gzip.NewReader(fileReader) + if err != nil { + return fmt.Errorf("unable to uncompress gzip file %s", filePath) + } + defer uncompressor.Close() + fileReader = uncompressor + } + var unmarshalMethod unmarshaller = yaml.Unmarshal + if strings.HasSuffix(filePath, ".json") { + unmarshalMethod = json.Unmarshal + } + return or.updateFromReader(fileReader, unmarshalMethod) +} + +func (or *MultiFilesOIDResolver) updateFromReader(reader io.Reader, unmarshalMethod unmarshaller) error { + fileContent, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + var trapData trapDBFileContent + err = unmarshalMethod(fileContent, &trapData) + if err != nil { + return err + } + + return or.updateResolverWithData(trapData) +} + +func (or *MultiFilesOIDResolver) updateResolverWithData(trapDB trapDBFileContent) error { + definedVariables := variableSpec{} + for variableOID, variableData := range trapDB.Variables { + variableOID := NormalizeOID(variableOID) + definedVariables[variableOID] = variableData + } + + for trapOID, trapData := range trapDB.Traps { + trapOID := NormalizeOID(trapOID) + if _, trapConflict := or.traps[trapOID]; trapConflict { + log.Debugf("a trap with OID %s is defined in multiple traps db files", trapOID) + } + or.traps[trapOID] = TrapMetadata{ + Name: trapData.Name, + Description: trapData.Description, + variableSpecPtr: definedVariables, + } + } + return nil +} diff --git a/pkg/snmp/traps/oid_resolver_test.go b/pkg/snmp/traps/oid_resolver_test.go new file mode 100644 index 00000000000000..00b4f3e0fa3da0 --- /dev/null +++ b/pkg/snmp/traps/oid_resolver_test.go @@ -0,0 +1,219 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2022-present Datadog, Inc. + +package traps + +import ( + "bytes" + "encoding/json" + "fmt" + "io/fs" + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +var dummyTrapDB = trapDBFileContent{ + Traps: TrapSpec{ + "1.3.6.1.6.3.1.1.5.3": TrapMetadata{Name: "ifDown"}, // v1 Trap + "1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "netSnmpExampleHeartbeatNotification"}, // v2+ + }, + Variables: variableSpec{ + "1.3.6.1.2.1.2.2.1.1": VariableMetadata{Name: "ifIndex"}, + "1.3.6.1.2.1.2.2.1.7": VariableMetadata{Name: "ifAdminStatus"}, + "1.3.6.1.2.1.2.2.1.8": VariableMetadata{Name: "ifOperStatus"}, + "1.3.6.1.4.1.8072.2.3.2.1": VariableMetadata{Name: "netSnmpExampleHeartbeatRate"}, + }, +} + +var resolverWithData = &MockedResolver{content: dummyTrapDB} + +type MockedResolver struct { + content trapDBFileContent +} + +func (r MockedResolver) GetTrapMetadata(trapOid string) (TrapMetadata, error) { + trapOid = NormalizeOID(trapOid) + trapData, ok := r.content.Traps[trapOid] + if !ok { + return TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOid) + } + return trapData, nil +} +func (r MockedResolver) GetVariableMetadata(string, varOid string) (VariableMetadata, error) { + varOid = NormalizeOID(varOid) + varData, ok := r.content.Variables[varOid] + if !ok { + return VariableMetadata{}, fmt.Errorf("variable OID %s is not defined", varOid) + } + return varData, nil +} + +type MockedDirEntry struct { + name string + isDir bool +} + +func (m MockedDirEntry) Info() (fs.FileInfo, error) { + return nil, nil +} + +func (m MockedDirEntry) IsDir() bool { + return m.isDir +} + +func (m MockedDirEntry) Name() string { + return m.name +} + +func (m MockedDirEntry) Type() fs.FileMode { + return 0 +} +func TestDecoding(t *testing.T) { + trapDBFile := &trapDBFileContent{ + Traps: TrapSpec{ + "foo": TrapMetadata{ + Name: "xx", + }, + }, + Variables: variableSpec{ + "bar": VariableMetadata{ + Name: "yy", + Description: "dummy description", + }, + }, + } + data, err := json.Marshal(trapDBFile) + require.NoError(t, err) + require.Equal(t, []byte("{\"traps\":{\"foo\":{\"name\":\"xx\",\"descr\":\"\"}},\"vars\":{\"bar\":{\"name\":\"yy\",\"descr\":\"dummy description\"}}}"), data) + err = json.Unmarshal([]byte("{\"traps\": {\"1.2\": {\"name\": \"dd\"}}}"), &trapDBFile) + require.NoError(t, err) +} + +func TestSortFiles(t *testing.T) { + files := []fs.DirEntry{ + MockedDirEntry{name: "totoro", isDir: false}, + MockedDirEntry{name: "porco", isDir: false}, + MockedDirEntry{name: "Nausicaa", isDir: false}, + MockedDirEntry{name: "kiki", isDir: false}, + MockedDirEntry{name: "mononoke", isDir: false}, + MockedDirEntry{name: "ponyo", isDir: false}, + MockedDirEntry{name: "chihiro", isDir: false}, + MockedDirEntry{name: "directory", isDir: true}, + MockedDirEntry{name: "dd_traps_db.json.gz", isDir: true}, + MockedDirEntry{name: "dd_traps_db.json", isDir: true}, + MockedDirEntry{name: "dd_traps_db.yaml.gz", isDir: true}, + MockedDirEntry{name: "dd_traps_db.yaml", isDir: true}, + MockedDirEntry{name: "dd_traps_db.json.gz", isDir: false}, + MockedDirEntry{name: "dd_traps_db.json", isDir: false}, + MockedDirEntry{name: "dd_traps_db.yaml.gz", isDir: false}, + MockedDirEntry{name: "dd_traps_db.yaml", isDir: false}, + } + sortedFiles := getSortedFileNames(files) + require.EqualValues(t, + []string{ + "dd_traps_db.yaml", + "dd_traps_db.yaml.gz", + "dd_traps_db.json", + "dd_traps_db.json.gz", + "chihiro", + "kiki", + "mononoke", + "Nausicaa", + "ponyo", + "porco", + "totoro", + }, + sortedFiles, + ) +} + +func TestResolverWithNonStandardOIDs(t *testing.T) { + resolver := &MultiFilesOIDResolver{traps: make(TrapSpec)} + trapData := trapDBFileContent{ + Traps: TrapSpec{"1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "netSnmpExampleHeartbeat"}}, + Variables: variableSpec{ + "1.3.6.1.4.1.8072.2.3.2.1": VariableMetadata{ + Name: "netSnmpExampleHeartbeatRate", + }, + }, + } + updateResolverWithIntermediateJSONReader(t, resolver, trapData) + + data, err := resolver.GetTrapMetadata(".1.3.6.1.4.1.8072.2.3.0.1") + require.NoError(t, err) + require.Equal(t, "netSnmpExampleHeartbeat", data.Name) + + data, err = resolver.GetTrapMetadata("1.3.6.1.4.1.8072.2.3.0.1.0") + require.NoError(t, err) + require.Equal(t, "netSnmpExampleHeartbeat", data.Name) + + data, err = resolver.GetTrapMetadata(".1.3.6.1.4.1.8072.2.3.0.1.0") + require.NoError(t, err) + require.Equal(t, "netSnmpExampleHeartbeat", data.Name) +} +func TestResolverWithConflictingTrapOID(t *testing.T) { + resolver := &MultiFilesOIDResolver{traps: make(TrapSpec)} + trapDataA := trapDBFileContent{ + Traps: TrapSpec{"1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "foo"}}, + } + trapDataB := trapDBFileContent{ + Traps: TrapSpec{"1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "bar"}}, + } + updateResolverWithIntermediateJSONReader(t, resolver, trapDataA) + updateResolverWithIntermediateYAMLReader(t, resolver, trapDataB) + data, err := resolver.GetTrapMetadata("1.3.6.1.4.1.8072.2.3.0.1") + require.NoError(t, err) + require.Equal(t, "bar", data.Name) +} + +func TestResolverWithConflictingVariables(t *testing.T) { + resolver := &MultiFilesOIDResolver{traps: make(TrapSpec)} + trapDataA := trapDBFileContent{ + Traps: TrapSpec{"1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{}}, + Variables: variableSpec{ + "1.3.6.1.4.1.8072.2.3.2.1": VariableMetadata{ + Name: "netSnmpExampleHeartbeatRate", + }, + }, + } + trapDataB := trapDBFileContent{ + Traps: TrapSpec{"1.3.6.1.4.1.8072.2.3.0.2": TrapMetadata{}}, + Variables: variableSpec{ + "1.3.6.1.4.1.8072.2.3.2.1": VariableMetadata{ + Name: "netSnmpExampleHeartbeatRate2", + }, + }, + } + updateResolverWithIntermediateJSONReader(t, resolver, trapDataA) + updateResolverWithIntermediateYAMLReader(t, resolver, trapDataB) + data, err := resolver.GetVariableMetadata("1.3.6.1.4.1.8072.2.3.0.1", "1.3.6.1.4.1.8072.2.3.2.1") + require.NoError(t, err) + require.Equal(t, "netSnmpExampleHeartbeatRate", data.Name) + + // Same variable OID, different trap OID + data, err = resolver.GetVariableMetadata("1.3.6.1.4.1.8072.2.3.0.2", "1.3.6.1.4.1.8072.2.3.2.1") + require.NoError(t, err) + require.Equal(t, "netSnmpExampleHeartbeatRate2", data.Name) +} + +func updateResolverWithIntermediateJSONReader(t *testing.T, oidResolver *MultiFilesOIDResolver, trapData trapDBFileContent) { + data, err := json.Marshal(trapData) + require.NoError(t, err) + + reader := bytes.NewReader(data) + err = oidResolver.updateFromReader(reader, json.Unmarshal) + require.NoError(t, err) +} + +func updateResolverWithIntermediateYAMLReader(t *testing.T, oidResolver *MultiFilesOIDResolver, trapData trapDBFileContent) { + data, err := yaml.Marshal(trapData) + require.NoError(t, err) + + reader := bytes.NewReader(data) + err = oidResolver.updateFromReader(reader, yaml.Unmarshal) + require.NoError(t, err) +} diff --git a/pkg/snmp/traps/server_test.go b/pkg/snmp/traps/server_test.go index e2b53c3338948d..0e50109db0e8d5 100644 --- a/pkg/snmp/traps/server_test.go +++ b/pkg/snmp/traps/server_test.go @@ -6,6 +6,8 @@ package traps import ( + "net" + "strconv" "testing" "github.com/gosnmp/gosnmp" @@ -13,8 +15,45 @@ import ( "github.com/stretchr/testify/require" ) +var serverPort = getFreePort() + +func getFreePort() uint16 { + var port uint16 + for i := 0; i < 5; i++ { + conn, err := net.ListenPacket("udp", ":0") + if err != nil { + continue + } + conn.Close() + port, err = parsePort(conn.LocalAddr().String()) + if err != nil { + continue + } + listener, err := startSNMPTrapListener(&Config{Port: port}, nil) + if err != nil { + continue + } + listener.Close() + return port + } + panic("unable to find free port for starting the trap listener") +} + +func parsePort(addr string) (uint16, error) { + _, portString, err := net.SplitHostPort(addr) + if err != nil { + return 0, err + } + + port, err := strconv.ParseUint(portString, 10, 16) + if err != nil { + return 0, err + } + return uint16(port), nil +} + func TestServerV1GenericTrap(t *testing.T) { - config := Config{Port: GetPort(t), CommunityStrings: []string{"public"}} + config := Config{Port: serverPort, CommunityStrings: []string{"public"}} Configure(t, config) err := StartServer("dummy_hostname") @@ -30,7 +69,7 @@ func TestServerV1GenericTrap(t *testing.T) { } func TestServerV1SpecificTrap(t *testing.T) { - config := Config{Port: GetPort(t), CommunityStrings: []string{"public"}} + config := Config{Port: serverPort, CommunityStrings: []string{"public"}} Configure(t, config) err := StartServer("dummy_hostname") @@ -45,7 +84,7 @@ func TestServerV1SpecificTrap(t *testing.T) { } func TestServerV2(t *testing.T) { - config := Config{Port: GetPort(t), CommunityStrings: []string{"public"}} + config := Config{Port: serverPort, CommunityStrings: []string{"public"}} Configure(t, config) err := StartServer("dummy_hostname") @@ -60,7 +99,7 @@ func TestServerV2(t *testing.T) { } func TestServerV2BadCredentials(t *testing.T) { - config := Config{Port: GetPort(t), CommunityStrings: []string{"public"}} + config := Config{Port: serverPort, CommunityStrings: []string{"public"}} Configure(t, config) err := StartServer("dummy_hostname") @@ -73,7 +112,7 @@ func TestServerV2BadCredentials(t *testing.T) { func TestServerV3(t *testing.T) { userV3 := UserV3{Username: "user", AuthKey: "password", AuthProtocol: "sha", PrivKey: "password", PrivProtocol: "aes"} - config := Config{Port: GetPort(t), Users: []UserV3{userV3}} + config := Config{Port: serverPort, Users: []UserV3{userV3}} Configure(t, config) err := StartServer("dummy_hostname") @@ -95,7 +134,7 @@ func TestServerV3(t *testing.T) { func TestServerV3BadCredentials(t *testing.T) { userV3 := UserV3{Username: "user", AuthKey: "password", AuthProtocol: "sha", PrivKey: "password", PrivProtocol: "aes"} - config := Config{Port: GetPort(t), Users: []UserV3{userV3}} + config := Config{Port: serverPort, Users: []UserV3{userV3}} Configure(t, config) err := StartServer("dummy_hostname") @@ -117,7 +156,7 @@ func TestStartFailure(t *testing.T) { /* Start two servers with the same config to trigger an "address already in use" error. */ - port := GetPort(t) + port := serverPort config := Config{Port: port, CommunityStrings: []string{"public"}} Configure(t, config) diff --git a/pkg/snmp/traps/testing.go b/pkg/snmp/traps/testing.go index cc94fef4748b17..eafbd3d467957a 100644 --- a/pkg/snmp/traps/testing.go +++ b/pkg/snmp/traps/testing.go @@ -9,8 +9,6 @@ package traps import ( - "net" - "strconv" "strings" "testing" "time" @@ -67,24 +65,6 @@ var ( } ) -func parsePort(t *testing.T, addr string) uint16 { - _, portString, err := net.SplitHostPort(addr) - require.NoError(t, err) - - port, err := strconv.ParseUint(portString, 10, 16) - require.NoError(t, err) - - return uint16(port) -} - -// GetPort requests a random UDP port number and makes sure it is available -func GetPort(t *testing.T) uint16 { - conn, err := net.ListenPacket("udp", ":0") - require.NoError(t, err) - defer conn.Close() - return parsePort(t, conn.LocalAddr().String()) -} - // Configure sets Datadog Agent configuration from a config object. func Configure(t *testing.T, trapConfig Config) { ConfigureWithGlobalNamespace(t, trapConfig, "") diff --git a/pkg/snmp/traps/traps_db.go b/pkg/snmp/traps/traps_db.go new file mode 100644 index 00000000000000..97cd84a9664646 --- /dev/null +++ b/pkg/snmp/traps/traps_db.go @@ -0,0 +1,33 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2022-present Datadog, Inc. + +package traps + +// VariableMetadata is the MIB-extracted information of a given trap variable +type VariableMetadata struct { + Name string `yaml:"name" json:"name"` + Description string `yaml:"descr" json:"descr"` +} + +// variableSpec contains the variableMetadata for each known variable of a given trap db file +type variableSpec map[string]VariableMetadata + +// TrapMetadata is the MIB-extracted information of a given trap OID. +// It also contains a reference to the variableSpec that was defined in the same trap db file. +// This is to prevent variable conflicts and to give precedence to the variable definitions located] +// in the same trap db file as the trap. +type TrapMetadata struct { + Name string `yaml:"name" json:"name"` + Description string `yaml:"descr" json:"descr"` + variableSpecPtr variableSpec +} + +// TrapSpec contains the variableMetadata for each known trap in all trap db files +type TrapSpec map[string]TrapMetadata + +type trapDBFileContent struct { + Traps TrapSpec `yaml:"traps" json:"traps"` + Variables variableSpec `yaml:"vars" json:"vars"` +} diff --git a/releasenotes/notes/Resolve-SNMP-Traps-OIDs-to-names-70de58eecc4892aa.yaml b/releasenotes/notes/Resolve-SNMP-Traps-OIDs-to-names-70de58eecc4892aa.yaml new file mode 100644 index 00000000000000..a0ff736edf21fc --- /dev/null +++ b/releasenotes/notes/Resolve-SNMP-Traps-OIDs-to-names-70de58eecc4892aa.yaml @@ -0,0 +1,11 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + Traps OIDs are now resolved to names using user-provided 'traps db' files in 'snmp.d/traps_db/'. From e570bcb987e57ee101ea44f8f6e0d76d283a3ddc Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Fri, 25 Feb 2022 16:01:30 +0100 Subject: [PATCH 2/8] Fix tests --- pkg/snmp/traps/formatter.go | 16 +++++++----- pkg/snmp/traps/formatter_test.go | 40 +++++++++++++++-------------- pkg/snmp/traps/oid_resolver_test.go | 9 ++++--- pkg/snmp/traps/traps_db.go | 1 + 4 files changed, 36 insertions(+), 30 deletions(-) diff --git a/pkg/snmp/traps/formatter.go b/pkg/snmp/traps/formatter.go index 7e5d3f1db47161..a19b755640b095 100644 --- a/pkg/snmp/traps/formatter.go +++ b/pkg/snmp/traps/formatter.go @@ -84,16 +84,17 @@ func (f JSONFormatter) formatV1Trap(packet *gosnmp.SnmpPacket) map[string]interf // Generic trap trapOID = fmt.Sprintf("%s.%d", genericTrapOid, genericTrap+1) } - data["trap_oid"] = trapOID + data["snmpTrapOID"] = trapOID trapMetadata, err := f.oidResolver.GetTrapMetadata(trapOID) if err != nil { log.Debugf("unable to resolve OID: %s", err) } else { - data["trap_name"] = trapMetadata.Name + data["snmpTrapName"] = trapMetadata.Name + data["snmpTrapMIB"] = trapMetadata.MIBName } - data["enterprise_oid"] = enterpriseOid - data["generic_trap"] = genericTrap - data["specific_trap"] = specificTrap + data["enterpriseOID"] = enterpriseOid + data["genericTrap"] = genericTrap + data["specificTrap"] = specificTrap variables := parseVariables(packet.Variables) data["variables_raw"] = variables for _, variable := range variables { @@ -130,13 +131,14 @@ func (f JSONFormatter) formatTrap(packet *gosnmp.SnmpPacket) (map[string]interfa if err != nil { return nil, err } - data["trap_oid"] = trapOID + data["snmpTrapOID"] = trapOID trapMetadata, err := f.oidResolver.GetTrapMetadata(trapOID) if err != nil { log.Debugf("unable to resolve OID: %s", err) } else { - data["trap_name"] = trapMetadata.Name + data["snmpTrapName"] = trapMetadata.Name + data["snmpTrapMIB"] = trapMetadata.MIBName } parsedVariables := parseVariables(variables[2:]) diff --git a/pkg/snmp/traps/formatter_test.go b/pkg/snmp/traps/formatter_test.go index 731022e5535a4a..ed04e16f6831c0 100644 --- a/pkg/snmp/traps/formatter_test.go +++ b/pkg/snmp/traps/formatter_test.go @@ -55,20 +55,20 @@ func TestFormatPacketV1Generic(t *testing.T) { err = json.Unmarshal(formattedPacket, &data) require.NoError(t, err) - assert.Equal(t, "1.3.6.1.6.3.1.1.5.3", data["trap_oid"]) + assert.Equal(t, "1.3.6.1.6.3.1.1.5.3", data["snmpTrapOID"]) assert.NotNil(t, data["uptime"]) - assert.NotNil(t, data["enterprise_oid"]) - assert.NotNil(t, data["generic_trap"]) - assert.NotNil(t, data["specific_trap"]) + assert.NotNil(t, data["enterpriseOID"]) + assert.NotNil(t, data["genericTrap"]) + assert.NotNil(t, data["specificTrap"]) variables := make([]map[string]interface{}, 3) for i := 0; i < 3; i++ { variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) } - assert.Equal(t, "1.3.6.1.6.3.1.1.5", data["enterprise_oid"]) - assert.EqualValues(t, 2, data["generic_trap"]) - assert.EqualValues(t, 0, data["specific_trap"]) + assert.Equal(t, "1.3.6.1.6.3.1.1.5", data["enterpriseOID"]) + assert.EqualValues(t, 2, data["genericTrap"]) + assert.EqualValues(t, 0, data["specificTrap"]) ifIndex := variables[0] assert.EqualValues(t, ifIndex["oid"], "1.3.6.1.2.1.2.2.1.1") @@ -94,20 +94,20 @@ func TestFormatPacketV1Specific(t *testing.T) { err = json.Unmarshal(formattedPacket, &data) require.NoError(t, err) - assert.Equal(t, "1.3.6.1.2.1.118.0.2", data["trap_oid"]) + assert.Equal(t, "1.3.6.1.2.1.118.0.2", data["snmpTrapOID"]) assert.NotNil(t, data["uptime"]) - assert.NotNil(t, data["enterprise_oid"]) - assert.NotNil(t, data["generic_trap"]) - assert.NotNil(t, data["specific_trap"]) + assert.NotNil(t, data["enterpriseOID"]) + assert.NotNil(t, data["genericTrap"]) + assert.NotNil(t, data["specificTrap"]) variables := make([]map[string]interface{}, 2) for i := 0; i < 2; i++ { variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) } - assert.Equal(t, "1.3.6.1.2.1.118", data["enterprise_oid"]) - assert.EqualValues(t, 6, data["generic_trap"]) - assert.EqualValues(t, 2, data["specific_trap"]) + assert.Equal(t, "1.3.6.1.2.1.118", data["enterpriseOID"]) + assert.EqualValues(t, 6, data["genericTrap"]) + assert.EqualValues(t, 2, data["specificTrap"]) alarmActiveModelPointer := variables[0] assert.Equal(t, alarmActiveModelPointer["oid"], "1.3.6.1.2.1.118.1.2.2.1.13") @@ -130,7 +130,7 @@ func TestFormatPacketToJSON(t *testing.T) { err = json.Unmarshal(formattedPacket, &data) require.NoError(t, err) - assert.Equal(t, "1.3.6.1.4.1.8072.2.3.0.1", data["trap_oid"]) + assert.Equal(t, "1.3.6.1.4.1.8072.2.3.0.1", data["snmpTrapOID"]) assert.NotNil(t, data["uptime"]) variables := make([]map[string]interface{}, 2) @@ -159,7 +159,7 @@ func TestFormatPacketToJSONShouldFailIfNotEnoughVariables(t *testing.T) { require.Error(t, err) packet.Content.Variables = []gosnmp.SnmpPDU{ - // sysUpTimeInstance and data, but no snmpTrapOID + // sysUpTimeInstance and data, but no snmpsnmpTrapOID {Name: "1.3.6.1.2.1.1.3.0", Type: gosnmp.TimeTicks, Value: uint32(1000)}, {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, @@ -168,7 +168,7 @@ func TestFormatPacketToJSONShouldFailIfNotEnoughVariables(t *testing.T) { require.Error(t, err) packet.Content.Variables = []gosnmp.SnmpPDU{ - // snmpTrapOID and data, but no sysUpTimeInstance + // snmpsnmpTrapOID and data, but no sysUpTimeInstance {Name: "1.3.6.1.6.3.1.1.4.1.0", Type: gosnmp.OctetString, Value: "1.3.6.1.4.1.8072.2.3.0.1"}, {Name: "1.3.6.1.4.1.8072.2.3.2.1", Type: gosnmp.Integer, Value: 1024}, {Name: "1.3.6.1.4.1.8072.2.3.2.2", Type: gosnmp.OctetString, Value: "test"}, @@ -217,7 +217,8 @@ func TestFormatterWithResolverAndTrapV2(t *testing.T) { content := make(map[string]interface{}) json.Unmarshal(data, &content) - assert.EqualValues(t, "netSnmpExampleHeartbeatNotification", content["trap_name"]) + assert.EqualValues(t, "netSnmpExampleHeartbeatNotification", content["snmpTrapName"]) + assert.EqualValues(t, "NET-SNMP-EXAMPLES-MIB", content["snmpTrapMIB"]) assert.EqualValues(t, 1024, content["netSnmpExampleHeartbeatRate"]) tags := formatter.GetTags(packet) @@ -236,7 +237,8 @@ func TestFormatterWithResolverAndTrapV1Generic(t *testing.T) { content := make(map[string]interface{}) json.Unmarshal(data, &content) - assert.EqualValues(t, "ifDown", content["trap_name"]) + assert.EqualValues(t, "ifDown", content["snmpTrapName"]) + assert.EqualValues(t, "IF-MIB", content["snmpTrapMIB"]) assert.EqualValues(t, 2, content["ifIndex"]) assert.EqualValues(t, 1, content["ifAdminStatus"]) assert.EqualValues(t, 2, content["ifOperStatus"]) diff --git a/pkg/snmp/traps/oid_resolver_test.go b/pkg/snmp/traps/oid_resolver_test.go index 00b4f3e0fa3da0..2143d8e82e84bb 100644 --- a/pkg/snmp/traps/oid_resolver_test.go +++ b/pkg/snmp/traps/oid_resolver_test.go @@ -18,8 +18,8 @@ import ( var dummyTrapDB = trapDBFileContent{ Traps: TrapSpec{ - "1.3.6.1.6.3.1.1.5.3": TrapMetadata{Name: "ifDown"}, // v1 Trap - "1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "netSnmpExampleHeartbeatNotification"}, // v2+ + "1.3.6.1.6.3.1.1.5.3": TrapMetadata{Name: "ifDown", MIBName: "IF-MIB"}, // v1 Trap + "1.3.6.1.4.1.8072.2.3.0.1": TrapMetadata{Name: "netSnmpExampleHeartbeatNotification", MIBName: "NET-SNMP-EXAMPLES-MIB"}, // v2+ }, Variables: variableSpec{ "1.3.6.1.2.1.2.2.1.1": VariableMetadata{Name: "ifIndex"}, @@ -76,7 +76,8 @@ func TestDecoding(t *testing.T) { trapDBFile := &trapDBFileContent{ Traps: TrapSpec{ "foo": TrapMetadata{ - Name: "xx", + Name: "xx", + MIBName: "yy", }, }, Variables: variableSpec{ @@ -88,7 +89,7 @@ func TestDecoding(t *testing.T) { } data, err := json.Marshal(trapDBFile) require.NoError(t, err) - require.Equal(t, []byte("{\"traps\":{\"foo\":{\"name\":\"xx\",\"descr\":\"\"}},\"vars\":{\"bar\":{\"name\":\"yy\",\"descr\":\"dummy description\"}}}"), data) + require.Equal(t, []byte("{\"traps\":{\"foo\":{\"name\":\"xx\",\"mib\":\"yy\",\"descr\":\"\"}},\"vars\":{\"bar\":{\"name\":\"yy\",\"descr\":\"dummy description\"}}}"), data) err = json.Unmarshal([]byte("{\"traps\": {\"1.2\": {\"name\": \"dd\"}}}"), &trapDBFile) require.NoError(t, err) } diff --git a/pkg/snmp/traps/traps_db.go b/pkg/snmp/traps/traps_db.go index 97cd84a9664646..690e41f36d1d45 100644 --- a/pkg/snmp/traps/traps_db.go +++ b/pkg/snmp/traps/traps_db.go @@ -20,6 +20,7 @@ type variableSpec map[string]VariableMetadata // in the same trap db file as the trap. type TrapMetadata struct { Name string `yaml:"name" json:"name"` + MIBName string `yaml:"mib" json:"mib"` Description string `yaml:"descr" json:"descr"` variableSpecPtr variableSpec } From 338e11f9b3720ae1abdc64df357c4ff0e693bc80 Mon Sep 17 00:00:00 2001 From: Florian Veaux Date: Wed, 9 Mar 2022 09:51:04 +0100 Subject: [PATCH 3/8] Update pkg/snmp/traps/formatter.go Co-authored-by: Alexandre Yang --- pkg/snmp/traps/formatter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/snmp/traps/formatter.go b/pkg/snmp/traps/formatter.go index a19b755640b095..12b35fc3a2999d 100644 --- a/pkg/snmp/traps/formatter.go +++ b/pkg/snmp/traps/formatter.go @@ -64,9 +64,9 @@ func (f JSONFormatter) FormatPacket(packet *SnmpPacket) ([]byte, error) { // GetTags returns a list of tags associated to an SNMP trap packet. func (f JSONFormatter) GetTags(packet *SnmpPacket) []string { return []string{ - fmt.Sprintf("snmp_version:%s", formatVersion(packet.Content)), - fmt.Sprintf("device_namespace:%s", f.namespace), - fmt.Sprintf("snmp_device:%s", packet.Addr.IP.String()), + "snmp_version:" + formatVersion(packet.Content), + "device_namespace:" + f.namespace, + "snmp_device:" + packet.Addr.IP.String(), } } From d3d17a02eebfb22228e8c2f3f637cb3bd1104142 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Wed, 9 Mar 2022 10:06:27 +0100 Subject: [PATCH 4/8] Address review --- pkg/snmp/traps/oid_resolver.go | 36 ++++++++++++++++------------- pkg/snmp/traps/oid_resolver_test.go | 4 ++-- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/pkg/snmp/traps/oid_resolver.go b/pkg/snmp/traps/oid_resolver.go index 0d5d3a2e8a8a3d..9b5e41411e8ef3 100644 --- a/pkg/snmp/traps/oid_resolver.go +++ b/pkg/snmp/traps/oid_resolver.go @@ -103,27 +103,31 @@ func (or *MultiFilesOIDResolver) GetVariableMetadata(trapOID string, varOID stri } func getSortedFileNames(files []fs.DirEntry) []string { - fileNames := make([]string, 0, len(files)) + // There should usually be one file provided by Datadog and zero or more provided by the user + userProvidedFileNames := make([]string, 0, len(files)-1) + // Using a slice for error-proofing but there will usually be only one dd provided file. + ddProvidedFileNames := make([]string, 0, 1) for _, file := range files { if file.IsDir() { log.Debugf("not loading traps data from path %s: file is directory", file.Name()) continue } - fileNames = append(fileNames, file.Name()) + fileName := file.Name() + if strings.HasPrefix(fileName, ddTrapDBFileNamePrefix) { + ddProvidedFileNames = append(ddProvidedFileNames, fileName) + } else { + userProvidedFileNames = append(userProvidedFileNames, file.Name()) + } } - // Sort files alphabetically but put first the one shipped with the agent - sort.Slice(fileNames, func(i, j int) bool { - fileNameI := strings.ToLower(fileNames[i]) - fileNameJ := strings.ToLower(fileNames[j]) - if strings.HasPrefix(fileNameI, ddTrapDBFileNamePrefix) { - return true - } else if strings.HasPrefix(fileNameJ, ddTrapDBFileNamePrefix) { - return false - } - return fileNameI < fileNameJ + sort.Slice(userProvidedFileNames, func(i, j int) bool { + return strings.ToLower(userProvidedFileNames[i]) < strings.ToLower(userProvidedFileNames[j]) + }) + sort.Slice(ddProvidedFileNames, func(i, j int) bool { + return strings.ToLower(ddProvidedFileNames[i]) < strings.ToLower(ddProvidedFileNames[j]) }) - return fileNames + + return append(ddProvidedFileNames, userProvidedFileNames...) } func (or *MultiFilesOIDResolver) updateFromFile(filePath string) error { @@ -160,10 +164,11 @@ func (or *MultiFilesOIDResolver) updateFromReader(reader io.Reader, unmarshalMet return err } - return or.updateResolverWithData(trapData) + or.updateResolverWithData(trapData) + return nil } -func (or *MultiFilesOIDResolver) updateResolverWithData(trapDB trapDBFileContent) error { +func (or *MultiFilesOIDResolver) updateResolverWithData(trapDB trapDBFileContent) { definedVariables := variableSpec{} for variableOID, variableData := range trapDB.Variables { variableOID := NormalizeOID(variableOID) @@ -181,5 +186,4 @@ func (or *MultiFilesOIDResolver) updateResolverWithData(trapDB trapDBFileContent variableSpecPtr: definedVariables, } } - return nil } diff --git a/pkg/snmp/traps/oid_resolver_test.go b/pkg/snmp/traps/oid_resolver_test.go index 2143d8e82e84bb..4cbcef330555fc 100644 --- a/pkg/snmp/traps/oid_resolver_test.go +++ b/pkg/snmp/traps/oid_resolver_test.go @@ -116,10 +116,10 @@ func TestSortFiles(t *testing.T) { sortedFiles := getSortedFileNames(files) require.EqualValues(t, []string{ - "dd_traps_db.yaml", - "dd_traps_db.yaml.gz", "dd_traps_db.json", "dd_traps_db.json.gz", + "dd_traps_db.yaml", + "dd_traps_db.yaml.gz", "chihiro", "kiki", "mononoke", From 220529e55145ccdc2e7577101db7cef27ee82dd3 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Wed, 9 Mar 2022 10:20:23 +0100 Subject: [PATCH 5/8] Fail-fast --- pkg/logs/internal/launchers/traps/launcher.go | 5 ++-- pkg/logs/internal/tailers/traps/tailer.go | 10 ++++--- .../internal/tailers/traps/tailer_test.go | 21 +++++++++++++-- pkg/snmp/traps/formatter.go | 8 +++--- pkg/snmp/traps/formatter_test.go | 27 +++++++++++++++---- pkg/snmp/traps/oid_resolver.go | 13 --------- 6 files changed, 55 insertions(+), 29 deletions(-) diff --git a/pkg/logs/internal/launchers/traps/launcher.go b/pkg/logs/internal/launchers/traps/launcher.go index 33f3f8e1d3f40b..32e5ba40f08f95 100644 --- a/pkg/logs/internal/launchers/traps/launcher.go +++ b/pkg/logs/internal/launchers/traps/launcher.go @@ -39,9 +39,10 @@ func (l *Launcher) startNewTailer(source *config.LogSource, inputChan chan *trap outputChan := l.pipelineProvider.NextPipelineChan() oidResolver, err := traps.NewMultiFilesOIDResolver() if err != nil { - log.Errorf("unable to load traps database: %w", err) + log.Errorf("unable to load traps database: %w. Will not listen for SNMP traps", err) + return } - l.tailer = tailer.NewTailer(oidResolver, source, inputChan, outputChan) + l.tailer, err = tailer.NewTailer(oidResolver, source, inputChan, outputChan) l.tailer.Start() } diff --git a/pkg/logs/internal/tailers/traps/tailer.go b/pkg/logs/internal/tailers/traps/tailer.go index 021a00e6134e6d..b6e566eb10f9d3 100644 --- a/pkg/logs/internal/tailers/traps/tailer.go +++ b/pkg/logs/internal/tailers/traps/tailer.go @@ -24,14 +24,18 @@ type Tailer struct { } // NewTailer returns a new Tailer -func NewTailer(oidResolver traps.OIDResolver, source *config.LogSource, inputChan traps.PacketsChannel, outputChan chan *message.Message) *Tailer { +func NewTailer(oidResolver traps.OIDResolver, source *config.LogSource, inputChan traps.PacketsChannel, outputChan chan *message.Message) (*Tailer, error) { + formatter, err := traps.NewJSONFormatter(oidResolver) + if err != nil { + return nil, err + } return &Tailer{ source: source, inputChan: inputChan, outputChan: outputChan, - formatter: traps.NewJSONFormatter(oidResolver), + formatter: formatter, done: make(chan interface{}, 1), - } + }, nil } // Start starts the tailer. diff --git a/pkg/logs/internal/tailers/traps/tailer_test.go b/pkg/logs/internal/tailers/traps/tailer_test.go index ca8f78f856c88b..87eae95bddc16a 100644 --- a/pkg/logs/internal/tailers/traps/tailer_test.go +++ b/pkg/logs/internal/tailers/traps/tailer_test.go @@ -6,6 +6,7 @@ package traps import ( + "fmt" "net" "testing" "time" @@ -13,16 +14,31 @@ import ( "github.com/gosnmp/gosnmp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/DataDog/datadog-agent/pkg/logs/config" "github.com/DataDog/datadog-agent/pkg/logs/message" "github.com/DataDog/datadog-agent/pkg/snmp/traps" ) +// NoOpOIDResolver is a dummy OIDResolver implementation that is unable to get any Trap or Variable metadata. +type NoOpOIDResolver struct{} + +// GetTrapMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetTrapMetadata(trapOID string) (traps.TrapMetadata, error) { + return traps.TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + +// GetVariableMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetVariableMetadata(trapOID string, varOID string) (traps.VariableMetadata, error) { + return traps.VariableMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + func TestTrapsShouldReceiveMessages(t *testing.T) { inputChan := make(traps.PacketsChannel, 1) outputChan := make(chan *message.Message) - tailer := NewTailer(&traps.NoOpOIDResolver{}, config.NewLogSource("test", &config.LogsConfig{}), inputChan, outputChan) + tailer, err := NewTailer(&NoOpOIDResolver{}, config.NewLogSource("test", &config.LogsConfig{}), inputChan, outputChan) + require.NoError(t, err) tailer.Start() p := &traps.SnmpPacket{ @@ -59,7 +75,8 @@ func TestTrapsShouldReceiveMessages(t *testing.T) { } func format(t *testing.T, p *traps.SnmpPacket) []byte { - formatter := traps.NewJSONFormatter(nil) + formatter, err := traps.NewJSONFormatter(NoOpOIDResolver{}) + require.NoError(t, err) formattedPacket, err := formatter.FormatPacket(p) assert.NoError(t, err) return formattedPacket diff --git a/pkg/snmp/traps/formatter.go b/pkg/snmp/traps/formatter.go index 12b35fc3a2999d..7831b41ca7186a 100644 --- a/pkg/snmp/traps/formatter.go +++ b/pkg/snmp/traps/formatter.go @@ -38,12 +38,12 @@ const ( ) // NewJSONFormatter creates a new JSONFormatter instance with an optional OIDResolver variable. -func NewJSONFormatter(oidResolver OIDResolver) JSONFormatter { - namespace := GetNamespace() +func NewJSONFormatter(oidResolver OIDResolver) (JSONFormatter, error) { if oidResolver == nil { - return JSONFormatter{NoOpOIDResolver{}, namespace} + return JSONFormatter{}, fmt.Errorf("NewJSONFormatter called with a nil OIDResolver") } - return JSONFormatter{oidResolver, namespace} + namespace := GetNamespace() + return JSONFormatter{oidResolver, namespace}, nil } // FormatPacket converts a raw SNMP trap packet to a FormattedSnmpPacket containing the JSON data and the tags to attach diff --git a/pkg/snmp/traps/formatter_test.go b/pkg/snmp/traps/formatter_test.go index ed04e16f6831c0..1a8223a45a2455 100644 --- a/pkg/snmp/traps/formatter_test.go +++ b/pkg/snmp/traps/formatter_test.go @@ -7,6 +7,7 @@ package traps import ( "encoding/json" + "fmt" "net" "testing" @@ -15,7 +16,20 @@ import ( "github.com/stretchr/testify/require" ) -var defaultFormatter = NewJSONFormatter(NoOpOIDResolver{}) +// NoOpOIDResolver is a dummy OIDResolver implementation that is unable to get any Trap or Variable metadata. +type NoOpOIDResolver struct{} + +// GetTrapMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetTrapMetadata(trapOID string) (TrapMetadata, error) { + return TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + +// GetVariableMetadata always return an error in this OIDResolver implementation +func (or NoOpOIDResolver) GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) { + return VariableMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) +} + +var defaultFormatter, _ = NewJSONFormatter(NoOpOIDResolver{}) func createTestV1GenericPacket() *SnmpPacket { examplePacket := &gosnmp.SnmpPacket{Version: gosnmp.Version1, SnmpTrap: LinkDownv1GenericTrap} @@ -197,9 +211,10 @@ func TestGetTagsForUnsupportedVersionShouldStillSucceed(t *testing.T) { } func TestNewJSONFormatterWithNilStillWorks(t *testing.T) { - var formatter Formatter = NewJSONFormatter(nil) + var formatter, err = NewJSONFormatter(NoOpOIDResolver{}) + require.NoError(t, err) packet := createTestPacket() - _, err := formatter.FormatPacket(packet) + _, err = formatter.FormatPacket(packet) require.NoError(t, err) tags := formatter.GetTags(packet) assert.Equal(t, tags, []string{ @@ -210,7 +225,8 @@ func TestNewJSONFormatterWithNilStillWorks(t *testing.T) { } func TestFormatterWithResolverAndTrapV2(t *testing.T) { - formatter := NewJSONFormatter(resolverWithData) + formatter, err := NewJSONFormatter(resolverWithData) + require.NoError(t, err) packet := createTestPacket() data, err := formatter.FormatPacket(packet) require.NoError(t, err) @@ -230,7 +246,8 @@ func TestFormatterWithResolverAndTrapV2(t *testing.T) { } func TestFormatterWithResolverAndTrapV1Generic(t *testing.T) { - formatter := NewJSONFormatter(resolverWithData) + formatter, err := NewJSONFormatter(resolverWithData) + require.NoError(t, err) packet := createTestV1GenericPacket() data, err := formatter.FormatPacket(packet) require.NoError(t, err) diff --git a/pkg/snmp/traps/oid_resolver.go b/pkg/snmp/traps/oid_resolver.go index 9b5e41411e8ef3..dcc8eec4f39ebc 100644 --- a/pkg/snmp/traps/oid_resolver.go +++ b/pkg/snmp/traps/oid_resolver.go @@ -32,19 +32,6 @@ type OIDResolver interface { GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) } -// NoOpOIDResolver is a dummy OIDResolver implementation that is unable to get any Trap or Variable metadata. -type NoOpOIDResolver struct{} - -// GetTrapMetadata always return an error in this OIDResolver implementation -func (or NoOpOIDResolver) GetTrapMetadata(trapOID string) (TrapMetadata, error) { - return TrapMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) -} - -// GetVariableMetadata always return an error in this OIDResolver implementation -func (or NoOpOIDResolver) GetVariableMetadata(trapOID string, varOID string) (VariableMetadata, error) { - return VariableMetadata{}, fmt.Errorf("trap OID %s is not defined", trapOID) -} - // MultiFilesOIDResolver is an OIDResolver implementation that can be configured with multiple input files. // Trap OIDs conflicts are resolved using the name of the source file in alphabetical order and by giving // the less priority to Datadog's own database shipped with the agent. From ad3d552de1722a3a248593451fddaff5c07b1442 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Wed, 9 Mar 2022 10:42:49 +0100 Subject: [PATCH 6/8] Fix ineff-assign --- pkg/logs/internal/launchers/traps/launcher.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/logs/internal/launchers/traps/launcher.go b/pkg/logs/internal/launchers/traps/launcher.go index 32e5ba40f08f95..50ff74e742fcdf 100644 --- a/pkg/logs/internal/launchers/traps/launcher.go +++ b/pkg/logs/internal/launchers/traps/launcher.go @@ -43,6 +43,10 @@ func (l *Launcher) startNewTailer(source *config.LogSource, inputChan chan *trap return } l.tailer, err = tailer.NewTailer(oidResolver, source, inputChan, outputChan) + if err != nil { + log.Errorf("unable to load traps database: %w. Will not listen for SNMP traps", err) + return + } l.tailer.Start() } From cfadc33c1fbaa6383c2cd3bde49d85b733489226 Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Wed, 9 Mar 2022 14:15:09 +0100 Subject: [PATCH 7/8] Rename variables_raw to variables --- pkg/snmp/traps/formatter.go | 4 ++-- pkg/snmp/traps/formatter_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/snmp/traps/formatter.go b/pkg/snmp/traps/formatter.go index 7831b41ca7186a..54af1ba922f96d 100644 --- a/pkg/snmp/traps/formatter.go +++ b/pkg/snmp/traps/formatter.go @@ -96,7 +96,7 @@ func (f JSONFormatter) formatV1Trap(packet *gosnmp.SnmpPacket) map[string]interf data["genericTrap"] = genericTrap data["specificTrap"] = specificTrap variables := parseVariables(packet.Variables) - data["variables_raw"] = variables + data["variables"] = variables for _, variable := range variables { varMetadata, err := f.oidResolver.GetVariableMetadata(trapOID, variable.OID) if err != nil { @@ -142,7 +142,7 @@ func (f JSONFormatter) formatTrap(packet *gosnmp.SnmpPacket) (map[string]interfa } parsedVariables := parseVariables(variables[2:]) - data["variables_raw"] = parsedVariables + data["variables"] = parsedVariables for _, variable := range parsedVariables { varMetadata, err := f.oidResolver.GetVariableMetadata(trapOID, variable.OID) if err != nil { diff --git a/pkg/snmp/traps/formatter_test.go b/pkg/snmp/traps/formatter_test.go index 1a8223a45a2455..e37aa196d46083 100644 --- a/pkg/snmp/traps/formatter_test.go +++ b/pkg/snmp/traps/formatter_test.go @@ -77,7 +77,7 @@ func TestFormatPacketV1Generic(t *testing.T) { variables := make([]map[string]interface{}, 3) for i := 0; i < 3; i++ { - variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + variables[i] = data["variables"].([]interface{})[i].(map[string]interface{}) } assert.Equal(t, "1.3.6.1.6.3.1.1.5", data["enterpriseOID"]) @@ -116,7 +116,7 @@ func TestFormatPacketV1Specific(t *testing.T) { variables := make([]map[string]interface{}, 2) for i := 0; i < 2; i++ { - variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + variables[i] = data["variables"].([]interface{})[i].(map[string]interface{}) } assert.Equal(t, "1.3.6.1.2.1.118", data["enterpriseOID"]) @@ -149,7 +149,7 @@ func TestFormatPacketToJSON(t *testing.T) { variables := make([]map[string]interface{}, 2) for i := 0; i < 2; i++ { - variables[i] = data["variables_raw"].([]interface{})[i].(map[string]interface{}) + variables[i] = data["variables"].([]interface{})[i].(map[string]interface{}) } heartBeatRate := variables[0] From 4af51f5d307bd4914f442faf41ca1fdd3908e30d Mon Sep 17 00:00:00 2001 From: FlorianVeaux Date: Thu, 10 Mar 2022 09:14:47 +0100 Subject: [PATCH 8/8] Fix service name --- pkg/logs/schedulers/traps/scheduler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/logs/schedulers/traps/scheduler.go b/pkg/logs/schedulers/traps/scheduler.go index 28fb47214ed2db..cfbdc23bbc25e3 100644 --- a/pkg/logs/schedulers/traps/scheduler.go +++ b/pkg/logs/schedulers/traps/scheduler.go @@ -32,7 +32,7 @@ func (s *Scheduler) Start(sourceMgr schedulers.SourceManager) { source := logsConfig.NewLogSource(logsConfig.SnmpTraps, &logsConfig.LogsConfig{ Type: logsConfig.SnmpTrapsType, Service: "snmp-traps", - Source: "snmp", + Source: "snmp-traps", }) log.Debug("Adding SNMPTraps source to the Logs Agent") sourceMgr.AddSource(source)