diff --git a/collector/collector.go b/collector/collector.go index 0ffe7ece..83438807 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -18,6 +18,7 @@ import ( "encoding/binary" "fmt" "net" + "regexp" "strconv" "strings" "time" @@ -170,7 +171,39 @@ func ScrapeTarget(ctx context.Context, target string, config *config.Module, log } defer snmp.Conn.Close() - getOids := config.Get + // Evaluate rules. + newGet := config.Get + newWalk := config.Walk + for _, filter := range config.Filters { + var pdus []gosnmp.SnmpPDU + allowedList := []string{} + + if snmp.Version == gosnmp.Version1 { + pdus, err = snmp.WalkAll(filter.Oid) + } else { + pdus, err = snmp.BulkWalkAll(filter.Oid) + } + // Do not try to filter anything if we had errors. + if err != nil { + level.Info(logger).Log("msg", "Error getting OID, won't do any filter on this oid", "oid", filter.Oid) + continue + } + + allowedList = filterAllowedIndices(logger, filter, pdus, allowedList) + + // Update config to get only index and not walk them. + newWalk = updateWalkConfig(newWalk, filter, logger) + + // Only Keep indices not involved in filters. + newCfg := updateGetConfig(newGet, filter, logger) + + // We now add each index from filter to the get list. + newCfg = addAllowedIndices(filter, allowedList, logger, newCfg) + + newGet = newCfg + } + + getOids := newGet maxOids := int(config.WalkParams.MaxRepetitions) // Max Repetition can be 0, maxOids cannot. SNMPv1 can only report one OID error per call. if maxOids == 0 || snmp.Version == gosnmp.Version1 { @@ -213,7 +246,7 @@ func ScrapeTarget(ctx context.Context, target string, config *config.Module, log getOids = getOids[oids:] } - for _, subtree := range config.Walk { + for _, subtree := range newWalk { var pdus []gosnmp.SnmpPDU level.Debug(logger).Log("msg", "Walking subtree", "oid", subtree) walkStart := time.Now() @@ -235,6 +268,77 @@ func ScrapeTarget(ctx context.Context, target string, config *config.Module, log return results, nil } +func filterAllowedIndices(logger log.Logger, filter config.DynamicFilter, pdus []gosnmp.SnmpPDU, allowedList []string) []string { + level.Debug(logger).Log("msg", "Evaluating rule for oid", "oid", filter.Oid) + for _, pdu := range pdus { + found := false + for _, val := range filter.Values { + snmpval := pduValueAsString(&pdu, "DisplayString") + level.Debug(logger).Log("config value", val, "snmp value", snmpval) + + if regexp.MustCompile(val).MatchString(snmpval) { + found = true + break + } + } + if found { + pduArray := strings.Split(pdu.Name, ".") + index := pduArray[len(pduArray)-1] + level.Debug(logger).Log("msg", "Caching index", "index", index) + allowedList = append(allowedList, index) + } + } + return allowedList +} + +func updateWalkConfig(walkConfig []string, filter config.DynamicFilter, logger log.Logger) []string { + newCfg := []string{} + for _, elem := range walkConfig { + found := false + for _, targetOid := range filter.Targets { + if elem == targetOid { + level.Debug(logger).Log("msg", "Deleting for walk configuration", "oid", targetOid) + found = true + break + } + } + // Oid not found in target, we walk it. + if !found { + newCfg = append(newCfg, elem) + } + } + return newCfg +} + +func updateGetConfig(getConfig []string, filter config.DynamicFilter, logger log.Logger) []string { + newCfg := []string{} + for _, elem := range getConfig { + found := false + for _, targetOid := range filter.Targets { + if strings.HasPrefix(elem, targetOid) { + found = true + break + } + } + // Oid not found in targets, we keep it. + if !found { + level.Debug(logger).Log("msg", "Keeping get configuration", "oid", elem) + newCfg = append(newCfg, elem) + } + } + return newCfg +} + +func addAllowedIndices(filter config.DynamicFilter, allowedList []string, logger log.Logger, newCfg []string) []string { + for _, targetOid := range filter.Targets { + for _, index := range allowedList { + level.Debug(logger).Log("msg", "Adding get configuration", "oid", targetOid+"."+index) + newCfg = append(newCfg, targetOid+"."+index) + } + } + return newCfg +} + type MetricNode struct { metric *config.Metric diff --git a/collector/collector_test.go b/collector/collector_test.go index f988e274..c4010cff 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -1008,3 +1008,148 @@ func TestIndexesToLabels(t *testing.T) { } } } + +func TestFilterAllowedIndices(t *testing.T) { + + pdus := []gosnmp.SnmpPDU{ + gosnmp.SnmpPDU{ + Name: "1.3.6.1.2.1.2.2.1.8.1", + Value: "2", + }, + gosnmp.SnmpPDU{ + Name: "1.3.6.1.2.1.2.2.1.8.2", + Value: "1", + }, + gosnmp.SnmpPDU{ + Name: "1.3.6.1.2.1.2.2.1.8.3", + Value: "1", + }, + gosnmp.SnmpPDU{ + Name: "1.3.6.1.2.1.2.2.1.8.4", + Value: "5", + }, + } + + cases := []struct { + filter config.DynamicFilter + allowedList []string + result []string + }{ + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.5"}, + Values: []string{"1"}, + }, + result: []string{"2", "3"}, + }, + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.5"}, + Values: []string{"5"}, + }, + result: []string{"4"}, + }, + } + for _, c := range cases { + got := filterAllowedIndices(log.NewNopLogger(), c.filter, pdus, c.allowedList) + if !reflect.DeepEqual(got, c.result) { + t.Errorf("filterAllowedIndices(%v): got %v, want %v", c.filter, got, c.result) + } + } +} + +func TestUpdateWalkConfig(t *testing.T) { + cases := []struct { + filter config.DynamicFilter + result []string + }{ + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.5", "1.3.6.1.2.1.2.2.1.7"}, + Values: []string{"1"}, + }, + result: []string{}, + }, + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.21"}, + Values: []string{"1"}, + }, + result: []string{"1.3.6.1.2.1.2.2.1.5", "1.3.6.1.2.1.2.2.1.7"}, + }, + } + walkConfig := []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.5", "1.3.6.1.2.1.2.2.1.7"} + for _, c := range cases { + got := updateWalkConfig(walkConfig, c.filter, log.NewNopLogger()) + if !reflect.DeepEqual(got, c.result) { + t.Errorf("updateWalkConfig(%v): got %v, want %v", c.filter, got, c.result) + } + } +} + +func TestUpdateGetConfig(t *testing.T) { + cases := []struct { + filter config.DynamicFilter + result []string + }{ + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1"}, + Values: []string{"1"}, + }, + result: []string{}, + }, + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.21"}, + Values: []string{"1"}, + }, + result: []string{"1.3.6.1.2.1.2.2.1.5", "1.3.6.1.2.1.2.2.1.7"}, + }, + } + getConfig := []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.5", "1.3.6.1.2.1.2.2.1.7"} + for _, c := range cases { + got := updateGetConfig(getConfig, c.filter, log.NewNopLogger()) + if !reflect.DeepEqual(got, c.result) { + t.Errorf("updateGetConfig(%v): got %v, want %v", c.filter, got, c.result) + } + } +} + +func TestAddAllowedIndices(t *testing.T) { + cases := []struct { + filter config.DynamicFilter + result []string + }{ + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1"}, + Values: []string{"1"}, + }, + result: []string{"1.3.6.1.2.1.31.1.1.1.10", "1.3.6.1.2.1.31.1.1.1.11", "1.3.6.1.2.1.2.2.1.2", "1.3.6.1.2.1.2.2.1.3"}, + }, + { + filter: config.DynamicFilter{ + Oid: "1.3.6.1.2.1.2.2.1.8", + Targets: []string{"1.3.6.1.2.1.2.2.1.3", "1.3.6.1.2.1.2.2.1.21"}, + Values: []string{"1"}, + }, + result: []string{"1.3.6.1.2.1.31.1.1.1.10", "1.3.6.1.2.1.31.1.1.1.11", "1.3.6.1.2.1.2.2.1.3.2", "1.3.6.1.2.1.2.2.1.3.3", "1.3.6.1.2.1.2.2.1.21.2", "1.3.6.1.2.1.2.2.1.21.3"}, + }, + } + allowedList := []string{"2", "3"} + newCfg := []string{"1.3.6.1.2.1.31.1.1.1.10", "1.3.6.1.2.1.31.1.1.1.11"} + for _, c := range cases { + got := addAllowedIndices(c.filter, allowedList, log.NewNopLogger(), newCfg) + if !reflect.DeepEqual(got, c.result) { + t.Errorf("addAllowedIndices(%v): got %v, want %v", c.filter, got, c.result) + } + } +} diff --git a/config/config.go b/config/config.go index f8fd56d1..8eb20912 100644 --- a/config/config.go +++ b/config/config.go @@ -77,10 +77,11 @@ type WalkParams struct { type Module struct { // A list of OIDs. - Walk []string `yaml:"walk,omitempty"` - Get []string `yaml:"get,omitempty"` - Metrics []*Metric `yaml:"metrics"` - WalkParams WalkParams `yaml:",inline"` + Walk []string `yaml:"walk,omitempty"` + Get []string `yaml:"get,omitempty"` + Metrics []*Metric `yaml:"metrics"` + WalkParams WalkParams `yaml:",inline"` + Filters []DynamicFilter `yaml:"filters,omitempty"` } func (c *Module) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -191,6 +192,21 @@ func (c WalkParams) ConfigureSNMP(g *gosnmp.GoSNMP) { g.SecurityParameters = usm } +type Filters struct { + Static []StaticFilter `yaml:"static,omitempty"` + Dynamic []DynamicFilter `yaml:"dynamic,omitempty"` +} + +type StaticFilter struct { + Targets []string `yaml:"targets,omitempty"` + Indices []string `yaml:"indices,omitempty"` +} +type DynamicFilter struct { + Oid string `yaml:"oid"` + Targets []string `yaml:"targets,omitempty"` + Values []string `yaml:"values,omitempty"` +} + type Metric struct { Name string `yaml:"name"` Oid string `yaml:"oid"` diff --git a/generator/Dockerfile-local b/generator/Dockerfile-local new file mode 100644 index 00000000..9dc82ac9 --- /dev/null +++ b/generator/Dockerfile-local @@ -0,0 +1,14 @@ +FROM golang:latest + +RUN apt-get update && \ + apt-get install -y libsnmp-dev p7zip-full + +COPY ./generator /bin/generator + +WORKDIR "/opt" + +ENTRYPOINT ["/bin/generator"] + +ENV MIBDIRS mibs + +CMD ["generate"] diff --git a/generator/README.md b/generator/README.md index 56eef662..c267cf12 100644 --- a/generator/README.md +++ b/generator/README.md @@ -17,6 +17,12 @@ git clone https://github.com/prometheus/snmp_exporter.git cd snmp_exporter/generator make generator mibs ``` +## Preparation + +It is recommended to have a directory per device family which contains the mibs dir for the device family, +a logical link to the generator executable and the generator.yml configuration file. This is to avoid name space collisions +in the MIB definition. Keep only the required MIBS in the mibs directory for the devices. +Then merge all the resulting snmp.yml files into one main file that will be used by the snmp_exporter collector. ## Running @@ -24,7 +30,8 @@ make generator mibs make generate ``` -The generator reads in from `generator.yml` and writes to `snmp.yml`. +The generator reads in the simplified collection instructions from `generator.yml` and writes to `snmp.yml`. Only the snmp.yml file is used +by the snmp_exporter executable to collect data from the snmp enabled devices. Additional command are available for debugging, use the `help` command to see them. @@ -42,7 +49,7 @@ make docker-generate ## File Format -`generator.yml` provides a list of modules. The simplest module is just a name +`generator.yml` provides a list of modules. Each module defines what to collect from a device type. The simplest module is just a name and a set of OIDs to walk. ```yaml @@ -52,6 +59,8 @@ modules: - 1.3.6.1.2.1.2 # Same as "interfaces" - sysUpTime # Same as "1.3.6.1.2.1.1.3" - 1.3.6.1.2.1.31.1.1.1.6.40 # Instance of "ifHCInOctets" with index "40" + - 1.3.6.1.2.1.2.2.1.4 # Same as ifMtu (used for filter example) + - bsnDot11EssSsid # Same as 1.3.6.1.4.1.14179.2.1.1.1.2 (used for filter example) version: 2 # SNMP version to use. Defaults to 2. # 1 will use GETNEXT, 2 and 3 use GETBULK. @@ -105,19 +114,19 @@ modules: - source_indexes: [cbQosConfigIndex] lookup: cbQosCMName - overrides: # Allows for per-module overrides of bits of MIBs - metricName: - ignore: true # Drops the metric from the output. - regex_extracts: - Temp: # A new metric will be created appending this to the metricName to become metricNameTemp. - - regex: '(.*)' # Regex to extract a value from the returned SNMP walks's value. - value: '$1' # The result will be parsed as a float64, defaults to $1. - Status: - - regex: '.*Example' - value: '1' # The first entry whose regex matches and whose value parses wins. - - regex: '.*' - value: '0' - type: DisplayString # Override the metric type, possible types are: + overrides: # Allows for per-module overrides of bits of MIBs + metricName: + ignore: true # Drops the metric from the output. + regex_extracts: + Temp: # A new metric will be created appending this to the metricName to become metricNameTemp. + - regex: '(.*)' # Regex to extract a value from the returned SNMP walks's value. + value: '$1' # The result will be parsed as a float64, defaults to $1. + Status: + - regex: '.*Example' + value: '1' # The first entry whose regex matches and whose value parses wins. + - regex: '.*' + value: '0' + type: DisplayString # Override the metric type, possible types are: # gauge: An integer with type gauge. # counter: An integer with type counter. # OctetString: A bit string, rendered as 0xff34. @@ -134,6 +143,30 @@ modules: # EnumAsInfo: An enum for which a single timeseries is created. Good for constant values. # EnumAsStateSet: An enum with a time series per state. Good for variable low-cardinality enums. # Bits: An RFC 2578 BITS construct, which produces a StateSet with a time series per bit. + + filters: # Define filters to collect only a subset of OID table indices + static: # static filters are handled in the generator. They will convert walks to multiple gets with the specified indices + # in the resulting snmp.yml output. + # the index filter will reduce a walk of a table to only the defined indices to get + # If one of the target OIDs is used in a lookup, the filter will apply ALL tables using this lookup + # For a network switch, this could be used to collect a subset of interfaces such as uplinks + # For a router, this could be used to collect all real ports but not vlans and other virtual interfaces + # Specifying ifAlias or ifName if they are used in lookups with ifIndex will apply to the filter to + # all the OIDs that depend on the lookup, such as ifSpeed, ifInHcOctets, etc. + # This feature applies to any table(s) OIDs using a common index + - targets: + - bsnDot11EssSsid + indices: ["2","3","4"] # List of interface indices to get + + dynamic: # dynamic filters are handed by the snmp exporter. The generator will simply pass on the configuration in the snmp.yml. + # The exporter will do a snmp walk of the oid and will restrict snmp walk made on the targets + # to the index matching the value in the values list. + # This would be typically used to specify a filter for interfaces with a certain name in ifAlias, ifSpeed or admin status. + # For example, only get interfaces that a gig and faster, or get interfaces that are named Up or interfaces that are admin Up + - oid: 1.3.6.1.2.1.2.2.1.7 + targets: + - "1.3.6.1.2.1.2.2.1.4" + values: ["1", "2"] ``` ### EnumAsInfo and EnumAsStateSet diff --git a/generator/config.go b/generator/config.go index 460808b1..dcf81493 100644 --- a/generator/config.go +++ b/generator/config.go @@ -15,8 +15,8 @@ package main import ( "fmt" - "github.com/prometheus/snmp_exporter/config" + "strconv" ) // The generator config. @@ -50,6 +50,27 @@ type ModuleConfig struct { Lookups []*Lookup `yaml:"lookups"` WalkParams config.WalkParams `yaml:",inline"` Overrides map[string]MetricOverrides `yaml:"overrides"` + Filters config.Filters `yaml:"filters,omitempty"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *ModuleConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain ModuleConfig + if err := unmarshal((*plain)(c)); err != nil { + return err + } + + // Ensure indices in static filters are integer for input validation. + for _, filter := range c.Filters.Static { + for _, index := range filter.Indices { + _, err := strconv.Atoi(index) + if err != nil { + return fmt.Errorf("invalid index '%s'. Index must be integer", index) + } + } + } + + return nil } type Lookup struct { diff --git a/generator/tree.go b/generator/tree.go index 9498ee31..5198659d 100644 --- a/generator/tree.go +++ b/generator/tree.go @@ -180,7 +180,7 @@ func metricAccess(a string) bool { case "ACCESS_READONLY", "ACCESS_READWRITE", "ACCESS_CREATE", "ACCESS_NOACCESS": return true default: - // the others are inaccessible metrics. + // The others are inaccessible metrics. return false } } @@ -376,6 +376,19 @@ func generateConfigModule(cfg *ModuleConfig, node *Node, nameToNode map[string]* }) } + // Build an map of all oid targeted by a filter to access it easily later. + filterMap := map[string][]string{} + + for _, filter := range cfg.Filters.Static { + for _, oid := range filter.Targets { + n, ok := nameToNode[oid] + if ok { + oid = n.Oid + } + filterMap[oid] = filter.Indices + } + } + // Apply lookups. for _, metric := range out.Metrics { toDelete := []string{} @@ -433,6 +446,14 @@ func generateConfigModule(cfg *ModuleConfig, node *Node, nameToNode map[string]* } else { needToWalk[indexNode.Oid] = struct{}{} } + // We apply the same filter to metric.Oid if the lookup oid is filtered. + indices, found := filterMap[indexNode.Oid] + if found { + delete(needToWalk, metric.Oid) + for _, index := range indices { + needToWalk[metric.Oid+"."+index+"."] = struct{}{} + } + } if lookup.DropSourceIndexes { // Avoid leaving the old labelname around. toDelete = append(toDelete, lookup.SourceIndexes...) @@ -453,8 +474,8 @@ func generateConfigModule(cfg *ModuleConfig, node *Node, nameToNode map[string]* } } - // Check that the object before an InetAddress is an InetAddressType, - // if not, change it to an OctetString. + // Check that the object before an InetAddress is an InetAddressType. + // If not, change it to an OctetString. for _, metric := range out.Metrics { if metric.Type == "InetAddress" || metric.Type == "InetAddressMissingSize" { // Get previous oid. @@ -486,6 +507,23 @@ func generateConfigModule(cfg *ModuleConfig, node *Node, nameToNode map[string]* } } + // Apply filters. + for _, filter := range cfg.Filters.Static { + // Delete the oid targeted by the filter, as we won't walk the whole table. + for _, oid := range filter.Targets { + n, ok := nameToNode[oid] + if ok { + oid = n.Oid + } + delete(needToWalk, oid) + for _, index := range filter.Indices { + needToWalk[oid+"."+index+"."] = struct{}{} + } + } + } + + out.Filters = cfg.Filters.Dynamic + oids := []string{} for k := range needToWalk { oids = append(oids, k)