Skip to content

Commit

Permalink
Extract numeric time series from string values
Browse files Browse the repository at this point in the history
  • Loading branch information
@conr
conr committed Aug 4, 2017
1 parent 7ae789a commit eb00fc5
Show file tree
Hide file tree
Showing 5 changed files with 183 additions and 58 deletions.
62 changes: 45 additions & 17 deletions collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,10 @@ PduLoop:
}
if head.metric != nil {
// Found a match.
ch <- pduToSample(oidList[i+1:], &pdu, head.metric, oidToPdu)
samples := pduToSamples(oidList[i+1:], &pdu, head.metric, oidToPdu)
for _, sample := range samples {
ch <- sample
}
break
}
}
Expand All @@ -174,13 +177,20 @@ func getPduValue(pdu *gosnmp.SnmpPDU) float64 {
}
}

func pduToSample(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, oidToPdu map[string]gosnmp.SnmpPDU) prometheus.Metric {
func pduToSamples(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, oidToPdu map[string]gosnmp.SnmpPDU) []prometheus.Metric {
var res []prometheus.Metric
// The part of the OID that is the indexes.
labels := indexesToLabels(indexOids, metric, oidToPdu)

value := getPduValue(pdu)
t := prometheus.UntypedValue
stringType := false

labelnames := make([]string, 0, len(labels)+1)
labelvalues := make([]string, 0, len(labels)+1)
for k, v := range labels {
labelnames = append(labelnames, k)
labelvalues = append(labelvalues, v)
}

switch metric.Type {
case "counter":
Expand All @@ -191,25 +201,43 @@ func pduToSample(indexOids []int, pdu *gosnmp.SnmpPDU, metric *config.Metric, oi
// It's some form of string.
t = prometheus.GaugeValue
value = 1.0
stringType = true
}

labelnames := make([]string, 0, len(labels)+1)
labelvalues := make([]string, 0, len(labels)+1)
for k, v := range labels {
labelnames = append(labelnames, k)
labelvalues = append(labelvalues, v)
}
// For strings we put the value as a label with the same name as the metric.
// If the name is already an index, we do not need to set it again.
if stringType {
if len(metric.RegexpExtracts) > 0 {
return generateOverrideMetrics(metric, pdu.Value.(string), labelnames, labelvalues)
}
// For strings we put the value as a label with the same name as the metric.
// If the name is already an index, we do not need to set it again.
if _, ok := labels[metric.Name]; !ok {
labelnames = append(labelnames, metric.Name)
labelvalues = append(labelvalues, pduValueAsString(pdu, metric.Type))
}
}
return prometheus.MustNewConstMetric(prometheus.NewDesc(metric.Name, metric.Help, labelnames, nil),
t, value, labelvalues...)

if len(res) == 0 {
res = append(res, prometheus.MustNewConstMetric(prometheus.NewDesc(metric.Name, metric.Help, labelnames, nil),
t, value, labelvalues...))
}
return res
}

func generateOverrideMetrics(metric *config.Metric, pduValue string, labelnames, labelvalues []string) []prometheus.Metric {
results := []prometheus.Metric{}
for name, strMetricSlice := range metric.RegexpExtracts {
for _, strMetric := range strMetricSlice {
if strMetric.Regex.MatchString(pduValue) {
value := strMetric.Regex.ReplaceAllString(pduValue, strMetric.Value)
v, err := strconv.ParseFloat(value, 64)
if err != nil {
log.Debugf("Error parsing float64 from value: %v", value)
continue
}
newMetric := prometheus.MustNewConstMetric(prometheus.NewDesc(metric.Name+name, metric.Help, labelnames, nil),
prometheus.GaugeValue, v, labelvalues...)
results = append(results, newMetric)
break
}
}
}
return results
}

// Right pad oid with zeros, and split at the given point.
Expand Down
85 changes: 59 additions & 26 deletions collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,23 +2,54 @@ package main

import (
"reflect"
"regexp"
"testing"

"github.com/prometheus/client_model/go"
"github.com/soniah/gosnmp"

"github.com/prometheus/client_model/go"
"github.com/prometheus/snmp_exporter/config"
)

func TestPduToSample(t *testing.T) {
strMetrics := make(map[string][]config.RegexpExtract)
var regexpFooBar config.Regexp
regexpFooBar.Regexp, _ = regexp.Compile(".*")
strMetrics["Status"] = []config.RegexpExtract{
{
Regex: regexpFooBar,
Value: "5",
},
}

cases := []struct {
pdu *gosnmp.SnmpPDU
indexOids []int
metric *config.Metric
oidToPdu map[string]gosnmp.SnmpPDU
expectedMetric string
expectedMetricDesc string
pdu *gosnmp.SnmpPDU
indexOids []int
metric *config.Metric
oidToPdu map[string]gosnmp.SnmpPDU
expectedMetrics []string
expectedMetricDescs []string
}{
{
pdu: &gosnmp.SnmpPDU{
Name: "1.1.1.1.1",
Value: "SomeStringValue",
},
indexOids: []int{},
metric: &config.Metric{
Name: "snIfOpticalLaneMonitoringTemperature",
Oid: "1.1.1.1.1",
Help: "HelpText",
RegexpExtracts: strMetrics,
},
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetrics: []string{
`gauge:<value:5 > `,
},
expectedMetricDescs: []string{
`Desc{fqName: "snIfOpticalLaneMonitoringTemperatureStatus", help: "HelpText", constLabels: {}, variableLabels: []}`,
},
},
{
pdu: &gosnmp.SnmpPDU{
Name: "1.1.1.1.1",
Expand All @@ -32,9 +63,9 @@ func TestPduToSample(t *testing.T) {
Type: "counter",
Help: "Help string",
},
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetric: "counter:<value:2 > ",
expectedMetricDesc: `Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: []}`,
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetrics: []string{"counter:<value:2 > "},
expectedMetricDescs: []string{`Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: []}`},
},
{
pdu: &gosnmp.SnmpPDU{
Expand All @@ -49,9 +80,9 @@ func TestPduToSample(t *testing.T) {
Type: "gauge",
Help: "Help string",
},
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetric: "gauge:<value:2 > ",
expectedMetricDesc: `Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: []}`,
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetrics: []string{"gauge:<value:2 > "},
expectedMetricDescs: []string{`Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: []}`},
},
{
pdu: &gosnmp.SnmpPDU{
Expand All @@ -65,24 +96,26 @@ func TestPduToSample(t *testing.T) {
Oid: "1.1.1.1.1",
Help: "Help string",
},
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetric: `label:<name:"test_metric" value:"-2" > gauge:<value:1 > `,
expectedMetricDesc: `Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: [test_metric]}`,
oidToPdu: make(map[string]gosnmp.SnmpPDU),
expectedMetrics: []string{`label:<name:"test_metric" value:"-2" > gauge:<value:1 > `},
expectedMetricDescs: []string{`Desc{fqName: "test_metric", help: "Help string", constLabels: {}, variableLabels: [test_metric]}`},
},
}

for _, c := range cases {
m := pduToSample(c.indexOids, c.pdu, c.metric, c.oidToPdu)
metrics := pduToSamples(c.indexOids, c.pdu, c.metric, c.oidToPdu)
metric := &io_prometheus_client.Metric{}
err := m.Write(metric)
if err != nil {
t.Fatalf("Error writing metric: %v", err)
}
if metric.String() != c.expectedMetric {
t.Fatalf("Unexpected metric: got %v, want %v", metric.String(), c.expectedMetric)
}
if m.Desc().String() != c.expectedMetricDesc {
t.Fatalf("Unexpected metric description: got %v, want %v", m.Desc().String(), c.expectedMetricDesc)
for i, m := range metrics {
err := m.Write(metric)
if err != nil {
t.Fatalf("Error writing metric: %v", err)
}
if metric.String() != c.expectedMetrics[i] {
t.Fatalf("Unexpected metric: got %v, want %v", metric.String(), c.expectedMetrics[i])
}
if m.Desc().String() != c.expectedMetricDescs[i] {
t.Fatalf("Unexpected metric description: got %v, want %v", m.Desc().String(), c.expectedMetricDescs[i])
}
}
}
}
Expand Down
60 changes: 49 additions & 11 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"fmt"
"io/ioutil"
"regexp"
"strings"
"time"

Expand Down Expand Up @@ -40,6 +41,9 @@ var (
DefaultModule = Module{
WalkParams: DefaultWalkParams,
}
DefaultRegexpExtract = RegexpExtract{
Value: "$1",
}
)

// Config for the snmp_exporter.
Expand Down Expand Up @@ -156,24 +160,20 @@ func (c WalkParams) ConfigureSNMP(g *gosnmp.GoSNMP) {
}

type Metric struct {
Name string `yaml:"name"`
Oid string `yaml:"oid"`
Type string `yaml:"type"`
Help string `yaml:"help"`
Indexes []*Index `yaml:"indexes,omitempty"`
Lookups []*Lookup `yaml:"lookups,omitempty"`

XXX map[string]interface{} `yaml:",inline"`
Name string `yaml:"name"`
Oid string `yaml:"oid"`
Type string `yaml:"type"`
Help string `yaml:"help"`
Indexes []*Index `yaml:"indexes,omitempty"`
Lookups []*Lookup `yaml:"lookups,omitempty"`
RegexpExtracts map[string][]RegexpExtract `yaml:"regexp_extracts,omitempty"`
}

func (c *Metric) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain Metric
if err := unmarshal((*plain)(c)); err != nil {
return err
}
if err := CheckOverflow(c.XXX, "module"); err != nil {
return err
}
return nil
}

Expand Down Expand Up @@ -256,6 +256,44 @@ func (c *Auth) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}

type RegexpExtract struct {
Regex Regexp `yaml:"regex"`
Value string `yaml:"value"`

XXX map[string]interface{} `yaml:",inline"`
}

func (c *RegexpExtract) UnmarshalYAML(unmarshal func(interface{}) error) error {
*c = DefaultRegexpExtract
type plain RegexpExtract
if err := unmarshal((*plain)(c)); err != nil {
return err
}
if err := CheckOverflow(c.XXX, "regexp_extract"); err != nil {
return err
}
return nil
}

// Regexp encapsulates a regexp.Regexp and makes it YAML marshalable.
type Regexp struct {
*regexp.Regexp
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (re *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error {
var s string
if err := unmarshal(&s); err != nil {
return err
}
regex, err := regexp.Compile("^(?:" + s + ")$")
if err != nil {
return err
}
re.Regexp = regex
return nil
}

func CheckOverflow(m map[string]interface{}, ctx string) error {
if len(m) > 0 {
var keys []string
Expand Down
25 changes: 21 additions & 4 deletions generator/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,28 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}

type ModuleConfig struct {
Walk []string `yaml:"walk"`
Lookups []*Lookup `yaml:"lookups"`
type MetricOverrides struct {
RegexpExtracts map[string][]config.RegexpExtract `yaml:"regexp_extract,omitempty"`

XXX map[string]interface{} `yaml:",inline"`
}

WalkParams config.WalkParams `yaml:",inline"`
func (c *MetricOverrides) UnmarshalYAML(unmarshal func(interface{}) error) error {
type plain MetricOverrides
if err := unmarshal((*plain)(c)); err != nil {
return err
}
if err := config.CheckOverflow(c.XXX, "overrides"); err != nil {
return err
}
return nil
}

type ModuleConfig struct {
Walk []string `yaml:"walk"`
Lookups []*Lookup `yaml:"lookups"`
WalkParams config.WalkParams `yaml:",inline"`
Overrides map[string]MetricOverrides `yaml:"overrides"`

XXX map[string]interface{} `yaml:",inline"`
}
Expand Down
9 changes: 9 additions & 0 deletions generator/tree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,15 @@ func generateConfigModule(cfg *ModuleConfig, node *Node, nameToNode map[string]*
}
}

// Apply module config overrides to their corresponding metrics.
for name, params := range cfg.Overrides {
for _, metric := range out.Metrics {
if name == metric.Name || name == metric.Oid {
metric.RegexpExtracts = params.RegexpExtracts
}
}
}

oids := []string{}
for k, _ := range needToWalk {
oids = append(oids, k)
Expand Down

0 comments on commit eb00fc5

Please sign in to comment.