Skip to content

Commit

Permalink
Merge pull request #314 from glightfoot/histogram-type
Browse files Browse the repository at this point in the history 
[307] Add support for histograms and distributions without unit conversion
  • Loading branch information
Matthias Rampke authored Jun 19, 2020
2 parents 2fc2dcd + 4a64979 commit 7ba3550
Show file tree
Hide file tree
Showing 13 changed files with 457 additions and 232 deletions.
200 changes: 115 additions & 85 deletions bridge_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,60 +83,60 @@ func TestHandlePacket(t *testing.T) {
name: "simple timer",
in: "foo:200|ms",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 200,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.2,
OLabels: map[string]string{},
},
},
}, {
name: "simple histogram",
in: "foo:200|h",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 200,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
}, {
name: "simple distribution",
in: "foo:200|d",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 200,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
}, {
name: "distribution with sampling",
in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
},
}, {
Expand Down Expand Up @@ -265,30 +265,30 @@ func TestHandlePacket(t *testing.T) {
name: "histogram with sampling",
in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 0.01,
TLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.01,
OLabels: map[string]string{"tag1": "bar", "tag2": "baz"},
},
},
}, {
Expand Down Expand Up @@ -321,15 +321,15 @@ func TestHandlePacket(t *testing.T) {
name: "combined multiline metrics",
in: "foo:200|ms:300|ms:5|c|@0.1:6|g\nbar:1|c:5|ms",
out: event.Events{
&event.TimerEvent{
TMetricName: "foo",
TValue: 200,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "foo",
OValue: .200,
OLabels: map[string]string{},
},
&event.TimerEvent{
TMetricName: "foo",
TValue: 300,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "foo",
OValue: .300,
OLabels: map[string]string{},
},
&event.CounterEvent{
CMetricName: "foo",
Expand All @@ -346,26 +346,26 @@ func TestHandlePacket(t *testing.T) {
CValue: 1,
CLabels: map[string]string{},
},
&event.TimerEvent{
TMetricName: "bar",
TValue: 5,
TLabels: map[string]string{},
&event.ObserverEvent{
OMetricName: "bar",
OValue: .005,
OLabels: map[string]string{},
},
},
}, {
name: "timings with sampling factor",
in: "foo.timing:0.5|ms|@0.1",
out: event.Events{
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
&event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}},
},
}, {
name: "bad line",
Expand Down Expand Up @@ -422,6 +422,36 @@ func TestHandlePacket(t *testing.T) {
CLabels: map[string]string{},
},
},
}, {
name: "ms timer with conversion to seconds",
in: "foo:200|ms",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 0.2,
OLabels: map[string]string{},
},
},
}, {
name: "histogram with no unit conversion",
in: "foo:200|h",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
}, {
name: "distribution with no unit conversion",
in: "foo:200|d",
out: event.Events{
&event.ObserverEvent{
OMetricName: "foo",
OValue: 200,
OLabels: map[string]string{},
},
},
},
}

Expand Down Expand Up @@ -554,9 +584,9 @@ mappings:
GValue: 200,
},
// event with ttl = 2s from a mapping
&event.TimerEvent{
TMetricName: "bazqux.main",
TValue: 42000,
&event.ObserverEvent{
OMetricName: "bazqux.main",
OValue: 42,
},
}

Expand Down
24 changes: 17 additions & 7 deletions exporter_benchmark_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"testing"

"github.com/go-kit/kit/log"

"github.com/prometheus/statsd_exporter/pkg/event"
"github.com/prometheus/statsd_exporter/pkg/exporter"
"github.com/prometheus/statsd_exporter/pkg/listener"
Expand All @@ -38,6 +39,7 @@ func benchmarkUDPListener(times int, b *testing.B) {
"some_very_useful_metrics_with_quite_a_log_name:13|c",
}
bytesInput := make([]string, len(input)*times)
logger := log.NewNopLogger()
for run := 0; run < times; run++ {
for i := 0; i < len(input); i++ {
bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i])
Expand All @@ -46,7 +48,15 @@ func benchmarkUDPListener(times int, b *testing.B) {
for n := 0; n < b.N; n++ {
// there are more events than input lines, need bigger buffer
events := make(chan event.Events, len(bytesInput)*times*2)
l := listener.StatsDUDPListener{EventHandler: &event.UnbufferedEventHandler{C: events}}

l := listener.StatsDUDPListener{
EventHandler: &event.UnbufferedEventHandler{C: events},
Logger: logger,
UDPPackets: udpPackets,
LinesReceived: linesReceived,
SamplesReceived: samplesReceived,
TagsReceived: tagsReceived,
}

for i := 0; i < times; i++ {
for _, line := range bytesInput {
Expand Down Expand Up @@ -76,13 +86,13 @@ func BenchmarkExporterListener(b *testing.B) {
GMetricName: "gauge",
GValue: 10,
},
&event.TimerEvent{ // simple timer
TMetricName: "timer",
TValue: 200,
&event.ObserverEvent{ // simple timer
OMetricName: "timer",
OValue: 200,
},
&event.TimerEvent{ // simple histogram
TMetricName: "histogram.test",
TValue: 200,
&event.ObserverEvent{ // simple histogram
OMetricName: "histogram.test",
OValue: 200,
},
&event.CounterEvent{ // simple_tags
CMetricName: "simple_tags",
Expand Down
16 changes: 8 additions & 8 deletions pkg/event/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,16 @@ func (g *GaugeEvent) Value() float64 { return g.GValue }
func (c *GaugeEvent) Labels() map[string]string { return c.GLabels }
func (c *GaugeEvent) MetricType() mapper.MetricType { return mapper.MetricTypeGauge }

type TimerEvent struct {
TMetricName string
TValue float64
TLabels map[string]string
type ObserverEvent struct {
OMetricName string
OValue float64
OLabels map[string]string
}

func (t *TimerEvent) MetricName() string { return t.TMetricName }
func (t *TimerEvent) Value() float64 { return t.TValue }
func (c *TimerEvent) Labels() map[string]string { return c.TLabels }
func (c *TimerEvent) MetricType() mapper.MetricType { return mapper.MetricTypeTimer }
func (t *ObserverEvent) MetricName() string { return t.OMetricName }
func (t *ObserverEvent) Value() float64 { return t.OValue }
func (c *ObserverEvent) Labels() map[string]string { return c.OLabels }
func (c *ObserverEvent) MetricType() mapper.MetricType { return mapper.MetricTypeObserver }

type Events []Event

Expand Down
28 changes: 14 additions & 14 deletions pkg/exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,38 +140,38 @@ func (b *Exporter) handleEvent(thisEvent event.Event) {
b.ConflictingEventStats.WithLabelValues("gauge").Inc()
}

case *event.TimerEvent:
t := mapper.TimerTypeDefault
case *event.ObserverEvent:
t := mapper.ObserverTypeDefault
if mapping != nil {
t = mapping.TimerType
t = mapping.ObserverType
}
if t == mapper.TimerTypeDefault {
t = b.Mapper.Defaults.TimerType
if t == mapper.ObserverTypeDefault {
t = b.Mapper.Defaults.ObserverType
}

switch t {
case mapper.TimerTypeHistogram:
case mapper.ObserverTypeHistogram:
histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount)
if err == nil {
histogram.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond
b.EventStats.WithLabelValues("timer").Inc()
histogram.Observe(thisEvent.Value())
b.EventStats.WithLabelValues("observer").Inc()
} else {
level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err)
b.ConflictingEventStats.WithLabelValues("timer").Inc()
b.ConflictingEventStats.WithLabelValues("observer").Inc()
}

case mapper.TimerTypeDefault, mapper.TimerTypeSummary:
case mapper.ObserverTypeDefault, mapper.ObserverTypeSummary:
summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount)
if err == nil {
summary.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond
b.EventStats.WithLabelValues("timer").Inc()
summary.Observe(thisEvent.Value())
b.EventStats.WithLabelValues("observer").Inc()
} else {
level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err)
b.ConflictingEventStats.WithLabelValues("timer").Inc()
b.ConflictingEventStats.WithLabelValues("observer").Inc()
}

default:
level.Error(b.Logger).Log("msg", "unknown timer type", "type", t)
level.Error(b.Logger).Log("msg", "unknown observer type", "type", t)
os.Exit(1)
}

Expand Down
Loading

0 comments on commit 7ba3550

Please sign in to comment.