From 5c0b206f6eb23121fad0c41ea399dd518d27536a Mon Sep 17 00:00:00 2001 From: glightfoot Date: Thu, 11 Jun 2020 10:17:09 -0400 Subject: [PATCH 01/10] add support for histograms and distributions without unit conversion Signed-off-by: glightfoot --- bridge_test.go | 60 ++++++++++++++++++++++++++--------- pkg/event/event.go | 1 + pkg/exporter/exporter.go | 4 +-- pkg/exporter/exporter_test.go | 8 ++--- pkg/line/line.go | 8 ++++- 5 files changed, 58 insertions(+), 23 deletions(-) diff --git a/bridge_test.go b/bridge_test.go index f45cefa8..58b8c31e 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -85,7 +85,7 @@ func TestHandlePacket(t *testing.T) { out: event.Events{ &event.TimerEvent{ TMetricName: "foo", - TValue: 200, + TValue: 0.2, // convert statsd ms to seconds in parsing TLabels: map[string]string{}, }, }, @@ -323,12 +323,12 @@ func TestHandlePacket(t *testing.T) { out: event.Events{ &event.TimerEvent{ TMetricName: "foo", - TValue: 200, + TValue: .200, TLabels: map[string]string{}, }, &event.TimerEvent{ TMetricName: "foo", - TValue: 300, + TValue: .300, TLabels: map[string]string{}, }, &event.CounterEvent{ @@ -348,7 +348,7 @@ func TestHandlePacket(t *testing.T) { }, &event.TimerEvent{ TMetricName: "bar", - TValue: 5, + TValue: .005, TLabels: map[string]string{}, }, }, @@ -356,16 +356,16 @@ func TestHandlePacket(t *testing.T) { name: "timings with sampling factor", in: "foo.timing:0.5|ms|@0.1", out: event.Events{ - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.5, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, }, }, { name: "bad line", @@ -422,6 +422,36 @@ func TestHandlePacket(t *testing.T) { CLabels: map[string]string{}, }, }, + }, { + name: "ms timer with conversion to seconds", + in: "foo:200|ms", + out: event.Events{ + &event.TimerEvent{ + TMetricName: "foo", + TValue: 0.2, + TLabels: map[string]string{}, + }, + }, + }, { + name: "histogram with no unit conversion", + in: "foo:200|h", + out: event.Events{ + &event.TimerEvent{ + TMetricName: "foo", + TValue: 200, + TLabels: map[string]string{}, + }, + }, + }, { + name: "distribution with no unit conversion", + in: "foo:200|d", + out: event.Events{ + &event.TimerEvent{ + TMetricName: "foo", + TValue: 200, // convert statsd ms to seconds in parsing + TLabels: map[string]string{}, + }, + }, }, } @@ -556,7 +586,7 @@ mappings: // event with ttl = 2s from a mapping &event.TimerEvent{ TMetricName: "bazqux.main", - TValue: 42000, + TValue: 42, }, } diff --git a/pkg/event/event.go b/pkg/event/event.go index 54139be4..108d1fbc 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -52,6 +52,7 @@ func (g *GaugeEvent) Value() float64 { return g.GValue } func (c *GaugeEvent) Labels() map[string]string { return c.GLabels } func (c *GaugeEvent) MetricType() mapper.MetricType { return mapper.MetricTypeGauge } +// TimerEvent now defaults to seconds units type TimerEvent struct { TMetricName string TValue float64 diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index 66876cc7..558949ea 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -153,7 +153,7 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { case mapper.TimerTypeHistogram: histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { - histogram.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond + histogram.Observe(thisEvent.Value()) b.EventStats.WithLabelValues("timer").Inc() } else { level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) @@ -163,7 +163,7 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { case mapper.TimerTypeDefault, mapper.TimerTypeSummary: summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { - summary.Observe(thisEvent.Value() / 1000) // prometheus presumes seconds, statsd millisecond + summary.Observe(thisEvent.Value()) b.EventStats.WithLabelValues("timer").Inc() } else { level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index 86cb040a..ec4ae013 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -721,7 +721,7 @@ func TestHistogramUnits(t *testing.T) { c := event.Events{ &event.TimerEvent{ TMetricName: name, - TValue: 300, + TValue: .300, }, } events <- c @@ -737,9 +737,7 @@ func TestHistogramUnits(t *testing.T) { if value == nil { t.Fatal("Histogram value should not be nil") } - if *value == 300 { - t.Fatalf("Histogram observations not scaled into Seconds") - } else if *value != .300 { + if *value != .300 { t.Fatalf("Received unexpected value for histogram observation %f != .300", *value) } } @@ -871,7 +869,7 @@ mappings: // event with ttl = 2s from a mapping &event.TimerEvent{ TMetricName: "bazqux.main", - TValue: 42000, + TValue: 42, }, } diff --git a/pkg/line/line.go b/pkg/line/line.go index 730dbcfa..50dd7aa3 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -42,7 +42,13 @@ func buildEvent(statType, metric string, value float64, relative bool, labels ma GRelative: relative, GLabels: labels, }, nil - case "ms", "h", "d": + case "ms": + return &event.TimerEvent{ + TMetricName: metric, + TValue: float64(value) / 1000, // prometheus presumes seconds, statsd millisecond + TLabels: labels, + }, nil + case "h", "d": return &event.TimerEvent{ TMetricName: metric, TValue: float64(value), From 063112b138c9a7a5a9521eefa0a99e53085d57a5 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Thu, 11 Jun 2020 10:49:48 -0400 Subject: [PATCH 02/10] fix benchmark test Signed-off-by: glightfoot --- exporter_benchmark_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/exporter_benchmark_test.go b/exporter_benchmark_test.go index 900941b2..480930b8 100644 --- a/exporter_benchmark_test.go +++ b/exporter_benchmark_test.go @@ -18,6 +18,7 @@ import ( "testing" "github.com/go-kit/kit/log" + "github.com/prometheus/statsd_exporter/pkg/event" "github.com/prometheus/statsd_exporter/pkg/exporter" "github.com/prometheus/statsd_exporter/pkg/listener" @@ -38,6 +39,7 @@ func benchmarkUDPListener(times int, b *testing.B) { "some_very_useful_metrics_with_quite_a_log_name:13|c", } bytesInput := make([]string, len(input)*times) + logger := log.NewNopLogger() for run := 0; run < times; run++ { for i := 0; i < len(input); i++ { bytesInput[run*len(input)+i] = fmt.Sprintf("run%d%s", run, input[i]) @@ -46,7 +48,15 @@ func benchmarkUDPListener(times int, b *testing.B) { for n := 0; n < b.N; n++ { // there are more events than input lines, need bigger buffer events := make(chan event.Events, len(bytesInput)*times*2) - l := listener.StatsDUDPListener{EventHandler: &event.UnbufferedEventHandler{C: events}} + + l := listener.StatsDUDPListener{ + EventHandler: &event.UnbufferedEventHandler{C: events}, + Logger: logger, + UDPPackets: udpPackets, + LinesReceived: linesReceived, + SamplesReceived: samplesReceived, + TagsReceived: tagsReceived, + } for i := 0; i < times; i++ { for _, line := range bytesInput { From 01b19722d78d051fa10a37fca4c1e191193d919b Mon Sep 17 00:00:00 2001 From: glightfoot Date: Fri, 12 Jun 2020 08:37:06 -0400 Subject: [PATCH 03/10] remove comments Signed-off-by: glightfoot --- bridge_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bridge_test.go b/bridge_test.go index 58b8c31e..7425bee3 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -85,7 +85,7 @@ func TestHandlePacket(t *testing.T) { out: event.Events{ &event.TimerEvent{ TMetricName: "foo", - TValue: 0.2, // convert statsd ms to seconds in parsing + TValue: 0.2, TLabels: map[string]string{}, }, }, @@ -448,7 +448,7 @@ func TestHandlePacket(t *testing.T) { out: event.Events{ &event.TimerEvent{ TMetricName: "foo", - TValue: 200, // convert statsd ms to seconds in parsing + TValue: 200, TLabels: map[string]string{}, }, }, From 374d202daa32dc8e1e7211bee0d38b5716854c7c Mon Sep 17 00:00:00 2001 From: glightfoot Date: Fri, 12 Jun 2020 08:42:09 -0400 Subject: [PATCH 04/10] rename TimerEvent to ObserverEvent Signed-off-by: glightfoot --- bridge_test.go | 194 +++++++++++++++++----------------- exporter_benchmark_test.go | 12 +-- pkg/event/event.go | 17 ++- pkg/exporter/exporter.go | 2 +- pkg/exporter/exporter_test.go | 92 ++++++++-------- pkg/line/line.go | 16 +-- 6 files changed, 166 insertions(+), 167 deletions(-) diff --git a/bridge_test.go b/bridge_test.go index 7425bee3..ecff423e 100644 --- a/bridge_test.go +++ b/bridge_test.go @@ -83,60 +83,60 @@ func TestHandlePacket(t *testing.T) { name: "simple timer", in: "foo:200|ms", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.2, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, }, }, }, { name: "simple histogram", in: "foo:200|h", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 200, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, }, }, }, { name: "simple distribution", in: "foo:200|d", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 200, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, }, }, }, { name: "distribution with sampling", in: "foo:0.01|d|@0.2|#tag1:bar,#tag2:baz", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, { @@ -265,30 +265,30 @@ func TestHandlePacket(t *testing.T) { name: "histogram with sampling", in: "foo:0.01|h|@0.2|#tag1:bar,#tag2:baz", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, - }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.01, - TLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, + }, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.01, + OLabels: map[string]string{"tag1": "bar", "tag2": "baz"}, }, }, }, { @@ -321,15 +321,15 @@ func TestHandlePacket(t *testing.T) { name: "combined multiline metrics", in: "foo:200|ms:300|ms:5|c|@0.1:6|g\nbar:1|c:5|ms", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: .200, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: .200, + OLabels: map[string]string{}, }, - &event.TimerEvent{ - TMetricName: "foo", - TValue: .300, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: .300, + OLabels: map[string]string{}, }, &event.CounterEvent{ CMetricName: "foo", @@ -346,26 +346,26 @@ func TestHandlePacket(t *testing.T) { CValue: 1, CLabels: map[string]string{}, }, - &event.TimerEvent{ - TMetricName: "bar", - TValue: .005, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "bar", + OValue: .005, + OLabels: map[string]string{}, }, }, }, { name: "timings with sampling factor", in: "foo.timing:0.5|ms|@0.1", out: event.Events{ - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, - &event.TimerEvent{TMetricName: "foo.timing", TValue: 0.0005, TLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, + &event.ObserverEvent{OMetricName: "foo.timing", OValue: 0.0005, OLabels: map[string]string{}}, }, }, { name: "bad line", @@ -426,30 +426,30 @@ func TestHandlePacket(t *testing.T) { name: "ms timer with conversion to seconds", in: "foo:200|ms", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 0.2, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 0.2, + OLabels: map[string]string{}, }, }, }, { name: "histogram with no unit conversion", in: "foo:200|h", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 200, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, }, }, }, { name: "distribution with no unit conversion", in: "foo:200|d", out: event.Events{ - &event.TimerEvent{ - TMetricName: "foo", - TValue: 200, - TLabels: map[string]string{}, + &event.ObserverEvent{ + OMetricName: "foo", + OValue: 200, + OLabels: map[string]string{}, }, }, }, @@ -584,9 +584,9 @@ mappings: GValue: 200, }, // event with ttl = 2s from a mapping - &event.TimerEvent{ - TMetricName: "bazqux.main", - TValue: 42, + &event.ObserverEvent{ + OMetricName: "bazqux.main", + OValue: 42, }, } diff --git a/exporter_benchmark_test.go b/exporter_benchmark_test.go index 480930b8..5d263ada 100644 --- a/exporter_benchmark_test.go +++ b/exporter_benchmark_test.go @@ -86,13 +86,13 @@ func BenchmarkExporterListener(b *testing.B) { GMetricName: "gauge", GValue: 10, }, - &event.TimerEvent{ // simple timer - TMetricName: "timer", - TValue: 200, + &event.ObserverEvent{ // simple timer + OMetricName: "timer", + OValue: 200, }, - &event.TimerEvent{ // simple histogram - TMetricName: "histogram.test", - TValue: 200, + &event.ObserverEvent{ // simple histogram + OMetricName: "histogram.test", + OValue: 200, }, &event.CounterEvent{ // simple_tags CMetricName: "simple_tags", diff --git a/pkg/event/event.go b/pkg/event/event.go index 108d1fbc..bfe0de37 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -52,17 +52,16 @@ func (g *GaugeEvent) Value() float64 { return g.GValue } func (c *GaugeEvent) Labels() map[string]string { return c.GLabels } func (c *GaugeEvent) MetricType() mapper.MetricType { return mapper.MetricTypeGauge } -// TimerEvent now defaults to seconds units -type TimerEvent struct { - TMetricName string - TValue float64 - TLabels map[string]string +type ObserverEvent struct { + OMetricName string + OValue float64 + OLabels map[string]string } -func (t *TimerEvent) MetricName() string { return t.TMetricName } -func (t *TimerEvent) Value() float64 { return t.TValue } -func (c *TimerEvent) Labels() map[string]string { return c.TLabels } -func (c *TimerEvent) MetricType() mapper.MetricType { return mapper.MetricTypeTimer } +func (t *ObserverEvent) MetricName() string { return t.OMetricName } +func (t *ObserverEvent) Value() float64 { return t.OValue } +func (c *ObserverEvent) Labels() map[string]string { return c.OLabels } +func (c *ObserverEvent) MetricType() mapper.MetricType { return mapper.MetricTypeTimer } type Events []Event diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index 558949ea..199cc7e4 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -140,7 +140,7 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { b.ConflictingEventStats.WithLabelValues("gauge").Inc() } - case *event.TimerEvent: + case *event.ObserverEvent: t := mapper.TimerTypeDefault if mapping != nil { t = mapping.TimerType diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index ec4ae013..a58b735c 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -228,25 +228,25 @@ func TestInconsistentLabelSets(t *testing.T) { GValue: 1, GLabels: secondLabelSet, }, - &event.TimerEvent{ - TMetricName: "histogram.test", - TValue: 1, - TLabels: firstLabelSet, + &event.ObserverEvent{ + OMetricName: "histogram.test", + OValue: 1, + OLabels: firstLabelSet, }, - &event.TimerEvent{ - TMetricName: "histogram.test", - TValue: 1, - TLabels: secondLabelSet, + &event.ObserverEvent{ + OMetricName: "histogram.test", + OValue: 1, + OLabels: secondLabelSet, }, - &event.TimerEvent{ - TMetricName: "summary_test", - TValue: 1, - TLabels: firstLabelSet, + &event.ObserverEvent{ + OMetricName: "summary_test", + OValue: 1, + OLabels: firstLabelSet, }, - &event.TimerEvent{ - TMetricName: "summary_test", - TValue: 1, - TLabels: secondLabelSet, + &event.ObserverEvent{ + OMetricName: "summary_test", + OValue: 1, + OLabels: secondLabelSet, }, } events <- c @@ -427,9 +427,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "histogram_test1", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "histogram.test1", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "histogram.test1", + OValue: 2, }, }, }, @@ -441,9 +441,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "histogram_test1_sum", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "histogram.test1", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "histogram.test1", + OValue: 2, }, }, }, @@ -455,9 +455,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "histogram_test2_count", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "histogram.test2", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "histogram.test2", + OValue: 2, }, }, }, @@ -469,9 +469,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "histogram_test3_bucket", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "histogram.test3", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "histogram.test3", + OValue: 2, }, }, }, @@ -483,9 +483,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "cvsq_test", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "cvsq_test", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "cvsq_test", + OValue: 2, }, }, }, @@ -497,9 +497,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "cvsc_count", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "cvsc", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "cvsc", + OValue: 2, }, }, }, @@ -511,9 +511,9 @@ func TestConflictingMetrics(t *testing.T) { CMetricName: "cvss_sum", CValue: 1, }, - &event.TimerEvent{ - TMetricName: "cvss", - TValue: 2, + &event.ObserverEvent{ + OMetricName: "cvss", + OValue: 2, }, }, }, @@ -672,9 +672,9 @@ func TestSummaryWithQuantilesEmptyMapping(t *testing.T) { name := "default_foo" c := event.Events{ - &event.TimerEvent{ - TMetricName: name, - TValue: 300, + &event.ObserverEvent{ + OMetricName: name, + OValue: 300, }, } events <- c @@ -719,9 +719,9 @@ func TestHistogramUnits(t *testing.T) { // Then close events channel to stop a listener. name := "foo" c := event.Events{ - &event.TimerEvent{ - TMetricName: name, - TValue: .300, + &event.ObserverEvent{ + OMetricName: name, + OValue: .300, }, } events <- c @@ -867,9 +867,9 @@ mappings: GValue: 200, }, // event with ttl = 2s from a mapping - &event.TimerEvent{ - TMetricName: "bazqux.main", - TValue: 42, + &event.ObserverEvent{ + OMetricName: "bazqux.main", + OValue: 42, }, } diff --git a/pkg/line/line.go b/pkg/line/line.go index 50dd7aa3..c1e4ab3a 100644 --- a/pkg/line/line.go +++ b/pkg/line/line.go @@ -43,16 +43,16 @@ func buildEvent(statType, metric string, value float64, relative bool, labels ma GLabels: labels, }, nil case "ms": - return &event.TimerEvent{ - TMetricName: metric, - TValue: float64(value) / 1000, // prometheus presumes seconds, statsd millisecond - TLabels: labels, + return &event.ObserverEvent{ + OMetricName: metric, + OValue: float64(value) / 1000, // prometheus presumes seconds, statsd millisecond + OLabels: labels, }, nil case "h", "d": - return &event.TimerEvent{ - TMetricName: metric, - TValue: float64(value), - TLabels: labels, + return &event.ObserverEvent{ + OMetricName: metric, + OValue: float64(value), + OLabels: labels, }, nil case "s": return nil, fmt.Errorf("no support for StatsD sets") From d95a53553e703e41f54625d84d708ee14011559b Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 10:14:11 -0400 Subject: [PATCH 05/10] change metric labels to observer Signed-off-by: glightfoot --- pkg/exporter/exporter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index 199cc7e4..89703670 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -154,24 +154,24 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { histogram.Observe(thisEvent.Value()) - b.EventStats.WithLabelValues("timer").Inc() + b.EventStats.WithLabelValues("observer").Inc() } else { level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) - b.ConflictingEventStats.WithLabelValues("timer").Inc() + b.ConflictingEventStats.WithLabelValues("observer").Inc() } case mapper.TimerTypeDefault, mapper.TimerTypeSummary: summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { summary.Observe(thisEvent.Value()) - b.EventStats.WithLabelValues("timer").Inc() + b.EventStats.WithLabelValues("observer").Inc() } else { level.Debug(b.Logger).Log("msg", regErrF, "metric", metricName, "error", err) - b.ConflictingEventStats.WithLabelValues("timer").Inc() + b.ConflictingEventStats.WithLabelValues("observer").Inc() } default: - level.Error(b.Logger).Log("msg", "unknown timer type", "type", t) + level.Error(b.Logger).Log("msg", "unknown observer type", "type", t) os.Exit(1) } From 32e9e58e32ca1965a0892cdf137155c49b66a99f Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 17:26:20 -0400 Subject: [PATCH 06/10] Rename timer to observer in mapper Signed-off-by: glightfoot --- pkg/event/event.go | 2 +- pkg/exporter/exporter.go | 12 ++++++------ pkg/exporter/exporter_test.go | 2 +- pkg/mapper/fsm/README.md | 10 +++++----- pkg/mapper/mapper.go | 18 +++++++++--------- pkg/mapper/mapper_test.go | 18 +++++++++--------- pkg/mapper/metric_type.go | 10 +++++----- pkg/mapper/{timer.go => observer.go} | 22 +++++++++++----------- 8 files changed, 47 insertions(+), 47 deletions(-) rename pkg/mapper/{timer.go => observer.go} (61%) diff --git a/pkg/event/event.go b/pkg/event/event.go index bfe0de37..9234db92 100644 --- a/pkg/event/event.go +++ b/pkg/event/event.go @@ -61,7 +61,7 @@ type ObserverEvent struct { func (t *ObserverEvent) MetricName() string { return t.OMetricName } func (t *ObserverEvent) Value() float64 { return t.OValue } func (c *ObserverEvent) Labels() map[string]string { return c.OLabels } -func (c *ObserverEvent) MetricType() mapper.MetricType { return mapper.MetricTypeTimer } +func (c *ObserverEvent) MetricType() mapper.MetricType { return mapper.MetricTypeObserver } type Events []Event diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index 89703670..e2f202dc 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -141,16 +141,16 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { } case *event.ObserverEvent: - t := mapper.TimerTypeDefault + t := mapper.ObserverTypeDefault if mapping != nil { - t = mapping.TimerType + t = mapping.ObserverType } - if t == mapper.TimerTypeDefault { - t = b.Mapper.Defaults.TimerType + if t == mapper.ObserverTypeDefault { + t = b.Mapper.Defaults.ObsereverType } switch t { - case mapper.TimerTypeHistogram: + case mapper.ObserverTypeHistogram: histogram, err := b.Registry.GetHistogram(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { histogram.Observe(thisEvent.Value()) @@ -160,7 +160,7 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { b.ConflictingEventStats.WithLabelValues("observer").Inc() } - case mapper.TimerTypeDefault, mapper.TimerTypeSummary: + case mapper.ObserverTypeDefault, mapper.ObserverTypeSummary: summary, err := b.Registry.GetSummary(metricName, prometheusLabels, help, mapping, b.MetricsCount) if err == nil { summary.Observe(thisEvent.Value()) diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index a58b735c..2802ac1d 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -711,7 +711,7 @@ func TestHistogramUnits(t *testing.T) { testMapper := mapper.MetricMapper{} testMapper.InitCache(0) ex := NewExporter(&testMapper, log.NewNopLogger(), eventsActions, eventsUnmapped, errorEventStats, eventStats, conflictingEventStats, metricsCount) - ex.Mapper.Defaults.TimerType = mapper.TimerTypeHistogram + ex.Mapper.Defaults.ObsereverType = mapper.ObserverTypeHistogram ex.Listen(events) }() diff --git a/pkg/mapper/fsm/README.md b/pkg/mapper/fsm/README.md index 722ee210..9d7da44e 100644 --- a/pkg/mapper/fsm/README.md +++ b/pkg/mapper/fsm/README.md @@ -40,7 +40,7 @@ At first, the FSM only contains three states, representing three possible metric / (start)---- [counter] \ - '--- [ timer ] + '--- [observer] Adding a rule `client.*.request.count` with type `counter` will make the FSM to be: @@ -50,7 +50,7 @@ Adding a rule `client.*.request.count` with type `counter` will make the FSM to / (start)---- [counter] -- [client] -- [*] -- [request] -- [count] -- {R1} \ - '--- [timer] + '--- [observer] `{R1}` is short for result 1, which is the match result for `client.*.request.count`. @@ -60,7 +60,7 @@ Adding a rule `client.*.*.size` with type `counter` will make the FSM to be: / / (start)---- [counter] -- [client] -- [*] \ \__ [*] -- [size] -- {R2} - '--- [timer] + '--- [observer] ### Finding a result state in FSM @@ -76,7 +76,7 @@ FSM, the `^1` to `^7` symbols indicate how FSM will traversal in its tree: / / ^5 ^6 ^7 (start)---- [counter] -- [client] -- [*] ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} - '--- [timer] ^4 + '--- [observer] ^4 To map `client.bbb.request.size`, FSM will do a backtracking: @@ -86,7 +86,7 @@ To map `client.bbb.request.size`, FSM will do a backtracking: / / ^5 ^6 (start)---- [counter] -- [client] -- [*] ^1 \ ^2 ^3 \__ [*] -- [size] -- {R2} - '--- [timer] ^4 + '--- [observer] ^4 ^7 ^8 ^9 diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 8aac9d1f..dc9a17bd 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -36,7 +36,7 @@ var ( ) type mapperConfigDefaults struct { - TimerType TimerType `yaml:"timer_type"` + ObsereverType ObserverType `yaml:"observer_type"` Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` MatchType MatchType `yaml:"match_type"` @@ -64,7 +64,7 @@ type MetricMapping struct { Labels prometheus.Labels `yaml:"labels"` labelKeys []string labelFormatters []*fsm.TemplateFormatter - TimerType TimerType `yaml:"timer_type"` + ObserverType ObserverType `yaml:"observer_type"` LegacyBuckets []float64 `yaml:"buckets"` LegacyQuantiles []metricObjective `yaml:"quantiles"` MatchType MatchType `yaml:"match_type"` @@ -119,7 +119,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op remainingMappingsCount := len(n.Mappings) - n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeTimer)}, + n.FSM = fsm.NewFSM([]string{string(MetricTypeCounter), string(MetricTypeGauge), string(MetricTypeObserver)}, remainingMappingsCount, n.Defaults.GlobDisableOrdering) for i := range n.Mappings { @@ -181,8 +181,8 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op n.doRegex = true } - if currentMapping.TimerType == "" { - currentMapping.TimerType = n.Defaults.TimerType + if currentMapping.ObserverType == "" { + currentMapping.ObserverType = n.Defaults.ObsereverType } if currentMapping.LegacyQuantiles != nil && @@ -207,9 +207,9 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op return fmt.Errorf("cannot use buckets in both the top level and histogram options at the same time in %s", currentMapping.Match) } - if currentMapping.TimerType == TimerTypeHistogram { + if currentMapping.ObserverType == ObserverTypeHistogram { if currentMapping.SummaryOptions != nil { - return fmt.Errorf("cannot use histogram timer and summary options at the same time") + return fmt.Errorf("cannot use histogram observer and summary options at the same time") } if currentMapping.HistogramOptions == nil { currentMapping.HistogramOptions = &HistogramOptions{} @@ -222,9 +222,9 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op } } - if currentMapping.TimerType == TimerTypeSummary { + if currentMapping.ObserverType == ObserverTypeSummary { if currentMapping.HistogramOptions != nil { - return fmt.Errorf("cannot use summary timer and histogram options at the same time") + return fmt.Errorf("cannot use summary observer and histogram options at the same time") } if currentMapping.SummaryOptions == nil { currentMapping.SummaryOptions = &SummaryOptions{} diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index dc8a45d5..f4de0b66 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -441,12 +441,12 @@ mappings: }, }, }, - // Config with good timer type. + // Config with good observer type. { config: `--- mappings: - match: test.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} quantiles: @@ -471,7 +471,7 @@ mappings: config: `--- mappings: - match: test1.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} `, @@ -488,12 +488,12 @@ mappings: }, }, }, - // Config with bad timer type. + // Config with bad observer type. { config: `--- mappings: - match: test.*.* - timer_type: wrong + observer_type: wrong name: "foo" labels: {} `, @@ -504,7 +504,7 @@ mappings: config: `--- mappings: - match: test.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} summary_options: @@ -531,7 +531,7 @@ mappings: config: `--- mappings: - match: test.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} summary_options: @@ -564,7 +564,7 @@ mappings: config: `--- mappings: - match: test.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} quantiles: @@ -638,7 +638,7 @@ mappings: config: `--- mappings: - match: foo.*.* - timer_type: summary + observer_type: summary name: "foo" labels: {} `, diff --git a/pkg/mapper/metric_type.go b/pkg/mapper/metric_type.go index 0a0810f8..d98d80a1 100644 --- a/pkg/mapper/metric_type.go +++ b/pkg/mapper/metric_type.go @@ -18,9 +18,9 @@ import "fmt" type MetricType string const ( - MetricTypeCounter MetricType = "counter" - MetricTypeGauge MetricType = "gauge" - MetricTypeTimer MetricType = "timer" + MetricTypeCounter MetricType = "counter" + MetricTypeGauge MetricType = "gauge" + MetricTypeObserver MetricType = "observer" ) func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -34,8 +34,8 @@ func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error { *m = MetricTypeCounter case MetricTypeGauge: *m = MetricTypeGauge - case MetricTypeTimer: - *m = MetricTypeTimer + case MetricTypeObserver: + *m = MetricTypeObserver default: return fmt.Errorf("invalid metric type '%s'", v) } diff --git a/pkg/mapper/timer.go b/pkg/mapper/observer.go similarity index 61% rename from pkg/mapper/timer.go rename to pkg/mapper/observer.go index f1d2fb70..3d5da7ea 100644 --- a/pkg/mapper/timer.go +++ b/pkg/mapper/observer.go @@ -15,27 +15,27 @@ package mapper import "fmt" -type TimerType string +type ObserverType string const ( - TimerTypeHistogram TimerType = "histogram" - TimerTypeSummary TimerType = "summary" - TimerTypeDefault TimerType = "" + ObserverTypeHistogram ObserverType = "histogram" + ObserverTypeSummary ObserverType = "summary" + ObserverTypeDefault ObserverType = "" ) -func (t *TimerType) UnmarshalYAML(unmarshal func(interface{}) error) error { +func (t *ObserverType) UnmarshalYAML(unmarshal func(interface{}) error) error { var v string if err := unmarshal(&v); err != nil { return err } - switch TimerType(v) { - case TimerTypeHistogram: - *t = TimerTypeHistogram - case TimerTypeSummary, TimerTypeDefault: - *t = TimerTypeSummary + switch ObserverType(v) { + case ObserverTypeHistogram: + *t = ObserverTypeHistogram + case ObserverTypeSummary, ObserverTypeDefault: + *t = ObserverTypeSummary default: - return fmt.Errorf("invalid timer type '%s'", v) + return fmt.Errorf("invalid observer type '%s'", v) } return nil } From 77277a5150af0f3c39e84c3d0b0ac60a9d49ed55 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 18:41:16 -0400 Subject: [PATCH 07/10] support deprecated timer_type in configs Signed-off-by: glightfoot --- pkg/exporter/exporter.go | 2 +- pkg/exporter/exporter_test.go | 2 +- pkg/mapper/mapper.go | 65 +++++++++++++++++++++++++++++++++-- pkg/mapper/mapper_test.go | 60 ++++++++++++++++++++++++++++++++ pkg/mapper/metric_type.go | 3 ++ 5 files changed, 127 insertions(+), 5 deletions(-) diff --git a/pkg/exporter/exporter.go b/pkg/exporter/exporter.go index e2f202dc..7967b8d5 100644 --- a/pkg/exporter/exporter.go +++ b/pkg/exporter/exporter.go @@ -146,7 +146,7 @@ func (b *Exporter) handleEvent(thisEvent event.Event) { t = mapping.ObserverType } if t == mapper.ObserverTypeDefault { - t = b.Mapper.Defaults.ObsereverType + t = b.Mapper.Defaults.ObserverType } switch t { diff --git a/pkg/exporter/exporter_test.go b/pkg/exporter/exporter_test.go index 2802ac1d..c92221bd 100644 --- a/pkg/exporter/exporter_test.go +++ b/pkg/exporter/exporter_test.go @@ -711,7 +711,7 @@ func TestHistogramUnits(t *testing.T) { testMapper := mapper.MetricMapper{} testMapper.InitCache(0) ex := NewExporter(&testMapper, log.NewNopLogger(), eventsActions, eventsUnmapped, errorEventStats, eventStats, conflictingEventStats, metricsCount) - ex.Mapper.Defaults.ObsereverType = mapper.ObserverTypeHistogram + ex.Mapper.Defaults.ObserverType = mapper.ObserverTypeHistogram ex.Listen(events) }() diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index dc9a17bd..3638b1fb 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -36,7 +36,8 @@ var ( ) type mapperConfigDefaults struct { - ObsereverType ObserverType `yaml:"observer_type"` + ObserverType ObserverType `yaml:"observer_type"` + TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty Buckets []float64 `yaml:"buckets"` Quantiles []metricObjective `yaml:"quantiles"` MatchType MatchType `yaml:"match_type"` @@ -64,7 +65,8 @@ type MetricMapping struct { Labels prometheus.Labels `yaml:"labels"` labelKeys []string labelFormatters []*fsm.TemplateFormatter - ObserverType ObserverType `yaml:"observer_type"` + ObserverType ObserverType `yaml:"observer_type,omitempty"` + TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty LegacyBuckets []float64 `yaml:"buckets"` LegacyQuantiles []metricObjective `yaml:"quantiles"` MatchType MatchType `yaml:"match_type"` @@ -98,6 +100,63 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } +// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys +// observer_type will override timer_type +func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error { + type mapperConfigDefaultsAlias mapperConfigDefaults + var tmp mapperConfigDefaultsAlias + if err := unmarshal(&tmp); err != nil { + return err + } + + // Copy defaults + d.ObserverType = tmp.ObserverType + d.Buckets = tmp.Buckets + d.Quantiles = tmp.Quantiles + d.MatchType = tmp.MatchType + d.GlobDisableOrdering = tmp.GlobDisableOrdering + d.Ttl = tmp.Ttl + + // Use deprecated TimerType if necessary + if tmp.ObserverType == "" { + d.ObserverType = tmp.TimerType + } + + return nil +} + +// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys +// observer_type will override timer_type +func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error { + type MetricMappingAlias MetricMapping + var tmp MetricMappingAlias + if err := unmarshal(&tmp); err != nil { + return err + } + + // Copy defaults + m.Match = tmp.Match + m.Name = tmp.Name + m.Labels = tmp.Labels + m.ObserverType = tmp.ObserverType + m.LegacyBuckets = tmp.LegacyBuckets + m.LegacyQuantiles = tmp.LegacyQuantiles + m.MatchType = tmp.MatchType + m.HelpText = tmp.HelpText + m.Action = tmp.Action + m.MatchMetricType = tmp.MatchMetricType + m.Ttl = tmp.Ttl + m.SummaryOptions = tmp.SummaryOptions + m.HistogramOptions = tmp.HistogramOptions + + // Use deprecated TimerType if necessary + if tmp.ObserverType == "" { + m.ObserverType = tmp.TimerType + } + + return nil +} + func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, options ...CacheOption) error { var n MetricMapper @@ -182,7 +241,7 @@ func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, op } if currentMapping.ObserverType == "" { - currentMapping.ObserverType = n.Defaults.ObsereverType + currentMapping.ObserverType = n.Defaults.ObserverType } if currentMapping.LegacyQuantiles != nil && diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index f4de0b66..e00ae8f5 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -449,6 +449,33 @@ mappings: observer_type: summary name: "foo" labels: {} + quantiles: + - quantile: 0.42 + error: 0.04 + - quantile: 0.7 + error: 0.002 + `, + mappings: mappings{ + { + statsdMetric: "test.*.*", + name: "foo", + labels: map[string]string{}, + quantiles: []metricObjective{ + {Quantile: 0.42, Error: 0.04}, + {Quantile: 0.7, Error: 0.002}, + }, + }, + }, + }, + // Config with good observer type and unused timer type + { + config: `--- +mappings: +- match: test.*.* + observer_type: summary + timer_type: histogram + name: "foo" + labels: {} quantiles: - quantile: 0.42 error: 0.04 @@ -474,6 +501,28 @@ mappings: observer_type: summary name: "foo" labels: {} + `, + mappings: mappings{ + { + statsdMetric: "test1.*.*", + name: "foo", + labels: map[string]string{}, + quantiles: []metricObjective{ + {Quantile: 0.5, Error: 0.05}, + {Quantile: 0.9, Error: 0.01}, + {Quantile: 0.99, Error: 0.001}, + }, + }, + }, + }, + // Config with good deprecated timer type + { + config: `--- +mappings: +- match: test1.*.* + timer_type: summary + name: "foo" + labels: {} `, mappings: mappings{ { @@ -495,6 +544,17 @@ mappings: - match: test.*.* observer_type: wrong name: "foo" + labels: {} + `, + configBad: true, + }, + // Config with bad deprecated timer type. + { + config: `--- +mappings: +- match: test.*.* + timer_type: wrong + name: "foo" labels: {} `, configBad: true, diff --git a/pkg/mapper/metric_type.go b/pkg/mapper/metric_type.go index d98d80a1..920c16ed 100644 --- a/pkg/mapper/metric_type.go +++ b/pkg/mapper/metric_type.go @@ -21,6 +21,7 @@ const ( MetricTypeCounter MetricType = "counter" MetricTypeGauge MetricType = "gauge" MetricTypeObserver MetricType = "observer" + MetricTypeTimer MetricType = "timer" // DEPRECATED ) func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error { @@ -36,6 +37,8 @@ func (m *MetricType) UnmarshalYAML(unmarshal func(interface{}) error) error { *m = MetricTypeGauge case MetricTypeObserver: *m = MetricTypeObserver + case MetricTypeTimer: + *m = MetricTypeObserver default: return fmt.Errorf("invalid metric type '%s'", v) } From bdd4a76348e202df731d8ae2120a7a27e96f2eb2 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 18:46:20 -0400 Subject: [PATCH 08/10] add more tests for metric type Signed-off-by: glightfoot --- pkg/mapper/mapper_test.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/mapper/mapper_test.go b/pkg/mapper/mapper_test.go index e00ae8f5..ca288970 100644 --- a/pkg/mapper/mapper_test.go +++ b/pkg/mapper/mapper_test.go @@ -644,6 +644,26 @@ mappings: - match: test.*.* match_metric_type: counter name: "foo" + labels: {} + `, + }, + // Config with good metric type observer. + { + config: `--- +mappings: +- match: test.*.* + match_metric_type: observer + name: "foo" + labels: {} + `, + }, + // Config with good metric type timer. + { + config: `--- +mappings: +- match: test.*.* + match_metric_type: timer + name: "foo" labels: {} `, }, From 1444824911543bdaf82244352d5f1432e748f879 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 18:47:17 -0400 Subject: [PATCH 09/10] fix yaml tags Signed-off-by: glightfoot --- pkg/mapper/mapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 3638b1fb..95d3b0c0 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -65,7 +65,7 @@ type MetricMapping struct { Labels prometheus.Labels `yaml:"labels"` labelKeys []string labelFormatters []*fsm.TemplateFormatter - ObserverType ObserverType `yaml:"observer_type,omitempty"` + ObserverType ObserverType `yaml:"observer_type"` TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty LegacyBuckets []float64 `yaml:"buckets"` LegacyQuantiles []metricObjective `yaml:"quantiles"` From 4a64979563442b948c6b929daee10f3b15493681 Mon Sep 17 00:00:00 2001 From: glightfoot Date: Mon, 15 Jun 2020 21:37:22 -0400 Subject: [PATCH 10/10] move mapping and mapper_defaults into their own files with their unmarshalers Signed-off-by: glightfoot --- pkg/mapper/mapper.go | 88 ----------------------------------- pkg/mapper/mapper_defaults.go | 51 ++++++++++++++++++++ pkg/mapper/mapping.go | 76 ++++++++++++++++++++++++++++++ 3 files changed, 127 insertions(+), 88 deletions(-) create mode 100644 pkg/mapper/mapper_defaults.go create mode 100644 pkg/mapper/mapping.go diff --git a/pkg/mapper/mapper.go b/pkg/mapper/mapper.go index 95d3b0c0..e44e5816 100644 --- a/pkg/mapper/mapper.go +++ b/pkg/mapper/mapper.go @@ -35,16 +35,6 @@ var ( labelNameRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]+$`) ) -type mapperConfigDefaults struct { - ObserverType ObserverType `yaml:"observer_type"` - TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty - Buckets []float64 `yaml:"buckets"` - Quantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` - GlobDisableOrdering bool `yaml:"glob_disable_ordering"` - Ttl time.Duration `yaml:"ttl"` -} - type MetricMapper struct { Defaults mapperConfigDefaults `yaml:"defaults"` Mappings []MetricMapping `yaml:"mappings"` @@ -57,27 +47,6 @@ type MetricMapper struct { MappingsCount prometheus.Gauge } -type MetricMapping struct { - Match string `yaml:"match"` - Name string `yaml:"name"` - nameFormatter *fsm.TemplateFormatter - regex *regexp.Regexp - Labels prometheus.Labels `yaml:"labels"` - labelKeys []string - labelFormatters []*fsm.TemplateFormatter - ObserverType ObserverType `yaml:"observer_type"` - TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty - LegacyBuckets []float64 `yaml:"buckets"` - LegacyQuantiles []metricObjective `yaml:"quantiles"` - MatchType MatchType `yaml:"match_type"` - HelpText string `yaml:"help"` - Action ActionType `yaml:"action"` - MatchMetricType MetricType `yaml:"match_metric_type"` - Ttl time.Duration `yaml:"ttl"` - SummaryOptions *SummaryOptions `yaml:"summary_options"` - HistogramOptions *HistogramOptions `yaml:"histogram_options"` -} - type SummaryOptions struct { Quantiles []metricObjective `yaml:"quantiles"` MaxAge time.Duration `yaml:"max_age"` @@ -100,63 +69,6 @@ var defaultQuantiles = []metricObjective{ {Quantile: 0.99, Error: 0.001}, } -// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys -// observer_type will override timer_type -func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error { - type mapperConfigDefaultsAlias mapperConfigDefaults - var tmp mapperConfigDefaultsAlias - if err := unmarshal(&tmp); err != nil { - return err - } - - // Copy defaults - d.ObserverType = tmp.ObserverType - d.Buckets = tmp.Buckets - d.Quantiles = tmp.Quantiles - d.MatchType = tmp.MatchType - d.GlobDisableOrdering = tmp.GlobDisableOrdering - d.Ttl = tmp.Ttl - - // Use deprecated TimerType if necessary - if tmp.ObserverType == "" { - d.ObserverType = tmp.TimerType - } - - return nil -} - -// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys -// observer_type will override timer_type -func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error { - type MetricMappingAlias MetricMapping - var tmp MetricMappingAlias - if err := unmarshal(&tmp); err != nil { - return err - } - - // Copy defaults - m.Match = tmp.Match - m.Name = tmp.Name - m.Labels = tmp.Labels - m.ObserverType = tmp.ObserverType - m.LegacyBuckets = tmp.LegacyBuckets - m.LegacyQuantiles = tmp.LegacyQuantiles - m.MatchType = tmp.MatchType - m.HelpText = tmp.HelpText - m.Action = tmp.Action - m.MatchMetricType = tmp.MatchMetricType - m.Ttl = tmp.Ttl - m.SummaryOptions = tmp.SummaryOptions - m.HistogramOptions = tmp.HistogramOptions - - // Use deprecated TimerType if necessary - if tmp.ObserverType == "" { - m.ObserverType = tmp.TimerType - } - - return nil -} - func (m *MetricMapper) InitFromYAMLString(fileContents string, cacheSize int, options ...CacheOption) error { var n MetricMapper diff --git a/pkg/mapper/mapper_defaults.go b/pkg/mapper/mapper_defaults.go new file mode 100644 index 00000000..7fba9fea --- /dev/null +++ b/pkg/mapper/mapper_defaults.go @@ -0,0 +1,51 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper + +import "time" + +type mapperConfigDefaults struct { + ObserverType ObserverType `yaml:"observer_type"` + TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty + Buckets []float64 `yaml:"buckets"` + Quantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` + GlobDisableOrdering bool `yaml:"glob_disable_ordering"` + Ttl time.Duration `yaml:"ttl"` +} + +// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys +// observer_type will override timer_type +func (d *mapperConfigDefaults) UnmarshalYAML(unmarshal func(interface{}) error) error { + type mapperConfigDefaultsAlias mapperConfigDefaults + var tmp mapperConfigDefaultsAlias + if err := unmarshal(&tmp); err != nil { + return err + } + + // Copy defaults + d.ObserverType = tmp.ObserverType + d.Buckets = tmp.Buckets + d.Quantiles = tmp.Quantiles + d.MatchType = tmp.MatchType + d.GlobDisableOrdering = tmp.GlobDisableOrdering + d.Ttl = tmp.Ttl + + // Use deprecated TimerType if necessary + if tmp.ObserverType == "" { + d.ObserverType = tmp.TimerType + } + + return nil +} diff --git a/pkg/mapper/mapping.go b/pkg/mapper/mapping.go new file mode 100644 index 00000000..cdba27af --- /dev/null +++ b/pkg/mapper/mapping.go @@ -0,0 +1,76 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either xpress or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapper + +import ( + "regexp" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/statsd_exporter/pkg/mapper/fsm" +) + +type MetricMapping struct { + Match string `yaml:"match"` + Name string `yaml:"name"` + nameFormatter *fsm.TemplateFormatter + regex *regexp.Regexp + Labels prometheus.Labels `yaml:"labels"` + labelKeys []string + labelFormatters []*fsm.TemplateFormatter + ObserverType ObserverType `yaml:"observer_type"` + TimerType ObserverType `yaml:"timer_type,omitempty"` // DEPRECATED - field only present to preserve backwards compatibility in configs. Always empty + LegacyBuckets []float64 `yaml:"buckets"` + LegacyQuantiles []metricObjective `yaml:"quantiles"` + MatchType MatchType `yaml:"match_type"` + HelpText string `yaml:"help"` + Action ActionType `yaml:"action"` + MatchMetricType MetricType `yaml:"match_metric_type"` + Ttl time.Duration `yaml:"ttl"` + SummaryOptions *SummaryOptions `yaml:"summary_options"` + HistogramOptions *HistogramOptions `yaml:"histogram_options"` +} + +// UnmarshalYAML is a custom unmarshal function to allow use of deprecated config keys +// observer_type will override timer_type +func (m *MetricMapping) UnmarshalYAML(unmarshal func(interface{}) error) error { + type MetricMappingAlias MetricMapping + var tmp MetricMappingAlias + if err := unmarshal(&tmp); err != nil { + return err + } + + // Copy defaults + m.Match = tmp.Match + m.Name = tmp.Name + m.Labels = tmp.Labels + m.ObserverType = tmp.ObserverType + m.LegacyBuckets = tmp.LegacyBuckets + m.LegacyQuantiles = tmp.LegacyQuantiles + m.MatchType = tmp.MatchType + m.HelpText = tmp.HelpText + m.Action = tmp.Action + m.MatchMetricType = tmp.MatchMetricType + m.Ttl = tmp.Ttl + m.SummaryOptions = tmp.SummaryOptions + m.HistogramOptions = tmp.HistogramOptions + + // Use deprecated TimerType if necessary + if tmp.ObserverType == "" { + m.ObserverType = tmp.TimerType + } + + return nil +}