From f0ed2ae2562dd5c4e457a769b90c6d0f2c47b755 Mon Sep 17 00:00:00 2001
From: Shreyas Srivatsan <shreyas@chronosphere.io>
Date: Fri, 28 Feb 2020 15:06:07 -0800
Subject: [PATCH 1/5] Add support for prometheus with exemplars - Pull in
 latest v1.4.1 prometheus client - Add WithExemplar methods to counters and
 histograms

Signed-off-by: Shreyas Srivatsan <shreyas@chronosphere.io>
---
 Gopkg.lock                    | 86 +++++++++++++++++++----------------
 Gopkg.toml                    |  2 +-
 metrics/counter.go            |  3 ++
 metrics/go-kit/metrics.go     | 15 ++++++
 metrics/histogram.go          |  3 ++
 metrics/metricstest/local.go  | 12 +++++
 metrics/multi/multi.go        | 18 ++++++++
 metrics/prometheus/factory.go | 56 +++++++++++++++++++++++
 metrics/tally/metrics.go      | 15 ++++++
 metrics/timer.go              |  3 ++
 10 files changed, 172 insertions(+), 41 deletions(-)

diff --git a/Gopkg.lock b/Gopkg.lock
index e2b31ec..94c6ab0 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -17,6 +17,14 @@
   revision = "37c8de3658fcb183f997c4e13e8337516ab753e6"
   version = "v1.0.1"
 
+[[projects]]
+  digest = "1:7743d465c4c7c9322a4c4da92c2ac38e15115014eb7aa4d367968055f1e64c57"
+  name = "github.com/cespare/xxhash"
+  packages = ["v2"]
+  pruneopts = "UT"
+  revision = "d7df74196a9e781ede915320c11c378c1b2f3a1f"
+  version = "v2.1.1"
+
 [[projects]]
   branch = "master"
   digest = "1:4c4c33075b704791d6a7f09dfb55c66769e8a1dc6adf87026292d274fe8ad113"
@@ -50,24 +58,30 @@
   version = "v0.9.0"
 
 [[projects]]
-  digest = "1:4062bc6de62d73e2be342243cf138cf499b34d558876db8d9430e2149388a4d8"
+  digest = "1:522a207de5459ab329732b5df6bb6dd7592da9fddf8a99e9f82609e370d82f9b"
   name = "github.com/go-logfmt/logfmt"
   packages = ["."]
   pruneopts = "UT"
-  revision = "07c9b44f60d7ffdfb7d8efe1ad539965737836dc"
-  version = "v0.4.0"
+  revision = "3be5f6aae7db841d31dc5e1b3bb7b4cff19200b3"
+  version = "v0.5.0"
 
 [[projects]]
-  digest = "1:573ca21d3669500ff845bdebee890eb7fc7f0f50c59f2132f2a0c6b03d85086a"
+  digest = "1:ecd73c8c5c5e48f9079e042ae733c3f3ab021218d6c4da3411d82727fd5a412a"
   name = "github.com/golang/protobuf"
-  packages = ["proto"]
+  packages = [
+    "proto",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/timestamp",
+  ]
   pruneopts = "UT"
-  revision = "6c65a5562fc06764971b7c5d05c76c75e84bdbf7"
-  version = "v1.3.2"
+  revision = "5d5b4c10bd43f85e63bd9e4a3fa9b1ea2ef88af2"
+  version = "v1.3.4"
 
 [[projects]]
   branch = "master"
-  digest = "1:50708c8fc92aec981df5c446581cf9f90ba9e2a5692118e0ce75d4534aaa14a2"
+  digest = "1:00e5ad58045d6d2a6c9e65d1809ff2594bc396e911712ae892a93976fdece115"
   name = "github.com/influxdata/influxdb1-client"
   packages = [
     "models",
@@ -75,15 +89,7 @@
     "v2",
   ]
   pruneopts = "UT"
-  revision = "fc22c7df067eefd070157f157893fbce961d6359"
-
-[[projects]]
-  branch = "master"
-  digest = "1:a64e323dc06b73892e5bb5d040ced475c4645d456038333883f58934abbf6f72"
-  name = "github.com/kr/logfmt"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b84e30acd515aadc4b783ad4ff83aff3299bdfe0"
+  revision = "8bf82d3c094dc06be9da8e5bf9d3589b6ea032ae"
 
 [[projects]]
   digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc"
@@ -102,26 +108,26 @@
   version = "v1.0.0"
 
 [[projects]]
-  digest = "1:7097829edd12fd7211fca0d29496b44f94ef9e6d72f88fb64f3d7b06315818ad"
+  digest = "1:f6aeae371ef9202229eafaed1bdd994725a9b6f2716e003e689097726aa970a9"
   name = "github.com/prometheus/client_golang"
   packages = [
     "prometheus",
     "prometheus/internal",
   ]
   pruneopts = "UT"
-  revision = "170205fb58decfd011f1550d4cfb737230d7ae4f"
-  version = "v1.1.0"
+  revision = "913f67ef0627596efc752f02d4014308e7345dbf"
+  version = "v1.4.1"
 
 [[projects]]
-  branch = "master"
-  digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
+  digest = "1:0db23933b8052702d980a3f029149b3f175f7c0eea0cff85b175017d0f2722c0"
   name = "github.com/prometheus/client_model"
   packages = ["go"]
   pruneopts = "UT"
-  revision = "14fe0d1b01d4d5fc031dd4bec1823bd3ebbe8016"
+  revision = "7bc5445566f0fe75b15de23e6b93886e982d7bf9"
+  version = "v0.2.0"
 
 [[projects]]
-  digest = "1:f119e3205d3a1f0f19dbd7038eb37528e2c6f0933269dc344e305951fb87d632"
+  digest = "1:c1139d84a6fab0d2ebfda1ab3fe5f822162be845045db230aa1c672927d320c8"
   name = "github.com/prometheus/common"
   packages = [
     "expfmt",
@@ -129,11 +135,11 @@
     "model",
   ]
   pruneopts = "UT"
-  revision = "287d3e634a1e550c9e463dd7e5a75a422c614505"
-  version = "v0.7.0"
+  revision = "d978bcb1309602d68bb4ba69cf3f8ed900e07308"
+  version = "v0.9.1"
 
 [[projects]]
-  digest = "1:a210815b437763623ecca8eb91e6a0bf4f2d6773c5a6c9aec0e28f19e5fd6deb"
+  digest = "1:6421995dc4b0cae1fc87b615c1b7eb22e87a6746b5ac1f73221879bd2f1f4fc1"
   name = "github.com/prometheus/procfs"
   packages = [
     ".",
@@ -141,43 +147,43 @@
     "internal/util",
   ]
   pruneopts = "UT"
-  revision = "499c85531f756d1129edd26485a5f73871eeb308"
-  version = "v0.0.5"
+  revision = "4850c197847aa06aecc37570077fc16e9215d565"
+  version = "v0.0.10"
 
 [[projects]]
-  digest = "1:99d32780e5238c2621fff621123997c3e3cca96db8be13179013aea77dfab551"
+  digest = "1:49b1087f45b1e49687da2f8eb49d9e07919d4bad3813344251c722141dcb31d6"
   name = "github.com/stretchr/testify"
   packages = [
     "assert",
     "require",
   ]
   pruneopts = "UT"
-  revision = "221dbe5ed46703ee255b1da0dec05086f5035f62"
-  version = "v1.4.0"
+  revision = "3ebf1ddaeb260c4b1ae502a01c7844fa8c1fa0e9"
+  version = "v1.5.1"
 
 [[projects]]
-  digest = "1:228597ec86f00b7270b9f9e230f38abb285765850d5b7dfbe38b38f6ea1c409a"
+  digest = "1:887075851de31ace8a57e7f418eb80f2bcb562d78f1c6b441f655e42d061068f"
   name = "github.com/uber-go/tally"
   packages = ["."]
   pruneopts = "UT"
-  revision = "3332297784e46cd346ab6d9894fd4ea027dc9368"
-  version = "v3.3.12"
+  revision = "85437d773ad6967edfdf75ce51d35c95133c2f6a"
+  version = "v3.3.15"
 
 [[projects]]
   branch = "master"
-  digest = "1:6f104e30a35d62427b90130710ff507d6b9fc95ca76ceafb0025cd2809d93232"
+  digest = "1:181b0f31060ef04339c16e7bba946726dc4801a547f06edbd26d2b72a09df98c"
   name = "golang.org/x/sys"
   packages = ["windows"]
   pruneopts = "UT"
-  revision = "14da1ac737ccc89e3a28bf770cbbd260ce7e190b"
+  revision = "d5e6a3e2c0ae16fc7480523ebcb7fd4dd3215489"
 
 [[projects]]
-  digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96"
+  digest = "1:55b110c99c5fdc4f14930747326acce56b52cfce60b24b1c03ef686ac0e46bb1"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
   pruneopts = "UT"
-  revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
-  version = "v2.2.2"
+  revision = "53403b58ad1b561927d19068c655246f2db79d48"
+  version = "v2.2.8"
 
 [solve-meta]
   analyzer-name = "dep"
diff --git a/Gopkg.toml b/Gopkg.toml
index b532541..3ce7360 100644
--- a/Gopkg.toml
+++ b/Gopkg.toml
@@ -12,7 +12,7 @@
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"
-  version = "1.1.0"
+  version = "1.4.1"
 
 [[constraint]]
   name = "github.com/stretchr/testify"
diff --git a/metrics/counter.go b/metrics/counter.go
index 2a6a43e..2651fa2 100644
--- a/metrics/counter.go
+++ b/metrics/counter.go
@@ -18,6 +18,7 @@ package metrics
 type Counter interface {
 	// Inc adds the given value to the counter.
 	Inc(int64)
+	IncWithExemplar(int64, string)
 }
 
 // NullCounter counter that does nothing
@@ -26,3 +27,5 @@ var NullCounter Counter = nullCounter{}
 type nullCounter struct{}
 
 func (nullCounter) Inc(int64) {}
+
+func (n nullCounter) IncWithExemplar(int64, string) {}
diff --git a/metrics/go-kit/metrics.go b/metrics/go-kit/metrics.go
index deb0fb6..1aa7542 100644
--- a/metrics/go-kit/metrics.go
+++ b/metrics/go-kit/metrics.go
@@ -35,6 +35,11 @@ func (c *Counter) Inc(delta int64) {
 	c.counter.Add(float64(delta))
 }
 
+// IncWithExemplar adds the given value alongwith the trace ID provided.
+func (c *Counter) IncWithExemplar(int64, string) {
+	panic("not implemented")
+}
+
 // Gauge is an adapter from go-kit Gauge to jaeger-lib Gauge
 type Gauge struct {
 	gauge kit.Gauge
@@ -65,6 +70,11 @@ func (t *Timer) Record(delta time.Duration) {
 	t.hist.Observe(delta.Seconds())
 }
 
+// RecordWithExemplar saves the time passed in with the trace ID provided.
+func (t *Timer) RecordWithExemplar(time.Duration, string) {
+	panic("not implemented")
+}
+
 // Histogram is an adapter from go-kit Histogram to jaeger-lib Histogram
 type Histogram struct {
 	hist kit.Histogram
@@ -79,3 +89,8 @@ func NewHistogram(hist kit.Histogram) *Histogram {
 func (t *Histogram) Record(value float64) {
 	t.hist.Observe(value)
 }
+
+// RecordWithExemplar saves the time passed in with the trace ID provided.
+func (t *Histogram) RecordWithExemplar(float64, string) {
+	panic("not implemented")
+}
diff --git a/metrics/histogram.go b/metrics/histogram.go
index d3bd617..72143f8 100644
--- a/metrics/histogram.go
+++ b/metrics/histogram.go
@@ -18,6 +18,7 @@ package metrics
 type Histogram interface {
 	// Records the value passed in.
 	Record(float64)
+	RecordWithExemplar(float64, string)
 }
 
 // NullHistogram that does nothing
@@ -26,3 +27,5 @@ var NullHistogram Histogram = nullHistogram{}
 type nullHistogram struct{}
 
 func (nullHistogram) Record(float64) {}
+
+func (n nullHistogram) RecordWithExemplar(float64, string) {}
diff --git a/metrics/metricstest/local.go b/metrics/metricstest/local.go
index a67265a..e497e43 100644
--- a/metrics/metricstest/local.go
+++ b/metrics/metricstest/local.go
@@ -276,6 +276,10 @@ func (l *localTimer) Record(d time.Duration) {
 	l.localBackend.RecordTimer(l.name, l.tags, d)
 }
 
+func (l *localTimer) RecordWithExemplar(d time.Duration, t string) {
+	l.localBackend.RecordTimer(l.name, l.tags, d)
+}
+
 type localHistogram struct {
 	stats
 }
@@ -284,6 +288,10 @@ func (l *localHistogram) Record(v float64) {
 	l.localBackend.RecordHistogram(l.name, l.tags, v)
 }
 
+func (l *localHistogram) RecordWithExemplar(v float64, t string) {
+	l.localBackend.RecordHistogram(l.name, l.tags, v)
+}
+
 type localCounter struct {
 	stats
 }
@@ -292,6 +300,10 @@ func (l *localCounter) Inc(delta int64) {
 	l.localBackend.IncCounter(l.name, l.tags, delta)
 }
 
+func (l *localCounter) IncWithExemplar(delta int64, t string) {
+	l.localBackend.IncCounter(l.name, l.tags, delta)
+}
+
 type localGauge struct {
 	stats
 }
diff --git a/metrics/multi/multi.go b/metrics/multi/multi.go
index ead14e3..5da4905 100644
--- a/metrics/multi/multi.go
+++ b/metrics/multi/multi.go
@@ -42,6 +42,12 @@ func (c *counter) Inc(delta int64) {
 	}
 }
 
+func (c *counter) IncWithExemplar(delta int64, t string) {
+	for _, counter := range c.counters {
+		counter.Inc(delta)
+	}
+}
+
 // Counter implements metrics.Factory interface
 func (f *Factory) Counter(options metrics.Options) metrics.Counter {
 	counter := &counter{
@@ -63,6 +69,12 @@ func (t *timer) Record(delta time.Duration) {
 	}
 }
 
+func (t *timer) RecordWithExemplar(delta time.Duration, traceID string) {
+	for _, timer := range t.timers {
+		timer.Record(delta)
+	}
+}
+
 // Timer implements metrics.Factory interface
 func (f *Factory) Timer(options metrics.TimerOptions) metrics.Timer {
 	timer := &timer{
@@ -84,6 +96,12 @@ func (h *histogram) Record(value float64) {
 	}
 }
 
+func (h *histogram) RecordWithExemplar(value float64, t string) {
+	for _, histogram := range h.histograms {
+		histogram.Record(value)
+	}
+}
+
 // Histogram implements metrics.Factory interface
 func (f *Factory) Histogram(options metrics.HistogramOptions) metrics.Histogram {
 	histogram := &histogram{
diff --git a/metrics/prometheus/factory.go b/metrics/prometheus/factory.go
index 44205bc..74eafad 100644
--- a/metrics/prometheus/factory.go
+++ b/metrics/prometheus/factory.go
@@ -49,6 +49,9 @@ const (
 
 	// SeparatorColon uses a colon as separator
 	SeparatorColon = ':'
+
+	traceIDTag = "trace_id"
+	spanIDTag  = "span_id"
 )
 
 // Option is a function that sets some option for the Factory constructor.
@@ -223,6 +226,20 @@ func (c *counter) Inc(v int64) {
 	c.counter.Add(float64(v))
 }
 
+func (c *counter) IncWithExemplar(v int64, traceID string) {
+	ctr, ok := c.counter.(prometheus.ExemplarAdder)
+	if ok {
+		labels := make(map[string]string, 2)
+		t, s := parseTraceID(traceID)
+		labels[traceIDTag] = t
+		if len(s) > 0 {
+			labels[spanIDTag] = s
+		}
+		ctr.AddWithExemplar(float64(v), labels)
+	}
+	c.Inc(v)
+}
+
 type gauge struct {
 	gauge prometheus.Gauge
 }
@@ -243,6 +260,21 @@ func (t *timer) Record(v time.Duration) {
 	t.histogram.Observe(float64(v.Nanoseconds()) / float64(time.Second/time.Nanosecond))
 }
 
+func (t *timer) RecordWithExemplar(v time.Duration, traceID string) {
+	hist, ok := t.histogram.(prometheus.ExemplarObserver)
+	if ok {
+		labels := make(map[string]string, 2)
+		t, s := parseTraceID(traceID)
+		labels[traceIDTag] = t
+		if len(s) > 0 {
+			labels[spanIDTag] = s
+		}
+		hist.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), labels)
+		return
+	}
+	t.Record(v)
+}
+
 type histogram struct {
 	histogram observer
 }
@@ -251,6 +283,21 @@ func (h *histogram) Record(v float64) {
 	h.histogram.Observe(v)
 }
 
+func (h *histogram) RecordWithExemplar(v float64, traceID string) {
+	hist, ok := h.histogram.(prometheus.ExemplarObserver)
+	if ok {
+		labels := make(map[string]string, 2)
+		t, s := parseTraceID(traceID)
+		labels[traceIDTag] = t
+		if len(s) > 0 {
+			labels[spanIDTag] = s
+		}
+		hist.ObserveWithExemplar(v, labels)
+		return
+	}
+	h.Record(v)
+}
+
 func (f *Factory) subScope(name string) string {
 	if f.scope == "" {
 		return f.normalize(name)
@@ -299,3 +346,12 @@ func counterNamingConvention(name string) string {
 	}
 	return name
 }
+
+func parseTraceID(combinedTraceID string) (traceID, spanID string) {
+	parts := strings.Split(combinedTraceID, ":")
+	if len(parts) == 0 || len(parts) == 1 {
+		return combinedTraceID, ""
+	}
+
+	return parts[0], parts[1]
+}
diff --git a/metrics/tally/metrics.go b/metrics/tally/metrics.go
index 5069212..2175ef1 100644
--- a/metrics/tally/metrics.go
+++ b/metrics/tally/metrics.go
@@ -35,6 +35,11 @@ func (c *Counter) Inc(delta int64) {
 	c.counter.Inc(delta)
 }
 
+// IncWithExemplar adds the given value to the counter.
+func (c *Counter) IncWithExemplar(delta int64, t string) {
+	c.counter.Inc(delta)
+}
+
 // Gauge is an adapter from go-tally Gauge to jaeger-lib Gauge
 type Gauge struct {
 	gauge tally.Gauge
@@ -65,6 +70,11 @@ func (t *Timer) Record(delta time.Duration) {
 	t.timer.Record(delta)
 }
 
+// RecordWithExemplar saves the time passed in.
+func (t *Timer) RecordWithExemplar(delta time.Duration, traceID string) {
+	t.timer.Record(delta)
+}
+
 // Histogram is an adapter from go-tally Histogram to jaeger-lib Histogram
 type Histogram struct {
 	histogram tally.Histogram
@@ -79,3 +89,8 @@ func NewHistogram(histogram tally.Histogram) *Histogram {
 func (h *Histogram) Record(value float64) {
 	h.histogram.RecordValue(value)
 }
+
+// RecordWithExemplar saves the value passed in.
+func (h *Histogram) RecordWithExemplar(value float64, t string) {
+	h.histogram.RecordValue(value)
+}
diff --git a/metrics/timer.go b/metrics/timer.go
index e18d222..3f1a6b3 100644
--- a/metrics/timer.go
+++ b/metrics/timer.go
@@ -23,6 +23,7 @@ import (
 type Timer interface {
 	// Records the time passed in.
 	Record(time.Duration)
+	RecordWithExemplar(time.Duration, string)
 }
 
 // NullTimer timer that does nothing
@@ -31,3 +32,5 @@ var NullTimer Timer = nullTimer{}
 type nullTimer struct{}
 
 func (nullTimer) Record(time.Duration) {}
+
+func (n nullTimer) RecordWithExemplar(time.Duration, string) {}

From f821e644161d35c0cf8f3a12f89afa624ce9dd19 Mon Sep 17 00:00:00 2001
From: Shreyas Srivatsan <shreyas@chronosphere.io>
Date: Fri, 28 Feb 2020 15:32:34 -0800
Subject: [PATCH 2/5] some cleanups

Signed-off-by: Shreyas Srivatsan <shreyas@chronosphere.io>
---
 metrics/go-kit/metrics.go     | 12 ++++++------
 metrics/multi/multi.go        | 10 +++++-----
 metrics/prometheus/factory.go | 24 ++++++------------------
 metrics/tally/metrics.go      |  6 +++---
 4 files changed, 20 insertions(+), 32 deletions(-)

diff --git a/metrics/go-kit/metrics.go b/metrics/go-kit/metrics.go
index 1aa7542..eb73dff 100644
--- a/metrics/go-kit/metrics.go
+++ b/metrics/go-kit/metrics.go
@@ -36,8 +36,8 @@ func (c *Counter) Inc(delta int64) {
 }
 
 // IncWithExemplar adds the given value alongwith the trace ID provided.
-func (c *Counter) IncWithExemplar(int64, string) {
-	panic("not implemented")
+func (c *Counter) IncWithExemplar(delta int64, _ string) {
+	c.counter.Add(float64(delta))
 }
 
 // Gauge is an adapter from go-kit Gauge to jaeger-lib Gauge
@@ -71,8 +71,8 @@ func (t *Timer) Record(delta time.Duration) {
 }
 
 // RecordWithExemplar saves the time passed in with the trace ID provided.
-func (t *Timer) RecordWithExemplar(time.Duration, string) {
-	panic("not implemented")
+func (t *Timer) RecordWithExemplar(delta time.Duration, _ string) {
+	t.hist.Observe(delta.Seconds())
 }
 
 // Histogram is an adapter from go-kit Histogram to jaeger-lib Histogram
@@ -91,6 +91,6 @@ func (t *Histogram) Record(value float64) {
 }
 
 // RecordWithExemplar saves the time passed in with the trace ID provided.
-func (t *Histogram) RecordWithExemplar(float64, string) {
-	panic("not implemented")
+func (t *Histogram) RecordWithExemplar(value float64, _ string) {
+	t.hist.Observe(value)
 }
diff --git a/metrics/multi/multi.go b/metrics/multi/multi.go
index 5da4905..cb7ef2e 100644
--- a/metrics/multi/multi.go
+++ b/metrics/multi/multi.go
@@ -42,9 +42,9 @@ func (c *counter) Inc(delta int64) {
 	}
 }
 
-func (c *counter) IncWithExemplar(delta int64, t string) {
+func (c *counter) IncWithExemplar(delta int64, traceID string) {
 	for _, counter := range c.counters {
-		counter.Inc(delta)
+		counter.IncWithExemplar(delta, traceID)
 	}
 }
 
@@ -71,7 +71,7 @@ func (t *timer) Record(delta time.Duration) {
 
 func (t *timer) RecordWithExemplar(delta time.Duration, traceID string) {
 	for _, timer := range t.timers {
-		timer.Record(delta)
+		timer.RecordWithExemplar(delta, traceID)
 	}
 }
 
@@ -96,9 +96,9 @@ func (h *histogram) Record(value float64) {
 	}
 }
 
-func (h *histogram) RecordWithExemplar(value float64, t string) {
+func (h *histogram) RecordWithExemplar(value float64, traceID string) {
 	for _, histogram := range h.histograms {
-		histogram.Record(value)
+		histogram.RecordWithExemplar(value, traceID)
 	}
 }
 
diff --git a/metrics/prometheus/factory.go b/metrics/prometheus/factory.go
index 74eafad..006d8e5 100644
--- a/metrics/prometheus/factory.go
+++ b/metrics/prometheus/factory.go
@@ -229,12 +229,8 @@ func (c *counter) Inc(v int64) {
 func (c *counter) IncWithExemplar(v int64, traceID string) {
 	ctr, ok := c.counter.(prometheus.ExemplarAdder)
 	if ok {
-		labels := make(map[string]string, 2)
-		t, s := parseTraceID(traceID)
-		labels[traceIDTag] = t
-		if len(s) > 0 {
-			labels[spanIDTag] = s
-		}
+		labels := make(map[string]string, 1)
+		labels[traceIDTag] = traceID
 		ctr.AddWithExemplar(float64(v), labels)
 	}
 	c.Inc(v)
@@ -263,12 +259,8 @@ func (t *timer) Record(v time.Duration) {
 func (t *timer) RecordWithExemplar(v time.Duration, traceID string) {
 	hist, ok := t.histogram.(prometheus.ExemplarObserver)
 	if ok {
-		labels := make(map[string]string, 2)
-		t, s := parseTraceID(traceID)
-		labels[traceIDTag] = t
-		if len(s) > 0 {
-			labels[spanIDTag] = s
-		}
+		labels := make(map[string]string, 1)
+		labels[traceIDTag] = traceID
 		hist.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), labels)
 		return
 	}
@@ -286,12 +278,8 @@ func (h *histogram) Record(v float64) {
 func (h *histogram) RecordWithExemplar(v float64, traceID string) {
 	hist, ok := h.histogram.(prometheus.ExemplarObserver)
 	if ok {
-		labels := make(map[string]string, 2)
-		t, s := parseTraceID(traceID)
-		labels[traceIDTag] = t
-		if len(s) > 0 {
-			labels[spanIDTag] = s
-		}
+		labels := make(map[string]string, 1)
+		labels[traceIDTag] = traceID
 		hist.ObserveWithExemplar(v, labels)
 		return
 	}
diff --git a/metrics/tally/metrics.go b/metrics/tally/metrics.go
index 2175ef1..37a6afe 100644
--- a/metrics/tally/metrics.go
+++ b/metrics/tally/metrics.go
@@ -36,7 +36,7 @@ func (c *Counter) Inc(delta int64) {
 }
 
 // IncWithExemplar adds the given value to the counter.
-func (c *Counter) IncWithExemplar(delta int64, t string) {
+func (c *Counter) IncWithExemplar(delta int64, _ string) {
 	c.counter.Inc(delta)
 }
 
@@ -71,7 +71,7 @@ func (t *Timer) Record(delta time.Duration) {
 }
 
 // RecordWithExemplar saves the time passed in.
-func (t *Timer) RecordWithExemplar(delta time.Duration, traceID string) {
+func (t *Timer) RecordWithExemplar(delta time.Duration, _ string) {
 	t.timer.Record(delta)
 }
 
@@ -91,6 +91,6 @@ func (h *Histogram) Record(value float64) {
 }
 
 // RecordWithExemplar saves the value passed in.
-func (h *Histogram) RecordWithExemplar(value float64, t string) {
+func (h *Histogram) RecordWithExemplar(value float64, _ string) {
 	h.histogram.RecordValue(value)
 }

From c02652719414c41e9acf6c35d94b8d0d8999e6a8 Mon Sep 17 00:00:00 2001
From: Shreyas Srivatsan <shreyas@chronosphere.io>
Date: Fri, 28 Feb 2020 16:35:38 -0800
Subject: [PATCH 3/5] cast to Adder/Observer interfaces only once

Signed-off-by: Shreyas Srivatsan <shreyas@chronosphere.io>
---
 metrics/metricstest/local.go  |  6 ++--
 metrics/prometheus/factory.go | 52 ++++++++++++++++++++++++-----------
 2 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/metrics/metricstest/local.go b/metrics/metricstest/local.go
index e497e43..36ac016 100644
--- a/metrics/metricstest/local.go
+++ b/metrics/metricstest/local.go
@@ -276,7 +276,7 @@ func (l *localTimer) Record(d time.Duration) {
 	l.localBackend.RecordTimer(l.name, l.tags, d)
 }
 
-func (l *localTimer) RecordWithExemplar(d time.Duration, t string) {
+func (l *localTimer) RecordWithExemplar(d time.Duration, _ string) {
 	l.localBackend.RecordTimer(l.name, l.tags, d)
 }
 
@@ -288,7 +288,7 @@ func (l *localHistogram) Record(v float64) {
 	l.localBackend.RecordHistogram(l.name, l.tags, v)
 }
 
-func (l *localHistogram) RecordWithExemplar(v float64, t string) {
+func (l *localHistogram) RecordWithExemplar(v float64, _ string) {
 	l.localBackend.RecordHistogram(l.name, l.tags, v)
 }
 
@@ -300,7 +300,7 @@ func (l *localCounter) Inc(delta int64) {
 	l.localBackend.IncCounter(l.name, l.tags, delta)
 }
 
-func (l *localCounter) IncWithExemplar(delta int64, t string) {
+func (l *localCounter) IncWithExemplar(delta int64, _ string) {
 	l.localBackend.IncCounter(l.name, l.tags, delta)
 }
 
diff --git a/metrics/prometheus/factory.go b/metrics/prometheus/factory.go
index 006d8e5..2e0fe9c 100644
--- a/metrics/prometheus/factory.go
+++ b/metrics/prometheus/factory.go
@@ -51,7 +51,6 @@ const (
 	SeparatorColon = ':'
 
 	traceIDTag = "trace_id"
-	spanIDTag  = "span_id"
 )
 
 // Option is a function that sets some option for the Factory constructor.
@@ -141,8 +140,16 @@ func (f *Factory) Counter(options metrics.Options) metrics.Counter {
 		Help: help,
 	}
 	cv := f.cache.getOrMakeCounterVec(opts, labelNames)
+
+	ctr := cv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...)
+	exemplarAdder, ok := ctr.(prometheus.ExemplarAdder)
+	if !ok {
+		exemplarAdder = nil
+	}
+
 	return &counter{
-		counter: cv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...),
+		counter:       ctr,
+		exemplarAdder: exemplarAdder,
 	}
 }
 
@@ -180,8 +187,14 @@ func (f *Factory) Timer(options metrics.TimerOptions) metrics.Timer {
 		Buckets: asFloatBuckets(options.Buckets),
 	}
 	hv := f.cache.getOrMakeHistogramVec(opts, labelNames)
+	hist := hv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...)
+	exemplarObserver, ok := hist.(prometheus.ExemplarObserver)
+	if !ok {
+		exemplarObserver = nil
+	}
 	return &timer{
-		histogram: hv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...),
+		histogram:        hist,
+		exemplarObserver: exemplarObserver,
 	}
 }
 
@@ -208,8 +221,14 @@ func (f *Factory) Histogram(options metrics.HistogramOptions) metrics.Histogram
 		Buckets: options.Buckets,
 	}
 	hv := f.cache.getOrMakeHistogramVec(opts, labelNames)
+	hist := hv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...)
+	exemplarObserver, ok := hist.(prometheus.ExemplarObserver)
+	if !ok {
+		exemplarObserver = nil
+	}
 	return &histogram{
-		histogram: hv.WithLabelValues(f.tagsAsLabelValues(labelNames, tags)...),
+		histogram:        hist,
+		exemplarObserver: exemplarObserver,
 	}
 }
 
@@ -219,7 +238,8 @@ func (f *Factory) Namespace(scope metrics.NSOptions) metrics.Factory {
 }
 
 type counter struct {
-	counter prometheus.Counter
+	counter       prometheus.Counter
+	exemplarAdder prometheus.ExemplarAdder
 }
 
 func (c *counter) Inc(v int64) {
@@ -227,11 +247,11 @@ func (c *counter) Inc(v int64) {
 }
 
 func (c *counter) IncWithExemplar(v int64, traceID string) {
-	ctr, ok := c.counter.(prometheus.ExemplarAdder)
-	if ok {
+	if c.exemplarAdder != nil {
 		labels := make(map[string]string, 1)
 		labels[traceIDTag] = traceID
-		ctr.AddWithExemplar(float64(v), labels)
+		c.exemplarAdder.AddWithExemplar(float64(v), labels)
+		return
 	}
 	c.Inc(v)
 }
@@ -249,7 +269,8 @@ type observer interface {
 }
 
 type timer struct {
-	histogram observer
+	histogram        observer
+	exemplarObserver prometheus.ExemplarObserver
 }
 
 func (t *timer) Record(v time.Duration) {
@@ -257,18 +278,18 @@ func (t *timer) Record(v time.Duration) {
 }
 
 func (t *timer) RecordWithExemplar(v time.Duration, traceID string) {
-	hist, ok := t.histogram.(prometheus.ExemplarObserver)
-	if ok {
+	if t.exemplarObserver != nil {
 		labels := make(map[string]string, 1)
 		labels[traceIDTag] = traceID
-		hist.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), labels)
+		t.exemplarObserver.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), labels)
 		return
 	}
 	t.Record(v)
 }
 
 type histogram struct {
-	histogram observer
+	histogram        observer
+	exemplarObserver prometheus.ExemplarObserver
 }
 
 func (h *histogram) Record(v float64) {
@@ -276,11 +297,10 @@ func (h *histogram) Record(v float64) {
 }
 
 func (h *histogram) RecordWithExemplar(v float64, traceID string) {
-	hist, ok := h.histogram.(prometheus.ExemplarObserver)
-	if ok {
+	if h.exemplarObserver != nil {
 		labels := make(map[string]string, 1)
 		labels[traceIDTag] = traceID
-		hist.ObserveWithExemplar(v, labels)
+		h.exemplarObserver.ObserveWithExemplar(v, labels)
 		return
 	}
 	h.Record(v)

From e67361fc2eca948f1c84347ba30f78fd100c5422 Mon Sep 17 00:00:00 2001
From: Shreyas Srivatsan <shreyas@chronosphere.io>
Date: Sat, 29 Feb 2020 10:36:48 -0800
Subject: [PATCH 4/5] separate out interfaces

Signed-off-by: Shreyas Srivatsan <shreyas@chronosphere.io>
---
 metrics/counter.go           |  8 +++++++-
 metrics/go-kit/metrics.go    | 15 ---------------
 metrics/histogram.go         | 10 ++++++++--
 metrics/metricstest/local.go | 12 ------------
 metrics/multi/multi.go       | 18 ------------------
 metrics/tally/metrics.go     | 15 ---------------
 metrics/timer.go             | 10 ++++++++--
 7 files changed, 23 insertions(+), 65 deletions(-)

diff --git a/metrics/counter.go b/metrics/counter.go
index 2651fa2..3c3796d 100644
--- a/metrics/counter.go
+++ b/metrics/counter.go
@@ -18,11 +18,17 @@ package metrics
 type Counter interface {
 	// Inc adds the given value to the counter.
 	Inc(int64)
+}
+
+// CounterWithExemplar tracks the number of times an event has occurred and
+// supports exemplars
+type CounterWithExemplar interface {
+	Counter
 	IncWithExemplar(int64, string)
 }
 
 // NullCounter counter that does nothing
-var NullCounter Counter = nullCounter{}
+var NullCounter CounterWithExemplar = nullCounter{}
 
 type nullCounter struct{}
 
diff --git a/metrics/go-kit/metrics.go b/metrics/go-kit/metrics.go
index eb73dff..deb0fb6 100644
--- a/metrics/go-kit/metrics.go
+++ b/metrics/go-kit/metrics.go
@@ -35,11 +35,6 @@ func (c *Counter) Inc(delta int64) {
 	c.counter.Add(float64(delta))
 }
 
-// IncWithExemplar adds the given value alongwith the trace ID provided.
-func (c *Counter) IncWithExemplar(delta int64, _ string) {
-	c.counter.Add(float64(delta))
-}
-
 // Gauge is an adapter from go-kit Gauge to jaeger-lib Gauge
 type Gauge struct {
 	gauge kit.Gauge
@@ -70,11 +65,6 @@ func (t *Timer) Record(delta time.Duration) {
 	t.hist.Observe(delta.Seconds())
 }
 
-// RecordWithExemplar saves the time passed in with the trace ID provided.
-func (t *Timer) RecordWithExemplar(delta time.Duration, _ string) {
-	t.hist.Observe(delta.Seconds())
-}
-
 // Histogram is an adapter from go-kit Histogram to jaeger-lib Histogram
 type Histogram struct {
 	hist kit.Histogram
@@ -89,8 +79,3 @@ func NewHistogram(hist kit.Histogram) *Histogram {
 func (t *Histogram) Record(value float64) {
 	t.hist.Observe(value)
 }
-
-// RecordWithExemplar saves the time passed in with the trace ID provided.
-func (t *Histogram) RecordWithExemplar(value float64, _ string) {
-	t.hist.Observe(value)
-}
diff --git a/metrics/histogram.go b/metrics/histogram.go
index 72143f8..0da9f0d 100644
--- a/metrics/histogram.go
+++ b/metrics/histogram.go
@@ -18,14 +18,20 @@ package metrics
 type Histogram interface {
 	// Records the value passed in.
 	Record(float64)
+}
+
+// HistogramWithExemplar that keeps track of a distribution of values and
+// supports exemplars.
+type HistogramWithExemplar interface {
+	Histogram
 	RecordWithExemplar(float64, string)
 }
 
 // NullHistogram that does nothing
-var NullHistogram Histogram = nullHistogram{}
+var NullHistogram HistogramWithExemplar = nullHistogram{}
 
 type nullHistogram struct{}
 
 func (nullHistogram) Record(float64) {}
 
-func (n nullHistogram) RecordWithExemplar(float64, string) {}
+func (nullHistogram) RecordWithExemplar(float64, string) {}
diff --git a/metrics/metricstest/local.go b/metrics/metricstest/local.go
index 36ac016..a67265a 100644
--- a/metrics/metricstest/local.go
+++ b/metrics/metricstest/local.go
@@ -276,10 +276,6 @@ func (l *localTimer) Record(d time.Duration) {
 	l.localBackend.RecordTimer(l.name, l.tags, d)
 }
 
-func (l *localTimer) RecordWithExemplar(d time.Duration, _ string) {
-	l.localBackend.RecordTimer(l.name, l.tags, d)
-}
-
 type localHistogram struct {
 	stats
 }
@@ -288,10 +284,6 @@ func (l *localHistogram) Record(v float64) {
 	l.localBackend.RecordHistogram(l.name, l.tags, v)
 }
 
-func (l *localHistogram) RecordWithExemplar(v float64, _ string) {
-	l.localBackend.RecordHistogram(l.name, l.tags, v)
-}
-
 type localCounter struct {
 	stats
 }
@@ -300,10 +292,6 @@ func (l *localCounter) Inc(delta int64) {
 	l.localBackend.IncCounter(l.name, l.tags, delta)
 }
 
-func (l *localCounter) IncWithExemplar(delta int64, _ string) {
-	l.localBackend.IncCounter(l.name, l.tags, delta)
-}
-
 type localGauge struct {
 	stats
 }
diff --git a/metrics/multi/multi.go b/metrics/multi/multi.go
index cb7ef2e..ead14e3 100644
--- a/metrics/multi/multi.go
+++ b/metrics/multi/multi.go
@@ -42,12 +42,6 @@ func (c *counter) Inc(delta int64) {
 	}
 }
 
-func (c *counter) IncWithExemplar(delta int64, traceID string) {
-	for _, counter := range c.counters {
-		counter.IncWithExemplar(delta, traceID)
-	}
-}
-
 // Counter implements metrics.Factory interface
 func (f *Factory) Counter(options metrics.Options) metrics.Counter {
 	counter := &counter{
@@ -69,12 +63,6 @@ func (t *timer) Record(delta time.Duration) {
 	}
 }
 
-func (t *timer) RecordWithExemplar(delta time.Duration, traceID string) {
-	for _, timer := range t.timers {
-		timer.RecordWithExemplar(delta, traceID)
-	}
-}
-
 // Timer implements metrics.Factory interface
 func (f *Factory) Timer(options metrics.TimerOptions) metrics.Timer {
 	timer := &timer{
@@ -96,12 +84,6 @@ func (h *histogram) Record(value float64) {
 	}
 }
 
-func (h *histogram) RecordWithExemplar(value float64, traceID string) {
-	for _, histogram := range h.histograms {
-		histogram.RecordWithExemplar(value, traceID)
-	}
-}
-
 // Histogram implements metrics.Factory interface
 func (f *Factory) Histogram(options metrics.HistogramOptions) metrics.Histogram {
 	histogram := &histogram{
diff --git a/metrics/tally/metrics.go b/metrics/tally/metrics.go
index 37a6afe..5069212 100644
--- a/metrics/tally/metrics.go
+++ b/metrics/tally/metrics.go
@@ -35,11 +35,6 @@ func (c *Counter) Inc(delta int64) {
 	c.counter.Inc(delta)
 }
 
-// IncWithExemplar adds the given value to the counter.
-func (c *Counter) IncWithExemplar(delta int64, _ string) {
-	c.counter.Inc(delta)
-}
-
 // Gauge is an adapter from go-tally Gauge to jaeger-lib Gauge
 type Gauge struct {
 	gauge tally.Gauge
@@ -70,11 +65,6 @@ func (t *Timer) Record(delta time.Duration) {
 	t.timer.Record(delta)
 }
 
-// RecordWithExemplar saves the time passed in.
-func (t *Timer) RecordWithExemplar(delta time.Duration, _ string) {
-	t.timer.Record(delta)
-}
-
 // Histogram is an adapter from go-tally Histogram to jaeger-lib Histogram
 type Histogram struct {
 	histogram tally.Histogram
@@ -89,8 +79,3 @@ func NewHistogram(histogram tally.Histogram) *Histogram {
 func (h *Histogram) Record(value float64) {
 	h.histogram.RecordValue(value)
 }
-
-// RecordWithExemplar saves the value passed in.
-func (h *Histogram) RecordWithExemplar(value float64, _ string) {
-	h.histogram.RecordValue(value)
-}
diff --git a/metrics/timer.go b/metrics/timer.go
index 3f1a6b3..0746074 100644
--- a/metrics/timer.go
+++ b/metrics/timer.go
@@ -19,10 +19,16 @@ import (
 )
 
 // Timer accumulates observations about how long some operation took,
-// and also maintains a historgam of percentiles.
+// and also maintains a histogram of percentiles.
 type Timer interface {
 	// Records the time passed in.
 	Record(time.Duration)
+}
+
+// TimerWithExemplar accumulates observations about how long some operation
+// took, maintains a histogram of percentiles and supports exemplars.
+type TimerWithExemplar interface {
+	Timer
 	RecordWithExemplar(time.Duration, string)
 }
 
@@ -33,4 +39,4 @@ type nullTimer struct{}
 
 func (nullTimer) Record(time.Duration) {}
 
-func (n nullTimer) RecordWithExemplar(time.Duration, string) {}
+func (nullTimer) RecordWithExemplar(time.Duration, string) {}

From c2b2b7e3245c505337acd63b0ce668e295627097 Mon Sep 17 00:00:00 2001
From: Shreyas Srivatsan <shreyas@chronosphere.io>
Date: Sat, 29 Feb 2020 10:42:38 -0800
Subject: [PATCH 5/5] take labels

---
 metrics/counter.go            |  4 ++--
 metrics/histogram.go          |  4 ++--
 metrics/prometheus/factory.go | 29 ++++++-----------------------
 metrics/timer.go              |  4 ++--
 4 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/metrics/counter.go b/metrics/counter.go
index 3c3796d..889cb67 100644
--- a/metrics/counter.go
+++ b/metrics/counter.go
@@ -24,7 +24,7 @@ type Counter interface {
 // supports exemplars
 type CounterWithExemplar interface {
 	Counter
-	IncWithExemplar(int64, string)
+	IncWithExemplar(int64, map[string]string)
 }
 
 // NullCounter counter that does nothing
@@ -34,4 +34,4 @@ type nullCounter struct{}
 
 func (nullCounter) Inc(int64) {}
 
-func (n nullCounter) IncWithExemplar(int64, string) {}
+func (nullCounter) IncWithExemplar(int64, map[string]string) {}
diff --git a/metrics/histogram.go b/metrics/histogram.go
index 0da9f0d..0824ea3 100644
--- a/metrics/histogram.go
+++ b/metrics/histogram.go
@@ -24,7 +24,7 @@ type Histogram interface {
 // supports exemplars.
 type HistogramWithExemplar interface {
 	Histogram
-	RecordWithExemplar(float64, string)
+	RecordWithExemplar(float64, map[string]string)
 }
 
 // NullHistogram that does nothing
@@ -34,4 +34,4 @@ type nullHistogram struct{}
 
 func (nullHistogram) Record(float64) {}
 
-func (nullHistogram) RecordWithExemplar(float64, string) {}
+func (nullHistogram) RecordWithExemplar(float64, map[string]string) {}
diff --git a/metrics/prometheus/factory.go b/metrics/prometheus/factory.go
index 2e0fe9c..86d0662 100644
--- a/metrics/prometheus/factory.go
+++ b/metrics/prometheus/factory.go
@@ -49,8 +49,6 @@ const (
 
 	// SeparatorColon uses a colon as separator
 	SeparatorColon = ':'
-
-	traceIDTag = "trace_id"
 )
 
 // Option is a function that sets some option for the Factory constructor.
@@ -246,11 +244,9 @@ func (c *counter) Inc(v int64) {
 	c.counter.Add(float64(v))
 }
 
-func (c *counter) IncWithExemplar(v int64, traceID string) {
+func (c *counter) IncWithExemplar(v int64, l map[string]string) {
 	if c.exemplarAdder != nil {
-		labels := make(map[string]string, 1)
-		labels[traceIDTag] = traceID
-		c.exemplarAdder.AddWithExemplar(float64(v), labels)
+		c.exemplarAdder.AddWithExemplar(float64(v), l)
 		return
 	}
 	c.Inc(v)
@@ -277,11 +273,9 @@ func (t *timer) Record(v time.Duration) {
 	t.histogram.Observe(float64(v.Nanoseconds()) / float64(time.Second/time.Nanosecond))
 }
 
-func (t *timer) RecordWithExemplar(v time.Duration, traceID string) {
+func (t *timer) RecordWithExemplar(v time.Duration, l map[string]string) {
 	if t.exemplarObserver != nil {
-		labels := make(map[string]string, 1)
-		labels[traceIDTag] = traceID
-		t.exemplarObserver.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), labels)
+		t.exemplarObserver.ObserveWithExemplar(float64(v.Nanoseconds())/float64(time.Second/time.Nanosecond), l)
 		return
 	}
 	t.Record(v)
@@ -296,11 +290,9 @@ func (h *histogram) Record(v float64) {
 	h.histogram.Observe(v)
 }
 
-func (h *histogram) RecordWithExemplar(v float64, traceID string) {
+func (h *histogram) RecordWithExemplar(v float64, l map[string]string) {
 	if h.exemplarObserver != nil {
-		labels := make(map[string]string, 1)
-		labels[traceIDTag] = traceID
-		h.exemplarObserver.ObserveWithExemplar(v, labels)
+		h.exemplarObserver.ObserveWithExemplar(v, l)
 		return
 	}
 	h.Record(v)
@@ -354,12 +346,3 @@ func counterNamingConvention(name string) string {
 	}
 	return name
 }
-
-func parseTraceID(combinedTraceID string) (traceID, spanID string) {
-	parts := strings.Split(combinedTraceID, ":")
-	if len(parts) == 0 || len(parts) == 1 {
-		return combinedTraceID, ""
-	}
-
-	return parts[0], parts[1]
-}
diff --git a/metrics/timer.go b/metrics/timer.go
index 0746074..6cec1fe 100644
--- a/metrics/timer.go
+++ b/metrics/timer.go
@@ -29,7 +29,7 @@ type Timer interface {
 // took, maintains a histogram of percentiles and supports exemplars.
 type TimerWithExemplar interface {
 	Timer
-	RecordWithExemplar(time.Duration, string)
+	RecordWithExemplar(time.Duration, map[string]string)
 }
 
 // NullTimer timer that does nothing
@@ -39,4 +39,4 @@ type nullTimer struct{}
 
 func (nullTimer) Record(time.Duration) {}
 
-func (nullTimer) RecordWithExemplar(time.Duration, string) {}
+func (nullTimer) RecordWithExemplar(time.Duration, map[string]string) {}