metrics: add prometheus collector

internal/debug/flags.go

@@ -91,9 +91,9 @@ var Flags = []cli.Flag{
 	//vmoduleFlag,
 	//backtraceAtFlag,
 	//debugFlag,
-	//pprofFlag,
-	//pprofAddrFlag,
-	//pprofPortFlag,
+	pprofFlag,
+	pprofAddrFlag,
+	pprofPortFlag,
 	//memprofilerateFlag,
 	//blockprofilerateFlag,
 	//cpuprofileFlag,

metrics/exp/exp.go

@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"sync"
 
+	"github.com/tomochain/tomochain/metrics/prometheus"
+
 	"github.com/tomochain/tomochain/metrics"
 )
 
@@ -42,6 +44,7 @@ func Exp(r metrics.Registry) {
 	// http.HandleFunc("/debug/vars", e.expHandler)
 	// haven't found an elegant way, so just use a different endpoint
 	http.Handle("/debug/metrics", h)
+	http.Handle("/debug/metrics/prometheus", prometheus.Handler(r))
 }
 
 // ExpHandler will return an expvar powered metrics handler.
@@ -134,6 +137,17 @@ func (exp *exp) publishTimer(name string, metric metrics.Timer) {
 	exp.getFloat(name + ".mean-rate").Set(t.RateMean())
 }
 
+func (exp *exp) publishResettingTimer(name string, metric metrics.ResettingTimer) {
+	t := metric.Snapshot()
+	ps := t.Percentiles([]float64{50, 75, 95, 99})
+	exp.getInt(name + ".count").Set(int64(len(t.Values())))
+	exp.getFloat(name + ".mean").Set(t.Mean())
+	exp.getInt(name + ".50-percentile").Set(ps[0])
+	exp.getInt(name + ".75-percentile").Set(ps[1])
+	exp.getInt(name + ".95-percentile").Set(ps[2])
+	exp.getInt(name + ".99-percentile").Set(ps[3])
+}
+
 func (exp *exp) syncToExpvar() {
 	exp.registry.Each(func(name string, i interface{}) {
 		switch i.(type) {
@@ -149,6 +163,8 @@ func (exp *exp) syncToExpvar() {
 			exp.publishMeter(name, i.(metrics.Meter))
 		case metrics.Timer:
 			exp.publishTimer(name, i.(metrics.Timer))
+		case metrics.ResettingTimer:
+			exp.publishResettingTimer(name, i.(metrics.ResettingTimer))
 		default:
 			panic(fmt.Sprintf("unsupported type for '%s': %T", name, i))
 		}

metrics/prometheus/collector.go

@@ -0,0 +1,115 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package prometheus
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/tomochain/tomochain/metrics"
+)
+
+var (
+	typeGaugeTpl           = "# TYPE %s gauge\n"
+	typeCounterTpl         = "# TYPE %s counter\n"
+	typeSummaryTpl         = "# TYPE %s summary\n"
+	keyValueTpl            = "%s %v\n\n"
+	keyQuantileTagValueTpl = "%s {quantile=\"%s\"} %v\n\n"
+)
+
+// collector is a collection of byte buffers that aggregate Prometheus reports
+// for different metric types.
+type collector struct {
+	buff *bytes.Buffer
+}
+
+// newCollector createa a new Prometheus metric aggregator.
+func newCollector() *collector {
+	return &collector{
+		buff: &bytes.Buffer{},
+	}
+}
+
+func (c *collector) addCounter(name string, m metrics.Counter) {
+	c.writeGaugeCounter(name, m.Count())
+}
+
+func (c *collector) addGauge(name string, m metrics.Gauge) {
+	c.writeGaugeCounter(name, m.Value())
+}
+
+func (c *collector) addGaugeFloat64(name string, m metrics.GaugeFloat64) {
+	c.writeGaugeCounter(name, m.Value())
+}
+
+func (c *collector) addHistogram(name string, m metrics.Histogram) {
+	pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}
+	ps := m.Percentiles(pv)
+	c.writeSummaryCounter(name, m.Count())
+	for i := range pv {
+		c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i])
+	}
+}
+
+func (c *collector) addMeter(name string, m metrics.Meter) {
+	c.writeGaugeCounter(name, m.Count())
+}
+
+func (c *collector) addTimer(name string, m metrics.Timer) {
+	pv := []float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}
+	ps := m.Percentiles(pv)
+	c.writeSummaryCounter(name, m.Count())
+	for i := range pv {
+		c.writeSummaryPercentile(name, strconv.FormatFloat(pv[i], 'f', -1, 64), ps[i])
+	}
+}
+
+func (c *collector) addResettingTimer(name string, m metrics.ResettingTimer) {
+	if len(m.Values()) <= 0 {
+		return
+	}
+	ps := m.Percentiles([]float64{50, 95, 99})
+	val := m.Values()
+	c.writeSummaryCounter(name, len(val))
+	c.writeSummaryPercentile(name, "0.50", ps[0])
+	c.writeSummaryPercentile(name, "0.95", ps[1])
+	c.writeSummaryPercentile(name, "0.99", ps[2])
+}
+
+func (c *collector) writeGaugeCounter(name string, value interface{}) {
+	name = mutateKey(name)
+	c.buff.WriteString(fmt.Sprintf(typeGaugeTpl, name))
+	c.buff.WriteString(fmt.Sprintf(keyValueTpl, name, value))
+}
+
+func (c *collector) writeSummaryCounter(name string, value interface{}) {
+	name = mutateKey(name + "_count")
+	c.buff.WriteString(fmt.Sprintf(typeCounterTpl, name))
+	c.buff.WriteString(fmt.Sprintf(keyValueTpl, name, value))
+}
+
+func (c *collector) writeSummaryPercentile(name, p string, value interface{}) {
+	name = mutateKey(name)
+	c.buff.WriteString(fmt.Sprintf(typeSummaryTpl, name))
+	c.buff.WriteString(fmt.Sprintf(keyQuantileTagValueTpl, name, p, value))
+}
+
+func mutateKey(key string) string {
+	return strings.Replace(key, "/", "_", -1)
+}

metrics/prometheus/prometheus.go

@@ -0,0 +1,68 @@
+// Copyright 2019 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+// Package prometheus exposes go-metrics into a Prometheus format.
+package prometheus
+
+import (
+	"fmt"
+	"net/http"
+	"sort"
+
+	"github.com/tomochain/tomochain/log"
+	"github.com/tomochain/tomochain/metrics"
+)
+
+// Handler returns an HTTP handler which dump metrics in Prometheus format.
+func Handler(reg metrics.Registry) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Gather and pre-sort the metrics to avoid random listings
+		var names []string
+		reg.Each(func(name string, i interface{}) {
+			names = append(names, name)
+		})
+		sort.Strings(names)
+
+		// Aggregate all the metris into a Prometheus collector
+		c := newCollector()
+
+		for _, name := range names {
+			i := reg.Get(name)
+
+			switch m := i.(type) {
+			case metrics.Counter:
+				c.addCounter(name, m.Snapshot())
+			case metrics.Gauge:
+				c.addGauge(name, m.Snapshot())
+			case metrics.GaugeFloat64:
+				c.addGaugeFloat64(name, m.Snapshot())
+			case metrics.Histogram:
+				c.addHistogram(name, m.Snapshot())
+			case metrics.Meter:
+				c.addMeter(name, m.Snapshot())
+			case metrics.Timer:
+				c.addTimer(name, m.Snapshot())
+			case metrics.ResettingTimer:
+				c.addResettingTimer(name, m.Snapshot())
+			default:
+				log.Warn("Unknown Prometheus metric type", "type", fmt.Sprintf("%T", i))
+			}
+		}
+		w.Header().Add("Content-Type", "text/plain")
+		w.Header().Add("Content-Length", fmt.Sprint(c.buff.Len()))
+		w.Write(c.buff.Bytes())
+	})
+}

metrics/resetting_timer.go

@@ -58,7 +58,11 @@ type NilResettingTimer struct {
 func (NilResettingTimer) Values() []int64 { return nil }
 
 // Snapshot is a no-op.
-func (NilResettingTimer) Snapshot() ResettingTimer { return NilResettingTimer{} }
+func (NilResettingTimer) Snapshot() ResettingTimer {
+	return &ResettingTimerSnapshot{
+		values: []int64{},
+	}
+}
 
 // Time is a no-op.
 func (NilResettingTimer) Time(func()) {}
@@ -210,7 +214,7 @@ func (t *ResettingTimerSnapshot) calc(percentiles []float64) {
 				// poor man's math.Round(x):
 				// math.Floor(x + 0.5)
 				indexOfPerc := int(math.Floor(((abs / 100.0) * float64(count)) + 0.5))
-				if pct >= 0 {
+				if pct >= 0 && indexOfPerc > 0 {
 					indexOfPerc -= 1 // index offset=0
 				}
 				thresholdBoundary = t.values[indexOfPerc]

metrics/resetting_timer_test.go

@@ -104,3 +104,113 @@ func TestResettingTimer(t *testing.T) {
 		}
 	}
 }
+
+func TestResettingTimerWithFivePercentiles(t *testing.T) {
+	tests := []struct {
+		values   []int64
+		start    int
+		end      int
+		wantP05  int64
+		wantP20  int64
+		wantP50  int64
+		wantP95  int64
+		wantP99  int64
+		wantMean float64
+		wantMin  int64
+		wantMax  int64
+	}{
+		{
+			values:  []int64{},
+			start:   1,
+			end:     11,
+			wantP05: 1, wantP20: 2, wantP50: 5, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+		{
+			values:  []int64{},
+			start:   1,
+			end:     101,
+			wantP05: 5, wantP20: 20, wantP50: 50, wantP95: 95, wantP99: 99,
+			wantMin: 1, wantMax: 100, wantMean: 50.5,
+		},
+		{
+			values:  []int64{1},
+			start:   0,
+			end:     0,
+			wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 1, wantP99: 1,
+			wantMin: 1, wantMax: 1, wantMean: 1,
+		},
+		{
+			values:  []int64{0},
+			start:   0,
+			end:     0,
+			wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{},
+			start:   0,
+			end:     0,
+			wantP05: 0, wantP20: 0, wantP50: 0, wantP95: 0, wantP99: 0,
+			wantMin: 0, wantMax: 0, wantMean: 0,
+		},
+		{
+			values:  []int64{1, 10},
+			start:   0,
+			end:     0,
+			wantP05: 1, wantP20: 1, wantP50: 1, wantP95: 10, wantP99: 10,
+			wantMin: 1, wantMax: 10, wantMean: 5.5,
+		},
+	}
+	for ind, tt := range tests {
+		timer := NewResettingTimer()
+
+		for i := tt.start; i < tt.end; i++ {
+			tt.values = append(tt.values, int64(i))
+		}
+
+		for _, v := range tt.values {
+			timer.Update(time.Duration(v))
+		}
+
+		snap := timer.Snapshot()
+
+		ps := snap.Percentiles([]float64{5, 20, 50, 95, 99})
+
+		val := snap.Values()
+
+		if len(val) > 0 {
+			if tt.wantMin != val[0] {
+				t.Fatalf("%d: min: got %d, want %d", ind, val[0], tt.wantMin)
+			}
+
+			if tt.wantMax != val[len(val)-1] {
+				t.Fatalf("%d: max: got %d, want %d", ind, val[len(val)-1], tt.wantMax)
+			}
+		}
+
+		if tt.wantMean != snap.Mean() {
+			t.Fatalf("%d: mean: got %.2f, want %.2f", ind, snap.Mean(), tt.wantMean)
+		}
+
+		if tt.wantP05 != ps[0] {
+			t.Fatalf("%d: p05: got %d, want %d", ind, ps[0], tt.wantP05)
+		}
+
+		if tt.wantP20 != ps[1] {
+			t.Fatalf("%d: p20: got %d, want %d", ind, ps[1], tt.wantP20)
+		}
+
+		if tt.wantP50 != ps[2] {
+			t.Fatalf("%d: p50: got %d, want %d", ind, ps[2], tt.wantP50)
+		}
+
+		if tt.wantP95 != ps[3] {
+			t.Fatalf("%d: p95: got %d, want %d", ind, ps[3], tt.wantP95)
+		}
+
+		if tt.wantP99 != ps[4] {
+			t.Fatalf("%d: p99: got %d, want %d", ind, ps[4], tt.wantP99)
+		}
+	}
+}