Feat: Add storage metrics

Refactor: metrics register method

go.mod

@@ -19,6 +19,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.8.0
 	github.com/prometheus/client_golang v0.9.2
+	github.com/prometheus/procfs v0.0.0-20190328153300-af7bedc223fb // indirect
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/viper v1.3.1
 	github.com/stretchr/testify v1.2.2

go.sum

@@ -90,6 +90,8 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
 github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190328153300-af7bedc223fb h1:LvNCMEj0FFZQYsxZb7o3xQPrtqOOB6lrTUOWshC+ZTs=
+github.com/prometheus/procfs v0.0.0-20190328153300-af7bedc223fb/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=

gossip/auditor/auditor.go

@@ -131,10 +131,7 @@ func (a Auditor) RegisterMetrics(srv *metrics.Server) {
 		QedAuditorBatchesReceivedTotal,
 		QedAuditorGetMembershipProofErrTotal,
 	}
-
-	for _, m := range metrics {
-		srv.Register(m)
-	}
+	srv.Register(metrics)
 }
 
 func (a Auditor) runTaskDispatcher() {

gossip/monitor/monitor.go

@@ -128,10 +128,7 @@ func (m Monitor) RegisterMetrics(srv *metrics.Server) {
 		QedMonitorBatchesProcessSeconds,
 		QedMonitorGetIncrementalProofErrTotal,
 	}
-
-	for _, m := range metrics {
-		srv.Register(m)
-	}
+	srv.Register(metrics)
 }
 
 func (m Monitor) Process(b *protocol.BatchSnapshots) {

gossip/publisher/publisher.go

@@ -112,10 +112,7 @@ func (p Publisher) RegisterMetrics(srv *metrics.Server) {
 		QedPublisherBatchesReceivedTotal,
 		QedPublisherBatchesProcessSeconds,
 	}
-
-	for _, m := range metrics {
-		srv.Register(m)
-	}
+	srv.Register(metrics)
 }
 
 func (p *Publisher) Process(b *protocol.BatchSnapshots) {
@@ -127,7 +124,7 @@ func (p *Publisher) Process(b *protocol.BatchSnapshots) {
 	for _, signedSnap := range b.Snapshots {
 		_, err := p.processed.Get(signedSnap.Signature)
 		if err != nil {
-			p.processed.Set(signedSnap.Signature, []byte{0x0}, 0)
+			_ = p.processed.Set(signedSnap.Signature, []byte{0x0}, 0)
 			batch.Snapshots = append(batch.Snapshots, signedSnap)
 		}
 	}
@@ -214,4 +211,3 @@ func (t PublishTask) Do() {
 		log.Infof("Publisher had an error getting response from snapStore saving a batch: %v", err)
 	}
 }
-

gossip/sender/sender.go

@@ -99,10 +99,7 @@ func (s Sender) RegisterMetrics(srv *metrics.Server) {
 		QedSenderInstancesCount,
 		QedSenderBatchesSentTotal,
 	}
-
-	for _, m := range metrics {
-		srv.Register(m)
-	}
+	srv.Register(metrics)
 }
 
 func (s Sender) newBatch() *protocol.BatchSnapshots {

metrics/metrics.go

@@ -78,11 +78,12 @@ func (m Server) Shutdown() {
 
 // Register a prometheus collector in the prometheus registry used
 // by the metrics server.
-func (m Server) Register(metric prometheus.Collector) {
-	if err := m.registry.Register(metric); err != nil {
-		log.Infof("metric not registered:", err)
-	} else {
-		log.Infof("metric registered.")
+func (m Server) Register(collectors []prometheus.Collector) {
+	for _, c := range collectors {
+		if err := m.registry.Register(c); err != nil {
+			log.Infof("metric not registered:", err)
+		} else {
+			log.Infof("metric registered.")
+		}
 	}
 }
-

server/server.go

@@ -120,9 +120,10 @@ func NewServer(conf *Config) (*Server, error) {
 
 	// create metrics server and register default qed metrics
 	server.metricsServer = metrics.NewServer(conf.MetricsAddr)
-	for _, m := range metrics.DefaultMetrics {
-		server.metricsServer.Register(m)
-	}
+	server.metricsServer.Register(metrics.DefaultMetrics)
+	server.metricsServer.Register(rocks.PrometheusCollectors())
+	// server.metricsServer.Register(raft.PrometheusCollectors())
+	// server.metricsServer.Register(balloon.PrometheusCollectors())
 
 	// Create gossip agent
 	config := gossip.DefaultConfig()

storage/rocks/metrics.go

@@ -45,10 +45,10 @@ func PrometheusCollectors() []prometheus.Collector {
 }
 
 // namespace is the leading part of all published metrics for the Storage service.
-const namespace = "storage"
+const namespace = "qed_storage"
 
-const blockCacheSubsystem = "" // sub-system associated with metrics for block cache.
-const ioSubsystem = "io"       // sub-system associated with metrics for I/O.
+const blockCacheSubsystem = "block" // sub-system associated with metrics for block cache.
+const ioSubsystem = "io"            // sub-system associated with metrics for I/O.
 
 type rocksDBMetrics struct {
 	*blockCacheMetrics
@@ -75,7 +75,7 @@ func newBlockCacheMetrics(stats *rocksdb.Statistics) *blockCacheMetrics {
 			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Subsystem: blockCacheSubsystem,
-				Name:      "block_cache_miss",
+				Name:      "cache_miss",
 				Help:      "Total block cache misses.",
 			},
 			extractMetric(stats, rocksdb.TickerBlockCacheMiss),
@@ -84,7 +84,7 @@ func newBlockCacheMetrics(stats *rocksdb.Statistics) *blockCacheMetrics {
 			prometheus.GaugeOpts{
 				Namespace: namespace,
 				Subsystem: blockCacheSubsystem,
-				Name:      "block_cache_hit",
+				Name:      "cache_hit",
 				Help:      "Total block cache hits.",
 			},
 			extractMetric(stats, rocksdb.TickerBlockCacheHit),