Merge pull request #102 from aalda/metrics

Metrics

api/apihttp/apihttp.go

@@ -23,7 +23,6 @@ import (
 	"time"
 
 	"github.com/bbva/qed/log"
-	"github.com/bbva/qed/metrics"
 	"github.com/bbva/qed/protocol"
 	"github.com/bbva/qed/raftwal"
 )
@@ -44,8 +43,6 @@ type HealthCheckResponse struct {
 // and no body.
 func HealthCheckHandler(w http.ResponseWriter, r *http.Request) {
 
-	metrics.QedAPIHealthcheckRequestsTotal.Inc()
-
 	// Make sure we can only be called with an HTTP POST request.
 	if r.Method != "HEAD" {
 		w.Header().Set("Allow", "HEAD")

balloon/balloon.go

@@ -182,11 +182,6 @@ func (b *Balloon) RefreshVersion() error {
 
 func (b *Balloon) Add(event []byte) (*Snapshot, []*storage.Mutation, error) {
 
-	// Metrics
-	metrics.QedBalloonAddTotal.Inc()
-	//timer := prometheus.NewTimer(metrics.QedBalloonAddDurationSeconds)
-	//defer timer.ObserveDuration()
-
 	// Activate metrics gathering
 	stats := metrics.Balloon
 
@@ -230,21 +225,13 @@ func (b *Balloon) Add(event []byte) (*Snapshot, []*storage.Mutation, error) {
 		Version:       version,
 	}
 
-	// Increment add hits and version
-	stats.AddFloat("add_hits", 1)
+	// Increment version
 	stats.Set("version", metrics.Uint64ToVar(version))
 
 	return snapshot, mutations, nil
 }
 
 func (b Balloon) QueryDigestMembership(keyDigest hashing.Digest, version uint64) (*MembershipProof, error) {
-	// Metrics
-	metrics.QedBalloonDigestMembershipTotal.Inc()
-	//timer := prometheus.NewTimer(metrics.QedBalloonDigestMembershipDurationSeconds)
-	//defer timer.ObserveDuration()
-
-	stats := metrics.Balloon
-	stats.AddFloat("QueryMembership", 1)
 
 	var proof MembershipProof
 	var err error
@@ -297,9 +284,6 @@ func (b Balloon) QueryMembership(event []byte, version uint64) (*MembershipProof
 
 func (b Balloon) QueryConsistency(start, end uint64) (*IncrementalProof, error) {
 
-	// Metrics
-	metrics.QedBalloonIncrementalTotal.Inc()
-
 	stats := metrics.Balloon
 	stats.AddFloat("QueryConsistency", 1)
 	var proof IncrementalProof

balloon/balloon_test.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/bbva/qed/hashing"
 	"github.com/bbva/qed/log"
+	metrics_utils "github.com/bbva/qed/testutils/metrics"
 	"github.com/bbva/qed/testutils/rand"
 	storage_utils "github.com/bbva/qed/testutils/storage"
 	"github.com/bbva/qed/util"
@@ -225,50 +226,6 @@ func TestGenIncrementalAndVerify(t *testing.T) {
 	assert.True(t, correct, "Unable to verify incremental proof")
 }
 
-func BenchmarkAddBadger(b *testing.B) {
-
-	log.SetLogger("BenchmarkAddBadger", log.SILENT)
-
-	store, closeF := storage_utils.OpenBadgerStore(b, "/var/tmp/balloon_bench.db")
-	defer closeF()
-
-	balloon, err := NewBalloon(store, hashing.NewSha256Hasher)
-	require.NoError(b, err)
-
-	b.ResetTimer()
-	b.N = 100000
-	for i := 0; i < b.N; i++ {
-		event := rand.Bytes(128)
-		_, mutations, _ := balloon.Add(event)
-		store.Mutate(mutations)
-	}
-
-}
-
-func BenchmarkQueryBadger(b *testing.B) {
-	var events [][]byte
-	log.SetLogger("BenchmarkAddBadger", log.SILENT)
-
-	store, closeF := storage_utils.OpenBadgerStore(b, "/var/tmp/ballon_bench.db")
-	defer closeF()
-
-	balloon, err := NewBalloon(store, hashing.NewSha256Hasher)
-	require.NoError(b, err)
-
-	b.N = 100000
-	for i := 0; i < b.N; i++ {
-		event := rand.Bytes(128)
-		events = append(events, event)
-		_, mutations, _ := balloon.Add(event)
-		store.Mutate(mutations)
-	}
-
-	b.ResetTimer()
-	for i, e := range events {
-		balloon.QueryMembership(e, uint64(i))
-	}
-
-}
 func BenchmarkAddRocksDB(b *testing.B) {
 
 	log.SetLogger("BenchmarkAddRocksDB", log.SILENT)
@@ -279,12 +236,18 @@ func BenchmarkAddRocksDB(b *testing.B) {
 	balloon, err := NewBalloon(store, hashing.NewSha256Hasher)
 	require.NoError(b, err)
 
+	balloonMetrics := metrics_utils.CustomRegister(AddTotal)
+	srvCloseF := metrics_utils.StartMetricsServer(balloonMetrics, store)
+	defer srvCloseF()
+
 	b.ResetTimer()
-	b.N = 1000000
+	b.N = 2000000
 	for i := 0; i < b.N; i++ {
 		event := rand.Bytes(128)
-		_, mutations, _ := balloon.Add(event)
-		store.Mutate(mutations)
+		_, mutations, err := balloon.Add(event)
+		require.NoError(b, err)
+		require.NoError(b, store.Mutate(mutations))
+		AddTotal.Inc()
 	}
 
 }

balloon/metrics.go

@@ -0,0 +1,57 @@
+/*
+   Copyright 2018-2019 Banco Bilbao Vizcaya Argentaria, S.A.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package balloon
+
+import "github.com/prometheus/client_golang/prometheus"
+
+const namespace = "qed"
+const subSystem = "balloon"
+
+var (
+	AddTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subSystem,
+			Name:      "add_total",
+			Help:      "Number of add operations",
+		},
+	)
+	MembershipTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subSystem,
+			Name:      "membership_total",
+			Help:      "Number of membership queries.",
+		},
+	)
+	DigestMembershipTotal = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subSystem,
+			Name:      "digest_membership_total",
+			Help:      "Number of membership by digest queries.",
+		},
+	)
+	IncrementalTotal = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subSystem,
+			Name:      "incremental_total",
+			Help:      "Number of incremental queries.",
+		},
+	)
+)

deploy/aws/provision/files/grafana/dashboards/QED.json

@@ -215,7 +215,7 @@
       "tableColumn": "Value",
       "targets": [
         {
-          "expr": "sum(qed_balloon_add_total) / sum(qed_server_instances)",
+          "expr": "sum(qed_raft_balloon_adds) / sum(qed_server_instances)",
           "format": "time_series",
           "hide": false,
           "instant": false,
@@ -296,7 +296,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "sum(qed_balloon_digest_membership_total)",
+          "expr": "sum(qed_raft_balloon_digest_membership_queries)",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -375,7 +375,7 @@
       "tableColumn": "",
       "targets": [
         {
-          "expr": "sum(qed_balloon_incremental_total)",
+          "expr": "sum(qed_raft_balloon_incremental_queries)",
           "format": "time_series",
           "intervalFactor": 1,
           "refId": "A"
@@ -443,7 +443,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(qed_balloon_add_total[$interval])) by (job)",
+          "expr": "sum(rate(qed_raft_balloon_adds[$interval])) by (job)",
           "format": "time_series",
           "hide": false,
           "interval": "",
@@ -530,7 +530,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(qed_balloon_digest_membership_total[$interval])) by (job)",
+          "expr": "sum(rate(qed_raft_balloon_digest_membership_queries[$interval])) by (job)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{job}}",
@@ -615,7 +615,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(qed_balloon_incremental_total[$interval])) by (job)",
+          "expr": "sum(rate(qed_raft_balloon_incremental_queries[$interval])) by (job)",
           "format": "time_series",
           "intervalFactor": 1,
           "legendFormat": "{{job}}",