Merge pull request #146 from hashicorp/fix-prometheus-panic

prometheus: prevent panic when incrmenting counter

prometheus/prometheus.go

@@ -372,6 +372,13 @@ func (p *PrometheusSink) IncrCounterWithLabels(parts []string, val float32, labe
 	key, hash := flattenKey(parts, labels)
 	pc, ok := p.counters.Load(hash)
 
+	// Prometheus Counter.Add() panics if val < 0. We don't want this to
+	// cause applications to crash, so log an error instead.
+	if val < 0 {
+		log.Printf("[ERR] Attempting to increment Prometheus counter %v with value negative value %v", key, val)
+		return
+	}
+
 	// Does the counter exist?
 	if ok {
 		localCounter := *pc.(*counter)

prometheus/prometheus_test.go

@@ -168,6 +168,9 @@ func TestDefinitions(t *testing.T) {
 	sink.AddSample(summaryDef.Name, 42)
 	sink.IncrCounter(counterDef.Name, 1)
 
+	// Prometheus panic should not be propagated
+	sink.IncrCounter(counterDef.Name, -1)
+
 	// Test that the expiry behavior works as expected. First pick a time which
 	// is after all the actual updates above.
 	timeAfterUpdates := time.Now()
@@ -380,6 +383,11 @@ func TestDefinitionsWithLabels(t *testing.T) {
 		}
 		return true
 	})
+
+	// Prometheus panic should not be propagated
+	sink.IncrCounterWithLabels(counterDef.Name, -1, []metrics.Label{
+		{Name: "version", Value: "some info"},
+	})
 }
 
 func TestMetricSinkInterface(t *testing.T) {