Add new metrics.

1. 'prometheus_tsdb_wal_truncate_fail' for failed WAL truncation.
2. 'prometheus_tsdb_checkpoint_delete_fail' for failed old checkpoint delete.

Signed-off-by: Ganesh Vernekar <[email protected]>

checkpoint.go

@@ -26,6 +26,7 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/tsdb/fileutil"
 	"github.com/prometheus/tsdb/wal"
 )
@@ -102,7 +103,8 @@ const checkpointPrefix = "checkpoint."
 // it with the original WAL.
 //
 // Non-critical errors are logged and not returned.
-func Checkpoint(logger log.Logger, w *wal.WAL, m, n int, keep func(id uint64) bool, mint int64) (*CheckpointStats, error) {
+func Checkpoint(logger log.Logger, w *wal.WAL, m, n int, keep func(id uint64) bool, mint int64,
+	walTruncationFail prometheus.Counter, checkpointDeleteFail prometheus.Counter) (*CheckpointStats, error) {
 	if logger == nil {
 		logger = log.NewNopLogger()
 	}
@@ -277,12 +279,14 @@ func Checkpoint(logger log.Logger, w *wal.WAL, m, n int, keep func(id uint64) bo
 		// Leftover segments will just be ignored in the future if there's a checkpoint
 		// that supersedes them.
 		level.Error(logger).Log("msg", "truncating segments failed", "err", err)
+		walTruncationFail.Add(float64(1))
 	}
 	if err := DeleteCheckpoints(w.Dir(), n); err != nil {
 		// Leftover old checkpoints do not cause problems down the line beyond
 		// occupying disk space.
 		// They will just be ignored since a higher checkpoint exists.
 		level.Error(logger).Log("msg", "delete old checkpoints", "err", err)
+		checkpointDeleteFail.Add(float64(1))
 	}
 	return stats, nil
 }

head.go

@@ -76,19 +76,21 @@ type Head struct {
 }
 
 type headMetrics struct {
-	activeAppenders     prometheus.Gauge
-	series              prometheus.Gauge
-	seriesCreated       prometheus.Counter
-	seriesRemoved       prometheus.Counter
-	seriesNotFound      prometheus.Counter
-	chunks              prometheus.Gauge
-	chunksCreated       prometheus.Counter
-	chunksRemoved       prometheus.Counter
-	gcDuration          prometheus.Summary
-	minTime             prometheus.GaugeFunc
-	maxTime             prometheus.GaugeFunc
-	samplesAppended     prometheus.Counter
-	walTruncateDuration prometheus.Summary
+	activeAppenders      prometheus.Gauge
+	series               prometheus.Gauge
+	seriesCreated        prometheus.Counter
+	seriesRemoved        prometheus.Counter
+	seriesNotFound       prometheus.Counter
+	chunks               prometheus.Gauge
+	chunksCreated        prometheus.Counter
+	chunksRemoved        prometheus.Counter
+	gcDuration           prometheus.Summary
+	minTime              prometheus.GaugeFunc
+	maxTime              prometheus.GaugeFunc
+	samplesAppended      prometheus.Counter
+	walTruncateDuration  prometheus.Summary
+	walTruncateFail      prometheus.Counter
+	checkpointDeleteFail prometheus.Counter
 }
 
 func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
@@ -146,10 +148,18 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
 		Name: "prometheus_tsdb_wal_truncate_duration_seconds",
 		Help: "Duration of WAL truncation.",
 	})
+	m.walTruncateFail = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_wal_truncate_fail",
+		Help: "Number of times WAL truncation failed.",
+	})
 	m.samplesAppended = prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "prometheus_tsdb_head_samples_appended_total",
 		Help: "Total number of appended samples.",
 	})
+	m.checkpointDeleteFail = prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "prometheus_tsdb_checkpoint_delete_fail",
+		Help: "Number of times deletion of old checkpoint failed.",
+	})
 
 	if r != nil {
 		r.MustRegister(
@@ -165,7 +175,9 @@ func newHeadMetrics(h *Head, r prometheus.Registerer) *headMetrics {
 			m.maxTime,
 			m.gcDuration,
 			m.walTruncateDuration,
+			m.walTruncateFail,
 			m.samplesAppended,
+			m.checkpointDeleteFail,
 		)
 	}
 	return m
@@ -469,7 +481,7 @@ func (h *Head) Truncate(mint int64) error {
 	keep := func(id uint64) bool {
 		return h.series.getByID(id) != nil
 	}
-	if _, err = Checkpoint(h.logger, h.wal, m, n, keep, mint); err != nil {
+	if _, err = Checkpoint(h.logger, h.wal, m, n, keep, mint, h.metrics.walTruncateFail, h.metrics.checkpointDeleteFail); err != nil {
 		return errors.Wrap(err, "create checkpoint")
 	}
 	h.metrics.walTruncateDuration.Observe(time.Since(start).Seconds())