add more labels to prometheus output

docs/manual.md

@@ -270,7 +270,7 @@ The `application/vnd.goss-{output format}` media type can be used in the `Accept
   * `silent` - No output. Avoids exposing system information (e.g. when serving tests as a healthcheck endpoint)
 * `--format-options`, `-o` (output format option)
   * `perfdata` - Outputs Nagios "performance data". Applies to `nagios` output
-  * `verbose`  - Gives verbose output. Applies to `nagios` output
+  * `verbose`  - Gives verbose output. Applies to `nagios` and `prometheus` output
   * `pretty`   - Pretty printing for the `json` output
   * `sort`     - Sorts the results
 * `--loglevel level`, `-L level` - Goss logging verbosity level (default: `INFO`). `level` can be one of `TRACE | DEBUG | INFO | WARN | ERROR | FATAL`. Lower levels of tracing include all upper levels traces also (ie. INFO include WARN, ERROR and FATAL outputs).

outputs/prometheus.go

@@ -13,67 +13,38 @@ import (
 )
 
 const (
-	labelType    = "type"
-	labelOutcome = "outcome"
+	labelType       = "type"
+	labelOutcome    = "outcome"
+	labelResourceId = "resource_id"
 )
 
 var (
-	testOutcomes = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "goss",
-		Subsystem: "tests",
-		Name:      "outcomes_total",
-		Help:      "The number of test-outcomes from this run.",
-	}, []string{labelType, labelOutcome})
-	testDurations = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "goss",
-		Subsystem: "tests",
-		Name:      "outcomes_duration_milliseconds",
-		Help:      "The duration of tests from this run. Note; tests run concurrently.",
-	}, []string{labelType, labelOutcome})
-	runOutcomes = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "goss",
-		Subsystem: "tests",
-		Name:      "run_outcomes_total",
-		Help:      "The outcomes of this run as a whole.",
-	}, []string{labelOutcome})
-	runDuration = promauto.NewCounterVec(prometheus.CounterOpts{
-		Namespace: "goss",
-		Subsystem: "tests",
-		Name:      "run_duration_milliseconds",
-		Help:      "The end-to-end duration of this run.",
-	}, []string{labelOutcome})
+	registry      *prometheus.Registry
+	testOutcomes  *prometheus.CounterVec
+	testDurations *prometheus.CounterVec
+	runOutcomes   *prometheus.CounterVec
+	runDuration   *prometheus.CounterVec
 )
 
 // Prometheus renders metrics in prometheus.io text-format https://prometheus.io/docs/instrumenting/exposition_formats/#text-based-format
 type Prometheus struct{}
 
-// NewPrometheus creates and initialises a new Prometheus Outputer (to avoid missing metrics)
-func NewPrometheus() *Prometheus {
-	outputer := &Prometheus{}
-	outputer.init()
-	return outputer
-}
-
-func (r *Prometheus) init() {
-	// Avoid missing metrics: https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics
-	for resourceType := range resource.Resources() {
-		for _, outcome := range resource.HumanOutcomes() {
-			testOutcomes.WithLabelValues(resourceType, outcome).Add(0)
-			testDurations.WithLabelValues(resourceType, outcome).Add(0)
-		}
-	}
-	runOutcomes.WithLabelValues(labelOutcome).Add(0)
-	runDuration.WithLabelValues(labelOutcome).Add(0)
-}
-
 // ValidOptions is a list of valid format options for prometheus
 func (r Prometheus) ValidOptions() []*formatOption {
-	return []*formatOption{}
+	return []*formatOption{
+		{name: foVerbose},
+	}
 }
 
 // Output converts the results into the prometheus text-format.
 func (r Prometheus) Output(w io.Writer, results <-chan []resource.TestResult,
 	outConfig util.OutputConfig) (exitCode int) {
+	verbose := util.IsValueInList(foVerbose, outConfig.FormatOptions)
+
+	if registry == nil {
+		setupMetrics(verbose)
+	}
+
 	overallOutcome := resource.OutcomeUnknown
 	var startTime time.Time
 	for resultGroup := range results {
@@ -83,8 +54,14 @@ func (r Prometheus) Output(w io.Writer, results <-chan []resource.TestResult,
 			}
 			resType := strings.ToLower(tr.ResourceType)
 			outcome := tr.ToOutcome()
-			testOutcomes.WithLabelValues(resType, outcome).Inc()
-			testDurations.WithLabelValues(resType, outcome).Add(float64(tr.Duration.Milliseconds()))
+			if verbose {
+				resId := tr.ResourceId
+				testOutcomes.WithLabelValues(resType, outcome, resId).Inc()
+				testDurations.WithLabelValues(resType, outcome, resId).Add(float64(tr.Duration.Milliseconds()))
+			} else {
+				testOutcomes.WithLabelValues(resType, outcome).Inc()
+				testDurations.WithLabelValues(resType, outcome).Add(float64(tr.Duration.Milliseconds()))
+			}
 			if i == 0 || canChangeOverallOutcome(overallOutcome, outcome) {
 				overallOutcome = outcome
 			}
@@ -94,7 +71,7 @@ func (r Prometheus) Output(w io.Writer, results <-chan []resource.TestResult,
 	runOutcomes.WithLabelValues(overallOutcome).Inc()
 	runDuration.WithLabelValues(overallOutcome).Add(float64(time.Since(startTime).Milliseconds()))
 
-	metricsFamilies, err := prometheus.DefaultGatherer.Gather()
+	metricsFamilies, err := registry.Gather()
 	if err != nil {
 		return -1
 	}
@@ -109,6 +86,43 @@ func (r Prometheus) Output(w io.Writer, results <-chan []resource.TestResult,
 	return 0
 }
 
+func setupMetrics(verbose bool) {
+	registry = prometheus.NewRegistry()
+	factory := promauto.With(registry)
+
+	var testLabels []string
+	if verbose {
+		testLabels = []string{labelType, labelOutcome, labelResourceId}
+	} else {
+		testLabels = []string{labelType, labelOutcome}
+	}
+
+	testOutcomes = factory.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "outcomes_total",
+		Help:      "The number of test-outcomes from this run.",
+	}, testLabels)
+	testDurations = factory.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "outcomes_duration_milliseconds",
+		Help:      "The duration of tests from this run. Note; tests run concurrently.",
+	}, testLabels)
+	runOutcomes = factory.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "run_outcomes_total",
+		Help:      "The outcomes of this run as a whole.",
+	}, []string{labelOutcome})
+	runDuration = factory.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "goss",
+		Subsystem: "tests",
+		Name:      "run_duration_milliseconds",
+		Help:      "The end-to-end duration of this run.",
+	}, []string{labelOutcome})
+}
+
 func canChangeOverallOutcome(current, result string) bool {
 	switch current {
 	case resource.OutcomeSkip:

outputs/prometheus_test.go

@@ -15,6 +15,7 @@ import (
 func TestPrometheusOutput(t *testing.T) {
 	testCases := map[string]struct {
 		results         []resource.TestResult
+		formatOptions   []string
 		expectedMetrics []string
 	}{
 		"all-success-single-type": {
@@ -462,16 +463,52 @@ func TestPrometheusOutput(t *testing.T) {
 				`goss_tests_run_outcomes_total{outcome="unknown"} 1`,
 			},
 		},
+		"verbose": {
+			results: []resource.TestResult{
+				{
+					ResourceType: "Command",
+					ResourceId:   "some command here",
+					Duration:     10 * time.Millisecond,
+					Result:       resource.SUCCESS,
+				},
+				{
+					ResourceType: "Command",
+					ResourceId:   "something else here",
+					Duration:     10 * time.Millisecond,
+					Result:       resource.SUCCESS,
+				},
+				{
+					ResourceType: "File",
+					ResourceId:   "/path/to/file",
+					Duration:     10 * time.Millisecond,
+					Result:       resource.FAIL,
+				},
+			},
+			formatOptions: []string{foVerbose},
+			expectedMetrics: []string{
+				`goss_tests_outcomes_duration_milliseconds{outcome="pass",resource_id="some command here",type="command"} 10`,
+				`goss_tests_outcomes_total{outcome="pass",resource_id="some command here",type="command"} 1`,
+				`goss_tests_outcomes_duration_milliseconds{outcome="pass",resource_id="something else here",type="command"} 10`,
+				`goss_tests_outcomes_total{outcome="pass",resource_id="something else here",type="command"} 1`,
+				`goss_tests_outcomes_duration_milliseconds{outcome="fail",resource_id="/path/to/file",type="file"} 10`,
+				`goss_tests_outcomes_total{outcome="fail",resource_id="/path/to/file",type="file"} 1`,
+				`goss_tests_run_duration_milliseconds{outcome="fail"}`,
+				`goss_tests_run_outcomes_total{outcome="fail"} 1`,
+			},
+		},
 	}
 
 	for name, testCase := range testCases {
 		t.Run(name, func(t *testing.T) {
 			buf := &bytes.Buffer{}
 			outputer := &Prometheus{}
+			config := util.OutputConfig{
+				FormatOptions: testCase.formatOptions,
+			}
 
 			defer resetMetrics()
 
-			exitCode := outputer.Output(buf, makeResults(testCase.results...), util.OutputConfig{})
+			exitCode := outputer.Output(buf, makeResults(testCase.results...), config)
 			assert.Equal(t, 0, exitCode)
 
 			output := buf.String()
@@ -500,10 +537,7 @@ func makeResults(results ...resource.TestResult) <-chan []resource.TestResult {
 }
 
 func resetMetrics() {
-	testOutcomes.Reset()
-	testDurations.Reset()
-	runOutcomes.Reset()
-	runDuration.Reset()
+	registry = nil
 }
 
 func TestCanChangeOverallOutcome(t *testing.T) {