Merge pull request #1150 from flipt-io/gm/evaluation-metrics

feat(server): add metrics around flag evaluations

go.mod

@@ -36,7 +36,7 @@ require (
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4
 	go.opentelemetry.io/otel v1.11.1
 	go.opentelemetry.io/otel/exporters/jaeger v1.11.1
-	go.opentelemetry.io/otel/exporters/prometheus v0.33.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede
 	go.opentelemetry.io/otel/metric v0.33.0
 	go.opentelemetry.io/otel/sdk v1.11.1
 	go.opentelemetry.io/otel/sdk/metric v0.33.0
@@ -84,7 +84,7 @@ require (
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/moby/sys/mount v0.3.3 // indirect
 	github.com/moby/sys/mountinfo v0.6.2 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect

go.sum

@@ -214,6 +214,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -918,6 +919,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
@@ -1350,6 +1353,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
 go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60=
 go.opentelemetry.io/otel/exporters/prometheus v0.33.0/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI=
+go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede h1:cHViU9YS+sHIhsSx/XPi+gEI6byWA9pM3yywAaqmAkw=
+go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E=
 go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI=

internal/server/evaluator.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	errs "go.flipt.io/flipt/errors"
 	"go.flipt.io/flipt/internal/storage"
@@ -60,15 +61,38 @@ func (s *Server) batchEvaluate(ctx context.Context, r *flipt.BatchEvaluationRequ
 	return &res, nil
 }
 
-func (s *Server) evaluate(ctx context.Context, r *flipt.EvaluationRequest) (*flipt.EvaluationResponse, error) {
+func (s *Server) evaluate(ctx context.Context, r *flipt.EvaluationRequest) (resp *flipt.EvaluationResponse, err error) {
 	var (
-		resp = &flipt.EvaluationResponse{
-			RequestId:      r.RequestId,
-			EntityId:       r.EntityId,
-			RequestContext: r.Context,
-			FlagKey:        r.FlagKey,
-		}
+		startTime = time.Now().UTC()
+		flagAttr  = attributeFlag.String(r.FlagKey)
 	)
+	evaluationsTotal.Add(ctx, 1, flagAttr)
+	defer func() {
+		if err == nil {
+			evaluationResultsTotal.Add(ctx, 1,
+				flagAttr,
+				attributeMatch.Bool(resp.Match),
+				attributeSegment.String(resp.SegmentKey),
+				attributeReason.String(resp.Reason.String()),
+				attributeValue.String(resp.Value),
+			)
+		} else {
+			evaluationErrorsTotal.Add(ctx, 1, flagAttr)
+		}
+
+		evaluationLatency.Record(
+			ctx,
+			float64(time.Since(startTime).Nanoseconds())/1e6,
+			flagAttr,
+		)
+	}()
+
+	resp = &flipt.EvaluationResponse{
+		RequestId:      r.RequestId,
+		EntityId:       r.EntityId,
+		RequestContext: r.Context,
+		FlagKey:        r.FlagKey,
+	}
 
 	flag, err := s.store.GetFlag(ctx, r.FlagKey)
 	if err != nil {

internal/server/metrics.go

@@ -3,19 +3,63 @@ package server
 import (
 	"github.com/prometheus/client_golang/prometheus"
 	"go.flipt.io/flipt/internal/metrics"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric/instrument"
+	"go.opentelemetry.io/otel/metric/instrument/syncfloat64"
+	"go.opentelemetry.io/otel/metric/unit"
 )
 
 const (
-	namespace = "flipt"
-	subsystem = "server"
+	namespace            = "flipt"
+	serverSubsystem      = "server"
+	evaluationsSubsystem = "evaluations"
 )
 
 // Prometheus metrics used throughout the server package
+var errorsTotal = metrics.MustSyncInt64().
+	Counter(
+		prometheus.BuildFQName(namespace, serverSubsystem, "errors"),
+		instrument.WithDescription("The total number of server errors"),
+	)
+
+// Evaluation specific metrics
 var (
-	errorsTotal = metrics.MustSyncInt64().
-		Counter(
-			prometheus.BuildFQName(namespace, subsystem, "errors_total"),
-			instrument.WithDescription("The total number of server errors"),
+	evaluationsTotal = metrics.MustSyncInt64().
+				Counter(
+			prometheus.BuildFQName(namespace, evaluationsSubsystem, "requests"),
+			instrument.WithDescription("The total number of requested evaluations"),
+		)
+
+	evaluationErrorsTotal = metrics.MustSyncInt64().
+				Counter(
+			prometheus.BuildFQName(namespace, evaluationsSubsystem, "errors"),
+			instrument.WithDescription("The total number of requested evaluations"),
+		)
+
+	evaluationResultsTotal = metrics.MustSyncInt64().
+				Counter(
+			prometheus.BuildFQName(namespace, evaluationsSubsystem, "results"),
+			instrument.WithDescription("Count of results including match, flag, segment, reason and value attributes"),
 		)
+
+	evaluationLatency syncfloat64.Histogram
+
+	attributeMatch   = attribute.Key("match")
+	attributeFlag    = attribute.Key("flag")
+	attributeSegment = attribute.Key("segment")
+	attributeReason  = attribute.Key("reason")
+	attributeValue   = attribute.Key("value")
 )
+
+func init() {
+	var err error
+	evaluationLatency, err = metrics.Meter.SyncFloat64().Histogram(
+		prometheus.BuildFQName(namespace, evaluationsSubsystem, "latency"),
+		instrument.WithDescription("The latency of inidividual evaluations in milliseconds"),
+		instrument.WithUnit(unit.Milliseconds),
+	)
+
+	if err != nil {
+		panic(err)
+	}
+}