From ad97f430b5ada2d3e040399447699bda7f72e6f3 Mon Sep 17 00:00:00 2001 From: George MacRorie Date: Mon, 21 Nov 2022 17:04:13 +0000 Subject: [PATCH] feat(server): add metrics around flag evaluations --- go.mod | 4 +-- go.sum | 5 ++++ internal/server/evaluator.go | 38 +++++++++++++++++++----- internal/server/metrics.go | 56 ++++++++++++++++++++++++++++++++---- 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 07951e9f12..c863cdf076 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 go.opentelemetry.io/otel v1.11.1 go.opentelemetry.io/otel/exporters/jaeger v1.11.1 - go.opentelemetry.io/otel/exporters/prometheus v0.33.0 + go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede go.opentelemetry.io/otel/metric v0.33.0 go.opentelemetry.io/otel/sdk v1.11.1 go.opentelemetry.io/otel/sdk/metric v0.33.0 @@ -84,7 +84,7 @@ require ( github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/moby/sys/mount v0.3.3 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect diff --git a/go.sum b/go.sum index 8c3623c206..aa0bb077a1 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -918,6 +919,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -1350,6 +1353,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE= go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60= go.opentelemetry.io/otel/exporters/prometheus v0.33.0/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI= +go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede h1:cHViU9YS+sHIhsSx/XPi+gEI6byWA9pM3yywAaqmAkw= +go.opentelemetry.io/otel/exporters/prometheus v0.33.1-0.20221021151223-ccbc38e66ede/go.mod h1:ZSmYfKdYWEdSDBB4njLBIwTf4AU2JNsH3n2quVQDebI= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E= go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= diff --git a/internal/server/evaluator.go b/internal/server/evaluator.go index debbb855f2..ec33e1c334 100644 --- a/internal/server/evaluator.go +++ b/internal/server/evaluator.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "time" errs "go.flipt.io/flipt/errors" "go.flipt.io/flipt/internal/storage" @@ -60,15 +61,38 @@ func (s *Server) batchEvaluate(ctx context.Context, r *flipt.BatchEvaluationRequ return &res, nil } -func (s *Server) evaluate(ctx context.Context, r *flipt.EvaluationRequest) (*flipt.EvaluationResponse, error) { +func (s *Server) evaluate(ctx context.Context, r *flipt.EvaluationRequest) (resp *flipt.EvaluationResponse, err error) { var ( - resp = &flipt.EvaluationResponse{ - RequestId: r.RequestId, - EntityId: r.EntityId, - RequestContext: r.Context, - FlagKey: r.FlagKey, - } + startTime = time.Now().UTC() + flagAttr = attributeFlag.String(r.FlagKey) ) + evaluationsTotal.Add(ctx, 1, flagAttr) + defer func() { + if err == nil { + evaluationResultsTotal.Add(ctx, 1, + flagAttr, + attributeMatch.Bool(resp.Match), + attributeSegment.String(resp.SegmentKey), + attributeReason.String(resp.Reason.String()), + attributeValue.String(resp.Value), + ) + } else { + evaluationErrorsTotal.Add(ctx, 1, flagAttr) + } + + evaluationLatency.Record( + ctx, + float64(time.Since(startTime).Nanoseconds())/1e6, + flagAttr, + ) + }() + + resp = &flipt.EvaluationResponse{ + RequestId: r.RequestId, + EntityId: r.EntityId, + RequestContext: r.Context, + FlagKey: r.FlagKey, + } flag, err := s.store.GetFlag(ctx, r.FlagKey) if err != nil { diff --git a/internal/server/metrics.go b/internal/server/metrics.go index 7bb5abcced..96ea9069bd 100644 --- a/internal/server/metrics.go +++ b/internal/server/metrics.go @@ -3,19 +3,63 @@ package server import ( "github.com/prometheus/client_golang/prometheus" "go.flipt.io/flipt/internal/metrics" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric/instrument" + "go.opentelemetry.io/otel/metric/instrument/syncfloat64" + "go.opentelemetry.io/otel/metric/unit" ) const ( - namespace = "flipt" - subsystem = "server" + namespace = "flipt" + serverSubsystem = "server" + evaluationsSubsystem = "evaluations" ) // Prometheus metrics used throughout the server package +var errorsTotal = metrics.MustSyncInt64(). + Counter( + prometheus.BuildFQName(namespace, serverSubsystem, "errors"), + instrument.WithDescription("The total number of server errors"), + ) + +// Evaluation specific metrics var ( - errorsTotal = metrics.MustSyncInt64(). - Counter( - prometheus.BuildFQName(namespace, subsystem, "errors_total"), - instrument.WithDescription("The total number of server errors"), + evaluationsTotal = metrics.MustSyncInt64(). + Counter( + prometheus.BuildFQName(namespace, evaluationsSubsystem, "requests"), + instrument.WithDescription("The total number of requested evaluations"), + ) + + evaluationErrorsTotal = metrics.MustSyncInt64(). + Counter( + prometheus.BuildFQName(namespace, evaluationsSubsystem, "errors"), + instrument.WithDescription("The total number of requested evaluations"), + ) + + evaluationResultsTotal = metrics.MustSyncInt64(). + Counter( + prometheus.BuildFQName(namespace, evaluationsSubsystem, "results"), + instrument.WithDescription("Count of results including match, flag, segment, reason and value attributes"), ) + + evaluationLatency syncfloat64.Histogram + + attributeMatch = attribute.Key("match") + attributeFlag = attribute.Key("flag") + attributeSegment = attribute.Key("segment") + attributeReason = attribute.Key("reason") + attributeValue = attribute.Key("value") ) + +func init() { + var err error + evaluationLatency, err = metrics.Meter.SyncFloat64().Histogram( + prometheus.BuildFQName(namespace, evaluationsSubsystem, "latency"), + instrument.WithDescription("The latency of inidividual evaluations in milliseconds"), + instrument.WithUnit(unit.Milliseconds), + ) + + if err != nil { + panic(err) + } +}