feat: Zipkin otel

README.md

@@ -90,6 +90,7 @@ Flipt supports use cases such as:
 - :rocket: **Speed** - Since Flipt is co-located with your existing services, you do not have to communicate across the internet which can add excessive latency and slow down your applications.
 - :white_check_mark: **Simplicity** - Flipt is a single binary with no external dependencies by default.
 - :thumbsup: **Compatibility** - REST, GRPC, MySQL, Postgres, CockroachDB, SQLite, Redis... Flipt supports it all.
+- :eyes: **Observability** - Flipt integrates with [Prometheus](https://prometheus.io/) and [OpenTelemetry](https://opentelemetry.io/) to provide metrics and tracing. We support sending trace data to [Jaeger](https://www.jaegertracing.io/) and [Zipkin](https://zipkin.io/) backends.
 
 <br clear="both"/>
 

config/flipt.schema.cue

@@ -132,14 +132,19 @@ import "strings"
 
 	#tracing: {
 		enabled?: bool | *false
-		backend?: "jaeger" | *"jaeger"
+		backend?: "jaeger" | "zipkin" | *"jaeger"
 
 		// Jaeger
 		jaeger?: {
 			enabled?: bool | *false
 			host?:    string | *"localhost"
 			port?:    int | *6831
 		}
+
+		// Zipkin
+		zipkin?: {
+			endpoint?:    string | *"http://localhost:9411/api/v2/spans"
+		}
 	}
 
 	#ui: enabled?: bool | *true

config/flipt.schema.json

@@ -441,7 +441,7 @@
         },
         "backend": {
           "type": "string",
-          "enum": ["jaeger"],
+          "enum": ["jaeger", "zipkin"],
           "default": "jaeger"
         },
         "jaeger": {
@@ -463,6 +463,17 @@
             }
           },
           "title": "Jaeger"
+        },
+        "zipkin": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "endpoint": {
+              "type": "string",
+              "default": "http://localhost:9411/api/v2/spans"
+            }
+          },
+          "title": "Zipkin"
         }
       },
       "title": "Tracing"

examples/tracing/README.md

@@ -1,22 +1,10 @@
-# Tracing Example
+# Tracing Examples
 
-This example shows how you can run Flipt with a Jaeger/Open Telemetry sidecar application in Docker.
+This directory contains examples of how to setup Flipt to export traces using the [OpenTelemetry](https://opentelemetry.io/) integration to configured backends.
 
-!['Jaeger Example'](../images/jaeger.png)
+For more information on how to setup and enable tracing, see the [Observability](https://www.flipt.io/docs/configuration/observability) documentation.
 
-## Requirements
+## Contents
 
-To run this example application you'll need:
-
-* [Docker](https://docs.docker.com/install/)
-* [docker-compose](https://docs.docker.com/compose/install/)
-
-## Running the Example
-
-1. Run `docker-compose up` from this directory
-1. Open the Flipt UI (default: [http://localhost:8080](http://localhost:8080))
-1. Create some sample data: Flags/Segments/etc.
-1. Open the Jaeger UI (default: [http://localhost:16686](http://localhost:16686))
-1. Select 'flipt' from the Service dropdown
-1. Click 'Find Traces'
-1. You should see a list of traces to explore
+* [Jaeger Backend](jaeger/README.md)
+* [Zipkin Backend](zipkin/README.md)

examples/tracing/jaeger/README.md

@@ -0,0 +1,22 @@
+# Jaeger Example
+
+This example shows how you can run Flipt with a Jaeger application in Docker.
+
+!['Jaeger Example'](../../images/jaeger.jpg)
+
+## Requirements
+
+To run this example application you'll need:
+
+* [Docker](https://docs.docker.com/install/)
+* [docker-compose](https://docs.docker.com/compose/install/)
+
+## Running the Example
+
+1. Run `docker-compose up` from this directory
+1. Open the Flipt UI (default: [http://localhost:8080](http://localhost:8080))
+1. Create some sample data: Flags/Segments/etc. Perform a few evaluations in the Console.
+1. Open the Jaeger UI (default: [http://localhost:16686](http://localhost:16686))
+1. Select 'flipt' from the Service dropdown
+1. Click 'Find Traces'
+1. You should see a list of traces to explore

examples/tracing/docker-compose.yml → examples/tracing/jaeger/docker-compose.yml

@@ -13,15 +13,13 @@ services:
       - "14250:14250"
       - "14268:14268"
       - "14269:14269"
-      - "9411:9411"
     networks:
       - flipt_network
     environment:
-      - "COLLECTOR_ZIPKIN_HOST_PORT=:9411"
       - "COLLECTOR_OTLP_ENABLED=true"
 
   flipt:
-    image: flipt/flipt:latest
+    build: ../../..
     depends_on:
       - jaeger
     ports:
@@ -30,7 +28,8 @@ services:
       - flipt_network
     environment:
       - "FLIPT_LOG_LEVEL=debug"
-      - "FLIPT_TRACING_JAEGER_ENABLED=true"
+      - "FLIPT_TRACING_ENABLED=true"
+      - "FLIPT_TRACING_BACKEND=jaeger"
       - "FLIPT_TRACING_JAEGER_HOST=jaeger"
 
 networks:

examples/tracing/zipkin/README.md

@@ -0,0 +1,22 @@
+# Zipkin Example
+
+This example shows how you can run Flipt with a Zipkin application in Docker.
+
+!['Zipkin Example'](../../images/zipkin.png)
+
+## Requirements
+
+To run this example application you'll need:
+
+* [Docker](https://docs.docker.com/install/)
+* [docker-compose](https://docs.docker.com/compose/install/)
+
+## Running the Example
+
+1. Run `docker-compose up` from this directory
+1. Open the Flipt UI (default: [http://localhost:8080](http://localhost:8080))
+1. Create some sample data: Flags/Segments/etc. Perform a few evaluations in the Console.
+1. Open the Zipkin UI (default: [http://localhost:9411](http://localhost:9411))
+1. Select `serviceName=flipt` from the search box
+1. Click 'Run Query'
+1. You should see a list of traces to explore

examples/tracing/zipkin/docker-compose.yml

@@ -0,0 +1,26 @@
+version: "3"
+
+services:
+  zipkin:
+    image: openzipkin/zipkin-slim
+    ports:
+      - "9411:9411"
+    networks:
+      - flipt_network
+
+  flipt:
+    build: ../../..
+    depends_on:
+      - zipkin
+    ports:
+      - "8080:8080"
+    networks:
+      - flipt_network
+    environment:
+      - "FLIPT_LOG_LEVEL=debug"
+      - "FLIPT_TRACING_ENABLED=true"
+      - "FLIPT_TRACING_BACKEND=zipkin"
+      - "FLIPT_TRACING_ZIPKIN_ENDPOINT=http://zipkin:9411/api/v2/spans"
+
+networks:
+  flipt_network:

go.mod

@@ -41,6 +41,7 @@ require (
 	go.opentelemetry.io/otel v1.12.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.12.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.34.0
+	go.opentelemetry.io/otel/exporters/zipkin v1.12.0
 	go.opentelemetry.io/otel/metric v0.34.0
 	go.opentelemetry.io/otel/sdk v1.12.0
 	go.opentelemetry.io/otel/sdk/metric v0.34.0
@@ -84,10 +85,10 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/klauspost/compress v1.15.11 // indirect
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
@@ -102,6 +103,7 @@ require (
 	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/opencontainers/runc v1.1.3 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/openzipkin/zipkin-go v0.4.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -123,7 +125,7 @@ require (
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect
+	golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect
 	golang.org/x/oauth2 v0.3.0 // indirect
 	golang.org/x/sys v0.4.0 // indirect
 	golang.org/x/text v0.6.0 // indirect

go.sum

@@ -711,8 +711,9 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX
 github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
 github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -822,8 +823,9 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs
 github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
-github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c=
+github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1020,6 +1022,8 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A=
+github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
@@ -1098,7 +1102,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
-github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1294,6 +1298,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
 go.opentelemetry.io/otel/exporters/prometheus v0.34.0 h1:L5D+HxdaC/ORB47ribbTBbkXRZs9JzPjq0EoIOMWncM=
 go.opentelemetry.io/otel/exporters/prometheus v0.34.0/go.mod h1:6gUoJyfhoWqF0tOLaY0ZmKgkQRcvEQx6p5rVlKHp3s4=
+go.opentelemetry.io/otel/exporters/zipkin v1.12.0 h1:0Pvqi4MAvQ28rGUFoH8uIaNKNCGVLNpn7TbRCkB1gP8=
+go.opentelemetry.io/otel/exporters/zipkin v1.12.0/go.mod h1:zrxX/glbLVD/EFi01JCn2W1KdWNrJb3FkBHF+oRPlXs=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
 go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8=
 go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8=
@@ -1359,8 +1365,8 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 h1:x8vtB3zMecnlqZIwJNUUpwYKYSqCz5jXbiyv0ZJJZeI=
+golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

internal/cmd/grpc.go

@@ -16,6 +16,7 @@ import (
 	"go.flipt.io/flipt/internal/server/cache/redis"
 	"go.flipt.io/flipt/internal/server/metadata"
 	middlewaregrpc "go.flipt.io/flipt/internal/server/middleware/grpc"
+	fliptotel "go.flipt.io/flipt/internal/server/otel"
 	"go.flipt.io/flipt/internal/storage"
 	authsql "go.flipt.io/flipt/internal/storage/auth/sql"
 	oplocksql "go.flipt.io/flipt/internal/storage/oplock/sql"
@@ -26,11 +27,11 @@ import (
 	"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/zipkin"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
-	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"google.golang.org/grpc"
@@ -133,15 +134,23 @@ func NewGRPCServer(
 
 	logger.Debug("store enabled", zap.Stringer("driver", driver))
 
-	var tracingProvider = trace.NewNoopTracerProvider()
+	var tracingProvider = fliptotel.NewNoopProvider()
+
+	if cfg.Tracing.Enabled {
+		var exp tracesdk.SpanExporter
+
+		switch cfg.Tracing.Backend {
+		case config.TracingJaeger:
+			exp, err = jaeger.New(jaeger.WithAgentEndpoint(
+				jaeger.WithAgentHost(cfg.Tracing.Jaeger.Host),
+				jaeger.WithAgentPort(strconv.FormatInt(int64(cfg.Tracing.Jaeger.Port), 10)),
+			))
+		case config.TracingZipkin:
+			exp, err = zipkin.New(cfg.Tracing.Zipkin.Endpoint)
+		}
 
-	if cfg.Tracing.Enabled && cfg.Tracing.Backend == config.TracingJaeger {
-		exp, err := jaeger.New(jaeger.WithAgentEndpoint(
-			jaeger.WithAgentHost(cfg.Tracing.Jaeger.Host),
-			jaeger.WithAgentPort(strconv.FormatInt(int64(cfg.Tracing.Jaeger.Port), 10)),
-		))
 		if err != nil {
-			return nil, err
+			return nil, fmt.Errorf("creating exporter: %w", err)
 		}
 
 		tracingProvider = tracesdk.NewTracerProvider(
@@ -157,7 +166,10 @@ func NewGRPCServer(
 			tracesdk.WithSampler(tracesdk.AlwaysSample()),
 		)
 
-		logger.Debug("otel tracing enabled", zap.String("backend", "jaeger"))
+		logger.Debug("otel tracing enabled", zap.String("backend", cfg.Tracing.Backend.String()))
+		server.onShutdown(func(ctx context.Context) error {
+			return tracingProvider.Shutdown(ctx)
+		})
 	}
 
 	otel.SetTracerProvider(tracingProvider)