feat: otel traces + interceptor (#2144)

otel interceptor that propagates spans and logs FTL attributes
block · Jul 29, 2024 · f31dd1f · f31dd1f
1 parent 4857e9c
commit f31dd1f
Show file tree

Hide file tree

Showing 8 changed files with 435 additions and 27 deletions.
diff --git a/backend/controller/controller.go b/backend/controller/controller.go
@@ -1017,6 +1017,7 @@ func (s *Service) callWithRequest(
 		}
 	}
 
+	ctx = rpc.WithRequestKey(ctx, requestKey)
 	ctx = rpc.WithVerbs(ctx, append(callers, verbRef))
 	headers.AddCaller(req.Header(), schema.RefFromProto(req.Msg.Verb))
 

diff --git a/examples/go/echo/go.mod b/examples/go/echo/go.mod
@@ -9,7 +9,6 @@ require github.com/TBD54566975/ftl v0.248.0
 require (
 	connectrpc.com/connect v1.16.2 // indirect
 	connectrpc.com/grpcreflect v1.2.0 // indirect
-	connectrpc.com/otelconnect v0.7.1 // indirect
 	github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
 	github.com/alecthomas/concurrency v0.0.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect

diff --git a/examples/go/echo/go.sum b/examples/go/echo/go.sum
diff --git a/go.mod b/go.mod
@@ -5,7 +5,6 @@ go 1.22.2
 require (
 	connectrpc.com/connect v1.16.2
 	connectrpc.com/grpcreflect v1.2.0
-	connectrpc.com/otelconnect v0.7.1
 	github.com/BurntSushi/toml v1.4.0
 	github.com/TBD54566975/golang-tools v0.2.1
 	github.com/TBD54566975/scaffolder v1.0.0
@@ -55,6 +54,7 @@ require (
 	go.opentelemetry.io/otel/metric v1.28.0
 	go.opentelemetry.io/otel/sdk v1.28.0
 	go.opentelemetry.io/otel/sdk/metric v1.28.0
+	go.opentelemetry.io/otel/trace v1.28.0
 	go.uber.org/automaxprocs v1.5.3
 	golang.org/x/exp v0.0.0-20240707233637-46b078467d37
 	golang.org/x/mod v0.19.0
@@ -101,7 +101,6 @@ require (
 	github.com/stretchr/objx v0.5.2 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 // indirect
-	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	golang.org/x/tools v0.23.0 // indirect
 	gotest.tools/v3 v3.5.1 // indirect

diff --git a/go.sum b/go.sum
diff --git a/internal/observability/client.go b/internal/observability/client.go
@@ -9,9 +9,10 @@ import (
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
+	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/metric"
 	"go.opentelemetry.io/otel/sdk/resource"
-	"go.opentelemetry.io/otel/sdk/trace"
+	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
 
 	"github.com/TBD54566975/ftl/internal/log"
@@ -68,8 +69,13 @@ func Init(ctx context.Context, serviceName, serviceVersion string, config Config
 	if err != nil {
 		return fmt.Errorf("failed to create OTEL trace exporter: %w", err)
 	}
-	traceProvider := trace.NewTracerProvider(trace.WithBatcher(otelTraceExporter), trace.WithResource(res))
+
+	traceProvider := sdktrace.NewTracerProvider(
+		sdktrace.WithBatcher(otelTraceExporter),
+		sdktrace.WithResource(res),
+	)
 	otel.SetTracerProvider(traceProvider)
+	otel.SetTextMapPropagator(propagation.TraceContext{})
 
 	return nil
 }
diff --git a/internal/rpc/context.go b/internal/rpc/context.go
@@ -7,7 +7,6 @@ import (
 	"runtime/debug"
 
 	"connectrpc.com/connect"
-	"connectrpc.com/otelconnect"
 	"github.com/alecthomas/types/optional"
 	"golang.org/x/mod/semver"
 
@@ -74,13 +73,13 @@ func RequestKeyFromContext(ctx context.Context) (optional.Option[model.RequestKe
 	return optional.Some(key), nil
 }
 
-// WithRequestName adds the request key to the context.
-func WithRequestName(ctx context.Context, key model.RequestKey) context.Context {
+// WithRequestKey adds the request key to the context.
+func WithRequestKey(ctx context.Context, key model.RequestKey) context.Context {
 	return context.WithValue(ctx, requestIDKey{}, key.String())
 }
 
 func DefaultClientOptions(level log.Level) []connect.ClientOption {
-	interceptors := []connect.Interceptor{PanicInterceptor(), MetadataInterceptor(log.Debug), otelInterceptor()}
+	interceptors := []connect.Interceptor{PanicInterceptor(), MetadataInterceptor(log.Debug), OtelInterceptor()}
 	if ftl.Version != "dev" {
 		interceptors = append(interceptors, versionInterceptor{})
 	}
@@ -91,21 +90,13 @@ func DefaultClientOptions(level log.Level) []connect.ClientOption {
 }
 
 func DefaultHandlerOptions() []connect.HandlerOption {
-	interceptors := []connect.Interceptor{PanicInterceptor(), MetadataInterceptor(log.Debug), otelInterceptor()}
+	interceptors := []connect.Interceptor{PanicInterceptor(), MetadataInterceptor(log.Debug), OtelInterceptor()}
 	if ftl.Version != "dev" {
 		interceptors = append(interceptors, versionInterceptor{})
 	}
 	return []connect.HandlerOption{connect.WithInterceptors(interceptors...)}
 }
 
-func otelInterceptor() connect.Interceptor {
-	otel, err := otelconnect.NewInterceptor(otelconnect.WithTrustRemote(), otelconnect.WithoutServerPeerAttributes())
-	if err != nil {
-		panic(err)
-	}
-	return otel
-}
-
 // PanicInterceptor intercepts panics and logs them.
 func PanicInterceptor() connect.Interceptor {
 	return &panicInterceptor{}
@@ -255,7 +246,7 @@ func propagateHeaders(ctx context.Context, isClient bool, header http.Header) (c
 		if key, ok, err := headers.GetRequestKey(header); err != nil {
 			return nil, err
 		} else if ok {
-			ctx = WithRequestName(ctx, key)
+			ctx = WithRequestKey(ctx, key)
 		}
 	}
 	return ctx, nil
-Original file line number
+Diff line change
@@ Expand Up / @@ -1017,6 +1017,7 @@ func (s *Service) callWithRequest( @@
     		}
     	}
+    	ctx = rpc.WithRequestKey(ctx, requestKey)
     	ctx = rpc.WithVerbs(ctx, append(callers, verbRef))
     	headers.AddCaller(req.Header(), schema.RefFromProto(req.Msg.Verb))
@@ Expand Down @@