persist OTel tracers in containers

coroot · Nov 29, 2024 · 6702469 · 6702469
1 parent 1dac238
commit 6702469
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 26 deletions.
diff --git a/containers/container.go b/containers/container.go
@@ -137,6 +137,8 @@ type Container struct {
 
 	logParsers map[string]*LogParser
 
+	tracer *tracing.Tracer
+
 	registry *Registry
 
 	lock sync.RWMutex
@@ -173,6 +175,8 @@ func NewContainer(id ContainerID, cg *cgroup.Cgroup, md *ContainerMetadata, pid
 
 		logParsers: map[string]*LogParser{},
 
+		tracer: tracing.GetContainerTracer(string(id)),
+
 		registry: registry,
 
 		done: make(chan struct{}),
@@ -649,7 +653,7 @@ func (c *Container) onL7Request(pid uint32, fd uint64, timestamp uint64, r *l7.R
 	}
 	stats := c.l7Stats.get(r.Protocol, conn.DestinationKey)
 
-	trace := tracing.NewTrace(string(c.id), conn.DestinationKey.ActualDestinationIfKnown())
+	trace := c.tracer.NewTrace(conn.DestinationKey.ActualDestinationIfKnown())
 	switch r.Protocol {
 	case l7.ProtocolHTTP:
 		stats.observe(r.Status.Http(), "", r.Duration)

diff --git a/tracing/tracing.go b/tracing/tracing.go
@@ -25,7 +25,10 @@ const (
 )
 
 var (
-	tracer func(containerId string) trace.Tracer
+	batcher             sdktrace.TracerProviderOption
+	commonResourceAttrs []attribute.KeyValue
+	agentVersion        string
+	initialized         bool
 )
 
 func Init(machineId, hostname, version string) {
@@ -54,43 +57,54 @@ func Init(machineId, hostname, version string) {
 		klog.Exitln(err)
 	}
 
-	batcher := sdktrace.WithBatcher(exporter)
-
-	tracer = func(containerId string) trace.Tracer {
-		provider := sdktrace.NewTracerProvider(
-			batcher,
-			sdktrace.WithResource(resource.NewWithAttributes(
-				semconv.SchemaURL,
-				semconv.HostName(hostname),
-				semconv.HostID(machineId),
-				semconv.ServiceName(common.ContainerIdToOtelServiceName(containerId)),
-				semconv.ContainerID(containerId),
-			)),
-		)
-		return provider.Tracer("coroot-node-agent", trace.WithInstrumentationVersion(version))
-	}
+	batcher = sdktrace.WithBatcher(exporter)
+	commonResourceAttrs = []attribute.KeyValue{semconv.HostName(hostname), semconv.HostID(machineId)}
+	agentVersion = version
+	initialized = true
 }
 
-type Trace struct {
-	containerId string
-	destination common.HostPort
-	commonAttrs []attribute.KeyValue
+type Tracer struct {
+	otel trace.Tracer
 }
 
-func NewTrace(containerId string, destination common.HostPort) *Trace {
-	if tracer == nil {
-		return nil
+func GetContainerTracer(containerId string) *Tracer {
+	if !initialized {
+		return &Tracer{otel: nil}
 	}
-	return &Trace{containerId: containerId, destination: destination, commonAttrs: []attribute.KeyValue{
+	provider := sdktrace.NewTracerProvider(
+		batcher,
+		sdktrace.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			append(
+				commonResourceAttrs,
+				semconv.ServiceName(common.ContainerIdToOtelServiceName(containerId)),
+				semconv.ContainerID(containerId),
+			)...,
+		)),
+	)
+	return &Tracer{otel: provider.Tracer("coroot-node-agent", trace.WithInstrumentationVersion(agentVersion))}
+}
+
+func (t *Tracer) NewTrace(destination common.HostPort) *Trace {
+	return &Trace{tracer: t, destination: destination, commonAttrs: []attribute.KeyValue{
 		semconv.NetPeerName(destination.Host()),
 		semconv.NetPeerPort(int(destination.Port())),
 	}}
 }
 
+type Trace struct {
+	tracer      *Tracer
+	destination common.HostPort
+	commonAttrs []attribute.KeyValue
+}
+
 func (t *Trace) createSpan(name string, duration time.Duration, error bool, attrs ...attribute.KeyValue) {
+	if t.tracer.otel == nil {
+		return
+	}
 	end := time.Now()
 	start := end.Add(-duration)
-	_, span := tracer(t.containerId).Start(nil, name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient))
+	_, span := t.tracer.otel.Start(nil, name, trace.WithTimestamp(start), trace.WithSpanKind(trace.SpanKindClient))
 	span.SetAttributes(attrs...)
 	span.SetAttributes(t.commonAttrs...)
 	if error {