tracing: fix lazytag export for Jaegar/otel

Previously, lazytags would not render in the otel export to
Jaeger. This was because otel doesn't support an on-demand
nature of the lazytags in its interface.

This change manually adds lazytags as otel tags before an otel
span is finished, allowing them to render in Jaeger after they
are exported.

Resolves: cockroachdb#85166

Release note: None

pkg/util/tracing/crdbspan.go

@@ -965,6 +965,38 @@ func (s *crdbSpan) getRecordingNoChildrenLocked(
 	return rs
 }
 
+// getLazyTagsKvLocked returns a list of lazy tags as a
+// slice of attribute.KeyValue.
+//
+// It flattens lazy tags with children and attaches the parent's
+// key as a prefix on each child.
+func (s *crdbSpan) getLazyTagsKvLocked() []attribute.KeyValue {
+	var lazyTagsKv []attribute.KeyValue
+	for _, kv := range s.mu.lazyTags {
+		switch v := kv.Value.(type) {
+		case LazyTag:
+			for _, tag := range v.Render() {
+				lazyTagsKv = append(lazyTagsKv, attribute.KeyValue{
+					// Format string with the parent key as prefix.
+					Key:   attribute.Key(fmt.Sprintf("%s-%s", kv.Key, tag.Key)),
+					Value: tag.Value,
+				})
+			}
+		case fmt.Stringer:
+			lazyTagsKv = append(lazyTagsKv, attribute.KeyValue{
+				Key:   attribute.Key(kv.Key),
+				Value: attribute.StringValue(v.String()),
+			})
+		default:
+			lazyTagsKv = append(lazyTagsKv, attribute.KeyValue{
+				Key:   attribute.Key(kv.Key),
+				Value: attribute.StringValue(fmt.Sprintf("<can't render %T>", kv.Value)),
+			})
+		}
+	}
+	return lazyTagsKv
+}
+
 // addChildLocked adds a child to the receiver.
 //
 // The receiver's lock must be held.

pkg/util/tracing/span_inner.go

@@ -104,8 +104,13 @@ func (s *spanInner) Finish() {
 	}
 
 	if s.otelSpan != nil {
+		// Serialize the lazy tags.
+		s.crdb.mu.Lock()
+		defer s.crdb.mu.Unlock()
+		s.otelSpan.SetAttributes(s.crdb.getLazyTagsKvLocked()...)
 		s.otelSpan.End()
 	}
+
 	if s.netTr != nil {
 		s.netTr.Finish()
 	}

pkg/util/tracing/tracer_test.go

@@ -397,14 +397,38 @@ func TestOtelTracer(t *testing.T) {
 	}
 
 	sp2 := tr.StartSpan("child", WithRemoteParentFromSpanMeta(wireSpanMeta))
-	defer sp2.Finish()
+	sp2.SetLazyTag("lazy expanding tag", testExpandingTag{})
+	sp2.SetLazyTag("lazy tag", testStringerLazyTag{})
 
 	rs := sr.Started()
 	require.Len(t, rs, 2)
 	require.Len(t, rs[0].Events(), 1)
 	require.Equal(t, "hello", rs[0].Events()[0].Name)
 	require.Equal(t, rs[0].SpanContext().TraceID(), rs[1].Parent().TraceID())
 	require.Equal(t, rs[0].SpanContext().SpanID(), rs[1].Parent().SpanID())
+
+	sp2.Finish()
+	rs2 := sr.Ended()
+
+	expectedAttributes := []attribute.KeyValue{
+		{
+			Key:   attribute.Key("lazy expanding tag-exp1"),
+			Value: attribute.IntValue(1),
+		},
+		{
+			Key:   attribute.Key("lazy expanding tag-exp2"),
+			Value: attribute.IntValue(2),
+		},
+		{
+			Key:   attribute.Key("lazy tag"),
+			Value: attribute.StringValue("lazy stringer"),
+		},
+	}
+	actualAttributes := rs2[0].Attributes()
+
+	require.Len(t, rs2, 1)
+	require.Len(t, actualAttributes, 3)
+	require.Equal(t, expectedAttributes, actualAttributes)
 }
 
 func TestTracer_RegistryMaxSize(t *testing.T) {