[pkg/otlp/logs] Refactor to use attributes.Translator

DataDog · Dec 20, 2023 · 3e3146b · 3e3146b
1 parent bffb6c3
commit 3e3146b
Show file tree

Hide file tree

Showing 5 changed files with 112 additions and 32 deletions.
diff --git a/.chloggen/mx-psi_logs-translator.yaml b/.chloggen/mx-psi_logs-translator.yaml
@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component (e.g. pkg/quantile)
+component: pkg/otlp/logs
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Pass `attributes.Translator` explicitly to `logs.Translator`
+
+# The PR related to this change
+issues: [231]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
diff --git a/pkg/otlp/logs/go.mod b/pkg/otlp/logs/go.mod
@@ -9,6 +9,7 @@ require (
 	go.opentelemetry.io/collector/component v0.91.0
 	go.opentelemetry.io/collector/pdata v1.0.0
 	go.opentelemetry.io/collector/semconv v0.91.0
+	go.opentelemetry.io/otel v1.21.0
 	go.uber.org/zap v1.26.0
 )
 
@@ -31,7 +32,6 @@ require (
 	go.opentelemetry.io/collector/config/configtelemetry v0.91.0 // indirect
 	go.opentelemetry.io/collector/confmap v0.91.0 // indirect
 	go.opentelemetry.io/collector/featuregate v1.0.0 // indirect
-	go.opentelemetry.io/otel v1.21.0 // indirect
 	go.opentelemetry.io/otel/metric v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect

diff --git a/pkg/otlp/logs/transform.go b/pkg/otlp/logs/transform.go
@@ -64,6 +64,10 @@ const (
 // Deprecated: use Translator instead.
 func Transform(lr plog.LogRecord, res pcommon.Resource, logger *zap.Logger) datadogV2.HTTPLogItem {
 	host, service := extractHostNameAndServiceName(res.Attributes(), lr.Attributes())
+	return transform(lr, host, service, res, logger)
+}
+
+func transform(lr plog.LogRecord, host, service string, res pcommon.Resource, logger *zap.Logger) datadogV2.HTTPLogItem {
 	l := datadogV2.HTTPLogItem{
 		AdditionalProperties: make(map[string]string),
 	}
@@ -198,8 +202,8 @@ func extractHostNameAndServiceName(resourceAttrs pcommon.Map, logAttrs pcommon.M
 	if src, ok := attributes.SourceFromAttrs(resourceAttrs); ok && src.Kind == source.HostnameKind {
 		host = src.Identifier
 	}
-	// hostName is blank from resource
-	// we need to derive from log attributes
+	// HACK: Check for host in log record attributes if not present in resource attributes.
+	// This is not aligned with the specification and will be removed in the future.
 	if host == "" {
 		if src, ok := attributes.SourceFromAttrs(logAttrs); ok && src.Kind == source.HostnameKind {
 			host = src.Identifier
@@ -208,8 +212,8 @@ func extractHostNameAndServiceName(resourceAttrs pcommon.Map, logAttrs pcommon.M
 	if s, ok := resourceAttrs.Get(conventions.AttributeServiceName); ok {
 		service = s.AsString()
 	}
-	// serviceName is blank from resource
-	// we need to derive from log attributes
+	// HACK: Check for service in log record attributes if not present in resource attributes.
+	// This is not aligned with the specification and will be removed in the future.
 	if service == "" {
 		if s, ok := logAttrs.Get(conventions.AttributeServiceName); ok {
 			service = s.AsString()

diff --git a/pkg/otlp/logs/transform_test.go b/pkg/otlp/logs/transform_test.go
@@ -15,20 +15,23 @@
 package logs
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
 	conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
 	"go.uber.org/zap/zaptest"
 )
 
-func TestTransform(t *testing.T) {
-	testLogger := zaptest.NewLogger(t)
+func TestTranslator(t *testing.T) {
 	traceID := [16]byte{0x08, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0, 0x0, 0x0, 0x0, 0x0a}
 	var spanID [8]byte
 	copy(spanID[:], traceID[8:])
@@ -57,7 +60,7 @@ func TestTransform(t *testing.T) {
 				res: pcommon.NewResource(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"app":              "test",
@@ -83,7 +86,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString("service:otlp_col"),
+				Ddtags:  datadog.PtrString("service:otlp_col,otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -112,7 +115,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString("service:otlp_col,foo:bar"),
+				Ddtags:  datadog.PtrString("service:otlp_col,foo:bar,otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -140,7 +143,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -169,7 +172,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -202,7 +205,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -235,7 +238,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -268,7 +271,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -301,7 +304,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -336,7 +339,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -371,7 +374,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -403,7 +406,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString("service:otlp_col"),
+				Ddtags:  datadog.PtrString("service:otlp_col,otel_source:test"),
 				Message: *datadog.PtrString(""),
 				Service: datadog.PtrString("otlp_col"),
 				AdditionalProperties: map[string]string{
@@ -432,7 +435,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"app":              "test",
@@ -471,7 +474,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"root.nest1.nest2":         "val",
@@ -495,7 +498,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"status": "",
@@ -545,7 +548,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"nest1.nest2.nest3.nest4.nest5.nest6.nest7.nest8.nest9.nest10": "{\"nest11\":{\"nest12\":\"ok\"}}",
@@ -569,7 +572,7 @@ func TestTransform(t *testing.T) {
 				}(),
 			},
 			want: datadogV2.HTTPLogItem{
-				Ddtags:  datadog.PtrString(""),
+				Ddtags:  datadog.PtrString("otel_source:test"),
 				Message: *datadog.PtrString(""),
 				AdditionalProperties: map[string]string{
 					"status":           "debug",
@@ -580,9 +583,24 @@ func TestTransform(t *testing.T) {
 			},
 		},
 	}
+
+	set := componenttest.NewNopTelemetrySettings()
+	set.Logger = zaptest.NewLogger(t)
+	attributesTranslator, err := attributes.NewTranslator(set)
+	require.NoError(t, err)
+	translator, err := NewTranslator(set, attributesTranslator, "test")
+	require.NoError(t, err)
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := Transform(tt.args.lr, tt.args.res, testLogger)
+			logs := plog.NewLogs()
+			rl := logs.ResourceLogs().AppendEmpty()
+			tt.args.res.MoveTo(rl.Resource())
+			tt.args.lr.CopyTo(rl.ScopeLogs().AppendEmpty().LogRecords().AppendEmpty())
+
+			payloads := translator.MapLogs(context.Background(), logs)
+			require.Len(t, payloads, 1)
+			got := payloads[0]
 
 			gs, err := got.MarshalJSON()
 			if err != nil {

diff --git a/pkg/otlp/logs/translator.go b/pkg/otlp/logs/translator.go
@@ -15,40 +15,82 @@
 package logs
 
 import (
+	"context"
+
 	"github.com/DataDog/datadog-api-client-go/v2/api/datadogV2"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
+	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
+	conventions "go.opentelemetry.io/collector/semconv/v1.6.1"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+var (
+	signalTypeSet = attribute.NewSet(attribute.String("signal", "logs"))
 )
 
 // Translator of OTLP logs to Datadog format
 type Translator struct {
-	set     component.TelemetrySettings
-	otelTag string
+	set                  component.TelemetrySettings
+	attributesTranslator *attributes.Translator
+	otelTag              string
 }
 
 // NewTranslator returns a new Translator
-func NewTranslator(set component.TelemetrySettings, otelSource string) (*Translator, error) {
+func NewTranslator(set component.TelemetrySettings, attributesTranslator *attributes.Translator, otelSource string) (*Translator, error) {
 	return &Translator{
-		set:     set,
-		otelTag: "otel_source:" + otelSource,
+		set:                  set,
+		attributesTranslator: attributesTranslator,
+		otelTag:              "otel_source:" + otelSource,
 	}, nil
 }
 
+func (t *Translator) hostNameAndServiceNameFromResource(ctx context.Context, res pcommon.Resource) (host string, service string) {
+	if src, ok := t.attributesTranslator.ResourceToSource(ctx, res, signalTypeSet); ok && src.Kind == source.HostnameKind {
+		host = src.Identifier
+	}
+	if s, ok := res.Attributes().Get(conventions.AttributeServiceName); ok {
+		service = s.AsString()
+	}
+	return host, service
+}
+
+func (t *Translator) hostFromAttributes(ctx context.Context, attrs pcommon.Map) string {
+	if src, ok := t.attributesTranslator.AttributesToSource(ctx, attrs); ok && src.Kind == source.HostnameKind {
+		return src.Identifier
+	}
+	return ""
+}
+
 // MapLogs from OTLP format to Datadog format.
-func (t *Translator) MapLogs(ld plog.Logs) []datadogV2.HTTPLogItem {
+func (t *Translator) MapLogs(ctx context.Context, ld plog.Logs) []datadogV2.HTTPLogItem {
 	rsl := ld.ResourceLogs()
 	var payloads []datadogV2.HTTPLogItem
 	for i := 0; i < rsl.Len(); i++ {
 		rl := rsl.At(i)
 		sls := rl.ScopeLogs()
 		res := rl.Resource()
+		host, service := t.hostNameAndServiceNameFromResource(ctx, res)
 		for j := 0; j < sls.Len(); j++ {
 			sl := sls.At(j)
 			lsl := sl.LogRecords()
 			// iterate over Logs
 			for k := 0; k < lsl.Len(); k++ {
 				log := lsl.At(k)
-				payload := Transform(log, res, t.set.Logger)
+				// HACK: Check for host and service in log record attributes
+				// This is not aligned with the specification and will be removed in the future.
+				if host == "" {
+					host = t.hostFromAttributes(ctx, log.Attributes())
+				}
+				if service == "" {
+					if s, ok := log.Attributes().Get(conventions.AttributeServiceName); ok {
+						service = s.AsString()
+					}
+				}
+
+				payload := transform(log, host, service, res, t.set.Logger)
 				ddtags := payload.GetDdtags()
 				if ddtags != "" {
 					payload.SetDdtags(ddtags + "," + t.otelTag)