Fixed esexporter serialized document to avoid conflicting keys

.chloggen/fix_esexp-mapping.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: elasticsearchexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Duplicate Key in JSON 
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33454]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

exporter/elasticsearchexporter/exporter_test.go

@@ -43,15 +43,9 @@ func TestExporterLogs(t *testing.T) {
 		server := newESTestServer(t, func(docs []itemRequest) ([]itemResponse, error) {
 			rec.Record(docs)
 
-			var expectedDoc, actualDoc map[string]any
-			expected := []byte(`{"attrKey1":"abc","attrKey2":"def","application":"myapp","service":{"name":"myservice"},"error":{"stacktrace":"no no no no"},"agent":{"name":"otlp"},"@timestamp":"1970-01-01T00:00:00.000000000Z","message":"hello world"}`)
-			err := json.Unmarshal(expected, &expectedDoc)
-			require.NoError(t, err)
-
-			actual := docs[0].Document
-			err = json.Unmarshal(actual, &actualDoc)
-			require.NoError(t, err)
-			assert.Equal(t, expectedDoc, actualDoc)
+			expected := `{"@timestamp":"1970-01-01T00:00:00.000000000Z","agent":{"name":"otlp"},"application":"myapp","attrKey1":"abc","attrKey2":"def","error":{"stacktrace":"no no no no"},"message":"hello world","service":{"name":"myservice"}}`
+			actual := string(docs[0].Document)
+			assert.Equal(t, expected, actual)
 
 			return itemsAllOK(docs)
 		})

exporter/elasticsearchexporter/model.go

@@ -82,6 +82,11 @@ func (m *encodeModel) encodeLog(resource pcommon.Resource, record plog.LogRecord
 	}
 
 	var buf bytes.Buffer
+	if m.dedup {
+		document.Dedup()
+	} else if m.dedot {
+		document.Sort()
+	}
 	err := document.Serialize(&buf, m.dedot)
 	return buf.Bytes(), err
 }
@@ -104,12 +109,6 @@ func (m *encodeModel) encodeLogDefaultMode(resource pcommon.Resource, record plo
 	document.AddAttributes("Resource", resource.Attributes())
 	document.AddAttributes("Scope", scopeToAttributes(scope))
 
-	if m.dedup {
-		document.Dedup()
-	} else if m.dedot {
-		document.Sort()
-	}
-
 	return document
 
 }

exporter/elasticsearchexporter/model_test.go

@@ -218,6 +218,48 @@ func TestEncodeEvents(t *testing.T) {
 	}
 }
 
+func TestEncodeLogECSModeDuplication(t *testing.T) {
+	resource := pcommon.NewResource()
+	err := resource.Attributes().FromRaw(map[string]any{
+		semconv.AttributeServiceName:    "foo.bar",
+		semconv.AttributeHostName:       "localhost",
+		semconv.AttributeServiceVersion: "1.1.0",
+		semconv.AttributeOSType:         "darwin",
+		semconv.AttributeOSDescription:  "Mac OS Mojave",
+		semconv.AttributeOSName:         "Mac OS X",
+		semconv.AttributeOSVersion:      "10.14.1",
+	})
+	require.NoError(t, err)
+
+	want := `{"@timestamp":"2024-03-12T20:00:41.123456789Z","agent":{"name":"otlp"},"container":{"image":{"tag":["v3.4.0"]}},"event":{"action":"user-password-change"},"host":{"hostname":"localhost","os":{"full":"Mac OS Mojave","name":"Mac OS X","platform":"darwin","type":"macos","version":"10.14.1"}},"service":{"name":"foo.bar","version":"1.1.0"}}`
+	require.NoError(t, err)
+
+	resourceContainerImageTags := resource.Attributes().PutEmptySlice(semconv.AttributeContainerImageTags)
+	err = resourceContainerImageTags.FromRaw([]any{"v3.4.0"})
+	require.NoError(t, err)
+
+	scope := pcommon.NewInstrumentationScope()
+
+	record := plog.NewLogRecord()
+	err = record.Attributes().FromRaw(map[string]any{
+		"event.name": "user-password-change",
+	})
+	require.NoError(t, err)
+	observedTimestamp := pcommon.Timestamp(1710273641123456789)
+	record.SetObservedTimestamp(observedTimestamp)
+
+	m := encodeModel{
+		mode:  MappingECS,
+		dedot: true,
+		dedup: true,
+	}
+	doc, err := m.encodeLog(resource, record, scope)
+	require.NoError(t, err)
+
+	assert.Equal(t, want, string(doc))
+
+}
+
 func TestEncodeLogECSMode(t *testing.T) {
 	resource := pcommon.NewResource()
 	err := resource.Attributes().FromRaw(map[string]any{