From cf1c93e0408871c9f9e42a683770934a54ad7a6d Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Fri, 21 Jun 2024 16:29:25 +0300 Subject: [PATCH] [exporter/elasticsearch] Add ecs mapping for k8s.deployment.name (#33622) **Description:** This PR adds a mapping for the `k8s.deployment.name` attribute which is enabled by default by the [`k8sattributesprocessor`](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/721c9e68de27d00a5ce0651f4b2f47761547e296/processor/k8sattributesprocessor/metadata.yaml#L37-L40). The idea is to have all the enabled by default attributes mapped to ECS properly, for consistency. **Link to tracking Issue:** ~ **Testing:** Tested locally with: ```yaml mode: daemonset presets: logsCollection: enabled: true storeCheckpoints: true kubernetesAttributes: enabled: true extraEnvs: - name: ES_ENDPOINT valueFrom: secretKeyRef: name: elastic-secret key: es_endpoint - name: ES_API_KEY valueFrom: secretKeyRef: name: elastic-secret key: es_api_key image: repository: otelcontribcol-dev tag: "0.0.1" pullPolicy: IfNotPresent command: name: otelcontribcol config: exporters: debug: verbosity: detailed elasticsearch/logs: endpoints: - ${env:ES_ENDPOINT} api_key: ${env:ES_API_KEY} logs_index: otel_logs_index mapping: mode: ecs sending_queue: enabled: true num_consumers: 20 queue_size: 1000 receivers: filelog: start_at: end include_file_name: false include_file_path: true exclude: - /var/log/pods/default_daemonset-opentelemetry-collector*_*/opentelemetry-collector/*.log include: - /var/log/pods/*/*/*.log operators: - id: container-parser type: container output: some processors: k8sattributes: filter: node_from_env_var: K8S_NODE_NAME passthrough: false pod_association: - sources: - from: resource_attribute name: k8s.pod.ip - sources: - from: resource_attribute name: k8s.pod.uid - sources: - from: connection extract: metadata: - "k8s.namespace.name" - "k8s.deployment.name" - "k8s.node.name" - "k8s.pod.name" - "k8s.pod.uid" service: pipelines: logs: receivers: [filelog] processors: [batch] exporters: [elasticsearch/logs] ``` The documents are stored in ES properly: ```json { "@timestamp": "2024-06-18T10:57:45.307790375Z", "agent": { "name": "otlp" }, "k8s": { "container": { "name": "busybox", "restart_count": "0" }, }, "kubernetes": { "deployment": { "name": "deploy-logs" }, "namespace": { "name": "default" }, "node": { "name": "kind-control-plane" }, "pod": { "name": "deploy-logs-595cdc6b8b-dptrr", "uid": "5166a2e2-9152-4d86-b8a4-668cc2f49a39" } }, "log": { "file": { "path": "/var/log/pods/default_deploy-logs-595cdc6b8b-dptrr_5166a2e2-9152-4d86-b8a4-668cc2f49a39/busybox/0.log" }, "iostream": "stdout" }, "logtag": "F", "message": "otel logs at 10:57:45", "time": "2024-06-18T10:57:45.307790375Z" } ``` **Documentation:** ~ /cc @carsonip @lahsivjar @andrzej-stencel --------- Signed-off-by: ChrsMark --- .../add_es_exporter_deploy_name_ecs.yaml | 27 +++++++ exporter/elasticsearchexporter/model.go | 9 ++- exporter/elasticsearchexporter/model_test.go | 78 ++++++++++--------- 3 files changed, 72 insertions(+), 42 deletions(-) create mode 100644 .chloggen/add_es_exporter_deploy_name_ecs.yaml diff --git a/.chloggen/add_es_exporter_deploy_name_ecs.yaml b/.chloggen/add_es_exporter_deploy_name_ecs.yaml new file mode 100644 index 000000000000..fd9691fedc91 --- /dev/null +++ b/.chloggen/add_es_exporter_deploy_name_ecs.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: elasticsearchexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add translation for k8s.deployment.name resource attribute + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33622] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/elasticsearchexporter/model.go b/exporter/elasticsearchexporter/model.go index 36702776e791..6f29f031a252 100644 --- a/exporter/elasticsearchexporter/model.go +++ b/exporter/elasticsearchexporter/model.go @@ -43,10 +43,11 @@ var resourceAttrsConversionMap = map[string]string{ semconv.AttributeOSType: "host.os.platform", semconv.AttributeOSDescription: "host.os.full", semconv.AttributeOSVersion: "host.os.version", - "k8s.namespace.name": "kubernetes.namespace", - "k8s.node.name": "kubernetes.node.name", - "k8s.pod.name": "kubernetes.pod.name", - "k8s.pod.uid": "kubernetes.pod.uid", + semconv.AttributeK8SDeploymentName: "kubernetes.deployment.name", + semconv.AttributeK8SNamespaceName: "kubernetes.namespace", + semconv.AttributeK8SNodeName: "kubernetes.node.name", + semconv.AttributeK8SPodName: "kubernetes.pod.name", + semconv.AttributeK8SPodUID: "kubernetes.pod.uid", } // resourceAttrsToPreserve contains conventions that should be preserved in ECS mode. diff --git a/exporter/elasticsearchexporter/model_test.go b/exporter/elasticsearchexporter/model_test.go index 5fb15b05f6fa..2c1d1434ce99 100644 --- a/exporter/elasticsearchexporter/model_test.go +++ b/exporter/elasticsearchexporter/model_test.go @@ -299,6 +299,7 @@ func TestEncodeLogECSMode(t *testing.T) { "k8s.node.name": "node-1", "k8s.pod.name": "opentelemetry-pod-autoconf", "k8s.pod.uid": "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff", + "k8s.deployment.name": "coredns", }) require.NoError(t, err) @@ -321,44 +322,45 @@ func TestEncodeLogECSMode(t *testing.T) { expectedDocFields := pcommon.NewMap() err = expectedDocFields.FromRaw(map[string]any{ - "service.name": "foo.bar", - "service.version": "1.1.0", - "service.node.name": "i-103de39e0a", - "agent.name": "opentelemetry/perl", - "agent.version": "7.9.12", - "cloud.provider": "gcp", - "cloud.account.id": "19347013", - "cloud.region": "us-west-1", - "cloud.availability_zone": "us-west-1b", - "cloud.service.name": "gke", - "container.name": "happy-seger", - "container.id": "e69cc5d3dda", - "container.image.name": "my-app", - "container.runtime": "docker", - "host.hostname": "i-103de39e0a.gke.us-west-1b.cloud.google.com", - "host.name": "i-103de39e0a.gke.us-west-1b.cloud.google.com", - "host.id": "i-103de39e0a", - "host.type": "t2.medium", - "host.architecture": "x86_64", - "process.pid": 9833, - "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", - "process.executable": "/usr/bin/ssh", - "service.runtime.name": "OpenJDK Runtime Environment", - "service.runtime.version": "14.0.2", - "host.os.platform": "darwin", - "host.os.full": "Mac OS Mojave", - "host.os.name": "Mac OS X", - "host.os.version": "10.14.1", - "host.os.type": "macos", - "device.id": "00000000-54b3-e7c7-0000-000046bffd97", - "device.model.identifier": "SM-G920F", - "device.model.name": "Samsung Galaxy S6", - "device.manufacturer": "Samsung", - "event.action": "user-password-change", - "kubernetes.namespace": "default", - "kubernetes.node.name": "node-1", - "kubernetes.pod.name": "opentelemetry-pod-autoconf", - "kubernetes.pod.uid": "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff", + "service.name": "foo.bar", + "service.version": "1.1.0", + "service.node.name": "i-103de39e0a", + "agent.name": "opentelemetry/perl", + "agent.version": "7.9.12", + "cloud.provider": "gcp", + "cloud.account.id": "19347013", + "cloud.region": "us-west-1", + "cloud.availability_zone": "us-west-1b", + "cloud.service.name": "gke", + "container.name": "happy-seger", + "container.id": "e69cc5d3dda", + "container.image.name": "my-app", + "container.runtime": "docker", + "host.hostname": "i-103de39e0a.gke.us-west-1b.cloud.google.com", + "host.name": "i-103de39e0a.gke.us-west-1b.cloud.google.com", + "host.id": "i-103de39e0a", + "host.type": "t2.medium", + "host.architecture": "x86_64", + "process.pid": 9833, + "process.command_line": "/usr/bin/ssh -l user 10.0.0.16", + "process.executable": "/usr/bin/ssh", + "service.runtime.name": "OpenJDK Runtime Environment", + "service.runtime.version": "14.0.2", + "host.os.platform": "darwin", + "host.os.full": "Mac OS Mojave", + "host.os.name": "Mac OS X", + "host.os.version": "10.14.1", + "host.os.type": "macos", + "device.id": "00000000-54b3-e7c7-0000-000046bffd97", + "device.model.identifier": "SM-G920F", + "device.model.name": "Samsung Galaxy S6", + "device.manufacturer": "Samsung", + "event.action": "user-password-change", + "kubernetes.namespace": "default", + "kubernetes.node.name": "node-1", + "kubernetes.pod.name": "opentelemetry-pod-autoconf", + "kubernetes.pod.uid": "275ecb36-5aa8-4c2a-9c47-d8bb681b9aff", + "kubernetes.deployment.name": "coredns", }) require.NoError(t, err)