diff --git a/.chloggen/k8sattributes-e2e-metrics.yaml b/.chloggen/k8sattributes-e2e-metrics.yaml new file mode 100755 index 000000000000..604c43bd31ee --- /dev/null +++ b/.chloggen/k8sattributes-e2e-metrics.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: k8sattributesprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: k8sattributesprocessor e2e tests support metric data type. + +# One or more tracking issues related to the change +issues: [18391] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 7feeef3563a0..b35c4c3c4e95 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -70,27 +70,33 @@ jobs: - name: check collector status run: | kubectl wait --for=condition=Ready --timeout=60s pod/test-opentelemetry-collector-0 - - name: start telemetrygen job + - name: start telemetrygen job for trace, metric run: | - kubectl -n default create -f .github/workflows/e2e/k8s/telemetrygen-job.yml + sed 's/DATA_TYPE/traces/g' .github/workflows/e2e/k8s/telemetrygen-job.yml | kubectl -n default create -f - + sed 's/DATA_TYPE/metrics/g' .github/workflows/e2e/k8s/telemetrygen-job.yml | kubectl -n default create -f - - name: check telemetrygen job status run: | - kubectl wait --for=condition=Complete --timeout=60s job/test-telemetrygen-job - - name: start telemetrygen statefulset + kubectl wait --for=condition=Complete --timeout=60s job/telemetrygen-traces-job + kubectl wait --for=condition=Complete --timeout=60s job/telemetrygen-metrics-job + - name: start telemetrygen statefulset for trace, metric run: | - kubectl -n default create -f .github/workflows/e2e/k8s/telemetrygen-statefulset.yml - - name: start telemetrygen deployment + sed 's/DATA_TYPE/traces/g' .github/workflows/e2e/k8s/telemetrygen-statefulset.yml | kubectl -n default create -f - + sed 's/DATA_TYPE/metrics/g' .github/workflows/e2e/k8s/telemetrygen-statefulset.yml | kubectl -n default create -f - + - name: start telemetrygen deployment for trace, metric run: | - kubectl -n default create -f .github/workflows/e2e/k8s/telemetrygen-deployment.yml - - name: start telemetrygen daemonset + sed 's/DATA_TYPE/traces/g' .github/workflows/e2e/k8s/telemetrygen-deployment.yml | kubectl -n default create -f - + sed 's/DATA_TYPE/metrics/g' .github/workflows/e2e/k8s/telemetrygen-deployment.yml | kubectl -n default create -f - + - name: start telemetrygen daemonset for trace, metric run: | - kubectl -n default create -f .github/workflows/e2e/k8s/telemetrygen-daemonset.yml - - name: wait telemetrygen statefulset/deployment/daemonset to generate trace + sed 's/DATA_TYPE/traces/g' .github/workflows/e2e/k8s/telemetrygen-daemonset.yml | kubectl -n default create -f - + sed 's/DATA_TYPE/metrics/g' .github/workflows/e2e/k8s/telemetrygen-daemonset.yml | kubectl -n default create -f - + - name: wait telemetrygen statefulset/deployment/daemonset to generate trace and metric run: | sleep 30 - - name: copy trace output + - name: copy telemetry trace, metric data output run: | kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/trace.json processor/k8sattributesprocessor/testdata/trace.json + kubectl cp -c filecp default/test-opentelemetry-collector-0:tmp/metric.json processor/k8sattributesprocessor/testdata/metric.json - name: run e2e test to check output run: | cd processor/k8sattributesprocessor diff --git a/.github/workflows/e2e/k8s/collector-helm-values.yml b/.github/workflows/e2e/k8s/collector-helm-values.yml index f264d4145d8c..d71927032300 100644 --- a/.github/workflows/e2e/k8s/collector-helm-values.yml +++ b/.github/workflows/e2e/k8s/collector-helm-values.yml @@ -39,12 +39,17 @@ config: logging: {} file/trace: path: /tmp/trace.json + file/metric: + path: /tmp/metric.json service: pipelines: traces: exporters: - file/trace - + metrics: + exporters: + - file/metric + image: repository: otel/opentelemetry-collector-contrib-dev tag: "latest" diff --git a/.github/workflows/e2e/k8s/telemetrygen-daemonset.yml b/.github/workflows/e2e/k8s/telemetrygen-daemonset.yml index 083954d0a772..da4a0006f812 100644 --- a/.github/workflows/e2e/k8s/telemetrygen-daemonset.yml +++ b/.github/workflows/e2e/k8s/telemetrygen-daemonset.yml @@ -1,27 +1,27 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: test-telemetrygen-daemonset + name: telemetrygen-DATA_TYPE-daemonset spec: selector: matchLabels: - app: test-telemetrygen-daemonset + app: telemetrygen-DATA_TYPE-daemonset template: metadata: annotations: workload: daemonset labels: - app: test-telemetrygen-daemonset + app: telemetrygen-DATA_TYPE-daemonset spec: containers: - command: - /telemetrygen - - traces + - DATA_TYPE - --otlp-insecure - --otlp-endpoint=test-opentelemetry-collector:4317 - - --service=test-daemonset - --duration=36000s - --rate=1 + - --otlp-attributes=service.name="test-daemonset" - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.container.restart_count="0" image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest diff --git a/.github/workflows/e2e/k8s/telemetrygen-deployment.yml b/.github/workflows/e2e/k8s/telemetrygen-deployment.yml index 49ac9b879cc8..7de27065706a 100644 --- a/.github/workflows/e2e/k8s/telemetrygen-deployment.yml +++ b/.github/workflows/e2e/k8s/telemetrygen-deployment.yml @@ -1,29 +1,29 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: test-telemetrygen-deployment + name: telemetrygen-DATA_TYPE-deployment namespace: default spec: replicas: 1 selector: matchLabels: - app: test-telemetrygen-deployment + app: telemetrygen-DATA_TYPE-deployment template: metadata: annotations: workload: deployment labels: - app: test-telemetrygen-deployment + app: telemetrygen-DATA_TYPE-deployment spec: containers: - command: - /telemetrygen - - traces + - DATA_TYPE - --otlp-insecure - --otlp-endpoint=test-opentelemetry-collector:4317 - - --service=test-deployment - --duration=36000s - --rate=1 + - --otlp-attributes=service.name="test-deployment" - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.container.restart_count="0" image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest diff --git a/.github/workflows/e2e/k8s/telemetrygen-job.yml b/.github/workflows/e2e/k8s/telemetrygen-job.yml index 75c6464b57b2..175641c958f9 100644 --- a/.github/workflows/e2e/k8s/telemetrygen-job.yml +++ b/.github/workflows/e2e/k8s/telemetrygen-job.yml @@ -1,7 +1,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: test-telemetrygen-job + name: telemetrygen-DATA_TYPE-job namespace: default spec: template: @@ -9,17 +9,17 @@ spec: annotations: workload: job labels: - app: test-telemetrygen + app: telemetrygen-DATA_TYPE spec: containers: - command: - /telemetrygen - - traces + - DATA_TYPE - --otlp-insecure - --otlp-endpoint=test-opentelemetry-collector:4317 - - --service=test-job - - --traces=5 + - --DATA_TYPE=5 - --rate=1 + - --otlp-attributes=service.name="test-job" - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.container.restart_count="0" image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest diff --git a/.github/workflows/e2e/k8s/telemetrygen-statefulset.yml b/.github/workflows/e2e/k8s/telemetrygen-statefulset.yml index 46e3be027829..af54e36e6eac 100644 --- a/.github/workflows/e2e/k8s/telemetrygen-statefulset.yml +++ b/.github/workflows/e2e/k8s/telemetrygen-statefulset.yml @@ -1,31 +1,31 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: test-telemetrygen-statefulset + name: telemetrygen-DATA_TYPE-statefulset namespace: default spec: - serviceName: test-telemetrygen-statefulset + serviceName: telemetrygen-DATA_TYPE-statefulset replicas: 1 selector: matchLabels: - app: test-telemetrygen-statefulset + app: telemetrygen-DATA_TYPE-statefulset template: metadata: metadata: annotations: workload: statefulset labels: - app: test-telemetrygen-statefulset + app: telemetrygen-DATA_TYPE-statefulset spec: containers: - command: - /telemetrygen - - traces + - DATA_TYPE - --otlp-insecure - --otlp-endpoint=test-opentelemetry-collector:4317 - - --service=test-statefulset - --duration=36000s - --rate=1 + - --otlp-attributes=service.name="test-statefulset" - --otlp-attributes=k8s.container.name="telemetrygen" - --otlp-attributes=k8s.container.restart_count="0" image: ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen:latest diff --git a/processor/k8sattributesprocessor/e2e_test.go b/processor/k8sattributesprocessor/e2e_test.go index d191b9c3e8f1..4237d368cf59 100644 --- a/processor/k8sattributesprocessor/e2e_test.go +++ b/processor/k8sattributesprocessor/e2e_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/assert" "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/multierr" ) @@ -58,16 +59,16 @@ func TestTraceE2E(t *testing.T) { name: "job", service: "test-job", attrs: map[string]*expectedValue{ - "k8s.pod.name": newExpectedValue(regex, "test-telemetrygen-job-[a-z0-9]*"), + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-traces-job-[a-z0-9]*"), "k8s.pod.ip": newExpectedValue(exist, ""), "k8s.pod.uid": newExpectedValue(exist, ""), "k8s.pod.start_time": newExpectedValue(exist, ""), "k8s.node.name": newExpectedValue(exist, ""), "k8s.namespace.name": newExpectedValue(equal, "default"), - "k8s.job.name": newExpectedValue(equal, "test-telemetrygen-job"), + "k8s.job.name": newExpectedValue(equal, "telemetrygen-traces-job"), "k8s.job.uid": newExpectedValue(exist, ""), "k8s.annotations.workload": newExpectedValue(equal, "job"), - "k8s.labels.app": newExpectedValue(equal, "test-telemetrygen"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-traces"), "k8s.container.name": newExpectedValue(equal, "telemetrygen"), "k8s.container.restart_count": newExpectedValue(equal, "0"), "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), @@ -79,16 +80,16 @@ func TestTraceE2E(t *testing.T) { name: "statefulset", service: "test-statefulset", attrs: map[string]*expectedValue{ - "k8s.pod.name": newExpectedValue(equal, "test-telemetrygen-statefulset-0"), + "k8s.pod.name": newExpectedValue(equal, "telemetrygen-traces-statefulset-0"), "k8s.pod.ip": newExpectedValue(exist, ""), "k8s.pod.uid": newExpectedValue(exist, ""), "k8s.pod.start_time": newExpectedValue(exist, ""), "k8s.node.name": newExpectedValue(exist, ""), "k8s.namespace.name": newExpectedValue(equal, "default"), - "k8s.statefulset.name": newExpectedValue(equal, "test-telemetrygen-statefulset"), + "k8s.statefulset.name": newExpectedValue(equal, "telemetrygen-traces-statefulset"), "k8s.statefulset.uid": newExpectedValue(exist, ""), "k8s.annotations.workload": newExpectedValue(equal, "statefulset"), - "k8s.labels.app": newExpectedValue(equal, "test-telemetrygen-statefulset"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-traces-statefulset"), "k8s.container.name": newExpectedValue(equal, "telemetrygen"), "k8s.container.restart_count": newExpectedValue(equal, "0"), "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), @@ -100,17 +101,17 @@ func TestTraceE2E(t *testing.T) { name: "deployment", service: "test-deployment", attrs: map[string]*expectedValue{ - "k8s.pod.name": newExpectedValue(regex, "test-telemetrygen-deployment-[a-z0-9]*-[a-z0-9]*"), + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-traces-deployment-[a-z0-9]*-[a-z0-9]*"), "k8s.pod.ip": newExpectedValue(exist, ""), "k8s.pod.uid": newExpectedValue(exist, ""), "k8s.pod.start_time": newExpectedValue(exist, ""), "k8s.node.name": newExpectedValue(exist, ""), "k8s.namespace.name": newExpectedValue(equal, "default"), - "k8s.deployment.name": newExpectedValue(equal, "test-telemetrygen-deployment"), - "k8s.replicaset.name": newExpectedValue(regex, "test-telemetrygen-deployment-[a-z0-9]*"), + "k8s.deployment.name": newExpectedValue(equal, "telemetrygen-traces-deployment"), + "k8s.replicaset.name": newExpectedValue(regex, "telemetrygen-traces-deployment-[a-z0-9]*"), "k8s.replicaset.uid": newExpectedValue(exist, ""), "k8s.annotations.workload": newExpectedValue(equal, "deployment"), - "k8s.labels.app": newExpectedValue(equal, "test-telemetrygen-deployment"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-traces-deployment"), "k8s.container.name": newExpectedValue(equal, "telemetrygen"), "k8s.container.restart_count": newExpectedValue(equal, "0"), "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), @@ -122,16 +123,16 @@ func TestTraceE2E(t *testing.T) { name: "daemonset", service: "test-daemonset", attrs: map[string]*expectedValue{ - "k8s.pod.name": newExpectedValue(regex, "test-telemetrygen-daemonset-[a-z0-9]*"), + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-traces-daemonset-[a-z0-9]*"), "k8s.pod.ip": newExpectedValue(exist, ""), "k8s.pod.uid": newExpectedValue(exist, ""), "k8s.pod.start_time": newExpectedValue(exist, ""), "k8s.node.name": newExpectedValue(exist, ""), "k8s.namespace.name": newExpectedValue(equal, "default"), - "k8s.daemonset.name": newExpectedValue(equal, "test-telemetrygen-daemonset"), + "k8s.daemonset.name": newExpectedValue(equal, "telemetrygen-traces-daemonset"), "k8s.daemonset.uid": newExpectedValue(exist, ""), "k8s.annotations.workload": newExpectedValue(equal, "daemonset"), - "k8s.labels.app": newExpectedValue(equal, "test-telemetrygen-daemonset"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-traces-daemonset"), "k8s.container.name": newExpectedValue(equal, "telemetrygen"), "k8s.container.restart_count": newExpectedValue(equal, "0"), "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), @@ -219,3 +220,138 @@ func resourceHasAttributes(resource pcommon.Resource, kvs map[string]*expectedVa } return err } + +func TestMetricE2E(t *testing.T) { + tcs := []struct { + name string + service string + attrs map[string]*expectedValue + }{ + { + name: "job", + service: "test-job", + attrs: map[string]*expectedValue{ + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-metrics-job-[a-z0-9]*"), + "k8s.pod.ip": newExpectedValue(exist, ""), + "k8s.pod.uid": newExpectedValue(exist, ""), + "k8s.pod.start_time": newExpectedValue(exist, ""), + "k8s.node.name": newExpectedValue(exist, ""), + "k8s.namespace.name": newExpectedValue(equal, "default"), + "k8s.job.name": newExpectedValue(equal, "telemetrygen-metrics-job"), + "k8s.job.uid": newExpectedValue(exist, ""), + "k8s.annotations.workload": newExpectedValue(equal, "job"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-metrics"), + "k8s.container.name": newExpectedValue(equal, "telemetrygen"), + "k8s.container.restart_count": newExpectedValue(equal, "0"), + "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), + "container.image.tag": newExpectedValue(equal, "latest"), + "container.id": newExpectedValue(exist, ""), + }, + }, + { + name: "statefulset", + service: "test-statefulset", + attrs: map[string]*expectedValue{ + "k8s.pod.name": newExpectedValue(equal, "telemetrygen-metrics-statefulset-0"), + "k8s.pod.ip": newExpectedValue(exist, ""), + "k8s.pod.uid": newExpectedValue(exist, ""), + "k8s.pod.start_time": newExpectedValue(exist, ""), + "k8s.node.name": newExpectedValue(exist, ""), + "k8s.namespace.name": newExpectedValue(equal, "default"), + "k8s.statefulset.name": newExpectedValue(equal, "telemetrygen-metrics-statefulset"), + "k8s.statefulset.uid": newExpectedValue(exist, ""), + "k8s.annotations.workload": newExpectedValue(equal, "statefulset"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-metrics-statefulset"), + "k8s.container.name": newExpectedValue(equal, "telemetrygen"), + "k8s.container.restart_count": newExpectedValue(equal, "0"), + "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), + "container.image.tag": newExpectedValue(equal, "latest"), + "container.id": newExpectedValue(exist, ""), + }, + }, + { + name: "deployment", + service: "test-deployment", + attrs: map[string]*expectedValue{ + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-metrics-deployment-[a-z0-9]*-[a-z0-9]*"), + "k8s.pod.ip": newExpectedValue(exist, ""), + "k8s.pod.uid": newExpectedValue(exist, ""), + "k8s.pod.start_time": newExpectedValue(exist, ""), + "k8s.node.name": newExpectedValue(exist, ""), + "k8s.namespace.name": newExpectedValue(equal, "default"), + "k8s.deployment.name": newExpectedValue(equal, "telemetrygen-metrics-deployment"), + "k8s.replicaset.name": newExpectedValue(regex, "telemetrygen-metrics-deployment-[a-z0-9]*"), + "k8s.replicaset.uid": newExpectedValue(exist, ""), + "k8s.annotations.workload": newExpectedValue(equal, "deployment"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-metrics-deployment"), + "k8s.container.name": newExpectedValue(equal, "telemetrygen"), + "k8s.container.restart_count": newExpectedValue(equal, "0"), + "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), + "container.image.tag": newExpectedValue(equal, "latest"), + "container.id": newExpectedValue(exist, ""), + }, + }, + { + name: "daemonset", + service: "test-daemonset", + attrs: map[string]*expectedValue{ + "k8s.pod.name": newExpectedValue(regex, "telemetrygen-metrics-daemonset-[a-z0-9]*"), + "k8s.pod.ip": newExpectedValue(exist, ""), + "k8s.pod.uid": newExpectedValue(exist, ""), + "k8s.pod.start_time": newExpectedValue(exist, ""), + "k8s.node.name": newExpectedValue(exist, ""), + "k8s.namespace.name": newExpectedValue(equal, "default"), + "k8s.daemonset.name": newExpectedValue(equal, "telemetrygen-metrics-daemonset"), + "k8s.daemonset.uid": newExpectedValue(exist, ""), + "k8s.annotations.workload": newExpectedValue(equal, "daemonset"), + "k8s.labels.app": newExpectedValue(equal, "telemetrygen-metrics-daemonset"), + "k8s.container.name": newExpectedValue(equal, "telemetrygen"), + "k8s.container.restart_count": newExpectedValue(equal, "0"), + "container.image.name": newExpectedValue(equal, "ghcr.io/open-telemetry/opentelemetry-collector-contrib/telemetrygen"), + "container.image.tag": newExpectedValue(equal, "latest"), + "container.id": newExpectedValue(exist, ""), + }, + }, + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + scanMetricForAttributes(t, tc.service, tc.attrs) + }) + } +} + +func scanMetricForAttributes(t *testing.T, expectedService string, kvs map[string]*expectedValue) { + + f, _ := os.Open("./testdata/metric.json") + defer f.Close() + + scanner := bufio.NewScanner(f) + const maxCapacity int = 8388608 + buf := make([]byte, maxCapacity) + scanner.Buffer(buf, maxCapacity) + + unmarshaler := pmetric.JSONUnmarshaler{} + var err error + for scanner.Scan() { + metrics, serr := unmarshaler.UnmarshalMetrics(scanner.Bytes()) + if serr != nil { + continue + } + + len := metrics.ResourceMetrics().Len() + for i := 0; i < len; i++ { + resourceMetrics := metrics.ResourceMetrics().At(i) + + service, exist := resourceMetrics.Resource().Attributes().Get("service.name") + assert.Equal(t, true, exist, "metric do not has 'service.name' attribute in resource") + + if service.AsString() != expectedService { + continue + } + err = resourceHasAttributes(resourceMetrics.Resource(), kvs) + } + } + assert.NoError(t, err) + +}