Skip to content

Commit

Permalink
feat(inputs.opentelemetry): Add profiles support (influxdata#15637)
Browse files Browse the repository at this point in the history
  • Loading branch information
srebhan authored Aug 9, 2024
1 parent ec1fc7f commit 8903637
Show file tree
Hide file tree
Showing 11 changed files with 1,142 additions and 70 deletions.
2 changes: 2 additions & 0 deletions docs/LICENSE_OF_DEPENDENCIES.md
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ following works:
- github.com/grafana/regexp [BSD 3-Clause "New" or "Revised" License](https://github.com/grafana/regexp/blob/main/LICENSE)
- github.com/grid-x/modbus [BSD 3-Clause "New" or "Revised" License](https://github.com/grid-x/modbus/blob/master/LICENSE)
- github.com/grid-x/serial [MIT License](https://github.com/grid-x/serial/blob/master/LICENSE)
- github.com/grpc-ecosystem/grpc-gateway [BSD 3-Clause "New" or "Revised" License](https://github.com/grpc-ecosystem/grpc-gateway/blob/main/LICENSE)
- github.com/gsterjov/go-libsecret [MIT License](https://github.com/gsterjov/go-libsecret/blob/master/LICENSE)
- github.com/gwos/tcg/sdk [MIT License](https://github.com/gwos/tcg/blob/master/LICENSE)
- github.com/hailocab/go-hostpool [MIT License](https://github.com/hailocab/go-hostpool/blob/master/LICENSE)
Expand Down Expand Up @@ -399,6 +400,7 @@ following works:
- go.opentelemetry.io/otel [Apache License 2.0](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE)
- go.opentelemetry.io/otel/metric [Apache License 2.0](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE)
- go.opentelemetry.io/otel/trace [Apache License 2.0](https://github.com/open-telemetry/opentelemetry-go/blob/main/LICENSE)
- go.opentelemetry.io/proto/otlp [Apache License 2.0](https://github.com/open-telemetry/opentelemetry-proto-go/blob/main/LICENSE)
- go.starlark.net [BSD 3-Clause "New" or "Revised" License](https://github.com/google/starlark-go/blob/master/LICENSE)
- go.step.sm/crypto [Apache License 2.0](https://github.com/smallstep/crypto/blob/master/LICENSE)
- go.uber.org/atomic [MIT License](https://pkg.go.dev/go.uber.org/atomic?tab=licenses)
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -205,9 +205,10 @@ require (
github.com/xdg/scram v1.0.5
github.com/yuin/goldmark v1.6.0
go.mongodb.org/mongo-driver v1.16.0
go.opentelemetry.io/collector/pdata v1.8.0
go.opentelemetry.io/collector/pdata v1.12.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.44.0
go.opentelemetry.io/otel/sdk/metric v1.27.0
go.opentelemetry.io/proto/otlp v1.3.1
go.starlark.net v0.0.0-20240520160348-046347dcd104
go.step.sm/crypto v0.50.0
golang.org/x/crypto v0.25.0
Expand Down Expand Up @@ -492,7 +493,6 @@ require (
go.opentelemetry.io/otel/metric v1.27.0 // indirect
go.opentelemetry.io/otel/sdk v1.27.0 // indirect
go.opentelemetry.io/otel/trace v1.27.0 // indirect
go.opentelemetry.io/proto/otlp v1.2.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2480,8 +2480,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
go.opentelemetry.io/collector v0.81.0 h1:pF+sB8xNXlg/W0a0QTLz4mUWyool1a9toVj8LmLoFqg=
go.opentelemetry.io/collector/consumer v0.101.0 h1:9tDxaeHe1+Uovf3fhdx7T4pV5mo/Dc0hniH7O5H3RBA=
go.opentelemetry.io/collector/consumer v0.101.0/go.mod h1:ud5k64on9m7hHTrhjEeLhWbLkd8+Gp06rDt3p86TKNs=
go.opentelemetry.io/collector/pdata v1.8.0 h1:d/QQgZxB4Y+d3mqLVh2ozvzujUhloD3P/fk7X+In764=
go.opentelemetry.io/collector/pdata v1.8.0/go.mod h1:/W7clu0wFC4WSRp94Ucn6Vm36Wkrt+tmtlDb1aiNZCY=
go.opentelemetry.io/collector/pdata v1.12.0 h1:Xx5VK1p4VO0md8MWm2icwC1MnJ7f8EimKItMWw46BmA=
go.opentelemetry.io/collector/pdata v1.12.0/go.mod h1:MYeB0MmMAxeM0hstCFrCqWLzdyeYySim2dG6pDT6nYI=
go.opentelemetry.io/collector/pdata/testdata v0.101.0 h1:JzeUtg5RN1iIFgY8DakGlqBkGxOTJlkaYlLausnEGKY=
go.opentelemetry.io/collector/pdata/testdata v0.101.0/go.mod h1:ZGobfCus4fWo5RduZ7ENI0+HD9BewgKuO6qU2rBVnUg=
go.opentelemetry.io/collector/semconv v0.101.0 h1:tOe9iTe9dDCnvz/bqgfNRr4w80kXG8505tQJ5h5v08Q=
Expand Down Expand Up @@ -2509,8 +2509,8 @@ go.opentelemetry.io/otel/trace v1.27.0/go.mod h1:6RiD1hkAprV4/q+yd2ln1HG9GoPx39S
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
go.opentelemetry.io/proto/otlp v1.2.0 h1:pVeZGk7nXDC9O2hncA6nHldxEjm6LByfA2aN8IOkz94=
go.opentelemetry.io/proto/otlp v1.2.0/go.mod h1:gGpR8txAl5M03pDhMC79G6SdqNV26naRm/KDsgaHD8A=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.starlark.net v0.0.0-20240520160348-046347dcd104 h1:3qhteRISupnJvaWshOmeqEUs2y9oc/+/ePPvDh3Eygg=
go.starlark.net v0.0.0-20240520160348-046347dcd104/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8=
go.step.sm/crypto v0.50.0 h1:BqI9sEgocoHDLLHiZnFqdqXl5FjdMvOWKMm/fKL/lrw=
Expand Down
38 changes: 31 additions & 7 deletions plugins/inputs/opentelemetry/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,25 +42,33 @@ See the [CONFIGURATION.md][CONFIGURATION.md] for more details.
## These are always included as tags:
## - trace ID
## - span ID
## The default values are strongly recommended for use with Jaeger:
## - service.name
## - span.name
## Other common attributes can be found here:
## Common attributes can be found here:
## - https://github.com/open-telemetry/opentelemetry-collector/tree/main/semconv
# span_dimensions = ["service.name", "span.name"]

## Override the default log record attributes to be used as line protocol tags.
## These are always included as tags, if available:
## - trace ID
## - span ID
## The default values:
## - service.name
## Other common attributes can be found here:
## Common attributes can be found here:
## - https://github.com/open-telemetry/opentelemetry-collector/tree/main/semconv
## When using InfluxDB for both logs and traces, be certain that log_record_dimensions
## matches the span_dimensions value.
# log_record_dimensions = ["service.name"]

## Override the default profile attributes to be used as line protocol tags.
## These are always included as tags, if available:
## - profile_id
## - address
## - sample
## - sample_name
## - sample_unit
## - sample_type
## - sample_type_unit
## Common attributes can be found here:
## - https://github.com/open-telemetry/opentelemetry-collector/tree/main/semconv
# profile_dimensions = []

## Override the default (prometheus-v1) metrics schema.
## Supports: "prometheus-v1", "prometheus-v2"
## For more information about the alternatives, read the Prometheus input
Expand Down Expand Up @@ -148,3 +156,19 @@ logs fluent.tag="fluent.info",pid=18i,ppid=9i,worker=0i 1613769568895331700
logs fluent.tag="fluent.debug",instance=1720i,queue_size=0i,stage_size=0i 1613769568895697200
logs fluent.tag="fluent.info",worker=0i 1613769568896515100
```

### Profiles

```text
profiles,address=95210353,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=0,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="fab9b8c848218405738c11a7ec4982e9",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=18694144u,filename="chromium",frame_type="native",location="",memory_limit=250413056u,memory_start=18698240u,stack_trace_id="hYmAzQVF8vy8MWbzsKpQNw",start_time_unix_nano=1721306050081621681u,value=1i 1721306048731622020
profiles,address=15945263,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=1,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="7dab4a2e0005d025e75cc72191f8d6bf",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=15638528u,filename="dockerd",frame_type="native",location="",memory_limit=47255552u,memory_start=15638528u,stack_trace_id="4N3KEcGylb5Qoi2905c1ZA",start_time_unix_nano=1721306050081621681u,value=1i 1721306049831718725
profiles,address=15952400,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=1,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="7dab4a2e0005d025e75cc72191f8d6bf",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=15638528u,filename="dockerd",frame_type="native",location="",memory_limit=47255552u,memory_start=15638528u,stack_trace_id="4N3KEcGylb5Qoi2905c1ZA",start_time_unix_nano=1721306050081621681u,value=1i 1721306049831718725
profiles,address=15953899,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=1,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="7dab4a2e0005d025e75cc72191f8d6bf",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=15638528u,filename="dockerd",frame_type="native",location="",memory_limit=47255552u,memory_start=15638528u,stack_trace_id="4N3KEcGylb5Qoi2905c1ZA",start_time_unix_nano=1721306050081621681u,value=1i 1721306049831718725
profiles,address=16148175,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=1,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="7dab4a2e0005d025e75cc72191f8d6bf",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=15638528u,filename="dockerd",frame_type="native",location="",memory_limit=47255552u,memory_start=15638528u,stack_trace_id="4N3KEcGylb5Qoi2905c1ZA",start_time_unix_nano=1721306050081621681u,value=1i 1721306049831718725
profiles,address=4770577,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="cfc3dc7d1638c1284a6b62d4b5c0d74e",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=0u,filename="",frame_type="kernel",location="do_epoll_wait",memory_limit=0u,memory_start=0u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
profiles,address=4773632,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="cfc3dc7d1638c1284a6b62d4b5c0d74e",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=0u,filename="",frame_type="kernel",location="__x64_sys_epoll_wait",memory_limit=0u,memory_start=0u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
profiles,address=14783666,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="cfc3dc7d1638c1284a6b62d4b5c0d74e",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=0u,filename="",frame_type="kernel",location="do_syscall_64",memory_limit=0u,memory_start=0u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
profiles,address=16777518,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="cfc3dc7d1638c1284a6b62d4b5c0d74e",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=0u,filename="",frame_type="kernel",location="entry_SYSCALL_64_after_hwframe",memory_limit=0u,memory_start=0u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
profiles,address=1139937,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="982ed6c7a77f99f0ae746be0187953bf",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=147456u,filename="libc.so.6",frame_type="native",location="",memory_limit=1638400u,memory_start=147456u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
profiles,address=117834912,host.name=testbox,profile_id=618098d29a6cefd6a4c0ea806880c2a8,sample=2,sample_name=cpu,sample_type=samples,sample_type_unit=count,sample_unit=nanoseconds build_id="fab9b8c848218405738c11a7ec4982e9",build_id_type="BUILD_ID_BINARY_HASH",end_time_unix_nano=1721306050081621681u,file_offset=18694144u,filename="chromium",frame_type="native",location="",memory_limit=250413056u,memory_start=18698240u,stack_trace_id="UaO9bysJnAYXFYobSdHXqg",start_time_unix_nano=1721306050081621681u,value=1i 1721306050081621681
```
134 changes: 134 additions & 0 deletions plugins/inputs/opentelemetry/grpc_service_profile.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
package opentelemetry

import (
"context"
"encoding/hex"
"fmt"
"strconv"
"strings"
"time"

service "go.opentelemetry.io/proto/otlp/collector/profiles/v1experimental"
"google.golang.org/protobuf/encoding/protojson"

"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/filter"
)

type profileService struct {
service.UnimplementedProfilesServiceServer

acc telegraf.Accumulator
filter filter.Filter
logger telegraf.Logger
}

func newProfileService(acc telegraf.Accumulator, logger telegraf.Logger, dimensions []string) (*profileService, error) {
// Check for duplicate dimensions
seen := make(map[string]bool, len(dimensions))
duplicates := make([]string, 0)
dims := make([]string, 0, len(dimensions))
for _, d := range dimensions {
if seen[d] {
duplicates = append(duplicates, d)
continue
}
dims = append(dims, d)
seen[d] = true
}
if len(duplicates) > 0 {
return nil, fmt.Errorf("duplicate profile dimension(s) configured: %s", strings.Join(duplicates, ","))
}
f, err := filter.Compile(dims)
if err != nil {
return nil, fmt.Errorf("compiling dimensions filter failed: %w", err)
}

return &profileService{
acc: acc,
filter: f,
logger: logger,
}, nil
}

func (s *profileService) Export(_ context.Context, req *service.ExportProfilesServiceRequest) (*service.ExportProfilesServiceResponse, error) {
// Output the received message for debugging
buf, err := protojson.Marshal(req)
if err != nil {
s.logger.Errorf("marshalling received profile failed: %v", err)
} else {
s.logger.Debugf("received profile: %s", string(buf))
}

for _, rp := range req.ResourceProfiles {
// Extract the requested attributes that should be added as tags
attrtags := make(map[string]string)
for _, attr := range rp.Resource.Attributes {
if s.filter.Match(attr.Key) {
attrtags[attr.Key] = attr.GetValue().GetStringValue()
}
}

for _, sp := range rp.ScopeProfiles {
for _, p := range sp.Profiles {
for i, sample := range p.Profile.Sample {
for j := sample.LocationsStartIndex; j < sample.LocationsStartIndex+sample.LocationsLength; j++ {
for validx, value := range sample.Value {
loc := p.Profile.Location[j]
locations := make([]string, 0, len(loc.Line))
for _, line := range loc.Line {
f := p.Profile.Function[line.FunctionIndex]
fileloc := p.Profile.StringTable[f.Filename]
if f.StartLine > 0 {
if fileloc != "" {
fileloc += " "
}
fileloc += "line " + strconv.FormatInt(f.StartLine, 10)
}
l := p.Profile.StringTable[f.Name]
if fileloc != "" {
l += "(" + fileloc + ")"
}
locations = append(locations, l)
}
mapping := p.Profile.Mapping[loc.MappingIndex]
tags := map[string]string{
"profile_id": hex.EncodeToString(p.ProfileId),
"sample": strconv.Itoa(i),
"sample_name": p.Profile.StringTable[p.Profile.PeriodType.Type],
"sample_unit": p.Profile.StringTable[p.Profile.PeriodType.Unit],
"sample_type": p.Profile.StringTable[p.Profile.SampleType[validx].Type],
"sample_type_unit": p.Profile.StringTable[p.Profile.SampleType[validx].Unit],
"address": "0x" + strconv.FormatUint(loc.Address, 16),
}
for k, v := range attrtags {
tags[k] = v
}
fields := map[string]interface{}{
"start_time_unix_nano": p.StartTimeUnixNano,
"end_time_unix_nano": p.EndTimeUnixNano,
"location": strings.Join(locations, ","),
"frame_type": p.Profile.StringTable[loc.TypeIndex],
"stack_trace_id": p.Profile.StringTable[sample.StacktraceIdIndex],
"memory_start": mapping.MemoryStart,
"memory_limit": mapping.MemoryLimit,
"filename": p.Profile.StringTable[mapping.Filename],
"file_offset": mapping.FileOffset,
"build_id": p.Profile.StringTable[mapping.BuildId],
"build_id_type": mapping.BuildIdKind.String(),
"value": value,
}
for _, idx := range sample.Attributes {
attr := p.Profile.AttributeTable[idx]
fields[attr.Key] = attr.GetValue().Value
}
ts := sample.TimestampsUnixNano[validx]
s.acc.AddFields("profiles", fields, tags, time.Unix(0, int64(ts)))
}
}
}
}
}
}
return &service.ExportProfilesServiceResponse{}, nil
}
Loading

0 comments on commit 8903637

Please sign in to comment.