Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pkg/query: Use hash of labels instead of marshaling labels to json #3719

Merged
merged 1 commit into from
Aug 30, 2023
Merged

pkg/query: Use hash of labels instead of marshaling labels to json #3719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ require (
github.com/prometheus/prometheus v0.46.0
github.com/stretchr/testify v1.8.4
github.com/thanos-io/objstore v0.0.0-20230804084840-c042a6a16c58
github.com/zeebo/xxh3 v1.0.2
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0
go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.42.0
go.opentelemetry.io/otel v1.16.0
Expand Down Expand Up @@ -214,7 +215,6 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/tencentyun/cos-go-sdk-v5 v0.7.40 // indirect
github.com/vultr/govultr/v2 v2.17.2 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
Expand Down
44 changes: 21 additions & 23 deletions pkg/query/flamegraph_arrow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ import (
"bytes"
"context"
"fmt"
"strings"
"unsafe"

"github.com/apache/arrow/go/v14/arrow"
"github.com/apache/arrow/go/v14/arrow/array"
"github.com/apache/arrow/go/v14/arrow/ipc"
"github.com/apache/arrow/go/v14/arrow/memory"
"github.com/polarsignals/frostdb/pqarrow/builder"
"github.com/zeebo/xxh3"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"

Expand Down Expand Up @@ -119,6 +119,7 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr
row := fb.builderCumulative.Len()

profileReader := profile.NewReader(p)
labelHasher := xxh3.New()
for _, r := range profileReader.RecordReaders {
if err := fb.ensureLabelColumns(r.LabelFields); err != nil {
return nil, 0, 0, 0, fmt.Errorf("ensure label columns: %w", err)
Expand All @@ -132,40 +133,39 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr
defer t.Release()

// This field compares the current sample with the already added values in the builders.
lsbytes := make([]byte, 0, 512)
for i := 0; i < int(r.Record.NumRows()); i++ {
beg, end := r.Locations.ValueOffsets(i)

var sampleLabels map[string]string
hasLabels := false
for j, labelColumn := range r.LabelColumns {
labelHash := uint64(0)
for _, labelColumn := range r.LabelColumns {
if labelColumn.Col.IsValid(i) {
if sampleLabels == nil {
sampleLabels = map[string]string{}
}
hasLabels = true
labelName := strings.TrimPrefix(r.LabelFields[j].Name, profile.ColumnPprofLabelsPrefix)
sampleLabels[labelName] = string(labelColumn.Dict.Value(labelColumn.Col.GetValueIndex(i)))
break
}
}

if aggregateLabels && hasLabels {
lsbytes = lsbytes[:0]
lsbytes = MarshalStringMapSorted(lsbytes, sampleLabels)

labelHasher.Reset()
for j, labelColumn := range r.LabelColumns {
if labelColumn.Col.IsValid(i) {
_, _ = labelHasher.WriteString(r.LabelFields[j].Name)
_, _ = labelHasher.Write(labelColumn.Dict.Value(labelColumn.Col.GetValueIndex(i)))
}
}
labelHash = labelHasher.Sum64()
sampleLabelRow := row
if rows, ok := fb.rootsRow[unsafeString(lsbytes)]; ok {
if rows, ok := fb.rootsRow[labelHash]; ok {
sampleLabelRow = rows[0]
// We want to compare against this found label root's children.
fb.copyChildren(fb.children[sampleLabelRow])
fb.addRowValues(r, sampleLabelRow, i) // adds the cumulative and diff values to the existing row
} else {
lsstring := string(lsbytes) // we want to cast the bytes to a string and thus copy them.
err := fb.AppendLabelRow(r, t, recordLabelIndex, sampleLabelRow, i)
if err != nil {
return nil, 0, 0, 0, fmt.Errorf("failed to inject label row: %w", err)
}
fb.rootsRow[lsstring] = []int{sampleLabelRow}
fb.rootsRow[labelHash] = []int{sampleLabelRow}
}
fb.maxHeight = max(fb.maxHeight, fb.height)
fb.height = 1
Expand Down Expand Up @@ -194,7 +194,7 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr
if !r.Lines.IsValid(j) || llOffsetEnd-llOffsetStart <= 0 {
// We only want to compare the rows if this is the root, and we don't aggregate the labels.
if isRoot {
fb.copyChildren(fb.rootsRow[unsafeString(lsbytes)])
fb.copyChildren(fb.rootsRow[labelHash])
// append this row afterward to not compare to itself
fb.parent.Reset()
fb.maxHeight = max(fb.maxHeight, fb.height)
Expand All @@ -218,8 +218,7 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr

if isRoot {
// We aren't merging this root, so we'll keep track of it as a new one.
lsstring := string(lsbytes) // we want to cast the bytes to a string and thus copy them.
fb.rootsRow[lsstring] = append(fb.rootsRow[lsstring], row)
fb.rootsRow[labelHash] = append(fb.rootsRow[labelHash], row)
}

err = fb.appendRow(r, t, builderToRecordIndexMapping, i, j, -1, row)
Expand All @@ -238,7 +237,7 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr

// We only want to compare the rows if this is the root, and we don't aggregate the labels.
if isRoot {
fb.copyChildren(fb.rootsRow[unsafeString(lsbytes)])
fb.copyChildren(fb.rootsRow[labelHash])
// append this row afterward to not compare to itself
fb.parent.Reset()
fb.maxHeight = max(fb.maxHeight, fb.height)
Expand All @@ -256,8 +255,7 @@ func generateFlamegraphArrowRecord(ctx context.Context, mem memory.Allocator, tr

if isRoot {
// We aren't merging this root, so we'll keep track of it as a new one.
lsstring := string(lsbytes) // we want to cast the bytes to a string and thus copy them.
fb.rootsRow[lsstring] = append(fb.rootsRow[lsstring], row)
fb.rootsRow[labelHash] = append(fb.rootsRow[labelHash], row)
}

err = fb.appendRow(r, t, recordLabelIndex, i, j, k, row)
Expand Down Expand Up @@ -725,7 +723,7 @@ type flamegraphBuilder struct {
// This keeps track of the root rows indexed by the labels string.
// If the stack trace has no labels, we use the empty string as the key.
// This will be the root row's children, which is always our row 0 in flame graphs.
rootsRow map[string][]int
rootsRow map[uint64][]int
// compareRows are the rows that we compare to the current location against and potentially merge.
compareRows []int
// height keeps track of the current stack trace's height of the flame graph.
Expand Down Expand Up @@ -776,7 +774,7 @@ func newFlamegraphBuilder(

parent: parent(-1),
children: make([][]int, rows),
rootsRow: map[string][]int{},
rootsRow: map[uint64][]int{},
labelNameIndex: map[string]int{},
compareRows: make([]int, 0, 32),

Expand Down
39 changes: 0 additions & 39 deletions pkg/query/flamegraph_arrow_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
"bytes"
"compress/gzip"
"context"
"encoding/json"
"io"
"os"
"sort"
Expand Down Expand Up @@ -776,41 +775,3 @@ func BenchmarkArrowFlamegraph(b *testing.B) {
require.NoError(b, err)
}
}

func TestMarshalMap(t *testing.T) {
m := map[string]string{
"test1": "something",
"test2": "something_else",
}

buf := make([]byte, 0, 1024)
buf = MarshalStringMap(buf, m)
res := string(buf)
expected := []string{
`{"test1":"something","test2":"something_else"}`,
`{"test2":"something_else","test1":"something"}`,
}
require.Contains(t, expected, res)
}

func BenchmarkMarshalMap(b *testing.B) {
m := map[string]string{
"test1": "something",
"test2": "something_else",
}

var err error
b.ResetTimer()
b.Run("stdlib", func(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err = json.Marshal(m)
}
})
_ = err
b.Run("ours", func(b *testing.B) {
buf := make([]byte, 0, 1024)
for i := 0; i < b.N; i++ {
buf = MarshalStringMap(buf, m)
}
})
}
196 changes: 0 additions & 196 deletions pkg/query/json.go
View file
Open in desktop

This file was deleted.