Skip to content

Commit

Permalink
Implement unmarshal traces with jsoniter, make 40x faster than jsonpb (…
Browse files Browse the repository at this point in the history 
…#4817)

goos: darwin
goarch: amd64
pkg: go.opentelemetry.io/collector/model/otlp
cpu: Intel(R) Core(TM) i7-9750H CPU @ 2.60GHz
BenchmarkTracesJSONUnmarshal-12            10000            102486 ns/op           78030 B/op       1151 allocs/op
BenchmarkTracesJSONiterUnmarshal-12       512875              2431 ns/op            4002 B/op        185 allocs/op
  • Loading branch information
@hanjm
hanjm committed Mar 13, 2022
1 parent 5ee716f commit 0ec4ace
Show file tree
Hide file tree
Showing 8 changed files with 464 additions and 167 deletions.
3 changes: 3 additions & 0 deletions cmd/otelcorecol/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.0 // indirect
github.com/knadh/koanf v1.4.0 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
Expand All @@ -37,6 +38,8 @@ require (
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/mostynb/go-grpc-compression v1.1.16 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
Expand Down
165 changes: 0 additions & 165 deletions cmd/otelcorecol/go.sum
View file

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,13 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
Expand Down Expand Up @@ -368,9 +369,11 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mostynb/go-grpc-compression v1.1.16 h1:D9tGUINmcII049pxOj9dl32Fzhp26TrDVQXECoKJqQg=
github.com/mostynb/go-grpc-compression v1.1.16/go.mod h1:xxa6UoYynYS2h+5HB/Hglu81iYAp87ARaNmhhwi0s1s=
Expand Down
5 changes: 4 additions & 1 deletion model/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,17 @@ go 1.17

require (
github.com/gogo/protobuf v1.3.2
github.com/json-iterator/go v1.1.12
github.com/stretchr/testify v1.7.0
google.golang.org/grpc v1.44.0
google.golang.org/protobuf v1.27.1
)

require (
github.com/davecgh/go-spew v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/golang/protobuf v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f // indirect
Expand Down
11 changes: 10 additions & 1 deletion model/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,9 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP
github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
Expand Down Expand Up @@ -44,15 +45,23 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand Down
110 changes: 110 additions & 0 deletions model/otlp/json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ package otlp

import (
"testing"
"time"

"github.com/stretchr/testify/assert"

Expand Down Expand Up @@ -235,3 +236,112 @@ func TestMetricsNil(t *testing.T) {

assert.EqualValues(t, pdata.Metrics{}, got)
}

var tracesOTLPFull = func() pdata.Traces {
traceID := pdata.NewTraceID([16]byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10})
spanID := pdata.NewSpanID([8]byte{0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18})
td := pdata.NewTraces()
// Add ResourceSpans.
rs := td.ResourceSpans().AppendEmpty()
rs.SetSchemaUrl("schemaURL")
// Add resource.
rs.Resource().Attributes().UpsertString("host.name", "testHost")
rs.Resource().Attributes().UpsertString("service.name", "testService")
rs.Resource().SetDroppedAttributesCount(1)
// Add InstrumentationLibrarySpans.
il := rs.InstrumentationLibrarySpans().AppendEmpty()
il.InstrumentationLibrary().SetName("instrumentation name")
il.InstrumentationLibrary().SetVersion("instrumentation version")
il.SetSchemaUrl("schemaURL")
// Add spans.
sp := il.Spans().AppendEmpty()
sp.SetName("testSpan")
sp.SetKind(pdata.SpanKindClient)
sp.SetDroppedAttributesCount(1)
sp.SetStartTimestamp(pdata.NewTimestampFromTime(time.Now()))
sp.SetTraceID(traceID)
sp.SetSpanID(spanID)
sp.SetDroppedEventsCount(1)
sp.SetDroppedLinksCount(1)
sp.SetEndTimestamp(pdata.NewTimestampFromTime(time.Now()))
sp.SetParentSpanID(spanID)
sp.SetTraceState("state")
sp.Status().SetCode(pdata.StatusCodeOk)
sp.Status().SetMessage("message")
// Add attributes.
sp.Attributes().UpsertString("string", "value")
sp.Attributes().UpsertBool("bool", true)
sp.Attributes().UpsertInt("int", 1)
sp.Attributes().UpsertDouble("double", 1.1)
sp.Attributes().UpsertBytes("bytes", []byte("foo"))
// Add events.
event := sp.Events().AppendEmpty()
event.SetName("eventName")
event.SetTimestamp(pdata.NewTimestampFromTime(time.Now()))
event.SetDroppedAttributesCount(1)
event.Attributes().UpsertString("string", "value")
event.Attributes().UpsertBool("bool", true)
event.Attributes().UpsertInt("int", 1)
event.Attributes().UpsertDouble("double", 1.1)
event.Attributes().UpsertBytes("bytes", []byte("foo"))
// Add links.
link := sp.Links().AppendEmpty()
link.SetTraceState("state")
link.SetTraceID(traceID)
link.SetSpanID(spanID)
link.SetDroppedAttributesCount(1)
link.Attributes().UpsertString("string", "value")
link.Attributes().UpsertBool("bool", true)
link.Attributes().UpsertInt("int", 1)
link.Attributes().UpsertDouble("double", 1.1)
link.Attributes().UpsertBytes("bytes", []byte("foo"))
// Add another span.
sp2 := il.Spans().AppendEmpty()
sp2.SetName("testSpan2")
return td
}()

func TestTracesJSONiter(t *testing.T) {
encoder := NewJSONTracesMarshaler()
jsonBuf, err := encoder.MarshalTraces(tracesOTLPFull)
assert.NoError(t, err)

decoder := newJSONiterUnmarshaler()
got, err := decoder.UnmarshalTraces(jsonBuf)
assert.NoError(t, err)
assert.EqualValues(t, tracesOTLPFull, got)
}

func BenchmarkTracesJSONUnmarshal(b *testing.B) {
b.ReportAllocs()

encoder := NewJSONTracesMarshaler()
jsonBuf, err := encoder.MarshalTraces(tracesOTLPFull)
assert.NoError(b, err)
decoder := newJSONUnmarshaler()

b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, err := decoder.UnmarshalTraces(jsonBuf)
assert.NoError(b, err)
}
})
}

func BenchmarkTracesJSONiterUnmarshal(b *testing.B) {
b.ReportAllocs()

encoder := NewJSONTracesMarshaler()
jsonBuf, err := encoder.MarshalTraces(tracesOTLPFull)
assert.NoError(b, err)
decoder := newJSONiterUnmarshaler()

b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
_, err := decoder.UnmarshalTraces(jsonBuf)
assert.NoError(b, err)
}
})
}
Loading

0 comments on commit 0ec4ace

Please sign in to comment.