diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 79e77acf55..c2daabf1fc 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -27,6 +27,7 @@ https://github.com/elastic/apm-server/compare/6.4\...master[View commits] - Rename `transaction.span_count.dropped.total` to `transaction.span_count.dropped` on Intake API v2 {pull}1351[1351] - Require either `message` or `type` for `error.exception` {pull}1354[1354]. - Require `span.parent_id`, forbid `null` for `span.trace_id`, `transaction.trace_id` {pull}1391[1391] +- Require `error.id` and changed description to 128 random bits ID {pull}1384[1384] [[release-notes-6.4]] == APM Server version 6.4 diff --git a/docs/spec/errors/v2_error.json b/docs/spec/errors/v2_error.json index f6cd570b63..21d9ce1412 100644 --- a/docs/spec/errors/v2_error.json +++ b/docs/spec/errors/v2_error.json @@ -8,8 +8,8 @@ { "properties": { "id": { - "type": ["string", "null"], - "description": "Hex encoded 64 random bits ID of the error.", + "type": ["string"], + "description": "Hex encoded 128 random bits ID of the error.", "maxLength": 1024 }, "trace_id": { @@ -27,7 +27,8 @@ "type": ["string", "null"], "maxLength": 1024 } - } + }, + "required": ["id"] } ] } diff --git a/model/error/generated/schema/error.go b/model/error/generated/schema/error.go index 84649368ff..5b718bb635 100644 --- a/model/error/generated/schema/error.go +++ b/model/error/generated/schema/error.go @@ -423,8 +423,8 @@ const ModelSchema = `{ { "properties": { "id": { - "type": ["string", "null"], - "description": "Hex encoded 64 random bits ID of the error.", + "type": ["string"], + "description": "Hex encoded 128 random bits ID of the error.", "maxLength": 1024 }, "trace_id": { @@ -442,7 +442,8 @@ const ModelSchema = `{ "type": ["string", "null"], "maxLength": 1024 } - } + }, + "required": ["id"] } ] } diff --git a/model/span/event.go b/model/span/event.go index cf785a885a..fc2a23beb3 100644 --- a/model/span/event.go +++ b/model/span/event.go @@ -19,6 +19,8 @@ package span import ( "errors" + "math" + "strconv" "time" "github.com/santhosh-tekuri/jsonschema" @@ -61,16 +63,16 @@ type Event struct { Stacktrace m.Stacktrace Timestamp time.Time - TransactionId *string + TransactionId string // new in v2 - HexId *string - ParentId *string - TraceId *string + HexId string + ParentId string + TraceId string // deprecated in v2 - Id *int - Parent *int + Id *int64 + Parent *int64 } func V1DecodeEvent(input interface{}, err error) (transform.Transformable, error) { @@ -79,8 +81,11 @@ func V1DecodeEvent(input interface{}, err error) (transform.Transformable, error return nil, err } decoder := utility.ManualDecoder{} - e.Id = decoder.IntPtr(raw, "id") - e.Parent = decoder.IntPtr(raw, "parent") + e.Id = decoder.Int64Ptr(raw, "id") + e.Parent = decoder.Int64Ptr(raw, "parent") + if tid := decoder.StringPtr(raw, "transaction_id"); tid != nil { + e.TransactionId = *tid + } return e, decoder.Err } @@ -90,12 +95,42 @@ func V2DecodeEvent(input interface{}, err error) (transform.Transformable, error return nil, err } decoder := utility.ManualDecoder{} - e.HexId = decoder.StringPtr(raw, "id") - e.ParentId = decoder.StringPtr(raw, "parent_id") - e.TraceId = decoder.StringPtr(raw, "trace_id") + e.HexId = decoder.String(raw, "id") + e.ParentId = decoder.String(raw, "parent_id") + e.TraceId = decoder.String(raw, "trace_id") + e.TransactionId = decoder.String(raw, "transaction_id") + if decoder.Err != nil { + return nil, decoder.Err + } + + // HexId must be a 64 bit hex encoded string. The id is set to the integer + // converted value of the hexId + if idInt, err := hexToInt(e.HexId, 64); err == nil { + e.Id = &idInt + } else { + return nil, err + } + + // set parent to parentId + if id, err := hexToInt(e.ParentId, 64); err == nil { + e.Parent = &id + } else { + return nil, err + } + return e, decoder.Err } +var shift = uint64(math.Pow(2, 63)) + +func hexToInt(s string, bitSize int) (int64, error) { + us, err := strconv.ParseUint(s, 16, bitSize) + if err != nil { + return 0, err + } + return int64(us - shift), nil +} + func decodeEvent(input interface{}, err error) (*Event, map[string]interface{}, error) { if err != nil { return nil, nil, err @@ -110,13 +145,12 @@ func decodeEvent(input interface{}, err error) (*Event, map[string]interface{}, decoder := utility.ManualDecoder{} event := Event{ - Name: decoder.String(raw, "name"), - Type: decoder.String(raw, "type"), - Start: decoder.Float64(raw, "start"), - Duration: decoder.Float64(raw, "duration"), - Context: decoder.MapStr(raw, "context"), - Timestamp: decoder.TimeRFC3339(raw, "timestamp"), - TransactionId: decoder.StringPtr(raw, "transaction_id"), + Name: decoder.String(raw, "name"), + Type: decoder.String(raw, "type"), + Start: decoder.Float64(raw, "start"), + Duration: decoder.Float64(raw, "duration"), + Context: decoder.MapStr(raw, "context"), + Timestamp: decoder.TimeRFC3339(raw, "timestamp"), } var stacktr *m.Stacktrace stacktr, err = m.DecodeStacktrace(raw["stacktrace"], decoder.Err) @@ -142,9 +176,9 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event { spanDocType: e.fields(tctx), "context": tctx.Metadata.MergeMinimal(e.Context), } - utility.AddId(fields, "transaction", e.TransactionId) - utility.AddId(fields, "parent", e.ParentId) - utility.AddId(fields, "trace", e.TraceId) + utility.AddId(fields, "transaction", &e.TransactionId) + utility.AddId(fields, "parent", &e.ParentId) + utility.AddId(fields, "trace", &e.TraceId) return []beat.Event{ beat.Event{ @@ -163,7 +197,9 @@ func (s *Event) fields(tctx *transform.Context) common.MapStr { utility.Add(tr, "id", s.Id) utility.Add(tr, "parent", s.Parent) // v2 - utility.Add(tr, "hex_id", s.HexId) + if s.HexId != "" { + utility.Add(tr, "hex_id", s.HexId) + } utility.Add(tr, "name", s.Name) utility.Add(tr, "type", s.Type) diff --git a/model/span/event_test.go b/model/span/event_test.go index 1335efa134..37256934a0 100644 --- a/model/span/event_test.go +++ b/model/span/event_test.go @@ -34,17 +34,7 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TestSpanEventDecode(t *testing.T) { - tid := "longid" - name, spType := "foo", "db" - start, duration := 1.2, 3.4 - context := map[string]interface{}{"a": "b"} - stacktrace := []interface{}{map[string]interface{}{ - "filename": "file", "lineno": 1.0, - }} - timestamp := "2017-05-30T18:53:27.154Z" - timestampParsed, _ := time.Parse(time.RFC3339, timestamp) - +func TestSpanEventDecodeFailures(t *testing.T) { for _, test := range []struct { input interface{} err, inpErr error @@ -58,95 +48,227 @@ func TestSpanEventDecode(t *testing.T) { err: errors.New("Error fetching field"), e: nil, }, + } { + for _, decodeFct := range []func(interface{}, error) (transform.Transformable, error){V1DecodeEvent, V2DecodeEvent} { + transformable, err := decodeFct(test.input, test.inpErr) + assert.Equal(t, test.err, err) + if test.e != nil { + event := transformable.(*Event) + assert.Equal(t, test.e, event) + } else { + assert.Nil(t, transformable) + } + } + } +} + +func TestDecodeSpanV1(t *testing.T) { + spanTime, _ := time.Parse(time.RFC3339, "2018-05-30T19:53:17.134Z") + id, parent, tid := int64(1), int64(12), "abc" + name, spType := "foo", "db" + start, duration := 1.2, 3.4 + context := map[string]interface{}{"a": "b"} + stacktrace := []interface{}{map[string]interface{}{ + "filename": "file", "lineno": 1.0, + }} + for idx, test := range []struct { + input interface{} + err, inpErr error + s *Event + }{ { + //minimal span payload input: map[string]interface{}{ "name": name, "type": spType, "start": start, "duration": duration, - "context": context, "stacktrace": stacktrace, - "transaction_id": tid, "timestamp": timestamp, + "timestamp": "2018-05-30T19:53:17.134Z", }, err: nil, - e: &Event{ + s: &Event{ + Name: name, + Type: spType, + Start: start, + Duration: duration, + Timestamp: spanTime, + }, + }, + { + // full valid payload + input: map[string]interface{}{ + "name": name, "id": 1.0, "type": spType, + "start": start, "duration": duration, + "context": context, "parent": 12.0, + "timestamp": "2018-05-30T19:53:17.134Z", + "stacktrace": stacktrace, "transaction_id": tid, + }, + err: nil, + s: &Event{ + Id: &id, Name: name, Type: spType, Start: start, Duration: duration, Context: context, - Timestamp: timestampParsed, - TransactionId: &tid, + Parent: &parent, + Timestamp: spanTime, + TransactionId: tid, Stacktrace: m.Stacktrace{ &m.StacktraceFrame{Filename: "file", Lineno: 1}, }, }, }, + { + // ignore distributed tracing data + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, + "timestamp": "2018-05-30T19:53:17.134Z", + "hex_id": "hexId", "parent_id": "parentId", "trace_id": "trace_id", + }, + err: nil, + s: &Event{ + Name: name, + Type: spType, + Start: start, + Duration: duration, + Timestamp: spanTime, + }, + }, } { - for _, decodeFct := range []func(interface{}, error) (transform.Transformable, error){V1DecodeEvent, V2DecodeEvent} { - transformable, err := decodeFct(test.input, test.inpErr) - if test.e != nil { - event := transformable.(*Event) - assert.Equal(t, test.e, event) - } else { - assert.Nil(t, transformable) - } + span, err := V1DecodeEvent(test.input, test.inpErr) + assert.Equal(t, test.err, err) + if test.err != nil { + assert.Error(t, err) assert.Equal(t, test.err, err) } + assert.Equal(t, test.s, span, fmt.Sprintf("Idx <%x>", idx)) } } -func TestVersionedSpanEvendDecode(t *testing.T) { - name, spType, start, duration := "foo", "db", 1.2, 3.4 - tid := "longid" - hexId, parentId := "0147258369012345", "abcdef0123456789" - traceId := "abcdef0123456789abcdef0123456789" - id, parent := 1, 12 - timestamp := "2017-05-30T18:53:27.154Z" - //timestampParsed, _ := time.Parse(time.RFC3339, timestamp) - - // test V1 - input := map[string]interface{}{ - "name": name, "type": spType, "start": start, "duration": duration, - "id": 1.0, "parent": 12.0, "parent_id": parentId, - "trace_id": traceId, - "transaction_id": tid, timestamp: "timestamp", - } - e := &Event{ - Name: name, - Type: spType, - Start: start, - Duration: duration, - Id: &id, - Parent: &parent, - TransactionId: &tid, - } - transformable, err := V1DecodeEvent(input, nil) - assert.NoError(t, err) - assert.Equal(t, e, transformable.(*Event)) - - // test V2 - input = map[string]interface{}{ - "name": name, "type": spType, "start": start, "duration": duration, - "id": hexId, "parent": 12.0, "parent_id": parentId, - "trace_id": traceId, - "transaction_id": tid, timestamp: "timestamp", - } - e = &Event{ - Name: name, - Type: spType, - Start: start, - Duration: duration, - HexId: &hexId, - ParentId: &parentId, - TraceId: &traceId, - TransactionId: &tid, +func TestDecodeSpanV2(t *testing.T) { + spanTime, _ := time.Parse(time.RFC3339, "2018-05-30T19:53:17.134Z") + id, parentId, invalidId := "0000000000000000", "FFFFFFFFFFFFFFFF", "invalidId" + idInt, parentIdInt := int64(-9223372036854775808), int64(9223372036854775807) + transactionId, traceId := "ABCDEF0123456789", "01234567890123456789abcdefABCDEF" + name, spType := "foo", "db" + start, duration := 1.2, 3.4 + context := map[string]interface{}{"a": "b"} + stacktrace := []interface{}{map[string]interface{}{ + "filename": "file", "lineno": 1.0, + }} + fmt.Println(invalidId) + for idx, test := range []struct { + input interface{} + err, inpErr error + e transform.Transformable + }{ + { + // invalid id + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, "parent_id": parentId, + "timestamp": "2018-05-30T19:53:17.134Z", "id": invalidId, "trace_id": traceId, "transaction_id": transactionId, + }, + err: errors.New("strconv.ParseUint: parsing \"invalidId\": invalid syntax"), + e: nil, + }, + { + // missing traceId + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, "parent_id": parentId, + "timestamp": "2018-05-30T19:53:17.134Z", "id": id, "transaction_id": transactionId, + }, + err: errors.New("Error fetching field"), + e: nil, + }, + { + // missing transactionId + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, "parent_id": parentId, + "timestamp": "2018-05-30T19:53:17.134Z", "id": id, "trace_id": traceId, + }, + err: errors.New("Error fetching field"), + e: nil, + }, + { + // missing id + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, "parent_id": parentId, + "timestamp": "2018-05-30T19:53:17.134Z", "trace_id": traceId, "transaction_id": transactionId, + }, + err: errors.New("Error fetching field"), + e: nil, + }, + { + // missing parent_id + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, + "timestamp": "2018-05-30T19:53:17.134Z", "id": id, "trace_id": traceId, "transaction_id": transactionId, + }, + err: errors.New("Error fetching field"), + e: nil, + }, + { + // minimal payload + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, "parent_id": parentId, + "timestamp": "2018-05-30T19:53:17.134Z", "id": id, "trace_id": traceId, "transaction_id": transactionId, + }, + err: nil, + e: &Event{ + Name: name, + Type: spType, + Start: start, + Duration: duration, + Timestamp: spanTime, + Id: &idInt, + ParentId: parentId, + Parent: &parentIdInt, + HexId: id, + TraceId: traceId, + TransactionId: transactionId, + }, + }, + { + // full valid payload + input: map[string]interface{}{ + "name": name, "type": spType, "start": start, "duration": duration, + "context": context, "timestamp": "2018-05-30T19:53:17.134Z", "stacktrace": stacktrace, + "id": id, "parent_id": parentId, "trace_id": traceId, "transaction_id": transactionId, + }, + err: nil, + e: &Event{ + Name: name, + Type: spType, + Start: start, + Duration: duration, + Context: context, + Timestamp: spanTime, + Stacktrace: m.Stacktrace{ + &m.StacktraceFrame{Filename: "file", Lineno: 1}, + }, + Id: &idInt, + HexId: id, + TraceId: traceId, + ParentId: parentId, + Parent: &parentIdInt, + TransactionId: transactionId, + }, + }, + } { + event, err := V2DecodeEvent(test.input, test.inpErr) + if test.err != nil { + if assert.Error(t, err) { + assert.Equal(t, test.err.Error(), err.Error()) + } + } else { + assert.NoError(t, err) + } + + assert.Equal(t, test.e, event, fmt.Sprintf("Idx <%x>", idx)) } - transformable, err = V2DecodeEvent(input, nil) - assert.NoError(t, err) - assert.Equal(t, e, transformable.(*Event)) } func TestSpanTransform(t *testing.T) { path := "test/path" - parent := 12 - tid := 1 + parent, tid := int64(12), int64(1) service := metadata.Service{Name: "myService"} hexId, parentId, traceId := "0147258369012345", "abcdef0123456789", "01234567890123456789abcdefa" @@ -168,9 +290,9 @@ func TestSpanTransform(t *testing.T) { { Event: Event{ Id: &tid, - HexId: &hexId, - TraceId: &traceId, - ParentId: &parentId, + HexId: hexId, + TraceId: traceId, + ParentId: parentId, Name: "myspan", Type: "myspantype", Start: 0.65, @@ -180,13 +302,13 @@ func TestSpanTransform(t *testing.T) { Parent: &parent, }, Output: common.MapStr{ + "id": tid, + "parent": parent, + "hex_id": hexId, "duration": common.MapStr{"us": 1200}, - "id": 1, "name": "myspan", "start": common.MapStr{"us": 650}, "type": "myspantype", - "parent": 12, - "hex_id": hexId, "stacktrace": []common.MapStr{{ "exclude_from_grouping": false, "abs_path": path, @@ -195,13 +317,12 @@ func TestSpanTransform(t *testing.T) { "sourcemap": common.MapStr{ "error": "Colno mandatory for sourcemapping.", "updated": false, - }, - }}, + }}}, }, Msg: "Full Span", }, { - Event: Event{HexId: &hexId, ParentId: &parentId}, + Event: Event{HexId: hexId, ParentId: parentId}, Output: common.MapStr{ "type": "", "start": common.MapStr{"us": 0}, @@ -219,10 +340,10 @@ func TestSpanTransform(t *testing.T) { Service: &service, }, } - for idx, test := range tests { + for _, test := range tests { output := test.Event.Transform(tctx) fields := output[0].Fields["span"] - assert.Equal(t, test.Output, fields, fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) + assert.Equal(t, test.Output, fields) } } @@ -237,3 +358,32 @@ func TestEventTransformUseReqTime(t *testing.T) { require.Len(t, beatEvent, 1) assert.Equal(t, reqTimestampParsed, beatEvent[0].Timestamp) } + +func TestHexToInt(t *testing.T) { + testData := []struct { + input string + bitSize int + valid bool + out int64 + }{ + {"", 16, false, 0}, + {"ffffffffffffffff0", 64, false, 0}, //value out of range + {"abcdefx123456789", 64, false, 0}, //invalid syntax + {"0123456789abcdef", 64, true, -9141386507638288913}, // 81985529216486895-9223372036854775808 + {"0123456789ABCDEF", 64, true, -9141386507638288913}, // 81985529216486895-9223372036854775808 + {"0000000000000000", 64, true, -9223372036854775808}, // 0-9223372036854775808 + {"ffffffffffffffff", 64, true, 9223372036854775807}, // 18446744073709551615-9223372036854775808 + {"ac03", 16, true, -9223372036854731773}, // 44035-9223372036854775808 + {"acde123456789", 64, true, -9220330920244582519}, //3041116610193289-9223372036854775808 + } + for _, dt := range testData { + out, err := hexToInt(dt.input, dt.bitSize) + if dt.valid { + assert.NoError(t, err) + } else { + assert.Error(t, err) + } + assert.Equal(t, dt.out, out, + fmt.Sprintf("Expected hexToInt(%v) to return %v", dt.input, dt.out)) + } +} diff --git a/model/transaction/event.go b/model/transaction/event.go index 80e107ceaf..bf9745eb9a 100644 --- a/model/transaction/event.go +++ b/model/transaction/event.go @@ -70,7 +70,7 @@ type Event struct { //v2 ParentId *string - TraceId *string + TraceId string // deprecated in V2 Spans []*span.Event @@ -102,8 +102,8 @@ func V1DecodeEvent(input interface{}, err error) (transform.Transformable, error sp.Timestamp = e.Timestamp } - if sp.TransactionId == nil || *sp.TransactionId == "" { - sp.TransactionId = &e.Id + if sp.TransactionId == "" { + sp.TransactionId = e.Id } } @@ -121,8 +121,11 @@ func V2DecodeEvent(input interface{}, err error) (transform.Transformable, error e.SpanCount = SpanCount{Dropped: decoder.IntPtr(raw, "dropped", "span_count"), Started: decoder.IntPtr(raw, "started", "span_count")} e.ParentId = decoder.StringPtr(raw, "parent_id") - e.TraceId = decoder.StringPtr(raw, "trace_id") - return e, decoder.Err + e.TraceId = decoder.String(raw, "trace_id") + if decoder.Err != nil { + return nil, decoder.Err + } + return e, nil } func decodeEvent(input interface{}, err error) (*Event, map[string]interface{}, error) { @@ -193,7 +196,7 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event { "context": tctx.Metadata.Merge(e.Context), } utility.AddId(fields, "parent", e.ParentId) - utility.AddId(fields, "trace", e.TraceId) + utility.AddId(fields, "trace", &e.TraceId) events = append(events, beat.Event{Fields: fields, Timestamp: e.Timestamp}) spanCounter.Add(int64(len(e.Spans))) diff --git a/model/transaction/event_test.go b/model/transaction/event_test.go index 5fe572102f..e7a8784aac 100644 --- a/model/transaction/event_test.go +++ b/model/transaction/event_test.go @@ -34,15 +34,7 @@ import ( "github.com/elastic/beats/libbeat/common" ) -func TestTransactionEventDecode(t *testing.T) { - id, trType, name, result := "123", "type", "foo()", "555" - duration := 1.67 - context := map[string]interface{}{"a": "b"} - marks := map[string]interface{}{"k": "b"} - timestamp := "2017-05-30T18:53:27.154Z" - timestampParsed, _ := time.Parse(time.RFC3339, timestamp) - sampled := true - +func TestTransactionEventDecodeFailure(t *testing.T) { for _, test := range []struct { input interface{} err, inpErr error @@ -56,82 +48,135 @@ func TestTransactionEventDecode(t *testing.T) { err: errors.New("Error fetching field"), e: nil, }, - { - input: map[string]interface{}{ - "id": id, "type": trType, "name": name, "result": result, - "duration": duration, "timestamp": timestamp, - "context": context, "marks": marks, "sampled": sampled}, - err: nil, - e: &Event{ - Id: id, Type: trType, Name: &name, Result: &result, - Duration: duration, Timestamp: timestampParsed, - Context: context, Marks: marks, Sampled: &sampled, - }, - }, } { for _, decodeFct := range []func(interface{}, error) (transform.Transformable, error){V1DecodeEvent, V2DecodeEvent} { transformable, err := decodeFct(test.input, test.inpErr) + assert.Equal(t, test.err, err) if test.e != nil { event := transformable.(*Event) assert.Equal(t, test.e, event) } else { assert.Nil(t, transformable) } - assert.Equal(t, test.err, err) } } } -func TestVersionedTransactionEventDecode(t *testing.T) { - id, trType, duration := "123", "type", 1.67 +func TestTransactionEventDecodeV1(t *testing.T) { + id, trType, name, result := "123", "type", "foo()", "555" timestamp := "2017-05-30T18:53:27.154Z" timestampParsed, _ := time.Parse(time.RFC3339, timestamp) traceId, parentId := "0147258369012345abcdef0123456789", "abcdef0123456789" - dropped, startedSpans := 12, 148 + dropped, duration := 12, 1.67 + context := map[string]interface{}{"a": "b"} + marks := map[string]interface{}{"k": "b"} + sampled := true - // test V1 - input := map[string]interface{}{ - "id": id, "type": trType, "duration": duration, "timestamp": timestamp, - "parent_id": parentId, "trace_id": traceId, - "spans": []interface{}{ - map[string]interface{}{ - "name": "span", "type": "db", "start": 1.2, "duration": 2.3, + for _, test := range []struct { + input interface{} + e *Event + }{ + // minimal event + {input: map[string]interface{}{ + "id": id, "type": trType, "duration": duration, "timestamp": timestamp, + }, + e: &Event{ + Id: id, Type: trType, Duration: duration, Timestamp: timestampParsed, }, }, - "span_count": map[string]interface{}{"dropped": map[string]interface{}{"total": 12.0}}, - } - e := &Event{ - Id: id, Type: trType, Duration: duration, Timestamp: timestampParsed, - Spans: []*span.Event{ - &span.Event{Name: "span", Type: "db", Start: 1.2, Duration: 2.3, - TransactionId: &id, Timestamp: timestampParsed}, + // full event, ignoring v2 attrs + { + input: map[string]interface{}{ + "id": id, "type": trType, "name": name, "result": result, + "duration": duration, "timestamp": timestamp, + "context": context, "marks": marks, "sampled": sampled, + "parent_id": parentId, "trace_id": traceId, + "spans": []interface{}{ + map[string]interface{}{ + "name": "span", "type": "db", "start": 1.2, "duration": 2.3, + }}, + "span_count": map[string]interface{}{"dropped": map[string]interface{}{"total": 12.0}}}, + e: &Event{ + Id: id, Type: trType, Name: &name, Result: &result, + Duration: duration, Timestamp: timestampParsed, + Context: context, Marks: marks, Sampled: &sampled, + SpanCount: SpanCount{Dropped: &dropped}, + Spans: []*span.Event{ + &span.Event{Name: "span", Type: "db", Start: 1.2, Duration: 2.3, TransactionId: id, Timestamp: timestampParsed}, + }, + }, }, - SpanCount: SpanCount{Dropped: &dropped}, + } { + transformable, err := V1DecodeEvent(test.input, nil) + assert.NoError(t, err) + assert.Equal(t, test.e, transformable.(*Event)) } - transformable, err := V1DecodeEvent(input, nil) - assert.NoError(t, err) - assert.Equal(t, e, transformable.(*Event)) +} + +func TestTransactionEventDecodeV2(t *testing.T) { + id, trType, name, result := "123", "type", "foo()", "555" + timestamp := "2017-05-30T18:53:27.154Z" + timestampParsed, _ := time.Parse(time.RFC3339, timestamp) + traceId, parentId := "0147258369012345abcdef0123456789", "abcdef0123456789" + dropped, started, duration := 12, 6, 1.67 + context := map[string]interface{}{"a": "b"} + marks := map[string]interface{}{"k": "b"} + sampled := true - // test V2 - input = map[string]interface{}{ - "id": id, "type": trType, "duration": duration, "timestamp": timestamp, - "parent_id": parentId, "trace_id": traceId, - "spans": []interface{}{ - map[string]interface{}{ - "name": "span", "type": "db", "start": 1.2, "duration": 2.3, + for _, test := range []struct { + input interface{} + err error + e *Event + }{ + // traceId missing + { + input: map[string]interface{}{ + "id": id, "type": trType, "duration": duration, "timestamp": timestamp, + "span_count": map[string]interface{}{"started": 6.0}}, + err: errors.New("Error fetching field"), + }, + // minimal event + { + input: map[string]interface{}{ + "id": id, "type": trType, "duration": duration, "timestamp": timestamp, + "trace_id": traceId, "span_count": map[string]interface{}{"started": 6.0}}, + e: &Event{ + Id: id, Type: trType, TraceId: traceId, + Duration: duration, Timestamp: timestampParsed, + SpanCount: SpanCount{Started: &started}, }, }, - "span_count": map[string]interface{}{"dropped": 12.0, "started": 148.0}, - } - e = &Event{ - Id: id, Type: trType, Duration: duration, Timestamp: timestampParsed, - TraceId: &traceId, ParentId: &parentId, - SpanCount: SpanCount{Started: &startedSpans, Dropped: &dropped}, + // full event, ignoring spans + { + input: map[string]interface{}{ + "id": id, "type": trType, "name": name, "result": result, + "duration": duration, "timestamp": timestamp, + "context": context, "marks": marks, "sampled": sampled, + "parent_id": parentId, "trace_id": traceId, + "spans": []interface{}{ + map[string]interface{}{ + "name": "span", "type": "db", "start": 1.2, "duration": 2.3, + }}, + "span_count": map[string]interface{}{"dropped": 12.0, "started": 6.0}}, + e: &Event{ + Id: id, Type: trType, Name: &name, Result: &result, + ParentId: &parentId, TraceId: traceId, + Duration: duration, Timestamp: timestampParsed, + Context: context, Marks: marks, Sampled: &sampled, + SpanCount: SpanCount{Dropped: &dropped, Started: &started}, + }, + }, + } { + transformable, err := V2DecodeEvent(test.input, nil) + assert.Equal(t, test.err, err) + if test.e != nil { + event := transformable.(*Event) + assert.Equal(t, test.e, event) + } else { + assert.Nil(t, transformable) + } } - transformable, err = V2DecodeEvent(input, nil) - assert.NoError(t, err) - assert.Equal(t, e, transformable.(*Event)) } func TestEventTransform(t *testing.T) { diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidEvent.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidEvent.approved.json index 4ba402a1c4..3e5120c62e 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidEvent.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidEvent.approved.json @@ -23,7 +23,9 @@ "us": 141581 }, "hex_id": "abcdef01234567", + "id": -9175013389437287000, "name": "GET /api/types", + "parent": 3156441702022342700, "start": { "us": 0 }, diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidJSONEvent.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidJSONEvent.approved.json index 4ba402a1c4..3e5120c62e 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidJSONEvent.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationInvalidJSONEvent.approved.json @@ -23,7 +23,9 @@ "us": 141581 }, "hex_id": "abcdef01234567", + "id": -9175013389437287000, "name": "GET /api/types", + "parent": 3156441702022342700, "start": { "us": 0 }, diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationMixedMinimalProcess.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationMixedMinimalProcess.approved.json index 3b95efe542..66112e8812 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationMixedMinimalProcess.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationMixedMinimalProcess.approved.json @@ -56,7 +56,9 @@ "us": 3564 }, "hex_id": "0123456a89012345", + "id": -9141386494764572000, "name": "GET /api/types", + "parent": 3108404491683177500, "start": { "us": 1845 }, diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationOptionalTimestamps.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationOptionalTimestamps.approved.json index 6d27f1dc73..8e17e76376 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationOptionalTimestamps.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationOptionalTimestamps.approved.json @@ -88,7 +88,9 @@ "us": 20000 }, "hex_id": "0147258369abcdef", + "id": -9131288473126581000, "name": "sp1", + "parent": -9218247941889585000, "start": { "us": 10000 }, diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationSpans.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationSpans.approved.json index 97f6d51084..726dff42f9 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationSpans.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationSpans.approved.json @@ -23,7 +23,9 @@ "us": 141581 }, "hex_id": "abcdef01234567", + "id": -9175013389437287000, "name": "GET /api/types", + "parent": 3156441702022342700, "start": { "us": 0 }, @@ -58,8 +60,10 @@ "duration": { "us": 32592 }, - "hex_id": "1234abcdef56789568", + "hex_id": "1234abcdef567895", + "id": -7911509744411052000, "name": "GET /api/types", + "parent": -9223372036568445000, "start": { "us": 0 }, @@ -95,7 +99,9 @@ "us": 3564 }, "hex_id": "0123456a89012345", + "id": -9141386494764572000, "name": "GET /api/types", + "parent": 3156442435030055000, "start": { "us": 1845 }, @@ -131,7 +137,9 @@ "us": 13980 }, "hex_id": "abcde56a89012345", + "id": 3156431159584433000, "name": "get /api/types", + "parent": 3146835049025875000, "start": { "us": 0 }, @@ -176,7 +184,9 @@ "us": 3781 }, "hex_id": "1234567890aaaade", + "id": -7911603569559950000, "name": "SELECT FROM product_types", + "parent": 3156441702022342700, "stacktrace": [ { "exclude_from_grouping": false, diff --git a/testdata/intake-v2/spans.ndjson b/testdata/intake-v2/spans.ndjson index 2916910934..87b99d6c81 100644 --- a/testdata/intake-v2/spans.ndjson +++ b/testdata/intake-v2/spans.ndjson @@ -1,6 +1,6 @@ {"metadata": {"user": {"id": "123", "email": "s@test.com", "username": "john"}, "process": {"ppid": 6789, "pid": 1234,"argv": ["node", "server.js"], "title": "node"}, "system": {"platform": "darwin", "hostname": "prod1.example.com", "architecture": "x64"}, "service": {"name": "backendspans", "language": {"version": "8", "name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}, "environment": "staging", "framework": {"version": "1.2.3", "name": "Express"}, "version": "5.1.3", "runtime": {"version": "8.0.0", "name": "node"}}}} {"span": {"trace_id": "fdedef0123456789abcdef9876543210", "parent_id": "abcdef0123456789","id": "abcdef01234567", "transaction_id": "01af25874dec69dd", "name": "GET /api/types", "type": "request","start": 0, "duration": 141.581 }} -{ "span": {"trace_id": "abcdef0123456789abcdef9876543210", "id": "1234abcdef56789568", "transaction_id": "ab45781d265894fe", "parent_id": "0000000011111111", "parent": 45, "name": "GET /api/types", "type": "request", "start": 0, "duration": 32.592981 } } +{ "span": {"trace_id": "abcdef0123456789abcdef9876543210", "id": "1234abcdef567895", "transaction_id": "ab45781d265894fe", "parent_id": "0000000011111111", "parent": 45, "name": "GET /api/types", "type": "request", "start": 0, "duration": 32.592981 } } { "span": {"trace_id": "abcdef0123456789abcdef9876543210", "id": "0123456a89012345", "transaction_id": "ab23456a89012345", "parent": 1, "parent_id": "abcdefabcdef7890", "name": "GET /api/types", "type": "request", "start": 1.845, "duration": 3.5642981, "stacktrace": [], "context": {} }} { "span": {"trace_id": "abcdef0123456789abcdef9876543210","id": "abcde56a89012345", "parent_id": "ababcdcdefefabde", "transaction_id": "bed3456a89012345", "name": "get /api/types", "type": "request", "start": 0, "duration": 13.9802981, "stacktrace": null, "context": null }} { "span": {"trace_id": "abcdef0123456789abcdef9876543210", "id": "1234567890aaaade", "parent_id": "abcdef0123456789", "transaction_id": "aff4567890aaaade", "parent": null, "name": "SELECT FROM product_types", "type": "db.postgresql.query", "start": 2.83092, "duration": 3.781912, "stacktrace": [{ "filename": "net.js", "lineno": 547},{"filename": "file2.js", "lineno": 12, "post_context": [ " ins.currentTransaction = prev", "}"]}, { "function": "onread", "abs_path": "net.js", "filename": "net.js", "lineno": 547, "library_frame": true, "vars": { "key": "value" }, "module": "some module", "colno": 4, "context_line": "line3", "pre_context": [ " var trans = this.currentTransaction", "" ], "post_context": [ " ins.currentTransaction = prev", " return result"] }], "context": { "db": { "instance": "customers", "statement": "SELECT * FROM product_types WHERE user_id=?", "type": "sql", "user": "readonly_user" }, "http": { "url": "http://localhost:8000" } } }} diff --git a/utility/data_fetcher.go b/utility/data_fetcher.go index d704214b17..173856db5d 100644 --- a/utility/data_fetcher.go +++ b/utility/data_fetcher.go @@ -93,6 +93,27 @@ func (d *ManualDecoder) IntPtr(base map[string]interface{}, key string, keys ... return nil } +func (d *ManualDecoder) Int64Ptr(base map[string]interface{}, key string, keys ...string) *int64 { + val := getDeep(base, keys...)[key] + if val == nil { + return nil + } else if valNumber, ok := val.(json.Number); ok { + if valInt, err := valNumber.Int64(); err != nil { + d.Err = err + } else { + i := int64(valInt) + return &i + } + } else if valFloat, ok := val.(float64); ok { + valInt := int64(valFloat) + if valFloat == float64(valInt) { + return &valInt + } + } + d.Err = fetchErr + return nil +} + func (d *ManualDecoder) Int(base map[string]interface{}, key string, keys ...string) int { if val := d.IntPtr(base, key, keys...); val != nil { return *val diff --git a/utility/data_fetcher_test.go b/utility/data_fetcher_test.go index cb2bac334f..d7682b706f 100644 --- a/utility/data_fetcher_test.go +++ b/utility/data_fetcher_test.go @@ -107,6 +107,21 @@ func TestIntPtr(t *testing.T) { } } +func TestInt64Ptr(t *testing.T) { + var outnil *int64 + int64Fl64 := int64(intFl64) + for _, test := range []testStr{ + {key: "intfl64", keys: []string{}, out: &int64Fl64, err: nil}, + {key: "missing", keys: []string{"a", "b"}, out: outnil, err: nil}, + {key: "str", keys: []string{"a", "b"}, out: outnil, err: fetchErr}, + } { + decoder := ManualDecoder{} + out := decoder.Int64Ptr(decoderBase, test.key, test.keys...) + assert.Equal(t, test.out, out) + assert.Equal(t, test.err, decoder.Err) + } +} + func TestInt(t *testing.T) { for _, test := range []testStr{ {key: "intfl32", keys: []string{}, out: intFl32, err: nil}, diff --git a/utility/map_str_enhancer.go b/utility/map_str_enhancer.go index e9839517e7..8bd3d473f1 100644 --- a/utility/map_str_enhancer.go +++ b/utility/map_str_enhancer.go @@ -47,6 +47,12 @@ func Add(m common.MapStr, key string, val interface{}) { } else { delete(m, key) } + case *int64: + if newVal := val.(*int64); newVal != nil { + m[key] = *newVal + } else { + delete(m, key) + } case *string: if value != nil { m[key] = *value