From 71abd6621dd8d6b1cbd03da303acc201512a5a3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 7 Sep 2020 15:52:55 +0200 Subject: [PATCH 1/8] Derive outcome from the HTTP status code when available --- ...TestPublishIntegrationEvents.approved.json | 4 +- .../TestPublishIntegrationSpans.approved.json | 2 +- model/modeldecoder/span.go | 18 +++++-- model/modeldecoder/span_test.go | 47 ++++++++++++++++++- model/modeldecoder/transaction.go | 12 ++++- model/modeldecoder/transaction_test.go | 47 +++++++++++++++++++ .../testIntakeIntegrationEvents.approved.json | 4 +- .../testIntakeIntegrationSpans.approved.json | 2 +- .../testIntakeRUMV3Events.approved.json | 8 ++-- 9 files changed, 128 insertions(+), 16 deletions(-) diff --git a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json index 364589cfcc5..75e5ddc9701 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json @@ -17,7 +17,7 @@ "version": "1.5.0" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "amd64", @@ -210,7 +210,7 @@ "version": "1.5.0" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "amd64", diff --git a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json index ccdd0439ba6..82a24027f00 100644 --- a/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json +++ b/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json @@ -555,7 +555,7 @@ "version": "1.5.0" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "x64", diff --git a/model/modeldecoder/span.go b/model/modeldecoder/span.go index 786ed251ed1..706759e4639 100644 --- a/model/modeldecoder/span.go +++ b/model/modeldecoder/span.go @@ -18,6 +18,7 @@ package modeldecoder import ( + "net/http" "strings" "time" @@ -93,10 +94,6 @@ func decodeSpan(input Input, schema *jsonschema.Schema) (_ *model.Span, parentIn decodeString(raw, fieldName("parent_id"), &event.ParentID) decodeString(raw, fieldName("trace_id"), &event.TraceID) decodeString(raw, fieldName("transaction_id"), &event.TransactionID) - decodeString(raw, fieldName("outcome"), &event.Outcome) - if event.Outcome == "" { - event.Outcome = "unknown" - } ctx := decoder.MapStr(raw, fieldName("context")) if ctx != nil { @@ -139,6 +136,19 @@ func decodeSpan(input Input, schema *jsonschema.Schema) (_ *model.Span, parentIn } } } + decodeString(raw, fieldName("outcome"), &event.Outcome) + if event.Outcome == "" { + if event.HTTP != nil && event.HTTP.StatusCode != nil { + statusCode := *event.HTTP.StatusCode + if statusCode >= http.StatusBadRequest { + event.Outcome = "failure" + } else { + event.Outcome = "success" + } + } else { + event.Outcome = "unknown" + } + } var stacktr *model.Stacktrace stacktr, decoder.Err = decodeStacktrace(raw[fieldName("stacktrace")], input.Config.HasShortFieldNames, decoder.Err) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index 763c6a37711..e3d63cdbf03 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/apm-server/model" + m "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/tests" ) @@ -41,14 +42,16 @@ func TestDecodeSpan(t *testing.T) { name, spType := "foo", "db" start, duration := 1.2, 3.4 method, statusCode, url := "get", 200, "http://localhost" + badRequestStatusCode := 400 instance, statement, dbType, user, link, rowsAffected := "db01", "select *", "sql", "joe", "other.db.com", 34 address, port := "localhost", 8080 destServiceType, destServiceName, destServiceResource := "db", "elasticsearch", "elasticsearch" outcome := "success" + httpCtx := map[string]interface{}{"method": "GET", "status_code": json.Number("200"), "url": url} context := map[string]interface{}{ "a": "b", "tags": map[string]interface{}{"a": "tag", "tag_key": 17}, - "http": map[string]interface{}{"method": "GET", "status_code": json.Number("200"), "url": url}, + "http": httpCtx, "db": map[string]interface{}{ "instance": instance, "statement": statement, "type": dbType, "user": user, "link": link, "rows_affected": json.Number("34")}, @@ -191,6 +194,48 @@ func TestDecodeSpan(t *testing.T) { }, cfg: Config{Experimental: true}, }, + "with derived success outcome": { + input: map[string]interface{}{ + "name": name, "type": "db.postgresql.query.custom", "duration": duration, "parent_id": parentID, + "timestamp": timestampEpoch, "id": id, "trace_id": traceID, + "context": map[string]interface{}{"http": httpCtx}, + }, + e: &m.Span{ + Metadata: metadata, + Name: name, + Type: "db", + Subtype: &subtype, + Action: &action2, + Duration: duration, + HTTP: &m.HTTP{Method: &method, StatusCode: &statusCode, URL: &url}, + Timestamp: spanTime, + ParentID: parentID, + ID: id, + TraceID: traceID, + Outcome: "success", + }, + }, + "with derived failure outcome": { + input: map[string]interface{}{ + "name": name, "type": "db.postgresql.query.custom", "duration": duration, "parent_id": parentID, + "timestamp": timestampEpoch, "id": id, "trace_id": traceID, + "context": map[string]interface{}{"http": map[string]interface{}{"status_code": json.Number("400")}}, + }, + e: &m.Span{ + Metadata: metadata, + Name: name, + Type: "db", + Subtype: &subtype, + Action: &action2, + Duration: duration, + HTTP: &m.HTTP{StatusCode: &badRequestStatusCode}, + Timestamp: spanTime, + ParentID: parentID, + ID: id, + TraceID: traceID, + Outcome: "failure", + }, + }, "full valid payload": { input: map[string]interface{}{ "name": name, "type": "messaging", "subtype": subtype, "action": action, "start": start, diff --git a/model/modeldecoder/transaction.go b/model/modeldecoder/transaction.go index 818fead35bc..63b32742b2f 100644 --- a/model/modeldecoder/transaction.go +++ b/model/modeldecoder/transaction.go @@ -19,6 +19,7 @@ package modeldecoder import ( "encoding/json" + "net/http" "github.com/pkg/errors" "github.com/santhosh-tekuri/jsonschema" @@ -166,7 +167,16 @@ func decodeTransaction(input Input, schema *jsonschema.Schema) (*model.Transacti decodeString(raw, fieldName("outcome"), &e.Outcome) decodeFloat64(raw, fieldName("duration"), &e.Duration) if e.Outcome == "" { - e.Outcome = "unknown" + if ctx.Http != nil && ctx.Http.Response != nil && ctx.Http.Response.StatusCode != nil { + statusCode := *ctx.Http.Response.StatusCode + if statusCode >= http.StatusInternalServerError { + e.Outcome = "failure" + } else { + e.Outcome = "success" + } + } else { + e.Outcome = "unknown" + } } if obj := getObject(raw, fieldName("experience")); obj != nil { diff --git a/model/modeldecoder/transaction_test.go b/model/modeldecoder/transaction_test.go index e2c15075a78..e53e2d7519c 100644 --- a/model/modeldecoder/transaction_test.go +++ b/model/modeldecoder/transaction_test.go @@ -191,6 +191,7 @@ func TestTransactionEventDecode(t *testing.T) { page := model.Page{URL: model.ParseURL(url, ""), Referer: &referer} request := model.Req{Method: "post", Socket: &model.Socket{}, Headers: http.Header{"User-Agent": []string{ua}}} response := model.Resp{Finished: new(bool), MinimalResp: model.MinimalResp{Headers: http.Header{"Content-Type": []string{"text/html"}}}} + badRequestResp, internalErrorResp := 400, 500 h := model.Http{Request: &request, Response: &response} ctxURL := model.URL{Original: &origURL} custom := model.Custom{"abc": 1} @@ -374,6 +375,52 @@ func TestTransactionEventDecode(t *testing.T) { }, }, }, + "with derived success outcome": { + input: map[string]interface{}{ + "context": map[string]interface{}{ + "response": map[string]interface{}{ + "status_code": json.Number("400"), + }, + }, + }, + e: &model.Transaction{ + Metadata: inputMetadata, + ID: id, + Type: trType, + Name: name, + TraceID: traceID, + Duration: duration, + HTTP: &model.Http{Response: &model.Resp{ + MinimalResp: model.MinimalResp{StatusCode: &badRequestResp}, + }}, + Timestamp: requestTime, + SpanCount: model.SpanCount{Dropped: &dropped, Started: &started}, + Outcome: "success", + }, + }, + "with derived failure outcome": { + input: map[string]interface{}{ + "context": map[string]interface{}{ + "response": map[string]interface{}{ + "status_code": json.Number("500"), + }, + }, + }, + e: &model.Transaction{ + Metadata: inputMetadata, + ID: id, + Type: trType, + Name: name, + TraceID: traceID, + Duration: duration, + Timestamp: requestTime, + HTTP: &model.Http{Response: &model.Resp{ + MinimalResp: model.MinimalResp{StatusCode: &internalErrorResp}, + }}, + SpanCount: model.SpanCount{Dropped: &dropped, Started: &started}, + Outcome: "failure", + }, + }, } { t.Run(name, func(t *testing.T) { input := make(map[string]interface{}) diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json index fbf46b069ec..b2c41c84ff1 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationEvents.approved.json @@ -14,7 +14,7 @@ "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "amd64", @@ -195,7 +195,7 @@ "id": "8ec7ceb990749e79b37f6dc6cd3628633618d6ce412553a552a0fa6b69419ad4" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "amd64", diff --git a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json index fd76bb9799e..a644de5a3ea 100644 --- a/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeIntegrationSpans.approved.json @@ -508,7 +508,7 @@ "port": 5432 }, "event": { - "outcome": "unknown" + "outcome": "success" }, "host": { "architecture": "x64", diff --git a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json index bcdbdb715be..25fc7bfa09f 100644 --- a/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json +++ b/processor/stream/test_approved_es_documents/testIntakeRUMV3Events.approved.json @@ -10,7 +10,7 @@ "ip": "192.0.0.1" }, "event": { - "outcome": "unknown" + "outcome": "success" }, "http": { "request": { @@ -451,7 +451,7 @@ "port": 8000 }, "event": { - "outcome": "unknown" + "outcome": "success" }, "labels": { "testTagKey": "testTagValue" @@ -541,7 +541,7 @@ "port": 8003 }, "event": { - "outcome": "unknown" + "outcome": "success" }, "labels": { "testTagKey": "testTagValue" @@ -631,7 +631,7 @@ "port": 8003 }, "event": { - "outcome": "unknown" + "outcome": "success" }, "labels": { "testTagKey": "testTagValue" From f81dff267dbaff7b93ff5151ab81930c8e308e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:04:21 +0200 Subject: [PATCH 2/8] Update model/modeldecoder/span_test.go Co-authored-by: Andrew Wilkins --- model/modeldecoder/span_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index e3d63cdbf03..6227bcddf3a 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -221,7 +221,7 @@ func TestDecodeSpan(t *testing.T) { "timestamp": timestampEpoch, "id": id, "trace_id": traceID, "context": map[string]interface{}{"http": map[string]interface{}{"status_code": json.Number("400")}}, }, - e: &m.Span{ + e: &model.Span{ Metadata: metadata, Name: name, Type: "db", From 64f79797308edf4a2039fb2055b05d1698ce52f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:04:34 +0200 Subject: [PATCH 3/8] Update model/modeldecoder/span_test.go Co-authored-by: Andrew Wilkins --- model/modeldecoder/span_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index 6227bcddf3a..2f23b17ae40 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -200,7 +200,7 @@ func TestDecodeSpan(t *testing.T) { "timestamp": timestampEpoch, "id": id, "trace_id": traceID, "context": map[string]interface{}{"http": httpCtx}, }, - e: &m.Span{ + e: &model.Span{ Metadata: metadata, Name: name, Type: "db", From 8d5d1ffdc117252ff23ae567639ef3ff87af2b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:04:43 +0200 Subject: [PATCH 4/8] Update model/modeldecoder/span_test.go Co-authored-by: Andrew Wilkins --- model/modeldecoder/span_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index 2f23b17ae40..c43f3b1eed6 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -228,7 +228,7 @@ func TestDecodeSpan(t *testing.T) { Subtype: &subtype, Action: &action2, Duration: duration, - HTTP: &m.HTTP{StatusCode: &badRequestStatusCode}, + HTTP: &model.HTTP{StatusCode: &badRequestStatusCode}, Timestamp: spanTime, ParentID: parentID, ID: id, From c497601536147f8055b5c2aaee853f3ee668b866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:04:51 +0200 Subject: [PATCH 5/8] Update model/modeldecoder/span_test.go Co-authored-by: Andrew Wilkins --- model/modeldecoder/span_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index c43f3b1eed6..23962b1864a 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -207,7 +207,7 @@ func TestDecodeSpan(t *testing.T) { Subtype: &subtype, Action: &action2, Duration: duration, - HTTP: &m.HTTP{Method: &method, StatusCode: &statusCode, URL: &url}, + HTTP: &model.HTTP{Method: &method, StatusCode: &statusCode, URL: &url}, Timestamp: spanTime, ParentID: parentID, ID: id, From 69a3c87f06f30fc10a3570517ec475272983dd75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:04:59 +0200 Subject: [PATCH 6/8] Update model/modeldecoder/span_test.go Co-authored-by: Andrew Wilkins --- model/modeldecoder/span_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/model/modeldecoder/span_test.go b/model/modeldecoder/span_test.go index 23962b1864a..603a0f6080f 100644 --- a/model/modeldecoder/span_test.go +++ b/model/modeldecoder/span_test.go @@ -29,7 +29,6 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/apm-server/model" - m "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/tests" ) From c805b355615df1fc8e249c8ef262bd8a0ae70716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:45:04 +0200 Subject: [PATCH 7/8] add comment --- model/modeldecoder/transaction_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/model/modeldecoder/transaction_test.go b/model/modeldecoder/transaction_test.go index e53e2d7519c..947d783c4a2 100644 --- a/model/modeldecoder/transaction_test.go +++ b/model/modeldecoder/transaction_test.go @@ -395,7 +395,8 @@ func TestTransactionEventDecode(t *testing.T) { }}, Timestamp: requestTime, SpanCount: model.SpanCount{Dropped: &dropped, Started: &started}, - Outcome: "success", + // a 4xx code is a success from the server perspective + Outcome: "success", }, }, "with derived failure outcome": { From 079d6d654a4834a392a1f12232adfa84d99e0580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 8 Sep 2020 08:45:23 +0200 Subject: [PATCH 8/8] add changelog --- changelogs/head.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 71796497e15..0f000f0a7a3 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -30,3 +30,4 @@ https://github.com/elastic/apm-server/compare/7.9\...master[View commits] * Add event.outcome to transactions and spans {pull}4064[4064] * Add event.outcome to aggregated transaction metrics {pull}4110[4110] * Set event.outcome for Jaeger spans based on http.status_code {pull}4127[4127] +* Set event.outcome for transactions and spans based on http.status_code {pull}4165[4165]