diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index e0745181f25..868c8b17ca4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -31,6 +31,7 @@ https://github.com/elastic/apm-server/compare/4daa36bd5c144cf9182afc62dc8042af66 - Update Go to 1.9.4 {pull}786[786]. - Adjust in-memory queue defaults {pull}791[791]. - Listen on unix domain socket with `host=unix:/path` {pull}768[768]. +- Make timestamp optional in the intake api {pull}819[819]. ==== Deprecated diff --git a/docs/data/intake-api/generated/error/minimal_payload_exception.json b/docs/data/intake-api/generated/error/minimal_payload_exception.json index 288d6bc25d0..f616c4bc552 100644 --- a/docs/data/intake-api/generated/error/minimal_payload_exception.json +++ b/docs/data/intake-api/generated/error/minimal_payload_exception.json @@ -8,7 +8,6 @@ }, "errors": [ { - "timestamp": "2017-05-09T15:04:05.999999Z", "exception": { "message": "" } diff --git a/docs/data/intake-api/generated/error/minimal_payload_log.json b/docs/data/intake-api/generated/error/minimal_payload_log.json index 60268f757cf..79f51e395ec 100644 --- a/docs/data/intake-api/generated/error/minimal_payload_log.json +++ b/docs/data/intake-api/generated/error/minimal_payload_log.json @@ -8,7 +8,6 @@ }, "errors": [ { - "timestamp": "2017-05-09T15:04:05.999999Z", "log": { "message": "" } diff --git a/docs/data/intake-api/generated/transaction/minimal_payload.json b/docs/data/intake-api/generated/transaction/minimal_payload.json index 2e4e4dc8450..bfdc2f54e87 100644 --- a/docs/data/intake-api/generated/transaction/minimal_payload.json +++ b/docs/data/intake-api/generated/transaction/minimal_payload.json @@ -10,8 +10,7 @@ { "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "type": "request", - "duration": 32.592981, - "timestamp": "2017-05-09T15:04:05.999999Z" + "duration": 32.592981 } ] } diff --git a/docs/spec/errors/error.json b/docs/spec/errors/error.json index 1f39372bb70..c99b5dcbf78 100644 --- a/docs/spec/errors/error.json +++ b/docs/spec/errors/error.json @@ -109,7 +109,6 @@ } } }, - "required": ["timestamp"], "anyOf": [ { "required": ["exception"] diff --git a/docs/spec/transactions/transaction.json b/docs/spec/transactions/transaction.json index 03059dae40e..bcd8b2d9b55 100644 --- a/docs/spec/transactions/transaction.json +++ b/docs/spec/transactions/transaction.json @@ -75,5 +75,5 @@ } } }, - "required": ["id", "duration", "type", "timestamp"] + "required": ["id", "duration", "type"] } diff --git a/processor/error/event.go b/processor/error/event.go index 9afc2937f72..ce70899d759 100644 --- a/processor/error/event.go +++ b/processor/error/event.go @@ -65,7 +65,7 @@ func DecodeEvent(input interface{}, err error) (*Event, error) { Id: decoder.StringPtr(raw, "id"), Culprit: decoder.StringPtr(raw, "culprit"), Context: decoder.MapStr(raw, "context"), - Timestamp: decoder.TimeRFC3339(raw, "timestamp"), + Timestamp: decoder.TimeRFC3339WithDefault(raw, "timestamp"), } transactionId := decoder.StringPtr(raw, "id", "transaction") if transactionId != nil { diff --git a/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json b/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json index ae603aee64a..78dde7dd9b6 100644 --- a/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json +++ b/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-05-09T15:04:05.999999Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json b/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json index ddb84b75637..ae945f83d00 100644 --- a/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json +++ b/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-05-09T15:04:05.999999Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/error/package_tests/processor_test.go b/processor/error/package_tests/processor_test.go index 931dcea660b..45c311cc6b4 100644 --- a/processor/error/package_tests/processor_test.go +++ b/processor/error/package_tests/processor_test.go @@ -20,8 +20,6 @@ import ( // ensure all valid documents pass through the whole validation and transformation process func TestProcessorBackendOK(t *testing.T) { requestInfo := []tests.RequestInfo{ - {Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"}, - {Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"}, {Name: "TestProcessErrorMinimalService", Path: "data/valid/error/minimal_service.json"}, {Name: "TestProcessErrorMinimalProcess", Path: "data/valid/error/minimal_process.json"}, {Name: "TestProcessErrorFull", Path: "data/valid/error/payload.json"}, @@ -32,6 +30,15 @@ func TestProcessorBackendOK(t *testing.T) { tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{}) } +func TestProcessorMinimalPayloadOK(t *testing.T) { + requestInfo := []tests.RequestInfo{ + {Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"}, + {Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"}, + } + conf := config.Config{ExcludeFromGrouping: nil} + tests.TestProcessRequests(t, er.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"}) +} + func TestProcessorFrontendOK(t *testing.T) { requestInfo := []tests.RequestInfo{ {Name: "TestProcessErrorFrontend", Path: "data/valid/error/frontend.json"}, diff --git a/processor/error/schema.go b/processor/error/schema.go index 056499eeaa1..ea728c3bea8 100644 --- a/processor/error/schema.go +++ b/processor/error/schema.go @@ -543,7 +543,6 @@ var errorSchema = `{ } } }, - "required": ["timestamp"], "anyOf": [ { "required": ["exception"] diff --git a/processor/transaction/event.go b/processor/transaction/event.go index b5bee1ba17c..748d9bd4909 100644 --- a/processor/transaction/event.go +++ b/processor/transaction/event.go @@ -43,7 +43,7 @@ func DecodeEvent(input interface{}, err error) (*Event, error) { Name: decoder.StringPtr(raw, "name"), Result: decoder.StringPtr(raw, "result"), Duration: decoder.Float64(raw, "duration"), - Timestamp: decoder.TimeRFC3339(raw, "timestamp"), + Timestamp: decoder.TimeRFC3339WithDefault(raw, "timestamp"), Context: decoder.MapStr(raw, "context"), Marks: decoder.MapStr(raw, "marks"), Sampled: decoder.BoolPtr(raw, "sampled"), diff --git a/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json b/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json index ce3929a97ca..68108d83004 100644 --- a/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json +++ b/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-05-09T15:04:05.999999Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json b/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json index c74a625e8a0..7c5c448e039 100644 --- a/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json +++ b/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-05-09T15:04:05.999999Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json b/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json index a17cc34e4a2..8dadfec23e4 100644 --- a/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json +++ b/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-12-08T12:52:53.681Z", + "@timestamp": "-", "context": { "_metrics": { "connectEnd": 14, @@ -50,7 +50,7 @@ } }, { - "@timestamp": "2017-12-08T12:52:53.681Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json b/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json index b55a4ee8ce9..b3a1dd6adbe 100644 --- a/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json +++ b/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json @@ -1,7 +1,7 @@ { "events": [ { - "@timestamp": "2017-05-09T15:04:05.999999Z", + "@timestamp": "-", "context": { "service": { "agent": { diff --git a/processor/transaction/package_tests/processor_test.go b/processor/transaction/package_tests/processor_test.go index f5aef3c3b4e..dba67d5390a 100644 --- a/processor/transaction/package_tests/processor_test.go +++ b/processor/transaction/package_tests/processor_test.go @@ -19,7 +19,6 @@ func TestTransactionProcessorOK(t *testing.T) { {Name: "TestProcessTransactionNullValues", Path: "data/valid/transaction/null_values.json"}, {Name: "TestProcessSystemNull", Path: "data/valid/transaction/system_null.json"}, {Name: "TestProcessProcessNull", Path: "data/valid/transaction/process_null.json"}, - {Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"}, {Name: "TestProcessTransactionMinimalSpan", Path: "data/valid/transaction/minimal_span.json"}, {Name: "TestProcessTransactionMinimalService", Path: "data/valid/transaction/minimal_service.json"}, {Name: "TestProcessTransactionMinimalProcess", Path: "data/valid/transaction/minimal_process.json"}, @@ -29,6 +28,13 @@ func TestTransactionProcessorOK(t *testing.T) { tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{}) } +func TestMinimalTransactionProcessorOK(t *testing.T) { + requestInfo := []tests.RequestInfo{ + {Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"}, + } + tests.TestProcessRequests(t, transaction.NewProcessor(), config.Config{}, requestInfo, map[string]string{"@timestamp": "-"}) +} + func TestProcessorFrontendOK(t *testing.T) { requestInfo := []tests.RequestInfo{ {Name: "TestProcessTransactionFrontend", Path: "data/valid/transaction/frontend.json"}, @@ -39,7 +45,7 @@ func TestProcessorFrontendOK(t *testing.T) { LibraryPattern: regexp.MustCompile("/test/e2e|~"), ExcludeFromGrouping: regexp.MustCompile("^~/test"), } - tests.TestProcessRequests(t, transaction.NewProcessor(), conf, requestInfo, map[string]string{}) + tests.TestProcessRequests(t, transaction.NewProcessor(), conf, requestInfo, map[string]string{"@timestamp": "-"}) } // ensure invalid documents fail the json schema validation already diff --git a/processor/transaction/schema.go b/processor/transaction/schema.go index bbd53144f2f..5c709d09b47 100644 --- a/processor/transaction/schema.go +++ b/processor/transaction/schema.go @@ -552,7 +552,7 @@ var transactionSchema = `{ } } }, - "required": ["id", "duration", "type", "timestamp"] + "required": ["id", "duration", "type"] }, "minItems": 1 } diff --git a/tests/data/invalid/error/no_timestamp.json b/tests/data/invalid/error/no_timestamp.json deleted file mode 100644 index 83489d33d74..00000000000 --- a/tests/data/invalid/error/no_timestamp.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "exception": { - "message": "SyntaxError: Something!" - } -} diff --git a/tests/data/invalid/transaction/no_timestamp.json b/tests/data/invalid/transaction/no_timestamp.json deleted file mode 100644 index 1562b450f01..00000000000 --- a/tests/data/invalid/transaction/no_timestamp.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "id": "85915e55-b43f-4340-a8e0-df1906ecbfa9", - "type": "request", - "duration": 1.0, - "spans": [] -} diff --git a/tests/data/valid/error/minimal_payload_exception.json b/tests/data/valid/error/minimal_payload_exception.json index 288d6bc25d0..f616c4bc552 100644 --- a/tests/data/valid/error/minimal_payload_exception.json +++ b/tests/data/valid/error/minimal_payload_exception.json @@ -8,7 +8,6 @@ }, "errors": [ { - "timestamp": "2017-05-09T15:04:05.999999Z", "exception": { "message": "" } diff --git a/tests/data/valid/error/minimal_payload_log.json b/tests/data/valid/error/minimal_payload_log.json index 60268f757cf..79f51e395ec 100644 --- a/tests/data/valid/error/minimal_payload_log.json +++ b/tests/data/valid/error/minimal_payload_log.json @@ -8,7 +8,6 @@ }, "errors": [ { - "timestamp": "2017-05-09T15:04:05.999999Z", "log": { "message": "" } diff --git a/tests/data/valid/transaction/augmented_payload_frontend.json b/tests/data/valid/transaction/augmented_payload_frontend.json index 1caa5749cfd..f8ae2c38040 100644 --- a/tests/data/valid/transaction/augmented_payload_frontend.json +++ b/tests/data/valid/transaction/augmented_payload_frontend.json @@ -11,7 +11,6 @@ "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "type": "request", "duration": 32.592981, - "timestamp": "2017-05-09T15:04:05.999999Z", "context": { "url": { "location": "http://localhost:8000/test/e2e/general-usecase/" diff --git a/tests/data/valid/transaction/augmented_payload_frontend_no_context.json b/tests/data/valid/transaction/augmented_payload_frontend_no_context.json index dd5dedf6ee9..8915467007a 100644 --- a/tests/data/valid/transaction/augmented_payload_frontend_no_context.json +++ b/tests/data/valid/transaction/augmented_payload_frontend_no_context.json @@ -10,8 +10,7 @@ { "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "type": "request", - "duration": 32.592981, - "timestamp": "2017-05-09T15:04:05.999999Z" + "duration": 32.592981 } ], "user": { diff --git a/tests/data/valid/transaction/frontend.json b/tests/data/valid/transaction/frontend.json index 5efcca327d5..73d2013164c 100644 --- a/tests/data/valid/transaction/frontend.json +++ b/tests/data/valid/transaction/frontend.json @@ -10,7 +10,6 @@ "transactions": [ { "id": "611f4fa9-50f0-4631-9306-43399c0dc3db", - "timestamp": "2017-12-08T12:52:53.681Z", "type": "page-load", "duration": 643, "spans": [ diff --git a/tests/data/valid/transaction/minimal_payload.json b/tests/data/valid/transaction/minimal_payload.json index 2e4e4dc8450..bfdc2f54e87 100644 --- a/tests/data/valid/transaction/minimal_payload.json +++ b/tests/data/valid/transaction/minimal_payload.json @@ -10,8 +10,7 @@ { "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", "type": "request", - "duration": 32.592981, - "timestamp": "2017-05-09T15:04:05.999999Z" + "duration": 32.592981 } ] } diff --git a/tests/json_schema_test.go b/tests/json_schema_test.go index 034bb3e4f8a..fcd767b3231 100644 --- a/tests/json_schema_test.go +++ b/tests/json_schema_test.go @@ -89,7 +89,6 @@ func TestTransactionSchema(t *testing.T) { {File: "no_id.json", Error: `missing properties: "id"`}, {File: "no_duration.json", Error: `missing properties: "duration"`}, {File: "no_type.json", Error: `missing properties: "type"`}, - {File: "no_timestamp.json", Error: `missing properties: "timestamp"`}, {File: "invalid_id.json", Error: "[#/properties/id/pattern] does not match pattern"}, {File: "invalid_timestamp.json", Error: "is not valid \"date-time\""}, {File: "invalid_timestamp2.json", Error: "I[#/timestamp] S[#/properties/timestamp/pattern] does not match pattern"}, @@ -120,7 +119,6 @@ func TestErrorSchema(t *testing.T) { {File: "no_exception_message.json", Error: "missing properties: \"message\""}, {File: "no_log_or_exception.json", Error: "missing properties: \"exception\""}, {File: "no_log_or_exception.json", Error: "missing properties: \"log\""}, - {File: "no_timestamp.json", Error: "missing properties: \"timestamp\""}, {File: "invalid_code.json", Error: "expected string or integer or null"}, } testDataAgainstSchema(t, testData, "errors/error", "error", `"$ref": "../docs/spec/errors/`) diff --git a/utility/data_fetcher.go b/utility/data_fetcher.go index 4c3d3d874af..0d00fc84592 100644 --- a/utility/data_fetcher.go +++ b/utility/data_fetcher.go @@ -143,8 +143,12 @@ func (d *ManualDecoder) MapStr(base map[string]interface{}, key string, keys ... return nil } -func (d *ManualDecoder) TimeRFC3339(base map[string]interface{}, key string, keys ...string) time.Time { +// if the looked up value doesn't exist, it returns now +func (d *ManualDecoder) TimeRFC3339WithDefault(base map[string]interface{}, key string, keys ...string) time.Time { val := getDeep(base, keys...)[key] + if val == nil { + return time.Now() + } if valStr, ok := val.(string); ok { if valTime, err := time.Parse(time.RFC3339, valStr); err == nil { return valTime diff --git a/utility/data_fetcher_test.go b/utility/data_fetcher_test.go index 2ccd763cfc0..6a6e6a51a6b 100644 --- a/utility/data_fetcher_test.go +++ b/utility/data_fetcher_test.go @@ -193,17 +193,17 @@ func TestMapStr(t *testing.T) { } func TestTimeRFC3339(t *testing.T) { - var outnil time.Time + var outZero time.Time tp, _ := time.Parse(time.RFC3339, "2017-05-30T18:53:27.154Z") for _, test := range []testStr{ {key: "time", keys: []string{}, out: tp, err: nil}, - {key: "missing", keys: []string{"a", "b"}, out: outnil, err: fetchErr}, - {key: "str", keys: []string{"a", "b"}, out: outnil, err: fetchErr}, - {key: "b", keys: []string{"a"}, out: outnil, err: fetchErr}, + {key: "missing", keys: []string{"a", "b"}, out: time.Now(), err: nil}, + {key: "str", keys: []string{"a", "b"}, out: outZero, err: fetchErr}, + {key: "b", keys: []string{"a"}, out: outZero, err: fetchErr}, } { decoder := ManualDecoder{} - out := decoder.TimeRFC3339(decoderBase, test.key, test.keys...) - assert.Equal(t, out, test.out) + out := decoder.TimeRFC3339WithDefault(decoderBase, test.key, test.keys...) + assert.InDelta(t, out.Unix(), test.out.(time.Time).Unix(), time.Millisecond.Seconds()*10) assert.Equal(t, decoder.Err, test.err) } }