makes timestamp optional, fixes elastic#723

jalvz · Apr 9, 2018 · bb711e4 · bb711e4
1 parent 06d8682
commit bb711e4
Show file tree

Hide file tree

Showing 31 changed files with 66 additions and 41 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -30,6 +30,7 @@ https://github.com/elastic/apm-server/compare/4daa36bd5c144cf9182afc62dc8042af66
 - Push spans to separate ES index {pull}774[774].
 - Update Go to 1.9.4 {pull}786[786].
 - Adjust in-memory queue defaults {pull}791[791].
+- Make timestamp optional in the intake api {pull}818[818].
 
 
 ==== Deprecated

diff --git a/docs/data/intake-api/generated/error/minimal_payload_exception.json b/docs/data/intake-api/generated/error/minimal_payload_exception.json
@@ -8,7 +8,6 @@
     },
     "errors": [
         {
-            "timestamp": "2017-05-09T15:04:05.999999Z",
             "exception": {
                 "message": ""
             }

diff --git a/docs/data/intake-api/generated/error/minimal_payload_log.json b/docs/data/intake-api/generated/error/minimal_payload_log.json
@@ -8,7 +8,6 @@
     },
     "errors": [
         {
-            "timestamp": "2017-05-09T15:04:05.999999Z",
             "log": {
                 "message": ""
             }

diff --git a/docs/data/intake-api/generated/transaction/minimal_payload.json b/docs/data/intake-api/generated/transaction/minimal_payload.json
@@ -10,8 +10,7 @@
         {
             "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79",
             "type": "request",
-            "duration": 32.592981,
-            "timestamp": "2017-05-09T15:04:05.999999Z"
+            "duration": 32.592981
         }
     ]
 }
diff --git a/docs/spec/errors/error.json b/docs/spec/errors/error.json
@@ -109,7 +109,6 @@
             }
         }
     },
-    "required": ["timestamp"],
     "anyOf": [
         {
             "required": ["exception"]

diff --git a/docs/spec/transactions/transaction.json b/docs/spec/transactions/transaction.json
@@ -75,5 +75,5 @@
             }
         }
     },
-    "required": ["id", "duration", "type", "timestamp"]
+    "required": ["id", "duration", "type"]
 }
diff --git a/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json b/processor/error/package_tests/TestProcessErrorMinimalPayloadException.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-05-09T15:04:05.999999Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json b/processor/error/package_tests/TestProcessErrorMinimalPayloadLog.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-05-09T15:04:05.999999Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/error/package_tests/processor_test.go b/processor/error/package_tests/processor_test.go
@@ -20,8 +20,6 @@ import (
 // ensure all valid documents pass through the whole validation and transformation process
 func TestProcessorBackendOK(t *testing.T) {
 	requestInfo := []tests.RequestInfo{
-		{Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"},
-		{Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"},
 		{Name: "TestProcessErrorMinimalService", Path: "data/valid/error/minimal_service.json"},
 		{Name: "TestProcessErrorMinimalProcess", Path: "data/valid/error/minimal_process.json"},
 		{Name: "TestProcessErrorFull", Path: "data/valid/error/payload.json"},
@@ -32,6 +30,15 @@ func TestProcessorBackendOK(t *testing.T) {
 	tests.TestProcessRequests(t, er.NewProcessor, conf, requestInfo, map[string]string{})
 }
 
+func TestProcessorMinimalPayloadOK(t *testing.T) {
+	requestInfo := []tests.RequestInfo{
+		{Name: "TestProcessErrorMinimalPayloadException", Path: "data/valid/error/minimal_payload_exception.json"},
+		{Name: "TestProcessErrorMinimalPayloadLog", Path: "data/valid/error/minimal_payload_log.json"},
+	}
+	conf := config.Config{ExcludeFromGrouping: nil}
+	tests.TestProcessRequests(t, er.NewProcessor, conf, requestInfo, map[string]string{"@timestamp": "-"})
+}
+
 func TestProcessorFrontendOK(t *testing.T) {
 	requestInfo := []tests.RequestInfo{
 		{Name: "TestProcessErrorFrontend", Path: "data/valid/error/frontend.json"},

diff --git a/processor/error/payload.go b/processor/error/payload.go
@@ -1,6 +1,8 @@
 package error
 
 import (
+	"time"
+
 	"github.com/elastic/apm-server/config"
 	m "github.com/elastic/apm-server/model"
 	"github.com/elastic/apm-server/utility"
@@ -63,6 +65,10 @@ func (pa *payload) transform(config config.Config) []beat.Event {
 	var events []beat.Event
 	for _, event := range pa.Events {
 		context := context.Transform(event.Context)
+		var zeroTime = time.Time{}
+		if event.Timestamp == zeroTime {
+			event.Timestamp = time.Now()
+		}
 		ev := beat.Event{
 			Fields: common.MapStr{
 				"processor":  processorEntry,

diff --git a/processor/error/payload_test.go b/processor/error/payload_test.go
@@ -175,3 +175,13 @@ func TestPayloadTransform(t *testing.T) {
 		}
 	}
 }
+
+func TestSetTimestamp(t *testing.T) {
+	p := payload{
+		Service: m.Service{Name: "myservice"},
+		Events:  []Event{{}},
+	}
+	for _, event := range p.transform(config.Config{}) {
+		assert.InDelta(t, time.Now().Unix(), event.Timestamp.Unix(), time.Millisecond.Seconds()*10)
+	}
+}
diff --git a/processor/error/schema.go b/processor/error/schema.go
@@ -543,7 +543,6 @@ var errorSchema = `{
             }
         }
     },
-    "required": ["timestamp"],
     "anyOf": [
         {
             "required": ["exception"]

diff --git a/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json b/processor/transaction/package_tests/TestProcessTransactionAugmented.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-05-09T15:04:05.999999Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json b/processor/transaction/package_tests/TestProcessTransactionAugmentedMerge.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-05-09T15:04:05.999999Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json b/processor/transaction/package_tests/TestProcessTransactionFrontend.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-12-08T12:52:53.681Z",
+            "@timestamp": "-",
             "context": {
                 "_metrics": {
                     "connectEnd": 14,
@@ -50,7 +50,7 @@
             }
         },
         {
-            "@timestamp": "2017-12-08T12:52:53.681Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json b/processor/transaction/package_tests/TestProcessTransactionMinimalPayload.approved.json
@@ -1,7 +1,7 @@
 {
     "events": [
         {
-            "@timestamp": "2017-05-09T15:04:05.999999Z",
+            "@timestamp": "-",
             "context": {
                 "service": {
                     "agent": {

diff --git a/processor/transaction/package_tests/processor_test.go b/processor/transaction/package_tests/processor_test.go
@@ -19,7 +19,6 @@ func TestTransactionProcessorOK(t *testing.T) {
 		{Name: "TestProcessTransactionNullValues", Path: "data/valid/transaction/null_values.json"},
 		{Name: "TestProcessSystemNull", Path: "data/valid/transaction/system_null.json"},
 		{Name: "TestProcessProcessNull", Path: "data/valid/transaction/process_null.json"},
-		{Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"},
 		{Name: "TestProcessTransactionMinimalSpan", Path: "data/valid/transaction/minimal_span.json"},
 		{Name: "TestProcessTransactionMinimalService", Path: "data/valid/transaction/minimal_service.json"},
 		{Name: "TestProcessTransactionMinimalProcess", Path: "data/valid/transaction/minimal_process.json"},
@@ -29,6 +28,13 @@ func TestTransactionProcessorOK(t *testing.T) {
 	tests.TestProcessRequests(t, transaction.NewProcessor, config.Config{}, requestInfo, map[string]string{})
 }
 
+func TestMinimalTransactionProcessorOK(t *testing.T) {
+	requestInfo := []tests.RequestInfo{
+		{Name: "TestProcessTransactionMinimalPayload", Path: "data/valid/transaction/minimal_payload.json"},
+	}
+	tests.TestProcessRequests(t, transaction.NewProcessor, config.Config{}, requestInfo, map[string]string{"@timestamp": "-"})
+}
+
 func TestProcessorFrontendOK(t *testing.T) {
 	requestInfo := []tests.RequestInfo{
 		{Name: "TestProcessTransactionFrontend", Path: "data/valid/transaction/frontend.json"},
@@ -39,7 +45,7 @@ func TestProcessorFrontendOK(t *testing.T) {
 		LibraryPattern:      regexp.MustCompile("/test/e2e|~"),
 		ExcludeFromGrouping: regexp.MustCompile("^~/test"),
 	}
-	tests.TestProcessRequests(t, transaction.NewProcessor, conf, requestInfo, map[string]string{})
+	tests.TestProcessRequests(t, transaction.NewProcessor, conf, requestInfo, map[string]string{"@timestamp": "-"})
 }
 
 // ensure invalid documents fail the json schema validation already

diff --git a/processor/transaction/payload.go b/processor/transaction/payload.go
@@ -1,6 +1,8 @@
 package transaction
 
 import (
+	"time"
+
 	"github.com/elastic/apm-server/config"
 	m "github.com/elastic/apm-server/model"
 	"github.com/elastic/apm-server/utility"
@@ -65,6 +67,10 @@ func (pa *payload) transform(config config.Config) []beat.Event {
 
 	var events []beat.Event
 	for _, event := range pa.Events {
+		var zeroTime = time.Time{}
+		if event.Timestamp == zeroTime {
+			event.Timestamp = time.Now()
+		}
 		ev := beat.Event{
 			Fields: common.MapStr{
 				"processor":        processorTransEntry,

diff --git a/processor/transaction/payload_test.go b/processor/transaction/payload_test.go
@@ -239,3 +239,13 @@ func TestPayloadTransform(t *testing.T) {
 		}
 	}
 }
+
+func TestSetTimestamp(t *testing.T) {
+	p := payload{
+		Service: m.Service{Name: "myservice"},
+		Events:  []Event{{}},
+	}
+	for _, event := range p.transform(config.Config{}) {
+		assert.InDelta(t, time.Now().Unix(), event.Timestamp.Unix(), time.Millisecond.Seconds()*10)
+	}
+}
diff --git a/processor/transaction/schema.go b/processor/transaction/schema.go
@@ -552,7 +552,7 @@ var transactionSchema = `{
             }
         }
     },
-    "required": ["id", "duration", "type", "timestamp"]
+    "required": ["id", "duration", "type"]
             },
             "minItems": 1
         }

diff --git a/tests/data/invalid/error/no_timestamp.json b/tests/data/invalid/error/no_timestamp.json
diff --git a/tests/data/invalid/transaction/no_timestamp.json b/tests/data/invalid/transaction/no_timestamp.json
diff --git a/tests/data/valid/error/minimal_payload_exception.json b/tests/data/valid/error/minimal_payload_exception.json
@@ -8,7 +8,6 @@
     },
     "errors": [
         {
-            "timestamp": "2017-05-09T15:04:05.999999Z",
             "exception": {
                 "message": ""
             }

diff --git a/tests/data/valid/error/minimal_payload_log.json b/tests/data/valid/error/minimal_payload_log.json
@@ -8,7 +8,6 @@
     },
     "errors": [
         {
-            "timestamp": "2017-05-09T15:04:05.999999Z",
             "log": {
                 "message": ""
             }

diff --git a/tests/data/valid/transaction/augmented_payload_frontend.json b/tests/data/valid/transaction/augmented_payload_frontend.json
@@ -11,7 +11,6 @@
             "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79",
             "type": "request",
             "duration": 32.592981,
-            "timestamp": "2017-05-09T15:04:05.999999Z",
             "context": {
                 "url": {
                     "location": "http://localhost:8000/test/e2e/general-usecase/"

diff --git a/tests/data/valid/transaction/augmented_payload_frontend_no_context.json b/tests/data/valid/transaction/augmented_payload_frontend_no_context.json
@@ -10,8 +10,7 @@
         {
             "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79",
             "type": "request",
-            "duration": 32.592981,
-            "timestamp": "2017-05-09T15:04:05.999999Z"
+            "duration": 32.592981
         }
     ],
     "user": {

diff --git a/tests/data/valid/transaction/frontend.json b/tests/data/valid/transaction/frontend.json
@@ -10,7 +10,6 @@
     "transactions": [
         {
             "id": "611f4fa9-50f0-4631-9306-43399c0dc3db",
-            "timestamp": "2017-12-08T12:52:53.681Z",
             "type": "page-load",
             "duration": 643,
             "spans": [

diff --git a/tests/data/valid/transaction/minimal_payload.json b/tests/data/valid/transaction/minimal_payload.json
@@ -10,8 +10,7 @@
         {
             "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79",
             "type": "request",
-            "duration": 32.592981,
-            "timestamp": "2017-05-09T15:04:05.999999Z"
+            "duration": 32.592981
         }
     ]
 }
diff --git a/tests/json_schema_test.go b/tests/json_schema_test.go
@@ -89,7 +89,6 @@ func TestTransactionSchema(t *testing.T) {
 		{File: "no_id.json", Error: `missing properties: "id"`},
 		{File: "no_duration.json", Error: `missing properties: "duration"`},
 		{File: "no_type.json", Error: `missing properties: "type"`},
-		{File: "no_timestamp.json", Error: `missing properties: "timestamp"`},
 		{File: "invalid_id.json", Error: "[#/properties/id/pattern] does not match pattern"},
 		{File: "invalid_timestamp.json", Error: "is not valid \"date-time\""},
 		{File: "invalid_timestamp2.json", Error: "I[#/timestamp] S[#/properties/timestamp/pattern] does not match pattern"},
@@ -120,7 +119,6 @@ func TestErrorSchema(t *testing.T) {
 		{File: "no_exception_message.json", Error: "missing properties: \"message\""},
 		{File: "no_log_or_exception.json", Error: "missing properties: \"exception\""},
 		{File: "no_log_or_exception.json", Error: "missing properties: \"log\""},
-		{File: "no_timestamp.json", Error: "missing properties: \"timestamp\""},
 		{File: "invalid_code.json", Error: "expected string or integer or null"},
 	}
 	testDataAgainstSchema(t, testData, "errors/error", "error", `"$ref": "../docs/spec/errors/`)

diff --git a/utility/data_fetcher.go b/utility/data_fetcher.go
@@ -145,6 +145,9 @@ func (d *ManualDecoder) MapStr(base map[string]interface{}, key string, keys ...
 
 func (d *ManualDecoder) TimeRFC3339(base map[string]interface{}, key string, keys ...string) time.Time {
 	val := getDeep(base, keys...)[key]
+	if val == nil {
+		return time.Time{}
+	}
 	if valStr, ok := val.(string); ok {
 		if valTime, err := time.Parse(time.RFC3339, valStr); err == nil {
 			return valTime

diff --git a/utility/data_fetcher_test.go b/utility/data_fetcher_test.go
@@ -197,7 +197,7 @@ func TestTimeRFC3339(t *testing.T) {
 	tp, _ := time.Parse(time.RFC3339, "2017-05-30T18:53:27.154Z")
 	for _, test := range []testStr{
 		{key: "time", keys: []string{}, out: tp, err: nil},
-		{key: "missing", keys: []string{"a", "b"}, out: outnil, err: fetchErr},
+		{key: "missing", keys: []string{"a", "b"}, out: outnil, err: nil},
 		{key: "str", keys: []string{"a", "b"}, out: outnil, err: fetchErr},
 		{key: "b", keys: []string{"a"}, out: outnil, err: fetchErr},
 	} {
-Original file line number
+Diff line change
@@ Expand Up / @@ -109,7 +109,6 @@ @@
                 }
             }
         },
-        "required": ["timestamp"],
         "anyOf": [
             {
                 "required": ["exception"]
@@ Expand Down @@