[v2] accept unix epoch timestamps in microseconds (elastic#1413)

* makes "start" optional if "timestamp" is given for v2 * if "timestamp" is not given, it is set to the sum of req-time + start if data arrives through v2 * adds `timestamp.us` (long) to span, transaction and error documents for v2
roncohen · Oct 15, 2018 · 83ffefc · 83ffefc
1 parent 8802c48
commit 83ffefc
Show file tree

Hide file tree

Showing 70 changed files with 2,252 additions and 1,793 deletions.
diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml
@@ -14,6 +14,16 @@
       type: keyword
       description: Processor event.
 
+    - name: timestamp
+      type: group
+      description:
+      fields:
+        - name: us
+          type: long
+          count: 1
+          description: >
+            Timestamp of the event in microseconds since Unix epoch.
+
     - name: context
       type: group
       description: >

diff --git a/beater/approved-stream-result/TestRequestIntegrationInvalidEvent.approved.json b/beater/approved-stream-result/TestRequestIntegrationInvalidEvent.approved.json
@@ -3,7 +3,7 @@
     "errors": [
         {
             "document": "{ \"transaction\": { \"id\": 12345, \"trace_id\": \"0123456789abcdef0123456789abcdef\", \"parent_id\": \"abcdefabcdef01234567\", \"type\": \"request\", \"duration\": 32.592981, \"span_count\": { \"started\": 21 } } }   ",
-            "message": "Problem validating JSON document against schema: I[#] S[#] doesn't validate with \"transaction#\"\n  I[#] S[#/allOf/1] allOf failed\n    I[#/id] S[#/allOf/1/properties/id/type] expected string, but got number"
+            "message": "Problem validating JSON document against schema: I[#] S[#] doesn't validate with \"transaction#\"\n  I[#] S[#/allOf/2] allOf failed\n    I[#/id] S[#/allOf/2/properties/id/type] expected string, but got number"
         }
     ]
 }
diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc
@@ -57,6 +57,22 @@ type: keyword
 
 Processor event.
 
+--
+
+[float]
+== timestamp fields
+
+None
+
+
+*`timestamp.us`*::
++
+--
+type: long
+
+Timestamp of the event in microseconds since Unix epoch.
+
+
 --
 
 [float]

diff --git a/docs/spec/errors/common_error.json b/docs/spec/errors/common_error.json
@@ -1,5 +1,5 @@
 {
-    "$id": "docs/spec/errors/error.json",
+    "$id": "docs/spec/errors/common_error.json",
     "type": "object",
     "description": "Data captured by an agent representing an event occurring in a monitored service",
     "properties": {
@@ -86,12 +86,6 @@
                 }
             },
             "required": ["message"]
-        },
-        "timestamp": {
-            "type": ["string","null"],
-            "format": "date-time",
-            "pattern": "Z$",
-            "description": "Recorded time of the error, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
         }
     },
     "anyOf": [

diff --git a/docs/spec/errors/v1_error.json b/docs/spec/errors/v1_error.json
@@ -17,7 +17,8 @@
                 "description": "Data captured by an agent representing an event occurring in a monitored service",
                 "allOf": [
 
-                    { "$ref": "common_error.json"  }, 
+                    { "$ref": "common_error.json"  },
+                    { "$ref": "../timestamp_rfc3339.json" },
                     {  
                         "properties": {
                             "id": {

diff --git a/docs/spec/errors/v2_error.json b/docs/spec/errors/v2_error.json
@@ -5,6 +5,7 @@
     "allOf": [
 
         { "$ref": "common_error.json"  }, 
+        { "$ref": "../timestamp_epoch.json" },
         {  
             "properties": {
                 "id": {
@@ -36,7 +37,6 @@
                   "then": { "required": ["parent_id"], "properties": {"parent_id": { "type": "string" }}} },
                 { "if": {"required": ["parent_id"], "properties": {"parent_id": { "type": "string" }}},
                   "then": { "required": ["transaction_id"], "properties": {"transaction_id": { "type": "string" }}} }
-
             ]
         }
     ]

diff --git a/docs/spec/metricsets/metricset.json → docs/spec/metricsets/common_metricset.json b/docs/spec/metricsets/metricset.json → docs/spec/metricsets/common_metricset.json
@@ -1,6 +1,6 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "$id": "docs/spec/metricsets/metricset.json",
+    "$id": "docs/spec/metricsets/common_metricset.json",
     "type": "object",
     "description": "Metric data captured by an APM agent",
     "properties": {
@@ -24,13 +24,7 @@
                 }
             },
             "additionalProperties": false
-        },
-        "timestamp": {
-            "type": "string",
-            "format": "date-time",
-            "pattern": "Z$",
-            "description": "Recorded time of the metric, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
         }
     },
-    "required": ["samples", "timestamp"]
+    "required": ["samples"]
 }
diff --git a/docs/spec/metricsets/payload.json b/docs/spec/metricsets/payload.json
@@ -8,7 +8,7 @@
         "metrics": {
             "type": "array",
             "items": {
-                "$ref": "metricset.json"
+                "$ref": "v1_metricset.json"
             },
             "minItems": 1
         },

diff --git a/docs/spec/metricsets/v1_metricset.json b/docs/spec/metricsets/v1_metricset.json
@@ -0,0 +1,12 @@
+{
+    "$id": "docs/spec/metricsets/v1_metricset.json",
+    "type": "object",
+    "description": "Data captured by an agent representing an event occurring in a monitored service",
+    "allOf": [
+
+        { "$ref": "common_metricset.json"  },
+        { "$ref": "../timestamp_rfc3339.json" },
+        {"required": ["timestamp"], "properties": {"timestamp": { "type": "string" }}}
+
+    ]
+}
diff --git a/docs/spec/metricsets/v2_metricset.json b/docs/spec/metricsets/v2_metricset.json
@@ -0,0 +1,11 @@
+{
+    "$id": "docs/spec/metricsets/v2_metricset.json",
+    "type": "object",
+    "description": "Data captured by an agent representing an event occurring in a monitored service",
+    "allOf": [
+
+        { "$ref": "common_metricset.json"  }, 
+        { "$ref": "../timestamp_epoch.json"},
+        {"required": ["timestamp"], "properties": {"timestamp": { "type": "integer" }}}
+    ]
+}
diff --git a/docs/spec/spans/common_span.json b/docs/spec/spans/common_span.json
@@ -60,15 +60,11 @@
             },
             "minItems": 0
         },
-        "start": {
-            "type": "number",
-            "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
-        },
         "type": {
             "type": "string",
             "description": "Keyword of specific relevance in the service's domain (eg: 'db.postgresql.query', 'template.erb', etc)",
             "maxLength": 1024
         }
     },
-    "required": ["duration", "name", "start", "type"]
+    "required": ["duration", "name", "type"]
 }
diff --git a/docs/spec/spans/v1_span.json b/docs/spec/spans/v1_span.json
@@ -18,8 +18,13 @@
                 "parent": {
                     "type": ["integer", "null"],
                     "description": "The ID of the parent of the span."
+                },
+                "start": {
+                    "type": "number",
+                    "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
                 }
             },
+            "required": ["start"],
             "dependencies": {
                 "parent": {
                     "required": ["id"]

diff --git a/docs/spec/spans/v2_span.json b/docs/spec/spans/v2_span.json
@@ -3,7 +3,8 @@
     "type": "object",
     "$ref": "common_span.json",
     "allOf": [
-        { "$ref": "common_span.json"  }, 
+        { "$ref": "common_span.json"  },
+        { "$ref": "../timestamp_epoch.json" },
         {  
             "properties": {
                 "id": {
@@ -26,14 +27,17 @@
                     "type": "string",
                     "maxLength": 1024
                 },
-                "timestamp": {
-                    "description": "Recorded time of the span, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ",
-                    "type": ["string", "null"],
-                    "pattern": "Z$",
-                    "format": "date-time"
+                "start": {
+                    "type": ["number", "null"],
+                    "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
                 }
             },
             "required": ["id", "transaction_id", "trace_id", "parent_id"]
+        },
+        { "anyOf":[
+                {"required": ["timestamp"], "properties": {"timestamp": { "type": "integer" }}},
+                {"required": ["start"], "properties": {"start": { "type": "number" }}}
+            ]
         }
     ]
 }
diff --git a/docs/spec/timestamp_epoch.json b/docs/spec/timestamp_epoch.json
@@ -0,0 +1,12 @@
+{
+    "$id": "doc/spec/timestamp_epoch.json",
+    "title": "Timestamp Epoch",
+    "description": "Object with 'timestamp' property.",
+    "type": ["object"],
+    "properties": {  
+        "timestamp": {
+            "description": "Recorded time of the event, UTC based and formatted as microseconds since Unix epoch",
+            "type": ["integer", "null"]
+        }
+    }
+}
diff --git a/docs/spec/timestamp_rfc3339.json b/docs/spec/timestamp_rfc3339.json
@@ -0,0 +1,14 @@
+{
+    "$id": "doc/spec/timestamp_epoch.json",
+    "title": "Timestamp Epoch",
+    "description": "Object with 'timestamp' property.",
+    "type": ["object"],
+    "properties": {  
+        "timestamp": {
+            "type": ["string", "null"],
+            "pattern": "Z$",
+            "format": "date-time",
+            "description": "Recorded time of the transaction, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
+        }
+    }
+}
diff --git a/docs/spec/transactions/common_transaction.json b/docs/spec/transactions/common_transaction.json
@@ -20,12 +20,6 @@
             "description": "The result of the transaction. For HTTP-related transactions, this should be the status code formatted like 'HTTP 2xx'.",
             "maxLength": 1024
         },
-        "timestamp": {
-            "type": ["string", "null"],
-            "pattern": "Z$",
-            "format": "date-time",
-            "description": "Recorded time of the transaction, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
-        },
         "type": {
             "type": "string",
             "description": "Keyword of specific relevance in the service's domain (eg: 'request', 'backgroundjob', etc)",

diff --git a/docs/spec/transactions/v1_transaction.json b/docs/spec/transactions/v1_transaction.json
@@ -20,6 +20,7 @@
                 "description": "Data captured by an agent representing an event occurring in a monitored service",
                 "allOf": [
                     { "$ref": "common_transaction.json"  }, 
+                    { "$ref": "../timestamp_rfc3339.json" },
                     {  
                         "properties": {
                             "id": {

diff --git a/docs/spec/transactions/v2_transaction.json b/docs/spec/transactions/v2_transaction.json
@@ -3,7 +3,8 @@
     "type": "object",
     "description": "Data captured by an agent representing an event occurring in a monitored service",
     "allOf": [
-        { "$ref": "common_transaction.json"  }, 
+        { "$ref": "common_transaction.json"  },
+        { "$ref": "../timestamp_epoch.json" },
         {  
             "properties": {
                 "id": {