[v2] accept unix epoch timestamps in microseconds

* makes "start" optional if "timestamp" is given for v2 * if "timestamp" is not given, it is set to the sum of req-time + start * adds timestamp.us (long) to span, transaction and error documents (both v1 and v2)
elastic · Oct 1, 2018 · 126e9b3 · 126e9b3
1 parent 388353b
commit 126e9b3
Show file tree

Hide file tree

Showing 85 changed files with 2,099 additions and 1,683 deletions.
diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml
@@ -14,6 +14,16 @@
       type: keyword
       description: Processor event.
 
+    - name: timestamp
+      type: group
+      description:
+      fields:
+        - name: us
+          type: long
+          count: 1
+          description: >
+            Timestamp of the event in microseconds since Unix epoch.
+
     - name: context
       type: group
       description: >

diff --git a/docs/data/elasticsearch/error.json b/docs/data/elasticsearch/error.json
@@ -247,6 +247,9 @@
         "event": "error",
         "name": "error"
     },
+    "timestamp": {
+        "us": 1494342245999999
+    },
     "transaction": {
         "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79"
     }

diff --git a/docs/data/elasticsearch/span.json b/docs/data/elasticsearch/span.json
@@ -71,6 +71,9 @@
         },
         "type": "db.postgresql.query"
     },
+    "timestamp": {
+        "us": 1496170407154000
+    },
     "transaction": {
         "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79"
     }

diff --git a/docs/data/elasticsearch/transaction.json b/docs/data/elasticsearch/transaction.json
@@ -111,6 +111,9 @@
         "event": "transaction",
         "name": "transaction"
     },
+    "timestamp": {
+        "us": 1496170407154000
+    },
     "transaction": {
         "duration": {
             "us": 32592

diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc
@@ -57,6 +57,22 @@ type: keyword
 
 Processor event.
 
+--
+
+[float]
+== timestamp fields
+
+None
+
+
+*`timestamp.us`*::
++
+--
+type: long
+
+Timestamp of the event in microseconds since Unix epoch.
+
+
 --
 
 [float]

diff --git a/docs/spec/errors/common_error.json b/docs/spec/errors/common_error.json
@@ -1,5 +1,5 @@
 {
-    "$id": "docs/spec/errors/error.json",
+    "$id": "docs/spec/errors/common_error.json",
     "type": "object",
     "description": "Data captured by an agent representing an event occurring in a monitored service",
     "properties": {
@@ -86,12 +86,6 @@
                 }
             },
             "required": ["message"]
-        },
-        "timestamp": {
-            "type": ["string","null"],
-            "format": "date-time",
-            "pattern": "Z$",
-            "description": "Recorded time of the error, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
         }
     },
     "anyOf": [

diff --git a/docs/spec/errors/v1_error.json b/docs/spec/errors/v1_error.json
@@ -35,6 +35,12 @@
                                         "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
                                     }
                                 }
+                            },
+                            "timestamp": {
+                                "type": ["string","null"],
+                                "format": "date-time",
+                                "pattern": "Z$",
+                                "description": "Recorded time of the error, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
                             }
                         }
                     }

diff --git a/docs/spec/errors/v2_error.json b/docs/spec/errors/v2_error.json
@@ -26,6 +26,10 @@
                     "description": "Hex encoded 64 random bits ID of the parent transaction or span. Must be present if trace_id and transaction_id are set.", 
                     "type": ["string", "null"],
                     "maxLength": 1024
+                },
+                "timestamp": {
+                    "description": "Recorded time of the span, UTC based and formatted as microseconds since Unix epoch",
+                    "type": ["integer", "null"]
                 }
             },
             "allOf": [
@@ -36,7 +40,6 @@
                   "then": { "required": ["parent_id"], "properties": {"parent_id": { "type": "string" }}} },
                 { "if": {"required": ["parent_id"], "properties": {"parent_id": { "type": "string" }}},
                   "then": { "required": ["transaction_id"], "properties": {"transaction_id": { "type": "string" }}} }
-
             ]
         }
     ]

diff --git a/docs/spec/metricsets/metricset.json → docs/spec/metricsets/common_metricset.json b/docs/spec/metricsets/metricset.json → docs/spec/metricsets/common_metricset.json
@@ -1,6 +1,6 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "$id": "docs/spec/metricsets/metricset.json",
+    "$id": "docs/spec/metricsets/common_metricset.json",
     "type": "object",
     "description": "Metric data captured by an APM agent",
     "properties": {
@@ -24,13 +24,7 @@
                 }
             },
             "additionalProperties": false
-        },
-        "timestamp": {
-            "type": "string",
-            "format": "date-time",
-            "pattern": "Z$",
-            "description": "Recorded time of the metric, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
         }
     },
-    "required": ["samples", "timestamp"]
+    "required": ["samples"]
 }
diff --git a/docs/spec/metricsets/payload.json b/docs/spec/metricsets/payload.json
@@ -8,7 +8,7 @@
         "metrics": {
             "type": "array",
             "items": {
-                "$ref": "metricset.json"
+                "$ref": "v1_metricset.json"
             },
             "minItems": 1
         },

diff --git a/docs/spec/metricsets/v1_metricset.json b/docs/spec/metricsets/v1_metricset.json
@@ -0,0 +1,20 @@
+{
+    "$id": "docs/spec/errors/v2_error.json",
+    "type": "object",
+    "description": "Data captured by an agent representing an event occurring in a monitored service",
+    "allOf": [
+
+        { "$ref": "common_metricset.json"  }, 
+        {  
+            "properties": {
+                "timestamp": {
+                    "type": "string",
+                    "format": "date-time",
+                    "pattern": "Z$",
+                    "description": "Recorded time of the metric, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
+                }
+            },
+            "required": ["timestamp"]
+        }
+    ]
+}
diff --git a/docs/spec/metricsets/v2_metricset.json b/docs/spec/metricsets/v2_metricset.json
@@ -0,0 +1,19 @@
+{
+    "$id": "docs/spec/errors/v2_error.json",
+    "type": "object",
+    "description": "Data captured by an agent representing an event occurring in a monitored service",
+    "allOf": [
+
+        { "$ref": "common_metricset.json"  }, 
+        {  
+            "properties": {
+
+                "timestamp": {
+                    "description": "Recorded time of the span, UTC based and formatted as microseconds since Unix epoch",
+                    "type": "integer"
+                }
+            },
+            "required": ["timestamp"]
+        }
+    ]
+}
diff --git a/docs/spec/spans/common_span.json b/docs/spec/spans/common_span.json
@@ -60,15 +60,11 @@
             },
             "minItems": 0
         },
-        "start": {
-            "type": "number",
-            "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
-        },
         "type": {
             "type": "string",
             "description": "Keyword of specific relevance in the service's domain (eg: 'db.postgresql.query', 'template.erb', etc)",
             "maxLength": 1024
         }
     },
-    "required": ["duration", "name", "start", "type"]
+    "required": ["duration", "name", "type"]
 }
diff --git a/docs/spec/spans/v1_span.json b/docs/spec/spans/v1_span.json
@@ -18,8 +18,13 @@
                 "parent": {
                     "type": ["integer", "null"],
                     "description": "The ID of the parent of the span."
+                },
+                "start": {
+                    "type": "number",
+                    "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
                 }
             },
+            "required": ["start"],
             "dependencies": {
                 "parent": {
                     "required": ["id"]

diff --git a/docs/spec/spans/v2_span.json b/docs/spec/spans/v2_span.json
@@ -27,13 +27,20 @@
                     "maxLength": 1024
                 },
                 "timestamp": {
-                    "description": "Recorded time of the span, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ",
-                    "type": ["string", "null"],
-                    "pattern": "Z$",
-                    "format": "date-time"
+                    "description": "Recorded time of the span, UTC based and formatted as microseconds since Unix epoch",
+                    "type": ["integer", "null"]
+                },
+                "start": {
+                    "type": ["number", "null"],
+                    "description": "Offset relative to the transaction's timestamp identifying the start of the span, in milliseconds"
                 }
             },
             "required": ["id", "transaction_id", "trace_id", "parent_id"]
+        },
+        { "anyOf":[
+                {"required": ["timestamp"], "properties": {"timestamp": { "type": "integer" }}},
+                {"required": ["start"], "properties": {"start": { "type": "number" }}}
+            ]
         }
     ]
 }
diff --git a/docs/spec/transactions/common_transaction.json b/docs/spec/transactions/common_transaction.json
@@ -20,12 +20,6 @@
             "description": "The result of the transaction. For HTTP-related transactions, this should be the status code formatted like 'HTTP 2xx'.",
             "maxLength": 1024
         },
-        "timestamp": {
-            "type": ["string", "null"],
-            "pattern": "Z$",
-            "format": "date-time",
-            "description": "Recorded time of the transaction, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
-        },
         "type": {
             "type": "string",
             "description": "Keyword of specific relevance in the service's domain (eg: 'request', 'backgroundjob', etc)",

diff --git a/docs/spec/transactions/v1_transaction.json b/docs/spec/transactions/v1_transaction.json
@@ -47,6 +47,12 @@
                                         }
                                     }
                                 }
+                            },
+                            "timestamp": {
+                                "type": ["string", "null"],
+                                "pattern": "Z$",
+                                "format": "date-time",
+                                "description": "Recorded time of the transaction, UTC based and formatted as YYYY-MM-DDTHH:mm:ss.sssZ"
                             }
                         },
                         "required": ["id"]

diff --git a/docs/spec/transactions/v2_transaction.json b/docs/spec/transactions/v2_transaction.json
@@ -36,6 +36,10 @@
                         }
                     },
                     "required": ["started"]
+                },
+                "timestamp": {
+                    "description": "Recorded time of the span, UTC based and formatted as microseconds since Unix epoch",
+                    "type": ["integer", "null"]
                 }
             },
             "required": ["id", "trace_id", "span_count"]