Map RUM PerformanceResourceTiming fields

Add field mappings for: - http.response.transfer_size - http.response.encoded_body_size - http.response.decoded_body_size The fields are mapped with `index: false`, to minimise storage overhead. They can still be used in searches, just a bit slower. The intake schema for these fields has been changed from float64 to int, as these are originally integer values. See https://w3c.github.io/resource-timing/#sec-performanceresourcetiming testdata, and hence approvals, have been updated to change the mock values from floats to integers.
elastic · Oct 25, 2022 · 8622378 · 8622378
1 parent e72f68b
commit 8622378
Show file tree

Hide file tree

Showing 25 changed files with 95 additions and 78 deletions.
diff --git a/apmpackage/apm/changelog.yml b/apmpackage/apm/changelog.yml
@@ -9,6 +9,9 @@
     - description: Enable synthetic source for metrics data streams
       type: enhancement
       link: https://github.com/elastic/apm-server/pull/9215
+    - description: Add mappings for RUM `http.response.*_size` fields
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/9429
 - version: "8.5.0"
   changes:
     - description: Add package settings to enable the experimental collection of service metrics

diff --git a/apmpackage/apm/data_stream/traces/fields/fields.yml b/apmpackage/apm/data_stream/traces/fields/fields.yml
@@ -58,6 +58,26 @@
   type: object
   description: |
     The canonical headers of the monitored HTTP response.
+- name: http.response.transfer_size
+  type: long
+  unit: byte
+  index: false
+  description: |
+    The size (in octets) of the fetched resource.
+    The size includes the response header fields plus the response payload body (as defined by RFC7230).
+- name: http.response.encoded_body_size
+  type: long
+  unit: byte
+  index: false
+  description: |
+    The size (in octets) received from the fetch (HTTP or cache), of the payload body, before removing any applied content-codings.
+- name: http.response.decoded_body_size
+  type: long
+  index: false
+  description: |
+    The size (in octets) received from the fetch (HTTP or cache) of the message body, after removing any applied content-codings.
+    If the resource is retrieved from an application cache or local resources,
+    it returns the size of the payload after removing any applied content-codings.
 - name: kubernetes.namespace
   type: keyword
   description: |

diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -7,6 +7,7 @@ https://github.com/elastic/apm-server/compare/8.5\...main[View commits]
 ==== Breaking Changes
 - `apm-server.decoder.*` stack monitoring metrics are no longer recorded {pull}9210[9210]
 - `ecs.version` is no longer added to document `_source`; it is added as a `constant_keyword` field {pull}9208[9208]
+- `context.http.response.*_size` fields now enforce integer values {pull}9429[9429]
 
 [float]
 ==== Deprecations
@@ -19,6 +20,7 @@ Set error.id for OpenTelemetry exception span events {pull}9372[9372]
 ==== Intake API Changes
 - experimental:[] Extend logs v2 intake API to support ECS logging fields {pull}9349[9349]
 - experimental:[] Add support for string timestamp format(`2006-01-02T15:04:05.999-0700`) {pull}9376[9376]
+- `context.http.response.*_size` fields have been changed from floating-point to integer types {pull}9429[9429]
 
 [float]
 ==== Added

diff --git a/docs/spec/rumv3/error.json b/docs/spec/rumv3/error.json
@@ -118,14 +118,14 @@
               "description": "DecodedBodySize holds the size of the decoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "ebs": {
               "description": "EncodedBodySize holds the size of the encoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "he": {
@@ -159,7 +159,7 @@
               "description": "TransferSize holds the total size of the payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             }
           }

diff --git a/docs/spec/rumv3/span.json b/docs/spec/rumv3/span.json
@@ -116,21 +116,21 @@
                   "description": "DecodedBodySize holds the size of the decoded payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 },
                 "ebs": {
                   "description": "EncodedBodySize holds the size of the encoded payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 },
                 "ts": {
                   "description": "TransferSize holds the total size of the payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 }
               }

diff --git a/docs/spec/rumv3/transaction.json b/docs/spec/rumv3/transaction.json
@@ -118,14 +118,14 @@
               "description": "DecodedBodySize holds the size of the decoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "ebs": {
               "description": "EncodedBodySize holds the size of the encoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "he": {
@@ -159,7 +159,7 @@
               "description": "TransferSize holds the total size of the payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             }
           }
@@ -720,21 +720,21 @@
                         "description": "DecodedBodySize holds the size of the decoded payload.",
                         "type": [
                           "null",
-                          "number"
+                          "integer"
                         ]
                       },
                       "ebs": {
                         "description": "EncodedBodySize holds the size of the encoded payload.",
                         "type": [
                           "null",
-                          "number"
+                          "integer"
                         ]
                       },
                       "ts": {
                         "description": "TransferSize holds the total size of the payload.",
                         "type": [
                           "null",
-                          "number"
+                          "integer"
                         ]
                       }
                     }

diff --git a/docs/spec/v2/error.json b/docs/spec/v2/error.json
@@ -355,14 +355,14 @@
               "description": "DecodedBodySize holds the size of the decoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "encoded_body_size": {
               "description": "EncodedBodySize holds the size of the encoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "finished": {
@@ -410,7 +410,7 @@
               "description": "TransferSize holds the total size of the payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             }
           }

diff --git a/docs/spec/v2/span.json b/docs/spec/v2/span.json
@@ -192,14 +192,14 @@
                   "description": "DecodedBodySize holds the size of the decoded payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 },
                 "encoded_body_size": {
                   "description": "EncodedBodySize holds the size of the encoded payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 },
                 "headers": {
@@ -233,7 +233,7 @@
                   "description": "TransferSize holds the total size of the payload.",
                   "type": [
                     "null",
-                    "number"
+                    "integer"
                   ]
                 }
               }

diff --git a/docs/spec/v2/transaction.json b/docs/spec/v2/transaction.json
@@ -354,14 +354,14 @@
               "description": "DecodedBodySize holds the size of the decoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "encoded_body_size": {
               "description": "EncodedBodySize holds the size of the encoded payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             },
             "finished": {
@@ -409,7 +409,7 @@
               "description": "TransferSize holds the total size of the payload.",
               "type": [
                 "null",
-                "number"
+                "integer"
               ]
             }
           }

diff --git a/internal/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json b/internal/beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json
@@ -333,7 +333,7 @@
                         ]
                     },
                     "status_code": 302,
-                    "transfer_size": 300.12
+                    "transfer_size": 30012
                 }
             },
             "kubernetes": {
@@ -518,8 +518,8 @@
                     "method": "POST"
                 },
                 "response": {
-                    "decoded_body_size": 401.9,
-                    "encoded_body_size": 356.9,
+                    "decoded_body_size": 40190,
+                    "encoded_body_size": 35690,
                     "finished": true,
                     "headers": {
                         "Content-Type": [

diff --git a/internal/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json b/internal/beater/test_approved_es_documents/TestPublishIntegrationSpans.approved.json
@@ -589,7 +589,7 @@
                     "decoded_body_size": 401,
                     "encoded_body_size": 356,
                     "status_code": 200,
-                    "transfer_size": 300.12
+                    "transfer_size": 30012
                 }
             },
             "kubernetes": {
@@ -806,7 +806,7 @@
                     "decoded_body_size": 401,
                     "encoded_body_size": 356,
                     "status_code": 200,
-                    "transfer_size": 300.12
+                    "transfer_size": 30012
                 }
             },
             "kubernetes": {
@@ -1023,7 +1023,7 @@
                     "decoded_body_size": 401,
                     "encoded_body_size": 356,
                     "status_code": 200,
-                    "transfer_size": 300.12
+                    "transfer_size": 30012
                 }
             },
             "kubernetes": {

diff --git a/internal/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json b/internal/beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json
@@ -226,8 +226,8 @@
                     "referrer": "http://localhost:8000/test/e2e/"
                 },
                 "response": {
-                    "decoded_body_size": 29.9,
-                    "encoded_body_size": 26.9,
+                    "decoded_body_size": 2990,
+                    "encoded_body_size": 2690,
                     "finished": true,
                     "headers": {
                         "Content-Type": [
@@ -236,7 +236,7 @@
                     },
                     "headers_sent": true,
                     "status_code": 200,
-                    "transfer_size": 25.8
+                    "transfer_size": 258
                 },
                 "version": "1.1"
             },

diff --git a/...ter/test_approved_es_documents/TestPublishIntegrationTransactionsHugeTraces.approved.json b/...ter/test_approved_es_documents/TestPublishIntegrationTransactionsHugeTraces.approved.json
@@ -96,8 +96,8 @@
                     "referrer": "http://localhost:8000/test/e2e/"
                 },
                 "response": {
-                    "decoded_body_size": 29.9,
-                    "encoded_body_size": 26.9,
+                    "decoded_body_size": 2990,
+                    "encoded_body_size": 2690,
                     "finished": true,
                     "headers": {
                         "Content-Type": [
@@ -106,7 +106,7 @@
                     },
                     "headers_sent": true,
                     "status_code": 200,
-                    "transfer_size": 25.8
+                    "transfer_size": 258
                 },
                 "version": "1.1"
             },

diff --git a/internal/model/http.go b/internal/model/http.go
@@ -50,9 +50,9 @@ type HTTPResponse struct {
 	Headers         mapstr.M
 	Finished        *bool
 	HeadersSent     *bool
-	TransferSize    *float64
-	EncodedBodySize *float64
-	DecodedBodySize *float64
+	TransferSize    *int
+	EncodedBodySize *int
+	DecodedBodySize *int
 }
 
 func (h *HTTP) fields() mapstr.M {
@@ -90,8 +90,8 @@ func (h *HTTPResponse) fields() mapstr.M {
 	fields.maybeSetMapStr("headers", h.Headers)
 	fields.maybeSetBool("finished", h.Finished)
 	fields.maybeSetBool("headers_sent", h.HeadersSent)
-	fields.maybeSetFloat64ptr("transfer_size", h.TransferSize)
-	fields.maybeSetFloat64ptr("encoded_body_size", h.EncodedBodySize)
-	fields.maybeSetFloat64ptr("decoded_body_size", h.DecodedBodySize)
+	fields.maybeSetIntptr("transfer_size", h.TransferSize)
+	fields.maybeSetIntptr("encoded_body_size", h.EncodedBodySize)
+	fields.maybeSetIntptr("decoded_body_size", h.DecodedBodySize)
 	return mapstr.M(fields)
 }
diff --git a/internal/model/mapstr.go b/internal/model/mapstr.go
@@ -52,14 +52,6 @@ func (m *mapStr) maybeSetIntptr(k string, v *int) bool {
 	return false
 }
 
-func (m *mapStr) maybeSetFloat64ptr(k string, v *float64) bool {
-	if v != nil {
-		m.set(k, *v)
-		return true
-	}
-	return false
-}
-
 func (m *mapStr) maybeSetMapStr(k string, v mapstr.M) bool {
 	if len(v) > 0 {
 		m.set(k, v)

diff --git a/internal/model/modeldecoder/rumv3/model.go b/internal/model/modeldecoder/rumv3/model.go
@@ -99,15 +99,15 @@ type contextRequest struct {
 
 type contextResponse struct {
 	// DecodedBodySize holds the size of the decoded payload.
-	DecodedBodySize nullable.Float64 `json:"dbs"`
+	DecodedBodySize nullable.Int `json:"dbs"`
 	// EncodedBodySize holds the size of the encoded payload.
-	EncodedBodySize nullable.Float64 `json:"ebs"`
+	EncodedBodySize nullable.Int `json:"ebs"`
 	// Headers holds the http headers sent in the http response.
 	Headers nullable.HTTPHeader `json:"he"`
 	// StatusCode sent in the http response.
 	StatusCode nullable.Int `json:"sc"`
 	// TransferSize holds the total size of the payload.
-	TransferSize nullable.Float64 `json:"ts"`
+	TransferSize nullable.Int `json:"ts"`
 }
 
 type contextService struct {
@@ -432,11 +432,11 @@ type spanContextHTTP struct {
 
 type spanContextHTTPResponse struct {
 	// DecodedBodySize holds the size of the decoded payload.
-	DecodedBodySize nullable.Float64 `json:"dbs"`
+	DecodedBodySize nullable.Int `json:"dbs"`
 	// EncodedBodySize holds the size of the encoded payload.
-	EncodedBodySize nullable.Float64 `json:"ebs"`
+	EncodedBodySize nullable.Int `json:"ebs"`
 	// TransferSize holds the total size of the payload.
-	TransferSize nullable.Float64 `json:"ts"`
+	TransferSize nullable.Int `json:"ts"`
 }
 
 type spanContextService struct {