Update network connection fields (#5671)

* Update OTel network connection type conventions * Add network.connection.subtype field Change network.connection_type to network.connection.type, and add network.connection.subtype. Rename model.Carrier to model.NetworkCarrier to clarify its position within the document hierarchy. * model/modeldecoder: add "network.connection.type" Support agents sending "network.connection.type" ("n.c.t" in RUMv3) in metadata, which will be added to all events in the stream. This is currently intended for RUM. In the future we may want to support setting this at the even level too, to cater for operation-specific network information. * Update changelog
elastic · Aug 4, 2021 · d1ebfd4 · d1ebfd4
1 parent 52cfec8
commit d1ebfd4
Show file tree

Hide file tree

Showing 34 changed files with 533 additions and 77 deletions.
diff --git a/apmpackage/apm/data_stream/app_metrics/fields/fields.yml b/apmpackage/apm/data_stream/app_metrics/fields/fields.yml
@@ -44,10 +44,14 @@
   type: keyword
   description: |
     Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
   type: keyword
   description: |
-    Cellular network technology, eg. 4G
+    Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+  type: keyword
+  description: |
+    Network connection type, eg. "wifi", "cell"
 - name: observer.listening
   type: keyword
   description: |

diff --git a/apmpackage/apm/data_stream/error_logs/fields/fields.yml b/apmpackage/apm/data_stream/error_logs/fields/fields.yml
@@ -85,10 +85,14 @@
   type: keyword
   description: |
     Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
   type: keyword
   description: |
-    Cellular network technology, eg. 4G
+    Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+  type: keyword
+  description: |
+    Network connection type, eg. "wifi", "cell"
 - name: observer.listening
   type: keyword
   description: |

diff --git a/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml
@@ -44,10 +44,14 @@
   type: keyword
   description: |
     Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
   type: keyword
   description: |
-    Cellular network technology, eg. 4G
+    Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+  type: keyword
+  description: |
+    Network connection type, eg. "wifi", "cell"
 - name: observer.listening
   type: keyword
   description: |

diff --git a/apmpackage/apm/data_stream/profile_metrics/fields/fields.yml b/apmpackage/apm/data_stream/profile_metrics/fields/fields.yml
@@ -34,10 +34,14 @@
   type: keyword
   description: |
     Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
   type: keyword
   description: |
-    Cellular network technology, eg. 4G
+    Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+  type: keyword
+  description: |
+    Network connection type, eg. "wifi", "cell"
 - name: observer.listening
   type: keyword
   description: |

diff --git a/apmpackage/apm/data_stream/traces/fields/fields.yml b/apmpackage/apm/data_stream/traces/fields/fields.yml
@@ -50,10 +50,14 @@
   type: keyword
   description: |
     Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
   type: keyword
   description: |
-    Cellular network technology, eg. 4G
+    Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+  type: keyword
+  description: |
+    Network connection type, eg. "wifi", "cell"
 - name: observer.listening
   type: keyword
   description: |

diff --git a/apmpackage/apm/docs/README.md b/apmpackage/apm/docs/README.md
@@ -102,7 +102,8 @@ Traces are written to `traces-apm.*` indices.
 |network.carrier.mcc|Mobile country code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.mnc|Mobile network code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.hostname|Hostname of the APM Server.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
 |observer.listening|Address the server is listening on.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.type|The type will be set to \`apm-server\`.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
@@ -465,7 +466,8 @@ Metrics are written to `metrics-apm.app.*`, `metrics-apm.internal.*`, and `metri
 |network.carrier.mcc|Mobile country code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.mnc|Mobile network code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.hostname|Hostname of the APM Server.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
 |observer.listening|Address the server is listening on.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.type|The type will be set to \`apm-server\`.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
@@ -647,7 +649,8 @@ Logs are written to `logs-apm.error.*` indices.
 |network.carrier.mcc|Mobile country code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.mnc|Mobile network code|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.hostname|Hostname of the APM Server.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |
 |observer.listening|Address the server is listening on.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png)  |
 |observer.type|The type will be set to \`apm-server\`.|keyword|  ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png)  |

diff --git a/apmpackage/cmd/gen-package/genfields.go b/apmpackage/cmd/gen-package/genfields.go
@@ -44,6 +44,10 @@ func generateFields() map[string][]field {
 
 	for streamType, inputFields := range inputFieldsFiles {
 		log.Printf("%s", streamType)
+		for _, field := range inputFields {
+			log.Printf(" - %s", field.Name)
+		}
+
 		var ecsFields []field
 		var nonECSFields []field
 		for _, fields := range populateECSInfo(ecsFlatFields, inputFields) {

diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -5,13 +5,16 @@ https://github.com/elastic/apm-server/compare/7.13\...master[View commits]
 
 [float]
 ==== Breaking Changes
+- `network.connection_type` is now `network.connection.type` {pull}5671[5671]
 
 [float]
 ==== Bug fixes
 - Fix apm_error_grouping_name and apm_convert_destination_address {pull}5876[5876]
+- corrected OTel attribute names for `net.host.connection.*` {pull}5671[5671]
 
 [float]
 ==== Intake API Changes
+- `network.connection.type` was added to stream metadata {pull}5671[5671]
 
 [float]
 ==== Added

diff --git a/docs/data/intake-api/generated/rum_v3_events.ndjson b/docs/data/intake-api/generated/rum_v3_events.ndjson
@@ -1,3 +1,3 @@
-{"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "[email protected]","un": "John Doe"},"l": {"testTagKey": "testTagValue"}}}
+{"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "[email protected]","un": "John Doe"},"l": {"testTagKey": "testTagValue"},"n":{"c":{"t":"5G"}}}}
 {"x": {"id": "ec2e280be8345240","tid": "286ac3ad697892c406528f13c82e0ce1","pid": "1ef08ac234fca23b455d9e27c660f1ab","n": "general-usecase-initial-p-load","t": "p-load","d": 295,"me": [{"sa": {"xdc": {"v": 1},"xds": {"v": 295},"xbc": {"v": 1}}},{"y": {"t": "Request"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}},{"y": {"t": "Response"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}}],"y": [{"id": "bbd8bcc3be14d814","n": "Requesting and receiving the document","t": "hard-navigation","su": "browser-timing","s": 4,"d": 2},{"id": "fc546e87a90a774f","n": "Parsing the document, executing sy. scripts","t": "hard-navigation","su": "browser-timing","s": 14,"d": 106},{"id": "fb8f717930697299","n": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js","t": "rc","su": "script","s": 22.53499999642372,"d": 35.060000023804605,"c": {"h": {"url": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED","r": {"ts": 677175,"ebs": 676864,"dbs": 676864}},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "rc"},"ad": "localhost","po": 8000}}},{"id": "9b80535c4403c9fb","n": "OpenTracing y","t": "cu","s": 96.92999999970198,"d": 198.07000000029802},{"id": "5ecb8ee030749715","n": "GET /test/e2e/common/data.json","t": "external","su": "h","sy": true,"s": 98.94000005442649,"d": 6.72499998472631,"c": {"h": {"mt": "GET","url": "http://localhost:8000/test/e2e/common/data.json?test=hamid","sc": 200},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "external"},"ad": "localhost","po": 8000}}},{"id": "27f45fd274f976d4","n": "POST http://localhost:8003/data","t": "external","su": "h","sy": true,"s": 106.52000003028661,"d": 11.584999971091747,"c": {"h": {"mt": "POST","url": "http://localhost:8003/data","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "a3c043330bc2015e","pi": 0,"n": "POST http://localhost:8003/fetch","t": "external","su": "h","ac": "action","sy": false,"s": 119.93500008247793,"d": 15.949999913573265,"c": {"h": {"mt": "POST","url": "http://localhost:8003/fetch","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "bc7665dc25629379","st": [{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "generateError","li": 7662,"co": 9},{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "<anonymous>","li": 7666,"co": 3}],"n": "Fire \"DOMContentLoaded\" event","t": "hard-navigation","su": "browser-timing","s": 120,"d": 2,"o":"success"}],"c": {"p": {"rf": "http://localhost:8000/test/e2e/","url": "http://localhost:8000/test/e2e/general-usecase/"},"r": {"sc": 200,"ts": 983,"ebs": 690,"dbs": 690,"he": {"Content-Type": "application/json"}},"q": {"he": {"Accept": "application/json"},"hve": "1.1","mt": "GET"},"u": {"id": "uId","un": "un","em": "em"},"cu": {"testContext": "testContext"},"g": {"testTagKey": "testTagValue"}},"k": {"a": {"lp": 131.03000004775822,"fb": 5,"di": 120,"dc": 138,"ds": 100,"de": 110,"fp": 70.82500003930181},"nt": {"fs": 0,"ls": 0,"le": 0,"cs": 0,"ce": 0,"qs": 4,"rs": 5,"re": 6,"dl": 14,"di": 120,"ds": 120,"de": 122,"dc": 138,"es": 138,"ee": 138}},"yc": {"sd": 8,"dd": 1},"sm": true,"exp":{"cls":1,"fid":2.0,"tbt":3.4,"ignored":5,"also":"ignored","lt":{"count":3,"sum":2.5,"max":1}}}}
 {"me": {"y": {"t": "Processing","su": "subtype"},"sa": {"ysc": {"v": 1},"yss": {"v": 124}},"g": {"tag1": "value1"}}}
diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc
@@ -569,10 +569,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
 +
 --
-Cellular network technology, eg. 4G
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
++
+--
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -1668,10 +1685,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -3007,10 +3041,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
 +
 --
-Cellular network technology, eg. 4G
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
++
+--
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -4627,10 +4678,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -5826,10 +5894,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword

diff --git a/docs/spec/rumv3/metadata.json b/docs/spec/rumv3/metadata.json
@@ -18,6 +18,30 @@
         "maxLength": 1024
       }
     },
+    "n": {
+      "description": "Network holds information about the network over which the monitored service is communicating.",
+      "type": [
+        "null",
+        "object"
+      ],
+      "properties": {
+        "c": {
+          "type": [
+            "null",
+            "object"
+          ],
+          "properties": {
+            "t": {
+              "type": [
+                "null",
+                "string"
+              ],
+              "maxLength": 1024
+            }
+          }
+        }
+      }
+    },
     "se": {
       "description": "Service metadata about the monitored service.",
       "type": "object",

diff --git a/docs/spec/v2/metadata.json b/docs/spec/v2/metadata.json
@@ -160,6 +160,30 @@
         "maxLength": 1024
       }
     },
+    "network": {
+      "description": "Network holds information about the network over which the monitored service is communicating.",
+      "type": [
+        "null",
+        "object"
+      ],
+      "properties": {
+        "connection": {
+          "type": [
+            "null",
+            "object"
+          ],
+          "properties": {
+            "type": {
+              "type": [
+                "null",
+                "string"
+              ],
+              "maxLength": 1024
+            }
+          }
+        }
+      }
+    },
     "process": {
       "description": "Process metadata about the monitored service.",
       "type": [

diff --git a/include/fields.go b/include/fields.go
diff --git a/model/error/_meta/fields.yml b/model/error/_meta/fields.yml
@@ -354,11 +354,23 @@
         Optional network fields
       fields:
 
-        - name: connection_type
-          type: keyword
+        - name: connection
+          type: group
           description: >
-            Cellular network technology, eg. 4G
-          overwrite: true
+            Network connection details
+          fields:
+
+            - name: type
+              type: keyword
+              description: >
+                Network connection type, eg. "wifi", "cell"
+              overwrite: true
+
+            - name: subtype
+              type: keyword
+              description: >
+                Detailed network connection sub-type, e.g. "LTE", "CDMA"
+              overwrite: true
 
         - name: carrier
           type: group