Update network connection fields (#5671)

* Update OTel network connection type conventions * Add network.connection.subtype field Change network.connection_type to network.connection.type, and add network.connection.subtype. Rename model.Carrier to model.NetworkCarrier to clarify its position within the document hierarchy. * model/modeldecoder: add "network.connection.type" Support agents sending "network.connection.type" ("n.c.t" in RUMv3) in metadata, which will be added to all events in the stream. This is currently intended for RUM. In the future we may want to support setting this at the even level too, to cater for operation-specific network information. * Update changelog
elastic · Aug 4, 2021 · d1ebfd4 · d1ebfd4
1 parent 52cfec8
commit d1ebfd4
Show file tree

Hide file tree

Showing 34 changed files with 533 additions and 77 deletions.
diff --git a/apmpackage/apm/data_stream/app_metrics/fields/fields.yml b/apmpackage/apm/data_stream/app_metrics/fields/fields.yml
@@ -44,10 +44,14 @@
  type: keyword
  description: |
  Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
  type: keyword
  description: |
- Cellular network technology, eg. 4G
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+ type: keyword
+ description: |
+ Network connection type, eg. "wifi", "cell"
 - name: observer.listening
  type: keyword
  description: |

diff --git a/apmpackage/apm/data_stream/error_logs/fields/fields.yml b/apmpackage/apm/data_stream/error_logs/fields/fields.yml
@@ -85,10 +85,14 @@
  type: keyword
  description: |
  Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
  type: keyword
  description: |
- Cellular network technology, eg. 4G
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+ type: keyword
+ description: |
+ Network connection type, eg. "wifi", "cell"
 - name: observer.listening
  type: keyword
  description: |

diff --git a/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml b/apmpackage/apm/data_stream/internal_metrics/fields/fields.yml
@@ -44,10 +44,14 @@
  type: keyword
  description: |
  Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
  type: keyword
  description: |
- Cellular network technology, eg. 4G
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+ type: keyword
+ description: |
+ Network connection type, eg. "wifi", "cell"
 - name: observer.listening
  type: keyword
  description: |

diff --git a/apmpackage/apm/data_stream/profile_metrics/fields/fields.yml b/apmpackage/apm/data_stream/profile_metrics/fields/fields.yml
@@ -34,10 +34,14 @@
  type: keyword
  description: |
  Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
  type: keyword
  description: |
- Cellular network technology, eg. 4G
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+ type: keyword
+ description: |
+ Network connection type, eg. "wifi", "cell"
 - name: observer.listening
  type: keyword
  description: |

diff --git a/apmpackage/apm/data_stream/traces/fields/fields.yml b/apmpackage/apm/data_stream/traces/fields/fields.yml
@@ -50,10 +50,14 @@
  type: keyword
  description: |
  Carrier name, eg. Vodafone, T-Mobile, etc.
-- name: network.connection_type
+- name: network.connection.subtype
  type: keyword
  description: |
- Cellular network technology, eg. 4G
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+- name: network.connection.type
+ type: keyword
+ description: |
+ Network connection type, eg. "wifi", "cell"
 - name: observer.listening
  type: keyword
  description: |

diff --git a/apmpackage/apm/docs/README.md b/apmpackage/apm/docs/README.md
@@ -102,7 +102,8 @@ Traces are written to `traces-apm.*` indices.
 |network.carrier.mcc|Mobile country code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.mnc|Mobile network code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.hostname|Hostname of the APM Server.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
 |observer.listening|Address the server is listening on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.type|The type will be set to \`apm-server\`.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
@@ -465,7 +466,8 @@ Metrics are written to `metrics-apm.app.*`, `metrics-apm.internal.*`, and `metri
 |network.carrier.mcc|Mobile country code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.mnc|Mobile network code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.hostname|Hostname of the APM Server.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
 |observer.listening|Address the server is listening on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.type|The type will be set to \`apm-server\`.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
@@ -647,7 +649,8 @@ Logs are written to `logs-apm.error.*` indices.
 |network.carrier.mcc|Mobile country code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.mnc|Mobile network code|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |network.carrier.name|Carrier name, eg. Vodafone, T-Mobile, etc.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
-|network.connection\_type|Cellular network technology, eg. 4G|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.subtype|Detailed network connection sub-type, e.g. "LTE", "CDMA"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
+|network.connection.type|Network connection type, eg. "wifi", "cell"|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.hostname|Hostname of the APM Server.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |
 |observer.listening|Address the server is listening on.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png) |
 |observer.type|The type will be set to \`apm-server\`.|keyword| ![](https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png) |

diff --git a/apmpackage/cmd/gen-package/genfields.go b/apmpackage/cmd/gen-package/genfields.go
@@ -44,6 +44,10 @@ func generateFields() map[string][]field {
 
  for streamType, inputFields := range inputFieldsFiles {
  log.Printf("%s", streamType)
+ for _, field := range inputFields {
+ log.Printf(" - %s", field.Name)
+ }
+
  var ecsFields []field
  var nonECSFields []field
  for _, fields := range populateECSInfo(ecsFlatFields, inputFields) {

diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc
@@ -5,13 +5,16 @@ https:/elastic/apm-server/compare/7.13\...master[View commits]
 
 [float]
 ==== Breaking Changes
+- `network.connection_type` is now `network.connection.type` {pull}5671[5671]
 
 [float]
 ==== Bug fixes
 - Fix apm_error_grouping_name and apm_convert_destination_address {pull}5876[5876]
+- corrected OTel attribute names for `net.host.connection.*` {pull}5671[5671]
 
 [float]
 ==== Intake API Changes
+- `network.connection.type` was added to stream metadata {pull}5671[5671]
 
 [float]
 ==== Added

diff --git a/docs/data/intake-api/generated/rum_v3_events.ndjson b/docs/data/intake-api/generated/rum_v3_events.ndjson
@@ -1,3 +1,3 @@
-{"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "[email protected]","un": "John Doe"},"l": {"testTagKey": "testTagValue"}}}
+{"m": {"se": {"n": "apm-a-rum-test-e2e-general-usecase","ve": "0.0.1","en": "prod","a": {"n": "js-base","ve": "4.8.1"},"ru": {"n": "v8","ve": "8.0"},"la": {"n": "javascript","ve": "6"},"fw": {"n": "angular","ve": "2"}},"u": {"id": 123,"em": "[email protected]","un": "John Doe"},"l": {"testTagKey": "testTagValue"},"n":{"c":{"t":"5G"}}}}
 {"x": {"id": "ec2e280be8345240","tid": "286ac3ad697892c406528f13c82e0ce1","pid": "1ef08ac234fca23b455d9e27c660f1ab","n": "general-usecase-initial-p-load","t": "p-load","d": 295,"me": [{"sa": {"xdc": {"v": 1},"xds": {"v": 295},"xbc": {"v": 1}}},{"y": {"t": "Request"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}},{"y": {"t": "Response"},"sa": {"ysc": {"v": 1},"yss": {"v": 1}}}],"y": [{"id": "bbd8bcc3be14d814","n": "Requesting and receiving the document","t": "hard-navigation","su": "browser-timing","s": 4,"d": 2},{"id": "fc546e87a90a774f","n": "Parsing the document, executing sy. scripts","t": "hard-navigation","su": "browser-timing","s": 14,"d": 106},{"id": "fb8f717930697299","n": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js","t": "rc","su": "script","s": 22.53499999642372,"d": 35.060000023804605,"c": {"h": {"url": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=REDACTED","r": {"ts": 677175,"ebs": 676864,"dbs": 676864}},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "rc"},"ad": "localhost","po": 8000}}},{"id": "9b80535c4403c9fb","n": "OpenTracing y","t": "cu","s": 96.92999999970198,"d": 198.07000000029802},{"id": "5ecb8ee030749715","n": "GET /test/e2e/common/data.json","t": "external","su": "h","sy": true,"s": 98.94000005442649,"d": 6.72499998472631,"c": {"h": {"mt": "GET","url": "http://localhost:8000/test/e2e/common/data.json?test=hamid","sc": 200},"dt": {"se": {"n": "http://localhost:8000","rc": "localhost:8000","t": "external"},"ad": "localhost","po": 8000}}},{"id": "27f45fd274f976d4","n": "POST http://localhost:8003/data","t": "external","su": "h","sy": true,"s": 106.52000003028661,"d": 11.584999971091747,"c": {"h": {"mt": "POST","url": "http://localhost:8003/data","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "a3c043330bc2015e","pi": 0,"n": "POST http://localhost:8003/fetch","t": "external","su": "h","ac": "action","sy": false,"s": 119.93500008247793,"d": 15.949999913573265,"c": {"h": {"mt": "POST","url": "http://localhost:8003/fetch","sc": 200},"dt": {"se": {"n": "http://localhost:8003","rc": "localhost:8003","t": "external"},"ad": "localhost","po": 8003}}},{"id": "bc7665dc25629379","st": [{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "generateError","li": 7662,"co": 9},{"ap": "http://localhost:8000/test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","f": "test/e2e/general-usecase/app.e2e-bundle.min.js?token=secret","fn": "<anonymous>","li": 7666,"co": 3}],"n": "Fire \"DOMContentLoaded\" event","t": "hard-navigation","su": "browser-timing","s": 120,"d": 2,"o":"success"}],"c": {"p": {"rf": "http://localhost:8000/test/e2e/","url": "http://localhost:8000/test/e2e/general-usecase/"},"r": {"sc": 200,"ts": 983,"ebs": 690,"dbs": 690,"he": {"Content-Type": "application/json"}},"q": {"he": {"Accept": "application/json"},"hve": "1.1","mt": "GET"},"u": {"id": "uId","un": "un","em": "em"},"cu": {"testContext": "testContext"},"g": {"testTagKey": "testTagValue"}},"k": {"a": {"lp": 131.03000004775822,"fb": 5,"di": 120,"dc": 138,"ds": 100,"de": 110,"fp": 70.82500003930181},"nt": {"fs": 0,"ls": 0,"le": 0,"cs": 0,"ce": 0,"qs": 4,"rs": 5,"re": 6,"dl": 14,"di": 120,"ds": 120,"de": 122,"dc": 138,"es": 138,"ee": 138}},"yc": {"sd": 8,"dd": 1},"sm": true,"exp":{"cls":1,"fid":2.0,"tbt":3.4,"ignored":5,"also":"ignored","lt":{"count":3,"sum":2.5,"max":1}}}}
 {"me": {"y": {"t": "Processing","su": "subtype"},"sa": {"ysc": {"v": 1},"yss": {"v": 124}},"g": {"tag1": "value1"}}}
diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc
@@ -569,10 +569,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
 +
 --
-Cellular network technology, eg. 4G
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
++
+--
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -1668,10 +1685,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -3007,10 +3041,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
 +
 --
-Cellular network technology, eg. 4G
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
++
+--
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -4627,10 +4678,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword
@@ -5826,10 +5894,27 @@ Optional network fields
 
 
 
-*`network.connection_type`*::
+[float]
+=== connection
+
+Network connection details
+
+
+
+*`network.connection.type`*::
++
+--
+Network connection type, eg. "wifi", "cell"
+
+
+type: keyword
+
+--
+
+*`network.connection.subtype`*::
 +
 --
-Cellular network technology, eg. 4G
+Detailed network connection sub-type, e.g. "LTE", "CDMA"
 
 
 type: keyword

diff --git a/docs/spec/rumv3/metadata.json b/docs/spec/rumv3/metadata.json
@@ -18,6 +18,30 @@
  "maxLength": 1024
  }
  },
+ "n": {
+ "description": "Network holds information about the network over which the monitored service is communicating.",
+ "type": [
+ "null",
+ "object"
+ ],
+ "properties": {
+ "c": {
+ "type": [
+ "null",
+ "object"
+ ],
+ "properties": {
+ "t": {
+ "type": [
+ "null",
+ "string"
+ ],
+ "maxLength": 1024
+ }
+ }
+ }
+ }
+ },
  "se": {
  "description": "Service metadata about the monitored service.",
  "type": "object",

diff --git a/docs/spec/v2/metadata.json b/docs/spec/v2/metadata.json
@@ -160,6 +160,30 @@
  "maxLength": 1024
  }
  },
+ "network": {
+ "description": "Network holds information about the network over which the monitored service is communicating.",
+ "type": [
+ "null",
+ "object"
+ ],
+ "properties": {
+ "connection": {
+ "type": [
+ "null",
+ "object"
+ ],
+ "properties": {
+ "type": {
+ "type": [
+ "null",
+ "string"
+ ],
+ "maxLength": 1024
+ }
+ }
+ }
+ }
+ },
  "process": {
  "description": "Process metadata about the monitored service.",
  "type": [

diff --git a/include/fields.go b/include/fields.go
diff --git a/model/error/_meta/fields.yml b/model/error/_meta/fields.yml
@@ -354,11 +354,23 @@
  Optional network fields
  fields:
 
- - name: connection_type
- type: keyword
+ - name: connection
+ type: group
  description: >
- Cellular network technology, eg. 4G
- overwrite: true
+ Network connection details
+ fields:
+
+ - name: type
+ type: keyword
+ description: >
+ Network connection type, eg. "wifi", "cell"
+ overwrite: true
+
+ - name: subtype
+ type: keyword
+ description: >
+ Detailed network connection sub-type, e.g. "LTE", "CDMA"
+ overwrite: true
 
  - name: carrier
  type: group