ingest: translate known OpenTelemetry JVM metrics

Copy well-known JVM metrics from their OpenTelemetry
names and associated labels to their counterparts
produced by the Elastic APM Java agent. This enables
users to visualise the metrics in the Elastic APM
app in Kibana.

Metrics are duplicated to avoid surprising users by
silently dropping the OpenTelemetry names. Users can
drop either one of the duplicates with ingest pipeline
customisation.

apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.app-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.app-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "logs-apm.error-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "logs-apm.error-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.internal-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.internal-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.profiling-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.profiling-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "traces-apm-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "traces-apm-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

ingest/pipeline/definition.json

@@ -29,6 +29,12 @@
  "name": "apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
  }
@@ -137,5 +143,49 @@
  }
  ]
  }
+ },
+ {
+ "id": "apm_opentelemetry_metrics",
+ "body": {
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+ }
  }
 ]

ingest/pipeline/definition.yml

@@ -12,6 +12,9 @@ apm:
  - pipeline:
  name: apm_error_grouping_name
  if: ctx.processor?.event == 'error'
+ - pipeline:
+ name: apm_opentelemetry_metrics
+ if: ctx.processor?.event == 'metric'
 
 apm_user_agent:
  description: Add user agent information for APM events
@@ -74,3 +77,35 @@ apm_error_grouping_name:
  field: error.grouping_name
  copy_from: error.log.message
  if: ctx.error?.log?.message != null
+
+apm_opentelemetry_metrics:
+ description: Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts
+ processors:
+ - set:
+ # Copy `runtime.jvm.memory.area` (OpenTelemetry) to `jvm.memory.{area}.{type}` (Elastic APM).
+ field: jvm.memory.{{labels.area}}.{{labels.type}}
+ copy_from: runtime.jvm.memory.area
+ if: ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null
+ - set:
+ # Copy `runtime.jvm.gc.time` (OpenTelemetry) to `jvm.gc.time` (Elastic APM).
+ # Both are defined in milliseconds.
+ field: jvm.gc.time
+ copy_from: runtime.jvm.gc.time
+ if: ctx.runtime?.jvm?.gc?.time != null
+ - set:
+ # Copy `runtime.jvm.gc.count` (OpenTelemetry) to `jvm.gc.count` (Elastic APM).
+ field: jvm.gc.count
+ copy_from: runtime.jvm.gc.count
+ if: ctx.runtime?.jvm?.gc?.count != null
+ - set:
+ # Copy `runtime.jvm.gc.collection` (OpenTelemetry) to `jvm.gc.time` (Elastic APM).
+ # Both are defined in milliseconds. This is the old name for `runtime.jvm.gc.time`.
+ field: jvm.gc.time
+ copy_from: runtime.jvm.gc.collection
+ if: ctx.runtime?.jvm?.gc?.collection != null
+ - set:
+ # Copy `labels.gc` (OpenTelemetry) to `labels.name` (Elastic APM), for jvm.gc.{time,count}.
+ field: labels.name
+ copy_from: labels.gc
+ override: false # don't replace existing labels.name field, if any
+ if: ctx.labels?.gc != null