ingest: translate known OpenTelemetry JVM metrics

apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.app-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.app-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "logs-apm.error-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "logs-apm.error-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.internal-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.internal-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "metrics-apm.profiling-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "metrics-apm.profiling-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json

@@ -0,0 +1,41 @@
+{
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+}

apmpackage/apm/0.1.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json

@@ -26,6 +26,12 @@
  "name": "traces-apm-0.1.0-apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "traces-apm-0.1.0-apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
 }

changelogs/head.asciidoc

@@ -29,6 +29,7 @@ https:/elastic/apm-server/compare/7.12\...master[View commits]
 * Set `client.ip` for events from the Elastic APM iOS agent {pull}4975[4975]
 * Calculate service destination metrics for OpenTelemetry spans {pull}4976[4976]
 * Add exponential retries to api key and tail sampling requests{pull}4991[4991]
+* Ingest pipeline for translating OpenTelemetry Java metrics to Elastic APM fields {pull}4986[4986]
 
 [float]
 ==== Deprecated

docs/configuring-ingest.asciidoc

@@ -35,6 +35,17 @@ that are already available on the parent transaction.
 In previous versions of APM Server, this functionality was hardcoded internally.
 Switching metadata cleanup from an internal process to a processor allows you to keep any span metadata that is important in your architecture.
 
+`apm_error_grouping_name`::
+added:[7.13]
+Adds `error.grouping_name` to error documents for use in the {kibana-ref}/xpack-apm.html[Kibana APM UI].
+
+`apm_opentelemetry_metrics`::
+added:[7.13]
+Copies well-known OpenTelemetry metrics to their Elastic APM counterparts, for vizualisation in the {kibana-ref}/xpack-apm.html[Kibana APM UI].
+For example, the OpenTelemetry metric field `runtime.jvm.gc.time` is copied to the Elastic APM metric field `jvm.gc.time`.
+
+Metrics are duplicated so you can refer to them by either the OpenTelemetry or Elastic APM metric name.
+
 See the complete pipeline definition by navigating to the APM Server's home directory,
 and then viewing `ingest/pipeline/definition.json`.
 

ingest/pipeline/definition.json

@@ -29,6 +29,12 @@
  "name": "apm_error_grouping_name",
  "if": "ctx.processor?.event == 'error'"
  }
+ },
+ {
+ "pipeline": {
+ "name": "apm_opentelemetry_metrics",
+ "if": "ctx.processor?.event == 'metric'"
+ }
  }
  ]
  }
@@ -137,5 +143,49 @@
  }
  ]
  }
+ },
+ {
+ "id": "apm_opentelemetry_metrics",
+ "body": {
+ "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+ "processors": [
+ {
+ "set": {
+ "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+ "copy_from": "runtime.jvm.memory.area",
+ "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.time",
+ "if": "ctx.runtime?.jvm?.gc?.time != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.count",
+ "copy_from": "runtime.jvm.gc.count",
+ "if": "ctx.runtime?.jvm?.gc?.count != null"
+ }
+ },
+ {
+ "set": {
+ "field": "jvm.gc.time",
+ "copy_from": "runtime.jvm.gc.collection",
+ "if": "ctx.runtime?.jvm?.gc?.collection != null"
+ }
+ },
+ {
+ "set": {
+ "field": "labels.name",
+ "copy_from": "labels.gc",
+ "override": false,
+ "if": "ctx.labels?.gc != null"
+ }
+ }
+ ]
+ }
  }
 ]