From 7e039931e68df8aba0c15080872acad2cd28fbac Mon Sep 17 00:00:00 2001 From: Carsten Lohmann Date: Wed, 20 Apr 2022 07:11:17 +0200 Subject: [PATCH] [#3190] Use OpenTelemetry for tracing (via OpenTracing shim). Signed-off-by: Carsten Lohmann --- .github/workflows/native-images-tests.yml | 4 +- adapters/base-quarkus/pom.xml | 71 ++++------ adapters/coap-vertx-quarkus/pom.xml | 7 - .../coap/CoapOptionInjectExtractAdapter.java | 124 ------------------ .../coap/TracingSupportingHonoResource.java | 18 +-- .../CoapOptionInjectExtractAdapterTest.java | 55 -------- .../TracingSupportingHonoResourceTest.java | 47 +------ bom/pom.xml | 83 ++---------- clients/amqp-connection/pom.xml | 12 +- .../connection/impl/HonoConnectionImpl.java | 4 +- ...geAnnotationsInjectExtractAdapterTest.java | 20 +-- clients/kafka-common/pom.xml | 14 +- .../KafkaHeadersInjectExtractAdapterTest.java | 22 ++-- core/pom.xml | 18 ++- .../eclipse/hono/tracing/TracingHelper.java | 17 ++- .../JsonObjectInjectExtractAdapterTest.java | 22 ++-- .../MultiMapInjectExtractAdapterTest.java | 22 ++-- create_dependencies_list.sh | 4 +- legal/src/main/resources/legal/DEPENDENCIES | 40 ++++-- legal/src/main/resources/legal/NOTICE.md | 80 ++++++----- .../src/main/resources/legal/hono-maven.deps | 40 ++++-- service-base/pom.xml | 20 +-- .../hono/service/AbstractEndpoint.java | 1 - .../hono/service/AbstractServiceBase.java | 1 - .../hono/service/http/TracingHandler.java | 3 +- .../quarkus/DropHttpRequestSpansSampler.java | 70 ++++++++++ .../QuarkusReflectionRegistration.java | 41 ------ .../hono/service/quarkus/SamplerProducer.java | 67 ++++++++++ .../quarkus/SamplingPrioritySampler.java | 63 +++++++++ .../hono/service/quarkus/TracerProducer.java | 14 +- services/base-quarkus/pom.xml | 57 ++++---- .../admin-guide/monitoring-tracing-config.md | 47 +++---- .../deployment/helm-based-deployment.md | 45 +------ .../content/dev-guide/building_hono.md | 10 +- tests/pom.xml | 92 ++++++++++++- tests/readme.md | 16 +-- .../tests/DownstreamMessageAssertions.java | 32 ++--- .../hono/tests/IntegrationTestSupport.java | 20 ++- tests/src/test/resources/amqp/application.yml | 14 +- tests/src/test/resources/coap/application.yml | 14 +- .../clustered-cache/application.yml | 14 +- .../embedded-cache/application.yml | 14 +- .../deviceregistry-jdbc-h2/application.yml | 14 +- .../application.yml | 14 +- .../deviceregistry-mongodb/application.yml | 14 +- tests/src/test/resources/http/application.yml | 14 +- tests/src/test/resources/mqtt/application.yml | 14 +- .../otel-collector/otel-collector-config.yaml | 24 ++++ 48 files changed, 727 insertions(+), 746 deletions(-) delete mode 100644 adapters/coap-vertx-quarkus/src/main/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapter.java delete mode 100644 adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapterTest.java create mode 100644 service-base/src/main/java/org/eclipse/hono/service/quarkus/DropHttpRequestSpansSampler.java delete mode 100644 service-base/src/main/java/org/eclipse/hono/service/quarkus/QuarkusReflectionRegistration.java create mode 100644 service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplerProducer.java create mode 100644 service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplingPrioritySampler.java create mode 100644 tests/src/test/resources/otel-collector/otel-collector-config.yaml diff --git a/.github/workflows/native-images-tests.yml b/.github/workflows/native-images-tests.yml index 915897b734..11d44d3ad1 100644 --- a/.github/workflows/native-images-tests.yml +++ b/.github/workflows/native-images-tests.yml @@ -42,14 +42,14 @@ jobs: run: | mvn install -B -e -DskipTests -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -Dquarkus.native.remote-container-build=false -Dquarkus.native.container-build=true \ - -Pjaeger,build-native-image -am \ + -Pbuild-native-image -am \ -pl :hono-service-auth-quarkus,:hono-service-command-router-quarkus,:hono-service-device-registry-mongodb-quarkus,\ :hono-adapter-mqtt-vertx-quarkus,:hono-adapter-amqp-vertx-quarkus,:hono-adapter-coap-vertx-quarkus,:hono-adapter-http-vertx-quarkus,\ :hono-tests - name: "Build JDBC registry image" run: | mvn install -B -e -DskipTests -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -Pjaeger,build-docker-image -am -pl :hono-service-device-registry-jdbc-quarkus + -Pbuild-docker-image -am -pl :hono-service-device-registry-jdbc-quarkus - name: "Run integration tests with Mongo DB and Kafka" run: | mvn verify -pl :hono-tests -B -e -DCI=$CI \ diff --git a/adapters/base-quarkus/pom.xml b/adapters/base-quarkus/pom.xml index d69ea434b1..39b16798aa 100644 --- a/adapters/base-quarkus/pom.xml +++ b/adapters/base-quarkus/pom.xml @@ -43,6 +43,30 @@ io.quarkus quarkus-micrometer + + io.quarkus + quarkus-opentelemetry + + + io.quarkus + quarkus-opentelemetry-exporter-otlp + + + com.google.code.findbugs + jsr305 + + + + + + jakarta.ws.rs + jakarta.ws.rs-api + + + + io.opentelemetry + opentelemetry-extension-trace-propagators + io.quarkus quarkus-smallrye-health @@ -77,26 +101,6 @@ io.quarkus quarkus-maven-plugin - - org.codehaus.mojo - properties-maven-plugin - - - initialize - - set-system-properties - - - - - quarkus.jaeger.metrics.enabled - ${jaeger.metrics.enabled} - - - - - - @@ -254,32 +258,5 @@ - - - jaeger - - - jaeger - - - - - io.jaegertracing - jaeger-client - - - io.jaegertracing - jaeger-micrometer - - - io.quarkus - quarkus-jaeger - - - jakarta.activation - jakarta.activation-api - - - diff --git a/adapters/coap-vertx-quarkus/pom.xml b/adapters/coap-vertx-quarkus/pom.xml index 923dd662ec..7b33769122 100644 --- a/adapters/coap-vertx-quarkus/pom.xml +++ b/adapters/coap-vertx-quarkus/pom.xml @@ -44,13 +44,6 @@ vertx-core - - - io.jaegertracing - jaeger-client - test - - diff --git a/adapters/coap-vertx-quarkus/src/main/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapter.java b/adapters/coap-vertx-quarkus/src/main/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapter.java deleted file mode 100644 index 6a448b4bd1..0000000000 --- a/adapters/coap-vertx-quarkus/src/main/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapter.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright (c) 2020 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ - - -package org.eclipse.hono.adapter.coap; - -import java.nio.ByteBuffer; -import java.util.Objects; -import java.util.Optional; - -import org.eclipse.californium.core.coap.Option; -import org.eclipse.californium.core.coap.OptionSet; - -import io.opentracing.propagation.Binary; - -/** - * An adapter for injecting/extracting an OpenTracing {@code SpanContext} to/from a CoAP option. - *

- * The option number being used to hold the context is {@link #OPTION_TRACE_CONTEXT}. - * - */ -public class CoapOptionInjectExtractAdapter implements Binary { - - /** - * The number of the CoAP option holding the binary encoding of a trace context. - *

- * Note that the value is from the experimental range to reflect its - * for internal use only character. As such, the CoAP adapter's - * capability to extract a trace context from this option remains undocumented. - *

- * The option is elective (bit 0 = 0), safe-to-forward (bit 1 = 1) and - * must not be used as a cache-key (bits 2-4 = 1). - * - * @see RFC 7252, Option Numbers - */ - public static final int OPTION_TRACE_CONTEXT = 0b1111110111111110; // 65022 - - private OptionSet options; - private Option optionToExtractFrom; - - private CoapOptionInjectExtractAdapter() { - } - - /** - * Creates a new carrier for extracting a trace context from CoAP options. - * - * @param options The CoAP options to extract the context from. - * @throws NullPointerException if options is {@code null}. - * @return The carrier to use for extraction. - */ - public static Optional forExtraction(final OptionSet options) { - Objects.requireNonNull(options); - return getTraceContextOption(options) - .map(option -> { - final CoapOptionInjectExtractAdapter adapter = new CoapOptionInjectExtractAdapter(); - adapter.optionToExtractFrom = option; - return Optional.of(adapter); - }) - .orElse(Optional.empty()); - } - - /** - * Creates a new carrier for injecting a trace context into CoAP options. - * - * @param options The CoAP options to inject the context to. - * @throws NullPointerException if options is {@code null}. - * @return The carrier to use for injection. - */ - public static CoapOptionInjectExtractAdapter forInjection(final OptionSet options) { - Objects.requireNonNull(options); - final CoapOptionInjectExtractAdapter adapter = new CoapOptionInjectExtractAdapter(); - adapter.options = options; - return adapter; - } - - /** - * Gets the CoAP option that contains the binary encoded trace context from - * a request's set of options. - * - * @param optionSet The request option set. - * @return The option. - */ - private static Optional

- * If the request contains the {@link CoapOptionInjectExtractAdapter#OPTION_TRACE_CONTEXT} option, its value - * is expected to be a binary encoded trace context and the {@link CoapOptionInjectExtractAdapter} - * is used to extract a {@code SpanContext} which is then used as the parent of the newly created - * {@code Span}. */ @Override public void handleRequest(final Exchange exchange) { diff --git a/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapterTest.java b/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapterTest.java deleted file mode 100644 index 787e9b1535..0000000000 --- a/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/CoapOptionInjectExtractAdapterTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Copyright (c) 2020 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ - - -package org.eclipse.hono.adapter.coap; - -import static com.google.common.truth.Truth.assertThat; - -import org.eclipse.californium.core.coap.OptionSet; -import org.junit.jupiter.api.Test; - -import io.jaegertracing.Configuration; -import io.opentracing.Span; -import io.opentracing.SpanContext; -import io.opentracing.Tracer; -import io.opentracing.propagation.Format; - - -/** - * Tests verifying the behavior of {@link CoapOptionInjectExtractAdapter}. - * - */ -class CoapOptionInjectExtractAdapterTest { - - /** - * Verifies that the Jaeger tracer implementation can successfully use the adapter to inject and extract - * a SpanContext. - */ - @Test - public void testJaegerTracerInjectsAndExtractsSpanContext() { - - final Configuration config = new Configuration("test"); - final Tracer tracer = config.getTracer(); - final Span span = tracer.buildSpan("do").start(); - - final OptionSet optionSet = new OptionSet(); - final CoapOptionInjectExtractAdapter injectAdapter = CoapOptionInjectExtractAdapter.forInjection(optionSet); - tracer.inject(span.context(), Format.Builtin.BINARY, injectAdapter); - - final SpanContext context = CoapOptionInjectExtractAdapter.forExtraction(optionSet) - .map(carrier -> tracer.extract(Format.Builtin.BINARY, carrier)) - .orElse(null); - assertThat(context.toSpanId()).isEqualTo(span.context().toSpanId()); - } -} diff --git a/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/TracingSupportingHonoResourceTest.java b/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/TracingSupportingHonoResourceTest.java index c545cbdd79..08fe6ff971 100644 --- a/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/TracingSupportingHonoResourceTest.java +++ b/adapters/coap-vertx-quarkus/src/test/java/org/eclipse/hono/adapter/coap/TracingSupportingHonoResourceTest.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2020, 2021 Contributors to the Eclipse Foundation + * Copyright (c) 2020, 2022 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -22,7 +22,6 @@ import static org.mockito.Mockito.RETURNS_SELF; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -31,7 +30,6 @@ import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.CoAP.Code; -import org.eclipse.californium.core.coap.Option; import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.coap.Response; import org.eclipse.californium.core.network.Endpoint; @@ -53,8 +51,6 @@ import io.opentracing.SpanContext; import io.opentracing.Tracer; import io.opentracing.Tracer.SpanBuilder; -import io.opentracing.propagation.Binary; -import io.opentracing.propagation.Format; import io.opentracing.tag.Tags; import io.vertx.core.Future; @@ -140,43 +136,6 @@ private Exchange newExchange(final Request request) { return exchange; } - /** - * Verifies that the resource uses the SpanContext extracted from a CoAP request - * as the parent of the newly created Span. - */ - @Test - public void testHandleRequestExtractsParentTraceContext() { - - final SpanContext extractedContext = mock(SpanContext.class); - when(tracer.extract(eq(Format.Builtin.BINARY), any(Binary.class))).thenReturn(extractedContext); - - final Request request = new Request(Code.POST); - request.getOptions().addOption(new Option(CoapOptionInjectExtractAdapter.OPTION_TRACE_CONTEXT)); - final Exchange exchange = newExchange(request); - resource.handleRequest(exchange); - - verify(tracer).buildSpan(eq(Code.POST.toString())); - verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER.toString())); - verify(spanBuilder).addReference(eq(References.CHILD_OF), eq(extractedContext)); - } - - /** - * Verifies that the resource does not set a parent on the newly created Span if the CoAP request - * does not contain a trace context option. - */ - @Test - public void testExtractFromEmptyOptionSet() { - - final Request request = new Request(Code.POST); - final Exchange exchange = newExchange(request); - resource.handleRequest(exchange); - - verify(tracer, never()).extract(eq(Format.Builtin.BINARY), any(Binary.class)); - verify(tracer).buildSpan(eq(Code.POST.toString())); - verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER.toString())); - verify(spanBuilder).addReference(eq(References.CHILD_OF), isNull()); - } - /** * Verifies that the resource sets the trace sampling priority on the newly created Span if the CoAP request * belongs to a tenant for which a specific sampling priority is configured. @@ -195,7 +154,7 @@ public void testApplyTenantTraceSamplingPriority() { resource.handleRequest(exchange); verify(tracer).buildSpan(eq(Code.POST.toString())); - verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER.toString())); + verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER)); verify(spanBuilder).addReference(eq(References.CHILD_OF), isNull()); // verify sampling prio has been set to 0 (corresponding to TracingSamplingMode.NONE) verify(span).setTag(eq(Tags.SAMPLING_PRIORITY.getKey()), eq(0)); @@ -220,7 +179,7 @@ public void testApplyTenantTraceSamplingPrioritySetForAuthId() { resource.handleRequest(exchange); verify(tracer).buildSpan(eq(Code.POST.toString())); - verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER.toString())); + verify(spanBuilder).withTag(eq(Tags.SPAN_KIND.getKey()), eq(Tags.SPAN_KIND_SERVER)); verify(spanBuilder).addReference(eq(References.CHILD_OF), isNull()); // verify sampling prio has been set to 1 (corresponding to TracingSamplingMode.ALL) verify(span).setTag(eq(Tags.SAMPLING_PRIORITY.getKey()), eq(1)); diff --git a/bom/pom.xml b/bom/pom.xml index 20324e0999..b91f34a570 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -34,12 +34,10 @@ 1.0.2 quay.io/interconnectedcloud/qdrouterd:1.17.1 30.1-jre - 2.9.0 quay.io/infinispan/server-native:13.0 jaegertracing/all-in-one:1.31 - false - 1.8.0 - quay.io/quarkus/ubi-quarkus-native-image:21.3-java17 4.1.74.Final + otel/opentelemetry-collector:0.48.0 postgres:12-alpine 4.0.3 4.4.1.Final @@ -95,13 +94,14 @@ quarkus.http.non-application-root-path=${health.check.non-application-root-path} quarkus.http.port=${health.check.port} # this needs to be configured at build time in order for native images to be able to log at TRACE level quarkus.log.min-level=TRACE -quarkus.micrometer.binder-enabled-default= false +quarkus.micrometer.binder-enabled-default=false quarkus.micrometer.binder.jvm=true # Kafka client metrics are being managed explicitly by MicrometerKafkaClientMetricsSupport quarkus.micrometer.binder.kafka.enabled=false quarkus.micrometer.binder.system=true quarkus.micrometer.binder.vertx.enabled=true quarkus.micrometer.export.prometheus.path=/prometheus +quarkus.opentelemetry.propagators=tracecontext,baggage,jaeger quarkus.smallrye-health.root-path=${health.check.root-path} quarkus.smallrye-health.liveness-path=${health.check.liveness-path} quarkus.smallrye-health.readiness-path=${health.check.readiness-path} @@ -151,21 +151,6 @@ quarkus.vertx.resolver.cache-max-time-to-live=0 - - io.quarkus - quarkus-jaeger - ${quarkus.platform.version} - - - io.quarkus - quarkus-ide-launcher - - - com.sun.activation - jakarta.activation - - - io.quarkus quarkus-kafka-client @@ -567,53 +552,6 @@ quarkus.vertx.resolver.cache-max-time-to-live=0 runtime - - - io.jaegertracing - jaeger-client - ${jaeger.version} - runtime - - - io.jaegertracing - jaeger-core - ${jaeger.version} - runtime - - - io.jaegertracing - jaeger-thrift - ${jaeger.version} - runtime - - - - org.apache.tomcat.embed - tomcat-embed-core - - - - javax.annotation - javax.annotation-api - - - - - io.jaegertracing - jaeger-micrometer - ${jaeger.version} - runtime - - - com.google.code.gson - gson - ${gson.version} - runtime - - com.github.ben-manes.caffeine @@ -650,10 +588,6 @@ quarkus.vertx.resolver.cache-max-time-to-live=0 guava ${guava.version} - - com.google.guava - failureaccess - com.google.guava listenablefuture @@ -685,6 +619,11 @@ quarkus.vertx.resolver.cache-max-time-to-live=0 cryptvault ${cryptvault.version} + + jakarta.ws.rs + jakarta.ws.rs-api + ${jakarta.jaxrs-api.version} + diff --git a/clients/amqp-connection/pom.xml b/clients/amqp-connection/pom.xml index 06a7b0aa9e..a4c3412cd4 100644 --- a/clients/amqp-connection/pom.xml +++ b/clients/amqp-connection/pom.xml @@ -56,18 +56,18 @@ test - io.jaegertracing - jaeger-client + io.opentelemetry + opentelemetry-opentracing-shim test - io.jaegertracing - jaeger-core + io.opentelemetry + opentelemetry-sdk test - io.jaegertracing - jaeger-thrift + io.opentelemetry + opentelemetry-extension-trace-propagators test diff --git a/clients/amqp-connection/src/main/java/org/eclipse/hono/client/amqp/connection/impl/HonoConnectionImpl.java b/clients/amqp-connection/src/main/java/org/eclipse/hono/client/amqp/connection/impl/HonoConnectionImpl.java index d2c3cbe913..aed31eff18 100644 --- a/clients/amqp-connection/src/main/java/org/eclipse/hono/client/amqp/connection/impl/HonoConnectionImpl.java +++ b/clients/amqp-connection/src/main/java/org/eclipse/hono/client/amqp/connection/impl/HonoConnectionImpl.java @@ -174,9 +174,7 @@ public Vertx getVertx() { * @throws NullPointerException if tracer is {@code null}. */ public void setTracer(final Tracer opentracingTracer) { - Objects.requireNonNull(opentracingTracer); - LOG.info("using OpenTracing implementation [{}]", opentracingTracer.getClass().getName()); - this.tracer = opentracingTracer; + this.tracer = Objects.requireNonNull(opentracingTracer); } /** diff --git a/clients/amqp-connection/src/test/java/org/eclipse/hono/client/amqp/tracing/MessageAnnotationsInjectExtractAdapterTest.java b/clients/amqp-connection/src/test/java/org/eclipse/hono/client/amqp/tracing/MessageAnnotationsInjectExtractAdapterTest.java index a3c1e1e8b0..00da8cd566 100644 --- a/clients/amqp-connection/src/test/java/org/eclipse/hono/client/amqp/tracing/MessageAnnotationsInjectExtractAdapterTest.java +++ b/clients/amqp-connection/src/test/java/org/eclipse/hono/client/amqp/tracing/MessageAnnotationsInjectExtractAdapterTest.java @@ -22,7 +22,11 @@ import org.apache.qpid.proton.message.Message; import org.junit.jupiter.api.Test; -import io.jaegertracing.Configuration; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.extension.trace.propagation.JaegerPropagator; +import io.opentelemetry.opentracingshim.OpenTracingShim; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentracing.Span; import io.opentracing.SpanContext; import io.opentracing.Tracer; @@ -51,9 +55,7 @@ public void testInjectAndExtract() { final Message message = ProtonHelper.message(); // inject the properties final MessageAnnotationsInjectAdapter injectAdapter = new MessageAnnotationsInjectAdapter(message, propertiesMapName); - testEntries.forEach((key, value) -> { - injectAdapter.put(key, value); - }); + testEntries.forEach(injectAdapter::put); // encode the message final WritableBuffer.ByteBufferWrapper buffer = WritableBuffer.ByteBufferWrapper.allocate(100); @@ -71,13 +73,15 @@ public void testInjectAndExtract() { /** - * Verifies that the Jaeger tracer implementation can successfully use the adapter to inject and extract + * Verifies that the OpenTelemetry Tracer shim can successfully use the adapter to inject and extract * a SpanContext. */ @Test - public void testJaegerTracerCanUseAdapter() { - final Configuration config = new Configuration("test"); - final Tracer tracer = config.getTracer(); + public void testTracerShimCanUseAdapter() { + final OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(JaegerPropagator.getInstance())) + .build(); + final Tracer tracer = OpenTracingShim.createTracerShim(openTelemetry); final Span span = tracer.buildSpan("do").start(); final Message message = ProtonHelper.message(); diff --git a/clients/kafka-common/pom.xml b/clients/kafka-common/pom.xml index cf3b9a38f8..895b38a2fa 100644 --- a/clients/kafka-common/pom.xml +++ b/clients/kafka-common/pom.xml @@ -101,8 +101,18 @@ test - io.jaegertracing - jaeger-core + io.opentelemetry + opentelemetry-opentracing-shim + test + + + io.opentelemetry + opentelemetry-sdk + test + + + io.opentelemetry + opentelemetry-extension-trace-propagators test diff --git a/clients/kafka-common/src/test/java/org/eclipse/hono/client/kafka/tracing/KafkaHeadersInjectExtractAdapterTest.java b/clients/kafka-common/src/test/java/org/eclipse/hono/client/kafka/tracing/KafkaHeadersInjectExtractAdapterTest.java index 51bcadfaf9..0e2b0b8db9 100644 --- a/clients/kafka-common/src/test/java/org/eclipse/hono/client/kafka/tracing/KafkaHeadersInjectExtractAdapterTest.java +++ b/clients/kafka-common/src/test/java/org/eclipse/hono/client/kafka/tracing/KafkaHeadersInjectExtractAdapterTest.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2021 Contributors to the Eclipse Foundation + * Copyright (c) 2022 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -22,7 +22,11 @@ import org.junit.jupiter.api.Test; -import io.jaegertracing.Configuration; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.extension.trace.propagation.JaegerPropagator; +import io.opentelemetry.opentracingshim.OpenTracingShim; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentracing.Span; import io.opentracing.SpanContext; import io.opentracing.Tracer; @@ -48,9 +52,7 @@ public void testInjectAndExtract() { final List headers = new ArrayList<>(); // inject the properties final KafkaHeadersInjectAdapter injectAdapter = new KafkaHeadersInjectAdapter(headers); - testEntries.forEach((key, value) -> { - injectAdapter.put(key, value); - }); + testEntries.forEach(injectAdapter::put); // extract the properties final KafkaHeadersExtractAdapter extractAdapter = new KafkaHeadersExtractAdapter(headers); @@ -61,13 +63,15 @@ public void testInjectAndExtract() { /** - * Verifies that the Jaeger tracer implementation can successfully use the adapters to inject and extract + * Verifies that the OpenTelemetry Tracer shim can successfully use the adapter to inject and extract * a SpanContext. */ @Test - public void testJaegerTracerCanUseAdapter() { - final Configuration config = new Configuration("test"); - final Tracer tracer = config.getTracer(); + public void testTracerShimCanUseAdapter() { + final OpenTelemetry openTelemetry = OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(JaegerPropagator.getInstance())) + .build(); + final Tracer tracer = OpenTracingShim.createTracerShim(openTelemetry); final Span span = tracer.buildSpan("do").start(); final List headers = new ArrayList<>(); diff --git a/core/pom.xml b/core/pom.xml index 11f6acafc1..21688b0f79 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -1,6 +1,6 @@ diff --git a/service-base/src/main/java/org/eclipse/hono/service/AbstractEndpoint.java b/service-base/src/main/java/org/eclipse/hono/service/AbstractEndpoint.java index 12d34d8f19..73fd572a07 100644 --- a/service-base/src/main/java/org/eclipse/hono/service/AbstractEndpoint.java +++ b/service-base/src/main/java/org/eclipse/hono/service/AbstractEndpoint.java @@ -66,7 +66,6 @@ protected AbstractEndpoint(final Vertx vertx) { * @param opentracingTracer The tracer. */ public final void setTracer(final Tracer opentracingTracer) { - logger.info("using OpenTracing Tracer implementation [{}]", opentracingTracer.getClass().getName()); this.tracer = Objects.requireNonNull(opentracingTracer); } diff --git a/service-base/src/main/java/org/eclipse/hono/service/AbstractServiceBase.java b/service-base/src/main/java/org/eclipse/hono/service/AbstractServiceBase.java index 9a98868201..93547f7aa8 100644 --- a/service-base/src/main/java/org/eclipse/hono/service/AbstractServiceBase.java +++ b/service-base/src/main/java/org/eclipse/hono/service/AbstractServiceBase.java @@ -63,7 +63,6 @@ public abstract class AbstractServiceBase ext * @param opentracingTracer The tracer. */ public final void setTracer(final Tracer opentracingTracer) { - log.info("using OpenTracing Tracer implementation [{}]", opentracingTracer.getClass().getName()); this.tracer = Objects.requireNonNull(opentracingTracer); } diff --git a/service-base/src/main/java/org/eclipse/hono/service/http/TracingHandler.java b/service-base/src/main/java/org/eclipse/hono/service/http/TracingHandler.java index 45df5a1f3c..9b9dc3affe 100644 --- a/service-base/src/main/java/org/eclipse/hono/service/http/TracingHandler.java +++ b/service-base/src/main/java/org/eclipse/hono/service/http/TracingHandler.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2019, 2021 Contributors to the Eclipse Foundation + * Copyright (c) 2019, 2022 Contributors to the Eclipse Foundation * * See the NOTICE file(s) distributed with this work for additional * information regarding copyright ownership. @@ -103,6 +103,7 @@ protected void handlerNormal(final RoutingContext routingContext) { final Span span = tracer.buildSpan(routingContext.request().method().toString()) .asChildOf(extractedContext) + .ignoreActiveSpan() .withTag(Tags.SPAN_KIND.getKey(), Tags.SPAN_KIND_SERVER) .start(); diff --git a/service-base/src/main/java/org/eclipse/hono/service/quarkus/DropHttpRequestSpansSampler.java b/service-base/src/main/java/org/eclipse/hono/service/quarkus/DropHttpRequestSpansSampler.java new file mode 100644 index 0000000000..7f25b2452e --- /dev/null +++ b/service-base/src/main/java/org/eclipse/hono/service/quarkus/DropHttpRequestSpansSampler.java @@ -0,0 +1,70 @@ +/** + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.hono.service.quarkus; + +import java.net.URI; +import java.util.List; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import io.opentelemetry.semconv.trace.attributes.SemanticAttributes; + +/** + * Sampler that drops all HTTP request spans having the request path as span name. + */ +public class DropHttpRequestSpansSampler implements Sampler { + + private final Sampler sampler; + + /** + * Creates a new DropHttpRequestSpansSampler. + * + * @param sampler Sampler to use if the span wasn't identified as an HTTP request span to be dropped. + */ + public DropHttpRequestSpansSampler(final Sampler sampler) { + this.sampler = sampler; + } + + @Override + public SamplingResult shouldSample(final Context parentContext, final String traceId, final String spanName, final SpanKind spanKind, + final Attributes attributes, final List parentLinks) { + if (spanKind.equals(SpanKind.SERVER)) { + final String httpTargetPath = getHttpTargetPath(attributes.get(SemanticAttributes.HTTP_TARGET)); + // span names don't include the leading slash + if (httpTargetPath != null && !httpTargetPath.isBlank() && httpTargetPath.substring(1).equals(spanName)) { + return SamplingResult.drop(); + } + } + return sampler.shouldSample(parentContext, traceId, spanName, spanKind, attributes, parentLinks); + } + + private String getHttpTargetPath(final String httpTarget) { + if (httpTarget != null) { + try { + return URI.create(httpTarget).getPath(); + } catch (final IllegalArgumentException e) { + // ignore + } + } + return null; + } + + @Override + public String getDescription() { + return sampler.getDescription(); + } +} diff --git a/service-base/src/main/java/org/eclipse/hono/service/quarkus/QuarkusReflectionRegistration.java b/service-base/src/main/java/org/eclipse/hono/service/quarkus/QuarkusReflectionRegistration.java deleted file mode 100644 index 52209470d7..0000000000 --- a/service-base/src/main/java/org/eclipse/hono/service/quarkus/QuarkusReflectionRegistration.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (c) 2022 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0 - * - * SPDX-License-Identifier: EPL-2.0 - */ - - -package org.eclipse.hono.service.quarkus; - -import io.quarkus.runtime.annotations.RegisterForReflection; - -/** - * A dummy class for registering third party classes for reflection with Quarkus. - * - */ -@RegisterForReflection( - // TODO: remove the Jaeger classes once the Quarkus Jaeger extension issue - // https://github.com/quarkusio/quarkus/issues/10402 - // has been fixed - classNames = { - "io.jaegertracing.internal.samplers.http.OperationSamplingParameters", - "io.jaegertracing.internal.samplers.http.PerOperationSamplingParameters", - "io.jaegertracing.internal.samplers.http.ProbabilisticSamplingStrategy", - "io.jaegertracing.internal.samplers.http.RateLimitingSamplingStrategy", - "io.jaegertracing.internal.samplers.http.SamplingStrategyResponse" - }, - fields = true, - methods = false) -public class QuarkusReflectionRegistration { - - private QuarkusReflectionRegistration() { - // prevent instantiation - } -} diff --git a/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplerProducer.java b/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplerProducer.java new file mode 100644 index 0000000000..2431af5684 --- /dev/null +++ b/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplerProducer.java @@ -0,0 +1,67 @@ +/** + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.hono.service.quarkus; + +import java.util.Optional; + +import javax.enterprise.context.ApplicationScoped; +import javax.enterprise.inject.Produces; +import javax.inject.Singleton; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.quarkus.opentelemetry.runtime.tracing.TracerRuntimeConfig; + +/** + * A producer for an OpenTelemetry Sampler. + */ +@ApplicationScoped +public class SamplerProducer { + + private static final Logger LOG = LoggerFactory.getLogger(SamplerProducer.class); + + @Singleton + @Produces + Sampler sampler(final TracerRuntimeConfig tracerRuntimeConfig) { + final TracerRuntimeConfig.SamplerConfig samplerConfig = tracerRuntimeConfig.sampler; + if (!tracerRuntimeConfig.suppressNonApplicationUris) { + LOG.info("'quarkus.opentelemetry.tracer.suppress-non-application-uris' set to 'false' - will be ignored"); + } + if (!samplerConfig.parentBased) { + LOG.info("'quarkus.opentelemetry.tracer.sampler.parent-based' set to 'false' - will be ignored"); + } + + Sampler sampler = getBaseSampler(samplerConfig.samplerName, samplerConfig.ratio); + sampler = Sampler.parentBased(sampler); + sampler = new SamplingPrioritySampler(sampler); + // Drop all HTTP request spans created by the quarkus OpenTelemetryVertxTracer for now. + // Without http-server metrics enabled, these spans will get the HTTP request path as span name, including + // variable parts like device-id, making span selection in the Tracing UI unusable. + // Dropping all requests spans also makes sure there are no liveness/readiness request spans created. + // Suppression of these spans via "quarkus.opentelemetry.tracer.suppress-non-application-uris" doesn't work + // if the non-application-root-path is the same as the overall root path. + return new DropHttpRequestSpansSampler(sampler); + } + + private static Sampler getBaseSampler(final String samplerName, final Optional ratio) { + LOG.info("using OpenTelemetry tracing sampler mode '{}' [ratio {}]", samplerName, ratio.orElse(null)); + return switch (samplerName) { + case "on" -> Sampler.alwaysOn(); + case "off" -> Sampler.alwaysOff(); + case "ratio" -> Sampler.traceIdRatioBased(ratio.orElse(1.0d)); + default -> throw new IllegalArgumentException("Unrecognized value for sampler: " + samplerName); + }; + } +} diff --git a/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplingPrioritySampler.java b/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplingPrioritySampler.java new file mode 100644 index 0000000000..4db4fafad2 --- /dev/null +++ b/service-base/src/main/java/org/eclipse/hono/service/quarkus/SamplingPrioritySampler.java @@ -0,0 +1,63 @@ +/** + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0 + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.hono.service.quarkus; + +import java.util.List; +import java.util.Optional; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; + +/** + * Sampler that respects the sampling.priority attribute to either always record and sample a span (in case + * of priority 1) or always drop the span (in case of priority 0). + */ +public class SamplingPrioritySampler implements Sampler { + + private static final String SAMPLING_PRIORITY_TAG = "sampling.priority"; + + private final Sampler sampler; + + /** + * Creates a new SamplingPrioritySampler. + * @param sampler Sampler to use for spans not containing a sampling.priority attribute. + */ + public SamplingPrioritySampler(final Sampler sampler) { + this.sampler = sampler; + } + + @Override + public SamplingResult shouldSample(final Context parentContext, final String traceId, final String name, final SpanKind spanKind, + final Attributes attributes, final List parentLinks) { + return Optional.ofNullable(attributes.get(AttributeKey.longKey(SAMPLING_PRIORITY_TAG))) + .map(samplingPriority -> { + if (samplingPriority == 1L) { + return SamplingResult.recordAndSample(); + } else if (samplingPriority == 0L) { + return SamplingResult.drop(); + } + return null; + }) + .orElseGet(() -> sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks)); + } + + @Override + public String getDescription() { + return "SamplingPrioritySampler"; + } +} diff --git a/service-base/src/main/java/org/eclipse/hono/service/quarkus/TracerProducer.java b/service-base/src/main/java/org/eclipse/hono/service/quarkus/TracerProducer.java index 567c580bf1..4fff59cac7 100644 --- a/service-base/src/main/java/org/eclipse/hono/service/quarkus/TracerProducer.java +++ b/service-base/src/main/java/org/eclipse/hono/service/quarkus/TracerProducer.java @@ -16,11 +16,9 @@ import javax.enterprise.inject.Produces; import javax.inject.Singleton; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.opentracingshim.OpenTracingShim; import io.opentracing.Tracer; -import io.opentracing.util.GlobalTracer; /** * A producer for an OpenTracing Tracer. @@ -28,13 +26,9 @@ @ApplicationScoped public class TracerProducer { - private static final Logger LOG = LoggerFactory.getLogger(TracerProducer.class); - @Singleton @Produces - Tracer jaegerTracer() { - final var tracer = GlobalTracer.get(); - LOG.info("using tracer instance: {}", tracer.toString()); - return tracer; + Tracer tracer(final OpenTelemetry openTelemetry) { + return OpenTracingShim.createTracerShim(openTelemetry); } } diff --git a/services/base-quarkus/pom.xml b/services/base-quarkus/pom.xml index bdabd85dda..5d51511ce9 100644 --- a/services/base-quarkus/pom.xml +++ b/services/base-quarkus/pom.xml @@ -47,6 +47,30 @@ io.quarkus quarkus-micrometer + + io.quarkus + quarkus-opentelemetry + + + io.quarkus + quarkus-opentelemetry-exporter-otlp + + + com.google.code.findbugs + jsr305 + + + + + + jakarta.ws.rs + jakarta.ws.rs-api + + + + io.opentelemetry + opentelemetry-extension-trace-propagators + io.quarkus quarkus-smallrye-health @@ -60,8 +84,6 @@ commons-logging-jboss-logging - - @@ -91,10 +113,6 @@ hono.auth ${hono.auth} - - quarkus.jaeger.metrics.enabled - ${jaeger.metrics.enabled} - @@ -258,32 +276,5 @@ - - - jaeger - - - jaeger - - - - - io.jaegertracing - jaeger-client - - - io.jaegertracing - jaeger-micrometer - - - io.quarkus - quarkus-jaeger - - - jakarta.activation - jakarta.activation-api - - - diff --git a/site/documentation/content/admin-guide/monitoring-tracing-config.md b/site/documentation/content/admin-guide/monitoring-tracing-config.md index 6d32a61203..5ff9b022b2 100644 --- a/site/documentation/content/admin-guide/monitoring-tracing-config.md +++ b/site/documentation/content/admin-guide/monitoring-tracing-config.md @@ -98,22 +98,33 @@ individual entries in the log with each other and thus *trace* the processing of However, this is usually a very tedious (and error prone) process and the relevant information is often only logged at a level (e.g. *DEBUG*) that is not used in production (often *INFO* or above). -In order to address this problem, Hono's service components are instrumented using [OpenTracing](https://opentracing.io/). -OpenTracing provides *Vendor-neutral APIs and instrumentation for distributed tracing*. The Hono container images all -include the [Jaeger Java client](https://www.jaegertracing.io/docs/1.32/client-libraries/) and can be configured to -send tracing information to a [Jaeger collector](https://www.jaegertracing.io/docs/1.32/architecture/). -Please refer to the -[Jaeger client documentation](https://github.com/jaegertracing/jaeger-client-java/blob/master/jaeger-core/README.md) -for details regarding configuration. +In order to address this problem, Hono's service components are instrumented to produce tracing data via +[OpenTelemetry](https://opentelemetry.io/) (with the instrumentation in the code using [OpenTracing](https://opentracing.io/) +as described [below]({{< relref "#opentracing-instrumentation" >}})). OpenTelemetry provides an end-to-end implementation +to generate, emit, collect, process and export tracing data. +The Hono container images all can be configured to send tracing data to an +[OpenTelemetry collector](https://opentelemetry.io/docs/collector/) from which data can be exported to a back-end like +Jaeger. The decision, whether a trace should be sampled and exported, controlling the amount of tracing data to be +collected, is configured by means of a sampler configuration. + +The table below provides an overview of the configuration properties regarding the sampler and the trace exporter, for +exporting to an OpenTelemetry collector. + +| OS Environment Variable
Java System Property | Type | Default Value | Description | +| :---------------------------------------------- | :------------ | :------------ | :------------| +| `QUARKUS_OPENTELEMETRY_TRACER_SAMPLER`
`quarkus.opentelemetry.tracer.sampler` | *string* | `on` | The sampler to use for tracing. Valid values are `off` to export no traces at all, `on` to export *all* traces and `ratio` to define a ratio of traces to be exported. | +| `QUARKUS_OPENTELEMETRY_TRACER_SAMPLER_RATIO`
`quarkus.opentelemetry.tracer.sampler.ratio` | *double* | | The ratio of traces to be exported. To be used if the sampler is set to `ratio`. Must be within `[0.0, 1.0]`. | +| `QUARKUS_OPENTELEMETRY_TRACER_EXPORTER_OTLP_ENDPOINT`
`quarkus.opentelemetry.tracer.exporter.otlp.endpoint` | *string* | | The OTLP endpoint of the OpenTelemetry Collector to connect to. The endpoint must start with either `http://` or `https://`. | -## Configuring usage of Jaeger tracing (included in Docker images) +Please refer to the +[Quarkus OpenTelemetry documentation](https://quarkus.io/guides/opentelemetry#quarkus-opentelemetry-exporter-otlp_configuration) +for further configuration trace exporter options. -The Maven based Hono build process does not include the Jaeger client libraries by default. In order to include them, -the `jaeger` Maven profile needs to be activated: +## OpenTracing instrumentation -~~~sh -mvn clean install -Pbuild-docker-image,jaeger -~~~ +[OpenTracing](https://opentracing.io/) is a predecessor to [OpenTelemetry](https://opentelemetry.io/). Hono components +still use OpenTracing APIs, but use the *OpenTracing shim* from the OpenTelemetry SDK as OpenTracing implementation, +letting traces be exported in the OpenTelemetry format. ## Enforcing the recording of traces for a tenant @@ -124,13 +135,3 @@ all traces concerning the processing of telemetry, event and command messages fo Furthermore, this enforced trace sampling can be restricted to only apply to messages sent in the context of a specific authentication identifier. Please refer to the [description of the `tracing` object]({{< ref "/api/tenant#tracing-format" >}}) in the Tenant Information for details. - -## Enabling Jaeger Client Metrics - -The Jaeger client contained in Hono components can be configured to report metrics via Micrometer. By default, -reporting of these metrics is disabled. In order to enable it, Hono needs to be compiled setting the -*jaeger.metrics.enabled* Maven property to `true`: - -```sh -mvn clean install -Djaeger.metrics.enabled=true -Pmetrics-prometheus,jaeger,build-docker-image -``` diff --git a/site/documentation/content/deployment/helm-based-deployment.md b/site/documentation/content/deployment/helm-based-deployment.md index 421df830c5..3de3196a9a 100644 --- a/site/documentation/content/deployment/helm-based-deployment.md +++ b/site/documentation/content/deployment/helm-based-deployment.md @@ -17,7 +17,7 @@ instructions regarding installation and configuration. ## Deploying custom Container Images The chart by default installs Hono's pre-built container images. In some cases it might be desirable to build Hono -from source, e.g. in order to use a different metrics back end or to [use Jaeger tracing]({{< relref "#using-jaeger-tracing" >}}). +from source, e.g. in order to use a different metrics back end. The container images created as part of the build process need to be made available to the Kubernetes cluster that Hono should be installed to. This usually requires the images to be pushed to a (private) container registry that @@ -31,7 +31,7 @@ for details. Once the source code has been retrieved, the build process can be s ~~~sh # in base directory of Hono working tree: -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger +mvn clean install -Pbuild-docker-image,metrics-prometheus ~~~ After the build process has finished, the custom container images need to be pushed to the registry so that the @@ -90,7 +90,7 @@ In any case the build process can be started using the following command: ~~~sh # in base directory of Hono working tree: -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger +mvn clean install -Pbuild-docker-image,metrics-prometheus ~~~ To obtain the used Hono version and write it in a variable, use: @@ -105,45 +105,6 @@ The newly built images can then be deployed using Helm: helm install --dependency-update -n hono --set honoImagesTag=$HONO_VERSION eclipse-hono eclipse-iot/hono ~~~ - -### Using Jaeger Tracing - -Hono's components are instrumented using OpenTracing to allow tracking of the distributed processing of messages -flowing through the system. The Hono chart can be configured to report tracing information to the -[Jaeger tracing system](https://www.jaegertracing.io/). The *Spans* reported by the components can then be viewed in a -web browser. - -In order for Hono's components to use the Jaeger client for reporting tracing information, the container images need -to be built with the `jaeger` Maven profile. Please refer to -[Monitoring & Tracing]({{< relref "/admin-guide/monitoring-tracing-config#configuring-usage-of-jaeger-tracing-included-in-docker-images" >}}) -for details. The newly built images also need to be made available to the target Kubernetes cluster as described in the -two previous sections. - -The chart can be configured to deploy and use an example Jaeger back end by means of setting the -*jaegerBackendExample.enabled* property to `true` when running Helm: - -~~~sh -helm install --dependency-update -n hono --set jaegerBackendExample.enabled=true eclipse-hono eclipse-iot/hono -~~~ - -This will create a Jaeger back end instance suitable for testing purposes and will configure all deployed Hono -components to use the Jaeger back end. - -The following command can then be used to return the IP address with which the Jaeger UI can be accessed in a -browser (ensure `minikube tunnel` is running when using minikube): - -~~~sh -kubectl get service eclipse-hono-jaeger-query --output="jsonpath={.status.loadBalancer.ingress[0]['hostname','ip']}" -n hono -~~~ - -If no example Jaeger back end should be deployed but instead an existing Jaeger installation should be used, -the chart's *jaegerAgentConf* property can be set to environment variables which are passed in to -the Jaeger Agent that is deployed with each of Hono's components. - -~~~sh -helm install --dependency-update -n hono --set jaegerAgentConf.REPORTER_TYPE=tchannel --set jaegerAgentConf.REPORTER_TCHANNEL_HOST_PORT=my-jaeger-collector:14267 eclipse-hono eclipse-iot/hono -~~~ - ### Deploying to Azure Kubernetes Service (AKS) The following chapter describes how to use Azure Kubernetes Service (AKS) as a deployment target that has diff --git a/site/documentation/content/dev-guide/building_hono.md b/site/documentation/content/dev-guide/building_hono.md index 9b7724912b..3792c7b551 100644 --- a/site/documentation/content/dev-guide/building_hono.md +++ b/site/documentation/content/dev-guide/building_hono.md @@ -62,7 +62,7 @@ Run the following from the source folder: ```sh export DOCKER_HOST # in the "hono" folder containing the source code -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger +mvn clean install -Pbuild-docker-image,metrics-prometheus ``` This will build all libraries, Docker images and example code. @@ -79,7 +79,7 @@ The plugin is disabled by default and can be enabled by setting the *jacoco.skip ```sh # in the "hono" folder containing the source code -mvn clean install -Djacoco.skip=false -Pbuild-docker-image,metrics-prometheus,jaeger +mvn clean install -Djacoco.skip=false -Pbuild-docker-image,metrics-prometheus ``` The plugin will produce a `target/jacoco.exec` file in each module which contains the (binary) coverage data. @@ -97,7 +97,7 @@ is `index.docker.io`. The default value for *docker.image.org-name* is `eclipse` Hono's images using the `quay.io` registry and the `custom` repository name: ```sh -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger -Ddocker.registry-name=quay.io -Ddocker.image.org-name=custom +mvn clean install -Pbuild-docker-image,metrics-prometheus -Ddocker.registry-name=quay.io -Ddocker.image.org-name=custom ``` #### Building native Images @@ -107,7 +107,7 @@ In order to do so, the `build-native-image` Maven profile needs to be activated: ```sh # in the "hono" folder containing the source code -mvn clean install -Pbuild-native-image,metrics-prometheus,jaeger +mvn clean install -Pbuild-native-image,metrics-prometheus ``` {{% notice info %}} @@ -121,7 +121,7 @@ The container images that are created as part of the build process can be automa using the `docker-push-image` Maven profile: ```sh -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger,docker-push-image +mvn clean install -Pbuild-docker-image,metrics-prometheus,docker-push-image ``` Note that the container registry might require authentication in order to push images. The build uses the Docker Maven diff --git a/tests/pom.xml b/tests/pom.xml index 3bcdc64f63..cfaf9aa1bf 100644 --- a/tests/pom.xml +++ b/tests/pom.xml @@ -178,12 +178,13 @@ true 0 + true hono-jaeger - ${jaeger.host}:6831 - ${jaeger.host}:5778 18080 - jaeger + + hono-otel-collector + @@ -330,8 +331,18 @@ test - io.jaegertracing - jaeger-core + io.opentelemetry + opentelemetry-opentracing-shim + test + + + io.opentelemetry + opentelemetry-sdk + test + + + io.opentelemetry + opentelemetry-extension-trace-propagators test @@ -385,6 +396,7 @@ jaeger false + http://${otel-collector.host}:4317 @@ -733,6 +745,73 @@ + + + ${docker.repository}/hono-opentelemetry-collector-test:${project.version} + + ${jaeger.disabled} + IfNotPresent + ${opentelemetry-collector.image.name} + + dir + / + + config + + ${project.build.directory}/resources/otel-collector + etc/hono + + * + + + + + + + ${jaeger.disabled} + + +otel-collector.ip:otel-collector.health.port:13133 + +otel-collector.ip:otel-collector.otlp.port:4317 + + ${project.build.directory}/docker/otel-collector.port.properties + + custom + ${custom.network.name} + ${otel-collector.host} + + + 83886080 + 83886080 + + + /otelcol + --config=/etc/hono/otel-collector-config.yaml + + + + + + OTEL + ${log.color.extra-services} + + + + + GET + http://${otel-collector.ip}:${otel-collector.health.port} + 200..299 + + + + ${docker.repository}/hono-service-auth-test:${project.version} @@ -1663,8 +1742,6 @@ ${trace.frames} - test - ${jaeger.propagation} ${auth.ip} @@ -1739,6 +1816,7 @@ true ${project.build.directory}/docker/jaeger.port.properties + ${project.build.directory}/docker/otel-collector.port.properties ${project.build.directory}/docker/jdbc-db.port.properties ${project.build.directory}/docker/mongodb.port.properties ${project.build.directory}/docker/auth.port.properties diff --git a/tests/readme.md b/tests/readme.md index 451a388f98..b51c6b6bb4 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -97,20 +97,10 @@ mvn verify -Prun-tests -Dhono.deviceregistry.type=file ### Running the Tests with the Jaeger tracing component -The tests can be run in such a way, that the OpenTracing trace spans created in the Hono components -as part of a test run can be inspected later on. The OpenTracing component used for this is Jaeger. +The tests can be run in such a way, that the OpenTelemetry trace spans created in the Hono components +as part of a test run can be inspected later on. The tracing component used for this is Jaeger. -To include the Jaeger client, build the Hono Docker images using the `jaeger` Maven profile: - -```sh -# in the "hono" folder containing the source code -mvn clean install -Pbuild-docker-image,metrics-prometheus,jaeger -``` - -(Add a `-Ddocker.host` property definition if needed, as described in the -[Developer Guide](https://www.eclipse.org/hono/docs/dev-guide/building_hono/).) - -Then run the tests using the `jaeger` Maven profile: +Run the tests using the `jaeger` Maven profile: ```sh # in directory: hono/tests/ diff --git a/tests/src/test/java/org/eclipse/hono/tests/DownstreamMessageAssertions.java b/tests/src/test/java/org/eclipse/hono/tests/DownstreamMessageAssertions.java index 8b2c268612..f669ddc0d8 100644 --- a/tests/src/test/java/org/eclipse/hono/tests/DownstreamMessageAssertions.java +++ b/tests/src/test/java/org/eclipse/hono/tests/DownstreamMessageAssertions.java @@ -102,7 +102,8 @@ public static void assertTelemetryMessageProperties( } /** - * Asserts that a downstream message contains a tracing context. + * Asserts that a downstream message contains a tracing context (if tracing support is enabled for the + * integration tests). * * @param msg The message to check. * @param expectedTraceId The trace ID that the tracing context is expected to have or {@code null} if the ID should @@ -113,21 +114,20 @@ public static void assertMessageContainsTracingContext( final DownstreamMessage msg, final String expectedTraceId) { - final SpanContext spanContext; - - if (msg.getMessageContext() instanceof AmqpMessageContext) { - final var ctx = (AmqpMessageContext) msg.getMessageContext(); - spanContext = AmqpUtils.extractSpanContext(IntegrationTestSupport.CLIENT_TRACER, ctx.getRawMessage()); - } else if (msg.getMessageContext() instanceof KafkaMessageContext) { - final var ctx = (KafkaMessageContext) msg.getMessageContext(); - spanContext = KafkaTracingHelper.extractSpanContext(IntegrationTestSupport.CLIENT_TRACER, ctx.getRecord()); - } else { - throw new AssertionError("unsupported DownstreamMessage type [%s]".formatted(msg.getClass().getName())); - } - assertWithMessage("message contains a tracing context").that(spanContext).isNotNull(); - if (expectedTraceId != null) { - assertWithMessage("message contains a tracing context with trace ID").that(spanContext.toTraceId()) - .isEqualTo(expectedTraceId); + if (IntegrationTestSupport.JAEGER_ENABLED) { + final SpanContext spanContext; + if (msg.getMessageContext() instanceof final AmqpMessageContext ctx) { + spanContext = AmqpUtils.extractSpanContext(IntegrationTestSupport.CLIENT_TRACER, ctx.getRawMessage()); + } else if (msg.getMessageContext() instanceof final KafkaMessageContext ctx) { + spanContext = KafkaTracingHelper.extractSpanContext(IntegrationTestSupport.CLIENT_TRACER, ctx.getRecord()); + } else { + throw new AssertionError("unsupported DownstreamMessage type [%s]".formatted(msg.getClass().getName())); + } + assertWithMessage("message contains a tracing context").that(spanContext).isNotNull(); + if (expectedTraceId != null) { + assertWithMessage("message contains a tracing context with trace ID").that(spanContext.toTraceId()) + .isEqualTo(expectedTraceId); + } } } } diff --git a/tests/src/test/java/org/eclipse/hono/tests/IntegrationTestSupport.java b/tests/src/test/java/org/eclipse/hono/tests/IntegrationTestSupport.java index 84c4721b4b..8e991a1997 100644 --- a/tests/src/test/java/org/eclipse/hono/tests/IntegrationTestSupport.java +++ b/tests/src/test/java/org/eclipse/hono/tests/IntegrationTestSupport.java @@ -86,7 +86,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import io.jaegertracing.Configuration; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.extension.trace.propagation.JaegerPropagator; +import io.opentelemetry.opentracingshim.OpenTracingShim; +import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentracing.Tracer; import io.opentracing.noop.NoopSpan; import io.vertx.core.CompositeFuture; @@ -482,12 +486,18 @@ public final class IntegrationTestSupport { public static final int KAFKA_TOPIC_CREATION_ADD_TO_TIMEOUT = 2; // seconds to add /** - * A Jaeger based OpenTracing tracer that can be used by devices and downstream clients to inject + * {@code true} if the tests are run with the Jaeger tracing component. + */ + public static final boolean JAEGER_ENABLED = !Boolean.getBoolean("jaeger.disabled"); + + private static final OpenTelemetry OPENTELEMETRY = OpenTelemetrySdk.builder() + .setPropagators(ContextPropagators.create(JaegerPropagator.getInstance())) + .build(); + /** + * An OpenTracing tracer that can be used by devices and downstream clients to inject * and/or extract a trace context into/from messages. - *

- * The tracer picks up its configuration from environment variables set by the Maven Failsafe plugin. */ - public static Tracer CLIENT_TRACER = Configuration.fromEnv().getTracer(); + public static Tracer CLIENT_TRACER = OpenTracingShim.createTracerShim(OPENTELEMETRY); private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestSupport.class); diff --git a/tests/src/test/resources/amqp/application.yml b/tests/src/test/resources/amqp/application.yml index 4e0b80c23e..e093bcd96c 100644 --- a/tests/src/test/resources/amqp/application.yml +++ b/tests/src/test/resources/amqp/application.yml @@ -83,19 +83,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.amqp-adapter.host}" + application: + name: "${hono.amqp-adapter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/coap/application.yml b/tests/src/test/resources/coap/application.yml index 90c0658d88..75769d9c47 100644 --- a/tests/src/test/resources/coap/application.yml +++ b/tests/src/test/resources/coap/application.yml @@ -82,19 +82,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.coap-adapter.host}" + application: + name: "${hono.coap-adapter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/commandrouter/clustered-cache/application.yml b/tests/src/test/resources/commandrouter/clustered-cache/application.yml index c5122a13ea..fa4d813e76 100644 --- a/tests/src/test/resources/commandrouter/clustered-cache/application.yml +++ b/tests/src/test/resources/commandrouter/clustered-cache/application.yml @@ -56,19 +56,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.commandrouter.host}" + application: + name: "${hono.commandrouter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/commandrouter/embedded-cache/application.yml b/tests/src/test/resources/commandrouter/embedded-cache/application.yml index 3dc563e9a0..3ceaa7490e 100644 --- a/tests/src/test/resources/commandrouter/embedded-cache/application.yml +++ b/tests/src/test/resources/commandrouter/embedded-cache/application.yml @@ -49,19 +49,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.commandrouter.host}" + application: + name: "${hono.commandrouter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/deviceregistry-jdbc-h2/application.yml b/tests/src/test/resources/deviceregistry-jdbc-h2/application.yml index a5b88de565..0e6ba74ca1 100644 --- a/tests/src/test/resources/deviceregistry-jdbc-h2/application.yml +++ b/tests/src/test/resources/deviceregistry-jdbc-h2/application.yml @@ -57,19 +57,19 @@ hono: bootstrap.servers: ${hono.kafka.bootstrap.servers} quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.registration.host}" + application: + name: "${hono.registration.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/deviceregistry-jdbc-postgres/application.yml b/tests/src/test/resources/deviceregistry-jdbc-postgres/application.yml index c7b386e701..96591968d4 100644 --- a/tests/src/test/resources/deviceregistry-jdbc-postgres/application.yml +++ b/tests/src/test/resources/deviceregistry-jdbc-postgres/application.yml @@ -65,19 +65,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.registration.host}" + application: + name: "${hono.registration.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/deviceregistry-mongodb/application.yml b/tests/src/test/resources/deviceregistry-mongodb/application.yml index 1807ab02b7..c8c619ccde 100644 --- a/tests/src/test/resources/deviceregistry-mongodb/application.yml +++ b/tests/src/test/resources/deviceregistry-mongodb/application.yml @@ -53,19 +53,19 @@ hono: bootstrap.servers: ${hono.kafka.bootstrap.servers} quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.registration.host}" + application: + name: "${hono.registration.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/http/application.yml b/tests/src/test/resources/http/application.yml index ece8bdbe25..f132c55b7c 100644 --- a/tests/src/test/resources/http/application.yml +++ b/tests/src/test/resources/http/application.yml @@ -76,19 +76,19 @@ hono: bootstrap.servers: ${hono.kafka.bootstrap.servers} quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.http-adapter.host}" + application: + name: "${hono.http-adapter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/mqtt/application.yml b/tests/src/test/resources/mqtt/application.yml index 9f115464bf..e14c63b72c 100644 --- a/tests/src/test/resources/mqtt/application.yml +++ b/tests/src/test/resources/mqtt/application.yml @@ -74,19 +74,19 @@ hono: bootstrap.servers: "${hono.kafka.bootstrap.servers}" quarkus: - jaeger: - agent-host-port: "${jaeger.agent-host-port}" - propagation: "${jaeger.propagation}" - sampler-manager-host-port: "${jaeger.sampler-manager-host-port}" - service-name: "${hono.mqtt-adapter.host}" + application: + name: "${hono.mqtt-adapter.host}" + opentelemetry: + tracer: + exporter: + otlp: + endpoint: "${otel-collector.endpoint}" log: console: color: true level: INFO min-level: TRACE category: - "io.quarkus.jaeger": - level: DEBUG "io.quarkus.vertx.core.runtime": level: DEBUG vertx: diff --git a/tests/src/test/resources/otel-collector/otel-collector-config.yaml b/tests/src/test/resources/otel-collector/otel-collector-config.yaml new file mode 100644 index 0000000000..64768cd656 --- /dev/null +++ b/tests/src/test/resources/otel-collector/otel-collector-config.yaml @@ -0,0 +1,24 @@ +receivers: + otlp: + protocols: + grpc: + +exporters: + jaeger: + endpoint: ${jaeger.host}:14250 + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + +service: + extensions: [health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [jaeger]