Prototype Log4j2 Appender

instrumentation/log4j/log4j-2-common/testing/build.gradle.kts

@@ -5,6 +5,8 @@ plugins {
 dependencies {
  api(project(":testing-common"))
 
+ api("io.opentelemetry:opentelemetry-sdk-logs:1.9.0-alpha")
+
  api("org.apache.logging.log4j:log4j-core:2.7")
 
  implementation("com.google.guava:guava")

instrumentation/log4j/log4j-2-common/testing/src/main/groovy/Log4j2Test.groovy

@@ -9,7 +9,7 @@ import io.opentelemetry.instrumentation.test.InstrumentationSpecification
 import org.apache.logging.log4j.LogManager
 
 abstract class Log4j2Test extends InstrumentationSpecification {
- def cleanup() {
+ def setup() {
  ListAppender.get().clearEvents()
  }
 

instrumentation/log4j/log4j-2.13.2/library/README.md

@@ -1,15 +1,8 @@
 # Log4j 2 Integration
 
-This module integrates instrumentation with Log4j 2 by injecting the trace ID and span ID from a
-mounted span into
-Log4j's [context data](https://logging.apache.org/log4j/2.x/manual/thread-context.html).
-
-**Note**: Depending on your application, you may run into a [critical bug](https://issues.apache.org/jira/browse/LOG4J2-2838)
-with Log4j 2.13.2. If log messages show a `NullPointerException` when adding this instrumentation,
-please update to 2.13.3 or higher. The only change between 2.13.2 and 2.13.3 is the fix to this
-issue.
+This module provides Log4j2 extensions related to OpenTelemetry.
 
-To use it, just add the module to your application's runtime classpath.
+To use it, add the module to your application's runtime classpath.
 
 **Maven**
 
@@ -33,14 +26,26 @@ dependencies {
 }
 ```
 
-Log4j will automatically pick up our integration and will have these keys added to the context when
-a log statement is made when a span is active.
+## OpenTelemetry Context Data Provider
+
+`OpenTelemetryContextDataProvider` implements the Log4j2 `ContextDataProvider` SPI, and injects the
+trace ID and span ID from a mounted span into
+Log4j's [context data](https://logging.apache.org/log4j/2.x/manual/thread-context.html).
+
+**Note**: Depending on your application, you may run into
+a [critical bug](https://issues.apache.org/jira/browse/LOG4J2-2838)
+with Log4j 2.13.2. If log messages show a `NullPointerException` when adding this instrumentation,
+please update to 2.13.3 or higher. The only change between 2.13.2 and 2.13.3 is the fix to this
+issue.
+
+Log4j will automatically pick up the integration when you include this module. The following keys
+will be added to the context when a log statement is made when a span is active:
 
 - `trace_id`
 - `span_id`
 - `trace_flags`
 
-You can use these keys when defining an appender in your `log4j.xml` configuration, for example
+You can use these keys when defining an appender in your `log4j.xml` configuration, for example:
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
@@ -58,3 +63,43 @@ You can use these keys when defining an appender in your `log4j.xml` configurati
  </Loggers>
 </Configuration>
 ```
+
+## OpenTelemetry Appender
+
+`OpenTelemetryAppender` is a Log4j2 [appender](https://logging.apache.org/log4j/2.x/manual/appenders.html) that can be used to forward log events to the [OpenTelemetry Log SDK](https:/open-telemetry/opentelemetry-java/tree/main/sdk/logs).
+
+The following demonstrates how you might configure the appender in your `log4j.xml` configuration:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN" packages="io.opentelemetry.instrumentation.log4j.v2_13_2">
+ <Appenders>
+ <Console name="Console" target="SYSTEM_OUT">
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} traceId: %X{trace_id} spanId: %X{span_id} flags: %X{trace_flags} - %msg%n" />
+ </Console>
+ <OpenTelemetry name="OpenTelemetryAppender" />
+ </Appenders>
+ <Loggers>
+ <Root>
+ <AppenderRef ref="OpenTelemetryAppender" level="All" />
+ <AppenderRef ref="Console" level="All" />
+ </Root>
+ </Loggers>
+</Configuration>
+```
+
+Next, associate the `OpenTelemetryAppender` with a `SdkLogEmitterProvider` in your application:
+
+```
+SdkLogEmitterProvider logEmitterProvider =
+ SdkLogEmitterProvider.builder()
+ .setResource(Resource.create(...))
+ .addLogProcessor(...)
+ .build();
+OpenTelemetryLog4j.initialize(logEmitterProvider);
+```
+
+In this example Log4j2 logs will be sent to both the console appender and
+the `OpenTelemetryAppender`, which will drop the logs until `OpenTelemetryLog4j.initialize(..)` is
+called. Once initialized, logs will be emitted to a `LogEmitter` obtained from
+the `SdkLogEmitterProvider`.

instrumentation/log4j/log4j-2.13.2/library/build.gradle.kts

@@ -3,11 +3,15 @@ plugins {
 }
 
 dependencies {
+ api("io.opentelemetry:opentelemetry-sdk-logs:1.9.0-alpha")
+
  library("org.apache.logging.log4j:log4j-core:2.13.2")
 
  // Library instrumentation cannot be applied to 2.13.2 due to a bug in Log4J. The agent works
  // around it.
  testLibrary("org.apache.logging.log4j:log4j-core:2.13.3")
 
  testImplementation(project(":instrumentation:log4j:log4j-2-common:testing"))
+
+ testImplementation("org.mockito:mockito-core")
 }

instrumentation/log4j/log4j-2.13.2/library/src/main/java/io/opentelemetry/instrumentation/log4j/v2_13_2/LogEventMapper.java

@@ -0,0 +1,139 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.log4j.v2_13_2;
+
+import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.SPAN_ID;
+import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_FLAGS;
+import static io.opentelemetry.instrumentation.api.log.LoggingContextConstants.TRACE_ID;
+
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributesBuilder;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.sdk.logs.LogBuilder;
+import io.opentelemetry.sdk.logs.data.Severity;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.time.Instant;
+import org.apache.logging.log4j.message.Message;
+import org.apache.logging.log4j.util.ReadOnlyStringMap;
+
+final class LogEventMapper {
+
+ // Visible for testing
+ static final AttributeKey<String> ATTR_LOGGER_NAME = AttributeKey.stringKey("logger.name");
+ static final AttributeKey<String> ATTR_THREAD_NAME = AttributeKey.stringKey("thread.name");
+ static final AttributeKey<Long> ATTR_THREAD_ID = AttributeKey.longKey("thread.id");
+ static final AttributeKey<Long> ATTR_THREAD_PRIORITY = AttributeKey.longKey("thread.priority");
+ static final AttributeKey<String> ATTR_THROWABLE_NAME = AttributeKey.stringKey("throwable.name");
+ static final AttributeKey<String> ATTR_THROWABLE_MESSAGE =
+ AttributeKey.stringKey("throwable.message");
+ static final AttributeKey<List<String>> ATTR_NDC = AttributeKey.stringArrayKey("ndc");
+ static final AttributeKey<String> ATTR_FQCN = AttributeKey.stringKey("fqcn");
+ static final AttributeKey<String> ATTR_MARKER = AttributeKey.stringKey("marker");
+
+ static void mapLogEvent(LogBuilder builder, LogEvent logEvent) {
+ AttributesBuilder attributes = Attributes.builder();
+
+ // message
+ Message message = logEvent.getMessage();
+ if (message != null) {
+ builder.setBody(message.getFormattedMessage());
+ }
+
+ // time
+ Instant instant = logEvent.getInstant();
+ if (instant != null) {
+ builder.setEpoch(
+ TimeUnit.MILLISECONDS.toNanos(instant.getEpochMillisecond())
+ + instant.getNanoOfMillisecond(),
+ TimeUnit.NANOSECONDS);
+ }
+
+ // level
+ Level level = logEvent.getLevel();
+ if (level != null) {
+ builder.setSeverity(levelToSeverity(level));
+ builder.setSeverityText(logEvent.getLevel().name());
+ }
+
+ // logger
+ attributes.put(ATTR_LOGGER_NAME, logEvent.getLoggerName());
+
+ // fully qualified class name
+ attributes.put(ATTR_FQCN, logEvent.getLoggerFqcn());
+
+ // thread
+ attributes.put(ATTR_THREAD_NAME, logEvent.getThreadName());
+ attributes.put(ATTR_THREAD_ID, logEvent.getThreadId());
+ attributes.put(ATTR_THREAD_PRIORITY, logEvent.getThreadPriority());
+
+ // throwable
+ Throwable throwable = logEvent.getThrown();
+ if (throwable != null) {
+ attributes.put(ATTR_THROWABLE_NAME, throwable.getClass().getName());
+ attributes.put(ATTR_THROWABLE_MESSAGE, throwable.getMessage());
+ }
+
+ // marker
+ Marker marker = logEvent.getMarker();
+ if (marker != null) {
+ attributes.put(ATTR_MARKER, marker.getName());
+ }
+
+ // nested diagnostic context
+ List<String> contextStackList = logEvent.getContextStack().asList();
+ if (contextStackList.size() > 0) {
+ attributes.put(ATTR_NDC, contextStackList);
+ }
+
+ // mapped diagnostic context (included span context)
+ ReadOnlyStringMap contextData = logEvent.getContextData();
+ if (contextData != null) {
+ Map<String, String> contextMap = contextData.toMap();
+ if (contextMap != null) {
+ // Remove context fields placed by OpenTelemetryContextDataProvider to avoid duplicating
+ // trace context fields in attributes
+ contextMap.remove(TRACE_ID);
+ contextMap.remove(SPAN_ID);
+ contextMap.remove(TRACE_FLAGS);
+ builder.setContext(Context.current());
+
+ contextMap.forEach(attributes::put);
+ }
+ }
+
+ builder.setAttributes(attributes.build());
+ }
+
+ private static Severity levelToSeverity(Level level) {
+ switch (level.getStandardLevel()) {
+ case ALL:
+ return Severity.TRACE;
+ case TRACE:
+ return Severity.TRACE2;
+ case DEBUG:
+ return Severity.DEBUG;
+ case INFO:
+ return Severity.INFO;
+ case WARN:
+ return Severity.WARN;
+ case ERROR:
+ return Severity.ERROR;
+ case FATAL:
+ return Severity.FATAL;
+ case OFF:
+ return Severity.UNDEFINED_SEVERITY_NUMBER;
+ }
+ throw new IllegalStateException("Unrecognized level " + level.name());
+ }
+
+ private LogEventMapper() {}
+}

instrumentation/log4j/log4j-2.13.2/library/src/main/java/io/opentelemetry/instrumentation/log4j/v2_13_2/OpenTelemetryAppender.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.instrumentation.log4j.v2_13_2;
+
+import io.opentelemetry.sdk.logs.LogBuilder;
+import io.opentelemetry.sdk.logs.LogEmitter;
+import java.io.Serializable;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.Core;
+import org.apache.logging.log4j.core.Filter;
+import org.apache.logging.log4j.core.Layout;
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.config.Property;
+import org.apache.logging.log4j.core.config.plugins.Plugin;
+import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory;
+
+@Plugin(
+ name = OpenTelemetryAppender.PLUGIN_NAME,
+ category = Core.CATEGORY_NAME,
+ elementType = Appender.ELEMENT_TYPE)
+public class OpenTelemetryAppender extends AbstractAppender {
+
+ static final String PLUGIN_NAME = "OpenTelemetry";
+
+ @PluginBuilderFactory
+ public static <B extends Builder<B>> B newBuilder() {
+ return new Builder<B>().asBuilder();
+ }
+
+ static class Builder<B extends Builder<B>> extends AbstractAppender.Builder<B>
+ implements org.apache.logging.log4j.core.util.Builder<OpenTelemetryAppender> {
+
+ @Override
+ public OpenTelemetryAppender build() {
+ OpenTelemetryAppender appender =
+ new OpenTelemetryAppender(
+ getName(), getLayout(), getFilter(), isIgnoreExceptions(), getPropertyArray());
+ OpenTelemetryLog4j.registerInstance(appender);
+ return appender;
+ }
+ }
+
+ private final AtomicReference<LogEmitter> logEmitterRef = new AtomicReference<>();
+
+ private OpenTelemetryAppender(
+ String name,
+ Layout<? extends Serializable> layout,
+ Filter filter,
+ boolean ignoreExceptions,
+ Property[] properties) {
+ super(name, filter, layout, ignoreExceptions, properties);
+ }
+
+ @Override
+ public void append(LogEvent event) {
+ LogEmitter logEmitter = logEmitterRef.get();
+ if (logEmitter == null) {
+ // appender hasn't been initialized
+ return;
+ }
+ LogBuilder builder = logEmitter.logBuilder();
+ LogEventMapper.mapLogEvent(builder, event);
+ builder.emit();
+ }
+
+ void initialize(LogEmitter logEmitter) {
+ if (!logEmitterRef.compareAndSet(null, logEmitter)) {
+ throw new IllegalStateException("OpenTelemetryAppender has already been initialized.");
+ }
+ }
+}