-
Notifications
You must be signed in to change notification settings - Fork 828
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Implementing Json log exporter #3744
Merged
Merged
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
7f4a4d4
update (#1)
ohadza 10add8b
Merge branch 'open-telemetry:main' into main
ohadza 5e19f35
Merge branch 'open-telemetry:main' into main
ohadza 9afb92c
Merge branch 'open-telemetry:main' into main
ohadza f1450c0
implement json log exporter
ohadza File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
56 changes: 56 additions & 0 deletions
56
...otlp/src/main/java/io/opentelemetry/exporter/logging/otlp/OtlpJsonLoggingLogExporter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.exporter.logging.otlp; | ||
|
||
import static io.opentelemetry.exporter.logging.otlp.JsonUtil.JSON_FACTORY; | ||
|
||
import com.fasterxml.jackson.core.JsonGenerator; | ||
import com.fasterxml.jackson.core.io.SegmentedStringWriter; | ||
import io.opentelemetry.exporter.otlp.internal.logs.ResourceLogsMarshaler; | ||
import io.opentelemetry.sdk.common.CompletableResultCode; | ||
import io.opentelemetry.sdk.logs.data.LogData; | ||
import io.opentelemetry.sdk.logs.export.LogExporter; | ||
import java.io.IOException; | ||
import java.util.Collection; | ||
import java.util.logging.Level; | ||
import java.util.logging.Logger; | ||
|
||
/** | ||
* A {@link LogExporter} which writes {@linkplain LogData logs} to a {@link Logger} in OTLP JSON | ||
* format. Each log line will include a single {@code ResourceLogs}. | ||
*/ | ||
public final class OtlpJsonLoggingLogExporter implements LogExporter { | ||
|
||
private static final Logger logger = Logger.getLogger(OtlpJsonLoggingLogExporter.class.getName()); | ||
|
||
/** Returns a new {@link OtlpJsonLoggingLogExporter}. */ | ||
public static LogExporter create() { | ||
return new OtlpJsonLoggingLogExporter(); | ||
} | ||
|
||
private OtlpJsonLoggingLogExporter() {} | ||
|
||
@Override | ||
public CompletableResultCode export(Collection<LogData> logs) { | ||
ResourceLogsMarshaler[] allResourceLogs = ResourceLogsMarshaler.create(logs); | ||
for (ResourceLogsMarshaler resourceLogs : allResourceLogs) { | ||
SegmentedStringWriter sw = new SegmentedStringWriter(JSON_FACTORY._getBufferRecycler()); | ||
try (JsonGenerator gen = JsonUtil.create(sw)) { | ||
resourceLogs.writeJsonTo(gen); | ||
} catch (IOException e) { | ||
// Shouldn't happen in practice, just skip it. | ||
continue; | ||
} | ||
logger.log(Level.INFO, sw.getAndClear()); | ||
} | ||
return CompletableResultCode.ofSuccess(); | ||
} | ||
|
||
@Override | ||
public CompletableResultCode shutdown() { | ||
return CompletableResultCode.ofSuccess(); | ||
} | ||
} |
160 changes: 160 additions & 0 deletions
160
.../src/test/java/io/opentelemetry/exporter/logging/otlp/OtlpJsonLoggingLogExporterTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,160 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.exporter.logging.otlp; | ||
|
||
import static io.opentelemetry.api.common.AttributeKey.booleanKey; | ||
import static io.opentelemetry.api.common.AttributeKey.longKey; | ||
import static io.opentelemetry.api.common.AttributeKey.stringKey; | ||
import static org.assertj.core.api.Assertions.assertThat; | ||
|
||
import io.github.netmikey.logunit.api.LogCapturer; | ||
import io.opentelemetry.api.common.Attributes; | ||
import io.opentelemetry.sdk.common.InstrumentationLibraryInfo; | ||
import io.opentelemetry.sdk.logs.data.LogData; | ||
import io.opentelemetry.sdk.logs.data.LogRecord; | ||
import io.opentelemetry.sdk.logs.data.Severity; | ||
import io.opentelemetry.sdk.logs.export.LogExporter; | ||
import io.opentelemetry.sdk.resources.Resource; | ||
import java.util.Arrays; | ||
import org.junit.jupiter.api.BeforeEach; | ||
import org.junit.jupiter.api.Test; | ||
import org.junit.jupiter.api.extension.RegisterExtension; | ||
import org.skyscreamer.jsonassert.JSONAssert; | ||
import org.slf4j.event.Level; | ||
|
||
class OtlpJsonLoggingLogExporterTest { | ||
|
||
private static final Resource RESOURCE = | ||
Resource.create(Attributes.builder().put("key", "value").build()); | ||
|
||
private static final LogData LOG1 = | ||
LogRecord.builder(RESOURCE, InstrumentationLibraryInfo.create("instrumentation", "1")) | ||
.setName("testLog1") | ||
.setBody("body1") | ||
.setFlags(0) | ||
.setSeverity(Severity.INFO) | ||
.setSeverityText("INFO") | ||
.setSpanId("8765432112345876") | ||
.setTraceId("12345678876543211234567887654322") | ||
.setEpochMillis(1631533710L) | ||
.setAttributes(Attributes.of(stringKey("animal"), "cat", longKey("lives"), 9L)) | ||
.build(); | ||
|
||
private static final LogData LOG2 = | ||
LogRecord.builder(RESOURCE, InstrumentationLibraryInfo.create("instrumentation2", "2")) | ||
.setName("testLog2") | ||
.setBody("body2") | ||
.setFlags(0) | ||
.setSeverity(Severity.INFO) | ||
.setSeverityText("INFO") | ||
.setSpanId("8765432112345875") | ||
.setTraceId("12345678876543211234567887654322") | ||
.setEpochMillis(1631533710L) | ||
.setAttributes(Attributes.of(booleanKey("important"), true)) | ||
.build(); | ||
|
||
@RegisterExtension | ||
LogCapturer logs = LogCapturer.create().captureForType(OtlpJsonLoggingLogExporter.class); | ||
|
||
LogExporter exporter; | ||
|
||
@BeforeEach | ||
void setUp() { | ||
exporter = OtlpJsonLoggingLogExporter.create(); | ||
} | ||
|
||
@Test | ||
void log() throws Exception { | ||
exporter.export(Arrays.asList(LOG1, LOG2)); | ||
|
||
assertThat(logs.getEvents()) | ||
.hasSize(1) | ||
.allSatisfy(log -> assertThat(log.getLevel()).isEqualTo(Level.INFO)); | ||
JSONAssert.assertEquals( | ||
"{\n" | ||
+ " \"resource\":{\n" | ||
+ " \"attributes\":[\n" | ||
+ " {\n" | ||
+ " \"key\":\"key\",\n" | ||
+ " \"value\":{\n" | ||
+ " \"stringValue\":\"value\"\n" | ||
+ " }\n" | ||
+ " }\n" | ||
+ " ]\n" | ||
+ " },\n" | ||
+ " \"instrumentationLibraryLogs\":[\n" | ||
+ " {\n" | ||
+ " \"instrumentationLibrary\":{\n" | ||
+ " \"name\":\"instrumentation2\",\n" | ||
+ " \"version\":\"2\"\n" | ||
+ " },\n" | ||
+ " \"logs\":[\n" | ||
+ " {\n" | ||
+ " \"timeUnixNano\":\"1631533710000000\",\n" | ||
+ " \"severityNumber\":\"SEVERITY_NUMBER_INFO\",\n" | ||
+ " \"severityText\":\"INFO\",\n" | ||
+ " \"name\":\"testLog2\",\n" | ||
+ " \"body\":{\n" | ||
+ " \"stringValue\":\"body2\"\n" | ||
+ " },\n" | ||
+ " \"attributes\":[\n" | ||
+ " {\n" | ||
+ " \"key\":\"important\",\n" | ||
+ " \"value\":{\n" | ||
+ " \"boolValue\":true\n" | ||
+ " }\n" | ||
+ " }\n" | ||
+ " ],\n" | ||
+ " \"traceId\":\"12345678876543211234567887654322\",\n" | ||
+ " \"spanId\":\"8765432112345875\"\n" | ||
+ " }\n" | ||
+ " ]\n" | ||
+ " },\n" | ||
+ " {\n" | ||
+ " \"instrumentationLibrary\":{\n" | ||
+ " \"name\":\"instrumentation\",\n" | ||
+ " \"version\":\"1\"\n" | ||
+ " },\n" | ||
+ " \"logs\":[\n" | ||
+ " {\n" | ||
+ " \"timeUnixNano\":\"1631533710000000\",\n" | ||
+ " \"severityNumber\":\"SEVERITY_NUMBER_INFO\",\n" | ||
+ " \"severityText\":\"INFO\",\n" | ||
+ " \"name\":\"testLog1\",\n" | ||
+ " \"body\":{\n" | ||
+ " \"stringValue\":\"body1\"\n" | ||
+ " },\n" | ||
+ " \"attributes\":[\n" | ||
+ " {\n" | ||
+ " \"key\":\"animal\",\n" | ||
+ " \"value\":{\n" | ||
+ " \"stringValue\":\"cat\"\n" | ||
+ " }\n" | ||
+ " },\n" | ||
+ " {\n" | ||
+ " \"key\":\"lives\",\n" | ||
+ " \"value\":{\n" | ||
+ " \"intValue\":\"9\"\n" | ||
+ " }\n" | ||
+ " }\n" | ||
+ " ],\n" | ||
+ " \"traceId\":\"12345678876543211234567887654322\",\n" | ||
+ " \"spanId\":\"8765432112345876\"\n" | ||
+ " }\n" | ||
+ " ]\n" | ||
+ " }\n" | ||
+ " ]\n" | ||
+ "}", | ||
logs.getEvents().get(0).getMessage(), | ||
/* strict= */ false); | ||
assertThat(logs.getEvents().get(0).getMessage()).doesNotContain("\n"); | ||
} | ||
|
||
@Test | ||
void shutdown() { | ||
assertThat(exporter.shutdown().isSuccess()).isTrue(); | ||
} | ||
} |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This will end up putting a whole lot of log lines on a single log output. I don't think that's what I'd expect to happen. I think this should only output one log per line, rather than cramming the whole json body into one line.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I remember reading a PR from a while back that part of the intent of these loggers is that another process could parse each log line as a
Resource{type}
message. If that is indeed part of the intended use case of the log exporters, we shouldn't break convention.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The idea in general of logging exported
LogRecord
s is cyclical when thinking about it from the perspective of application logs: an application logs a message -> an appender feeds to the otel log sink -> the log exporter logs the message in json format -> an appender feeds to the otel log sinkI think its only really practical to use one of these log exporters if you're collecting events as
LogRecord
s rather than application logs.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Hmm. I see the point. We should have the equivalent of the LoggingSpanExporter that will just mirror the logs back to JUL, one per line, which can be very useful for debugging. But, of course, problematic if used with a JUL appender! Perhaps that one (which I know is not this one) should just use stdout.