Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[backport] fix: disable nanosecond timestamp when only EnableTimestampNanosecond is enabled (#1528) #1552

Merged
merged 2 commits into from
Jun 20, 2024
Merged

[backport] fix: disable nanosecond timestamp when only EnableTimestampNanosecond is enabled (#1528) #1552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
18be2ae
fix: disable nanosecond timestamp when only EnableTimestampNanosecond…
henryzhx8 Jun 7, 2024
41335f3
polish
henryzhx8 Jun 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions core/common/TimeUtil.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,10 +302,11 @@ void SetLogTime(sls_logs::Log* log, time_t second) {
log->set_time(second);
}

void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond) {
// Usage: set nanosecond first, and then discard at LogProcess@ProcessBufferLegacy
void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond) {
log->set_time(second);
log->set_time_ns(nanosecond);
if (nanosecond) {
log->set_time_ns(nanosecond.value());
}
}

LogtailTime GetCurrentLogtailTime() {
Expand Down
3 changes: 2 additions & 1 deletion core/common/TimeUtil.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#pragma once
#include <ctime>
#include <optional>
#include <string>
#include <thread>

Expand Down Expand Up @@ -83,7 +84,7 @@ uint64_t GetPreciseTimestampFromLogtailTime(LogtailTime logTime, const PreciseTi

void SetLogTime(sls_logs::Log* log, time_t second);

void SetLogTimeWithNano(sls_logs::Log* log, time_t second, long nanosecond);
void SetLogTimeWithNano(sls_logs::Log* log, time_t second, std::optional<uint32_t> nanosecond);

LogtailTime GetCurrentLogtailTime();

Expand Down
4 changes: 3 additions & 1 deletion core/models/LogEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,9 @@ Json::Value LogEvent::ToJson() const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
}
if (!Empty()) {
Json::Value contents;
for (const auto& content : *this) {
Expand Down
4 changes: 3 additions & 1 deletion core/models/MetricEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ Json::Value MetricEvent::ToJson() const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
}
return root;
}

Expand Down
21 changes: 13 additions & 8 deletions core/models/PipelineEvent.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include <ctime>
#include <map>
#include <memory>
#include <optional>
#include <string>

#include "models/StringView.h"
Expand All @@ -38,12 +39,16 @@ class PipelineEvent {
virtual ~PipelineEvent() = default;

Type GetType() const { return mType; }
time_t GetTimestamp() const { return timestamp; }
long GetTimestampNanosecond() const { return timestampNanosecond; }
void SetTimestamp(time_t t) { timestamp = t; }
void SetTimestamp(time_t t, long ns) {
timestamp = t;
timestampNanosecond = ns; // Only nanosecond part
time_t GetTimestamp() const { return mTimestamp; }
std::optional<uint32_t> GetTimestampNanosecond() const { return mTimestampNanosecond; }
void SetTimestamp(time_t t) { mTimestamp = t; }
void SetTimestamp(time_t t, uint32_t ns) {
mTimestamp = t;
mTimestampNanosecond = ns; // Only nanosecond part
}
void SetTimestamp(time_t t, std::optional<uint32_t> ns) {
mTimestamp = t;
mTimestampNanosecond = ns; // Only nanosecond part
}
void ResetPipelineEventGroup(PipelineEventGroup* ptr) { mPipelineEventGroupPtr = ptr; }
std::shared_ptr<SourceBuffer>& GetSourceBuffer();
Expand All @@ -61,8 +66,8 @@ class PipelineEvent {
PipelineEvent(Type type, PipelineEventGroup* ptr);

Type mType = Type::NONE;
time_t timestamp = 0;
long timestampNanosecond = 0;
time_t mTimestamp = 0;
std::optional<uint32_t> mTimestampNanosecond;
PipelineEventGroup* mPipelineEventGroupPtr = nullptr;
};

Expand Down
4 changes: 3 additions & 1 deletion core/models/SpanEvent.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ Json::Value SpanEvent::ToJson() const {
Json::Value root;
root["type"] = static_cast<int>(GetType());
root["timestamp"] = GetTimestamp();
root["timestampNanosecond"] = GetTimestampNanosecond();
if (GetTimestampNanosecond()) {
root["timestampNanosecond"] = static_cast<int32_t>(GetTimestampNanosecond().value());
}
return root;
}

Expand Down
2 changes: 1 addition & 1 deletion core/spl/PipelineEventGroupInput.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ void PipelineEventGroupInput::getTimeColumns(std::vector<uint32_t>& times,
for (const auto &event : mLogGroup->GetEvents()) {
const LogEvent& sourceEvent = event.Cast<LogEvent>();
times.emplace_back(sourceEvent.GetTimestamp());
timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond());
timeNanos.emplace_back(sourceEvent.GetTimestampNanosecond() ? sourceEvent.GetTimestampNanosecond().value() : 0);
}
}

Expand Down
10 changes: 0 additions & 10 deletions core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -492,15 +492,13 @@ void ProcessorParseApsaraNativeUnittest::TestMultipleLines() {
"__raw__": "[2023-09-04 13:15"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
"contents": {
"__raw__": ":50.1]\t[ERROR]\t[1]\t/ilogtail/AppConfigBase.cpp:1\t\tAppConfigBase AppConfigBase:1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
},
{
Expand Down Expand Up @@ -864,7 +862,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessKeyOverwritten() {
"rawLog": "value1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down Expand Up @@ -940,7 +937,6 @@ void ProcessorParseApsaraNativeUnittest::TestUploadRawLog() {
"rawLog": "value1"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down Expand Up @@ -1051,7 +1047,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1060,7 +1055,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1069,7 +1063,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1078,7 +1071,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
},
{
Expand All @@ -1087,7 +1079,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventKeepUnmatch() {
"rawLog" : "value1"
},
"timestamp" : 12345678901,
"timestampNanosecond": 0,
"type" : 1
}
]
Expand Down Expand Up @@ -1293,7 +1284,6 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() {
"rawLog": "[2023-09-04 13:18:04"
},
"timestamp": 12345678901,
"timestampNanosecond": 0,
"type": 1
}
]
Expand Down
Loading
Loading