diff --git a/core/processor/ProcessorParseApsaraNative.cpp b/core/processor/ProcessorParseApsaraNative.cpp index fd85de7d57..2e7d06383b 100644 --- a/core/processor/ProcessorParseApsaraNative.cpp +++ b/core/processor/ProcessorParseApsaraNative.cpp @@ -60,10 +60,10 @@ void ProcessorParseApsaraNative::Process(PipelineEventGroup& logGroup) { const StringView& logPath = logGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH_RESOLVED); EventsContainer& events = logGroup.MutableEvents(); StringView timeStrCache; - LogtailTime lastLogTime; + LogtailTime cachedLogTime; // works good normally. poor performance if most data need to be discarded. for (auto it = events.begin(); it != events.end();) { - if (ProcessEvent(logPath, *it, lastLogTime, timeStrCache)) { + if (ProcessEvent(logPath, *it, cachedLogTime, timeStrCache)) { ++it; } else { it = events.erase(it); @@ -76,13 +76,13 @@ void ProcessorParseApsaraNative::Process(PipelineEventGroup& logGroup) { * 处理单个日志事件。 * @param logPath - 日志文件的路径。 * @param e - 指向待处理日志事件的智能指针。 - * @param lastLogTime - 上一条日志的时间戳（秒）。 + * @param cachedLogTime - 上一条日志的时间戳（秒）。 * @param timeStrCache - 缓存时间字符串，用于比较和更新。 * @return 如果事件被处理且保留，则返回true，如果事件被丢弃，则返回false。 */ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, PipelineEventPtr& e, - LogtailTime& lastLogTime, + LogtailTime& cachedLogTime, StringView& timeStrCache) { if (!IsSupportedEvent(e)) { return true; @@ -97,7 +97,7 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, } mProcParseInSizeBytes->Add(buffer.size()); int64_t logTime_in_micro = 0; - time_t logTime = ApsaraEasyReadLogTimeParser(buffer, timeStrCache, lastLogTime, logTime_in_micro); + time_t logTime = ApsaraEasyReadLogTimeParser(buffer, timeStrCache, cachedLogTime, logTime_in_micro); if (logTime <= 0) // this case will handle empty apsara log line { StringView bufOut(buffer); @@ -210,18 +210,19 @@ bool ProcessorParseApsaraNative::ProcessEvent(const StringView& logPath, /* * 解析Apsara格式日志的时间。 * @param buffer - 包含日志数据的字符串视图。 - * @param timeStr - 解析后的时间字符串。 - * @param lastLogTime - 上一条日志的时间戳（秒）。 + * @param cachedTimeStr - 缓存的时间字符串。 + * @param cachedLogTime - 缓存的时间字符串的时间戳（秒），必须与cachedTimeStr同时修改。 * @param microTime - 解析出的微秒时间戳。 * @return 解析出的时间戳（秒），如果解析失败，则返回0。 */ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffer, - StringView& timeStr, - LogtailTime& lastLogTime, + StringView& cachedTimeStr, + LogtailTime& cachedLogTime, int64_t& microTime) { if (buffer[0] != '[') { return 0; } + LogtailTime logTime = {}; if (buffer[1] == '1') // for normal time, e.g 1378882630, starts with '1' { int nanosecondLength = 0; @@ -232,13 +233,13 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe } // strTime is the content between '[' and ']' and ends with '\0' std::string strTime = buffer.substr(1, pos).to_string(); - auto strptimeResult = Strptime(strTime.c_str(), "%s", &lastLogTime, nanosecondLength); + auto strptimeResult = Strptime(strTime.c_str(), "%s", &logTime, nanosecondLength); if (NULL == strptimeResult || strptimeResult[0] != ']') { LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)("timeformat", "%s")); return 0; } - microTime = (int64_t)lastLogTime.tv_sec * 1000000 + lastLogTime.tv_nsec / 1000; - return lastLogTime.tv_sec; + microTime = (int64_t)logTime.tv_sec * 1000000 + logTime.tv_nsec / 1000; + return logTime.tv_sec; } // test other date format case { @@ -249,15 +250,22 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe } // strTime is the content between '[' and ']' and ends with '\0' std::string strTime = buffer.substr(1, pos).to_string(); - if (IsPrefixString(strTime.c_str(), timeStr) == true) { - microTime = (int64_t)lastLogTime.tv_sec * 1000000 + lastLogTime.tv_nsec / 1000; - return lastLogTime.tv_sec; - } - struct tm tm; - memset(&tm, 0, sizeof(tm)); int nanosecondLength = 0; + if (IsPrefixString(strTime, cachedTimeStr) == true) { + if (strTime.size() > cachedTimeStr.size()) { + auto strptimeResult + = Strptime(strTime.c_str() + cachedTimeStr.size() + 1, "%f", &logTime, nanosecondLength); + if (NULL == strptimeResult) { + LOG_WARNING(sLogger, + ("parse apsara log time microsecond", + "fail")("string", buffer)("timeformat", "%Y-%m-%d %H:%M:%S.%f")); + } + } + microTime = (int64_t)cachedLogTime.tv_sec * 1000000 + logTime.tv_nsec / 1000; + return cachedLogTime.tv_sec; + } // parse second part - auto strptimeResult = Strptime(strTime.c_str(), "%Y-%m-%d %H:%M:%S", &lastLogTime, nanosecondLength); + auto strptimeResult = Strptime(strTime.c_str(), "%Y-%m-%d %H:%M:%S", &logTime, nanosecondLength); if (NULL == strptimeResult) { LOG_WARNING(sLogger, ("parse apsara log time", "fail")("string", buffer)("timeformat", "%Y-%m-%d %H:%M:%S")); @@ -265,21 +273,23 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe } // parse nanosecond part (optional) if (*strptimeResult != '\0') { - strptimeResult = Strptime(strptimeResult + 1, "%f", &lastLogTime, nanosecondLength); + strptimeResult = Strptime(strptimeResult + 1, "%f", &logTime, nanosecondLength); if (NULL == strptimeResult) { LOG_WARNING(sLogger, - ("parse apsara log time", "fail")("string", buffer)("timeformat", "%Y-%m-%d %H:%M:%S.%f")); + ("parse apsara log time microsecond", "fail")("string", buffer)("timeformat", + "%Y-%m-%d %H:%M:%S.%f")); } } + logTime.tv_sec = logTime.tv_sec - mLogTimeZoneOffsetSecond; + microTime = (int64_t)logTime.tv_sec * 1000000 + logTime.tv_nsec / 1000; // if the time is valid (strptime not return NULL), the date value size must be 19 ,like '2013-09-11 03:11:05' - timeStr = StringView(buffer.data() + 1, 19); - lastLogTime.tv_sec = lastLogTime.tv_sec - mLogTimeZoneOffsetSecond; - microTime = (int64_t)lastLogTime.tv_sec * 1000000 + lastLogTime.tv_nsec / 1000; + cachedTimeStr = StringView(buffer.data() + 1, 19); + cachedLogTime = logTime; // TODO: deprecated if (!mAdjustApsaraMicroTimezone) { microTime = (int64_t)microTime + (int64_t)mLogTimeZoneOffsetSecond * (int64_t)1000000; } - return lastLogTime.tv_sec; + return logTime.tv_sec; } } @@ -289,16 +299,8 @@ time_t ProcessorParseApsaraNative::ApsaraEasyReadLogTimeParser(StringView& buffe * @param prefix - 要检查的前缀。 * @return 如果字符串以指定前缀开头，则返回true；否则返回false。 */ -bool ProcessorParseApsaraNative::IsPrefixString(const char* all, const StringView& prefix) { - if (prefix.size() == 0) - return false; - for (size_t i = 0; i < prefix.size(); ++i) { - if (all[i] == '\0') - return false; - if (all[i] != prefix[i]) - return false; - } - return true; +bool ProcessorParseApsaraNative::IsPrefixString(const std::string& all, const StringView& prefix) { + return !prefix.empty() && std::equal(prefix.begin(), prefix.end(), all.begin()); } /* @@ -314,14 +316,14 @@ static int32_t FindBaseFields(const StringView& buffer, int32_t beginIndexArray[ if (buffer[i] == '[') { beginIndexArray[baseFieldNum] = i + 1; } else if (buffer[i] == ']') { - if (buffer[i + 1] == '\t' || buffer[i + 1] == '\n') { + if (i + 1 == buffer.size() || buffer[i + 1] == '\t' || buffer[i + 1] == '\n') { endIndexArray[baseFieldNum] = i; baseFieldNum++; } if (baseFieldNum >= LogParser::MAX_BASE_FIELD_NUM) { break; } - if (buffer[i + 1] == '\t' && buffer[i + 2] != '[') { + if (buffer[i + 1] == '\t' && (i + 2 == buffer.size() || buffer[i + 2] != '[')) { break; } } diff --git a/core/processor/ProcessorParseApsaraNative.h b/core/processor/ProcessorParseApsaraNative.h index 43ee3c951f..f3bac213c3 100644 --- a/core/processor/ProcessorParseApsaraNative.h +++ b/core/processor/ProcessorParseApsaraNative.h @@ -34,9 +34,9 @@ class ProcessorParseApsaraNative : public Processor { private: bool ProcessEvent(const StringView& logPath, PipelineEventPtr& e, LogtailTime& lastLogTime, StringView& timeStrCache); void AddLog(const StringView& key, const StringView& value, LogEvent& targetEvent); - time_t ApsaraEasyReadLogTimeParser(StringView& buffer, StringView& timeStr, LogtailTime& lastLogTime, int64_t& microTime); - int32_t GetApsaraLogMicroTime(StringView& buffer); - bool IsPrefixString(const char* all, const StringView& prefix); + time_t + ApsaraEasyReadLogTimeParser(StringView& buffer, StringView& timeStr, LogtailTime& lastLogTime, int64_t& microTime); + bool IsPrefixString(const std::string& all, const StringView& prefix); int32_t ParseApsaraBaseFields(const StringView& buffer, LogEvent& sourceEvent); std::string mSourceKey; diff --git a/core/reader/SourceBuffer.h b/core/reader/SourceBuffer.h index 57926ea4d3..269b61242f 100644 --- a/core/reader/SourceBuffer.h +++ b/core/reader/SourceBuffer.h @@ -82,7 +82,7 @@ class SourceBuffer { virtual ~SourceBuffer() {} StringBuffer AllocateStringBuffer(size_t size) { if (!mAllocator.IsInited()) { - if (!mAllocator.Init(size * 1.2)) { + if (!mAllocator.Init(std::max(4096UL, size_t(size * 1.2)))) { return StringBuffer(nullptr, 0); }; // TODO: better allocate strategy } diff --git a/core/unittest/processor/LogFilterUnittest.cpp b/core/unittest/processor/LogFilterUnittest.cpp index 1b7ce223f4..f18c12f4a0 100644 --- a/core/unittest/processor/LogFilterUnittest.cpp +++ b/core/unittest/processor/LogFilterUnittest.cpp @@ -433,14 +433,15 @@ class LogFilterUnittest : public ::testing::Test { file << config; file.close(); APSARA_TEST_TRUE_DESC(filterPtr->InitFilter("logtail_filter.json"), "Can't parse the filter config"); - APSARA_TEST_EQUAL_DESC(filterPtr->mFilters.size(), 2, "The filter size should be 2"); + APSARA_TEST_EQUAL_DESC(filterPtr->mFilters.size(), 2UL, "The filter size should be 2"); APSARA_TEST_TRUE_DESC(filterPtr->mFilters.find("123_proj_test") != filterPtr->mFilters.end(), "The 123_proj_test should be a key of the filter"); APSARA_TEST_TRUE_DESC(filterPtr->mFilters.find("456_proj_test_1") != filterPtr->mFilters.end(), "The 456_test_1 should be a key of the filter"); LogFilterRule* filterRulePtr = filterPtr->mFilters["123_proj_test"]; - APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterKeys.size(), 2, "The filter keys size should be 2"); - APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterRegs.size(), 2, "The filter regs size should be 2"); + APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterKeys.size(), 2UL, "The filter keys size should be 2"); + APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterRegs.size(), 2UL, "The filter regs size should be 2"); + APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterRegs.size(), 2UL, "The filter regs size should be 2"); APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterKeys[0], "key1", "The filter key should be key1"); APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterKeys[1], "key2", "The filter key should be key2"); APSARA_TEST_EQUAL_DESC(filterRulePtr->FilterRegs[0], regex("value1"), "The filter reg should be value1"); @@ -543,7 +544,7 @@ class LogFilterUnittest : public ::testing::Test { logContentPtr->set_value("value2xxxxx"); indexs = filterPtr->Filter("999_proj", "", logGroup); - APSARA_TEST_EQUAL_DESC(indexs.size(), 2, "the filtered size should be 2"); + APSARA_TEST_EQUAL_DESC(indexs.size(), 2UL, "the filtered size should be 2"); APSARA_TEST_EQUAL_DESC(indexs[0], 0, "the first item should be contained"); APSARA_TEST_EQUAL_DESC(indexs[1], 1, "the second item should be contained"); logGroup.Clear(); @@ -652,7 +653,7 @@ class LogFilterUnittest : public ::testing::Test { if (log.find("key") != string::npos) lines.push_back(log); } - APSARA_TEST_EQUAL(lines.size(), 6); + APSARA_TEST_EQUAL(lines.size(), 6UL); if (lines.size() == 6) { for (int i = 0; i < 2; ++i) { APSARA_TEST_TRUE_DESC(lines[i].find("value1") != string::npos, lines[i]); @@ -1142,7 +1143,7 @@ class LogFilterUnittest : public ::testing::Test { static LogFilter* filter = LogFilter::Instance(); std::vector index = filter->Filter( logGroup, root, LogGroupContext("default_region", "ant-test-project", "test-logstore")); - APSARA_TEST_EQUAL_FATAL(index.size(), 1); + APSARA_TEST_EQUAL_FATAL(index.size(), 1UL); APSARA_TEST_EQUAL(index[0], 1); } diff --git a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp index c32e0db2f8..fbb4f25669 100644 --- a/core/unittest/processor/ProcessorFilterNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorFilterNativeUnittest.cpp @@ -666,7 +666,7 @@ void ProcessorFilterNativeUnittest::TestBaseFilter() { })"; APSARA_TEST_STREQ_FATAL(CompactJson(expectJson).c_str(), CompactJson(outJson).c_str()); - APSARA_TEST_EQUAL_FATAL(2, processor.mProcFilterRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processor.mProcFilterRecordsTotal->GetValue()); } { const char* jsonStr = "{\n" diff --git a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp index 3f2665dc61..62cf10665f 100644 --- a/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseApsaraNativeUnittest.cpp @@ -45,6 +45,8 @@ class ProcessorParseApsaraNativeUnittest : public ::testing::Test { void TestProcessEventDiscardUnmatch(); void TestMultipleLines(); void TestProcessEventMicrosecondUnmatch(); + void TestApsaraEasyReadLogTimeParser(); + void TestApsaraLogLineParser(); PipelineContext mContext; }; @@ -59,6 +61,8 @@ UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventKeepUnmatch); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventDiscardUnmatch); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestMultipleLines); UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestProcessEventMicrosecondUnmatch); +UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestApsaraEasyReadLogTimeParser); +UNIT_TEST_CASE(ProcessorParseApsaraNativeUnittest, TestApsaraLogLineParser); void ProcessorParseApsaraNativeUnittest::TestMultipleLines() { // 第一个contents 测试多行下的解析，第二个contents测试多行下time的解析 @@ -950,6 +954,328 @@ void ProcessorParseApsaraNativeUnittest::TestProcessEventMicrosecondUnmatch() { APSARA_TEST_EQUAL_FATAL(uint64_t(0), processor->mProcDiscardRecordsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(uint64_t(1), processor->mProcParseErrorTotal->GetValue()); } + +struct LogTimeCase { + LogTimeCase(const char* buffer, int64_t secTime, int64_t microTime) + : buffer(buffer), secTime(secTime), microTime(microTime) {} + std::string buffer; + int64_t secTime{}; + int64_t microTime{}; +}; + +void ProcessorParseApsaraNativeUnittest::TestApsaraEasyReadLogTimeParser() { + // make config + Config config; + config.mDiscardUnmatch = false; + config.mUploadRawLog = false; + config.mTimeZoneAdjust = true; + config.mLogTimeZoneOffsetSecond = 8 * 3600; + config.mAdvancedConfig.mAdjustApsaraMicroTimezone = true; + + StringView cacheStr; + LogtailTime cacheTime = {0, 0}; + int64_t secTime{}; + int64_t microTime{}; + LogTimeCase cases[] = { + {"[1378972170425093]\tA:B", 1378972170, 1378972170425093}, + {"[1378972170093]\tA:B", 1378972170, 1378972170093000}, + {"[1378972172]", 1378972172, 1378972172000000}, + {"[2013-09-12 22:18:28.819129]\tA:B", 1378995508, 1378995508819129}, + {"[2013-09-12 22:18:28.819139]\tA:B", 1378995508, 1378995508819139}, + {"[2013-09-12 22:18:29.819139]", 1378995509, 1378995509819139}, + {"[2013-09-12 22:18:29.819]\tA:B", 1378995509, 1378995509819000}, + {"[2013-09-12 22:18:30,666666]\tA:B", 1378995510, 1378995510666666}, + {"[2013-09-12 22:18:30,,]", 1378995510, 1378995510000000}, + {"[2013-09-12 22:18:31]\t", 1378995511, 1378995511000000}, + }; + + // run function + std::string pluginId = "testID"; + ProcessorParseApsaraNative* processor = new ProcessorParseApsaraNative; + ProcessorInstance processorInstance(processor, pluginId); + ComponentConfig componentConfig(pluginId, config); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + for (auto& acase : cases) { + auto buffer = StringView(acase.buffer); + secTime = processor->ApsaraEasyReadLogTimeParser(buffer, cacheStr, cacheTime, microTime); + APSARA_TEST_EQUAL_FATAL(acase.secTime, secTime); + APSARA_TEST_EQUAL_FATAL(acase.microTime, microTime); + } +} + +void ProcessorParseApsaraNativeUnittest::TestApsaraLogLineParser() { + const char* logLine[] = { + "[2013-03-13 18:05:09.493309]\t[WARNING]\t[13000]\t[build/debug64/ilogtail/core/ilogtail.cpp:1753]", // 1 + "[2013-03-13 18:05:09.493309]\t[WARNING]\t[13000]\t[build/debug64/ilogtail/core/ilogtail.cpp:1753]\t", // 2 + "[2013-03-13 18:05:09.493309]\t[WARNING]\t[13000]\t[build/debug64/ilogtail/core/ilogtail.cpp:1754]\tsomestring", // 3 + "[2013-03-13 " + "18:05:09.493309]\t[WARNING]\t[13000]\t[build/debug64/ilogtail/core/ilogtail.cpp:1755]\tRealRecycle#Command:rm " + "-rf /apsara/tubo/.fuxi_tubo_trash/*", // 4 + "[2013-03-13 " + "18:14:57.365716]\t[ERROR]\t[12835]\t[build/debug64/ilogtail/core/" + "ilogtail.cpp:1945]\tParseWhiteListOK:{\n\\\"sys/" + "pangu/ChunkServerRole\\\": \\\"\\\",\n\\\"sys/pangu/PanguMasterRole\\\": \\\"\\\"}", // 5 + "[2013-03-13 18:14:57.365716]\t[12835]\t[ERROR]\t[build/debug64/ilogtail/core/ilogtail.cpp:1945]", // 6 + "[2013-03-13 18:14:57.365716]\t[build/debug64/ilogtail/core/ilogtail.cpp:1945]\t[12835]\t[ERROR]", // 7 + "[2013-03-13 18:14:57.365716]\t[build/debug64/ilogtail/core/ilogtail.cpp:1945]\t[ERROR]", // 8 + "[2013-03-13 18:14:57.365716]\t[build/debug64/ilogtail/core/ilogtail.cpp:1945]\t[12835]\t[ERROR]\t[5432187]", // 9 + "[2013-03-13 " + "18:14:57.365716]\t[build/debug64/ilogtail/core/ilogtail.cpp:1945]\t[12835]\t[ERROR]\t[5432187]\tcount:55", // 10 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]", // 11 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\t", // 12 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\n", // 13 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\tother\tcount:45", // 14 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\tother:\tcount:45", // 15 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\tcount:45", // 16 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\tcount:45\tnum:88\tjob:ss", // 17 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\t[corrupt\tcount:45\tnum:88\tjob:ss", // 18 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\t[corruptcount:45\tnum:88\tjob:ss", // 19 + "[2013-03-13 18:14:57.365716]\t[trace_id:787]\t[ERROR]\t[corrupt]count:45\tnum:88\tjob:ss", // 20 + "[2013-03-13 18:14:57.365716]\t[build/debug64]\t[ERROR]\tcount:45\tnum:88\tjob:ss", // 21 + "[2013-03-13 18:14:57.365716]\t[build/debug64:]\t[ERROR]\tcount:45\tnum:88\tjob:ss", // 22 + "[2013-03-13 18:14:57.365716]\t[build/debug64:]\t[ERROR]\tcount:45\t:88\tjob:ss", // 23 + "[2013-03-13 18:14:57.365716]", // 24 + "[2013-03-13 18:14:57.365716]\t", // 25 + "[2013-03-13 18:14:57.365716]\n", // 26 + "[2013-03-13 18:14:57.365716]\t\t\t", // 27 + "", // 28 + "[2013-03-13 " + "18:05:09.493309]\t[WARNING]\t[13000]\t[13003]\t[ERROR]\t[build/debug64/ilogtail/core/ilogtail.cpp:1753]", // 29 + "[2013-03-13 18:05:09.493309]\t[WARNING]\t[13000]\t[13003]\t[ERROR]\t[tubo.cpp:1753]", // 30 + "[2013-03-13 18:05:09.493309" // 31 + }; + static const char* APSARA_FIELD_LEVEL = "__LEVEL__"; + static const char* APSARA_FIELD_THREAD = "__THREAD__"; + static const char* APSARA_FIELD_FILE = "__FILE__"; + static const char* APSARA_FIELD_LINE = "__LINE__"; + const char* logParseResult[][16] = { + {"microtime", + "1363169109493309", + APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1753", + NULL}, // 1 + {"microtime", + "1363169109493309", + APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1753", + NULL}, // 2 + {"microtime", + "1363169109493309", + APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1754", + NULL}, // 3 + {APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1755", + "RealRecycle#Command", + "rm -rf /apsara/tubo/.fuxi_tubo_trash/*", + NULL}, // 4 + {APSARA_FIELD_LEVEL, + "ERROR", + APSARA_FIELD_THREAD, + "12835", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + "ParseWhiteListOK", + "{\n\"sys/pangu/ChunkServerRole\": \"\",\n\"sys/pangu/PanguMasterRole\": \"\"}", + NULL}, // 5 + {APSARA_FIELD_THREAD, + "12835", + APSARA_FIELD_LEVEL, + "ERROR", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + NULL}, // 6 + {APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + APSARA_FIELD_THREAD, + "12835", + APSARA_FIELD_LEVEL, + "ERROR", + NULL}, // 7 + {APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + APSARA_FIELD_LEVEL, + "ERROR", + NULL}, // 8 + {APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + APSARA_FIELD_THREAD, + "12835", + APSARA_FIELD_LEVEL, + "ERROR", + NULL}, // 9 + {APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1945", + APSARA_FIELD_THREAD, + "12835", + APSARA_FIELD_LEVEL, + "ERROR", + "count", + "55", + NULL}, // 10 + {APSARA_FIELD_LEVEL, "ERROR", NULL}, // 11 + {APSARA_FIELD_LEVEL, "ERROR", NULL}, // 12 + {APSARA_FIELD_LEVEL, "ERROR", NULL}, // 13 + {APSARA_FIELD_LEVEL, "ERROR", "count", "45", NULL}, // 14 + {APSARA_FIELD_LEVEL, "ERROR", "other", "", "count", "45", NULL}, // 15 + {APSARA_FIELD_LEVEL, "ERROR", "count", "45", NULL}, // 16 + {APSARA_FIELD_LEVEL, "ERROR", "count", "45", "num", "88", "job", "ss", NULL}, // 17 + {APSARA_FIELD_LEVEL, "ERROR", "count", "45", "num", "88", "job", "ss", NULL}, // 18 + {APSARA_FIELD_LEVEL, "ERROR", "[corruptcount", "45", "num", "88", "job", "ss", NULL}, // 19 + {APSARA_FIELD_LEVEL, "ERROR", "[corrupt]count", "45", "num", "88", "job", "ss", NULL}, // 20 + {APSARA_FIELD_FILE, + "build/debug64", + APSARA_FIELD_LEVEL, + "ERROR", + "count", + "45", + "num", + "88", + "job", + "ss", + NULL}, // 21 + {APSARA_FIELD_FILE, + "build/debug64", + APSARA_FIELD_LINE, + "", + APSARA_FIELD_LEVEL, + "ERROR", + "count", + "45", + "num", + "88", + "job", + "ss", + NULL}, // 22 + {APSARA_FIELD_FILE, + "build/debug64", + APSARA_FIELD_LINE, + "", + APSARA_FIELD_LEVEL, + "ERROR", + "count", + "45", + "", + "88", + "job", + "ss", + NULL}, // 23 + {"microtime", "1363169697365716", NULL}, // 24 + {"microtime", "1363169697365716", NULL}, // 25 + {"microtime", "1363169697365716", NULL}, // 26 + {"microtime", "1363169697365716", NULL}, // 27 + {"content", "", NULL}, // 28 + {APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "build/debug64/ilogtail/core/ilogtail.cpp", + APSARA_FIELD_LINE, + "1753", + NULL}, // 29 + {APSARA_FIELD_LEVEL, + "WARNING", + APSARA_FIELD_THREAD, + "13000", + APSARA_FIELD_FILE, + "tubo.cpp", + APSARA_FIELD_LINE, + "1753", + NULL}, // 30 + {NULL} // 31 + }; + + // make config + Config config; + config.mDiscardUnmatch = true; + config.mUploadRawLog = false; + config.mTimeZoneAdjust = true; + config.mLogTimeZoneOffsetSecond = 8 * 3600; + config.mAdvancedConfig.mAdjustApsaraMicroTimezone = true; + + std::string pluginId = "testID"; + ProcessorInstance processorInstance(new ProcessorParseApsaraNative, pluginId); + ComponentConfig componentConfig(pluginId, config); + APSARA_TEST_TRUE_FATAL(processorInstance.Init(componentConfig, mContext)); + + for (uint32_t i = 0; i < sizeof(logLine) / sizeof(logLine[0]); i++) { + auto sourceBuffer = std::make_shared(); + PipelineEventGroup eventGroup(sourceBuffer); + std::string inJson = R"({ + "events" : + [ + { + "contents" : + { + "content" : ")" + + std::string(logLine[i]) + + R"(" + }, + "timestamp" : 12345678901, + "type" : 1 + } + ] + })"; + eventGroup.FromJsonString(inJson); + processorInstance.Process(eventGroup); + + Json::Value outJson = eventGroup.ToJson(); + if (logParseResult[i][0] == NULL) { + APSARA_TEST_EQUAL(eventGroup.ToJsonString(), "null"); + continue; + } + for (int j = 0; j < 10 && logParseResult[i][j] != NULL; j++) { + if (j % 2 == 0) { + APSARA_TEST_TRUE(outJson.isMember("events")); + APSARA_TEST_TRUE(outJson["events"].isArray()); + APSARA_TEST_TRUE(outJson["events"][0].isObject()); + APSARA_TEST_TRUE(outJson["events"][0].isMember("contents")); + APSARA_TEST_TRUE(outJson["events"][0]["contents"].isMember(logParseResult[i][j])); + APSARA_TEST_EQUAL(outJson["events"][0]["contents"][logParseResult[i][j]], + std::string(logParseResult[i][j + 1])); + } else { + continue; + } + } + } +} } // namespace logtail UNIT_TEST_MAIN \ No newline at end of file diff --git a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp index 67bfcf9574..a10bfad798 100644 --- a/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorParseTimestampNativeUnittest.cpp @@ -423,15 +423,15 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormat() { })"; APSARA_TEST_EQUAL_FATAL(CompactJson(expectJsonSs.str()), CompactJson(outJson)); // check observablity - APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); + APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mProcInRecordsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(strlen(timebuff)*2, processor.mProcParseInSizeBytes->GetValue()); // discard history, so output is 0 - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mProcOutRecordsTotal->GetValue()); // size of one timestamp and one nanosecond equals to 8 byte, respectively - APSARA_TEST_EQUAL_FATAL(8*4, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(8 * 4UL, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormatFailed() { @@ -487,15 +487,15 @@ void ProcessorParseTimestampNativeUnittest::TestProcessRegularFormatFailed() { std::string outJson = eventGroup.ToJsonString(); APSARA_TEST_STREQ_FATAL(CompactJson(inJson).c_str(), CompactJson(outJson).c_str()); // check observablity - APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0, processor.GetContext().GetProcessProfile().historyFailures); + APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mProcInRecordsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(strlen(timebuff)*2, processor.mProcParseInSizeBytes->GetValue()); // discard history, so output is 0 - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mProcOutRecordsTotal->GetValue()); // size of one timestamp and one nanosecond equals to 8 byte, respectively - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(2, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseTimestampNativeUnittest::TestProcessHistoryDiscard() { @@ -545,13 +545,13 @@ void ProcessorParseTimestampNativeUnittest::TestProcessHistoryDiscard() { processorInstance.Process(eventGroup); // check observablity APSARA_TEST_EQUAL_FATAL(2, processor.GetContext().GetProcessProfile().historyFailures); - APSARA_TEST_EQUAL_FATAL(2, processorInstance.mProcInRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processorInstance.mProcInRecordsTotal->GetValue()); APSARA_TEST_EQUAL_FATAL(strlen(timebuff)*2, processor.mProcParseInSizeBytes->GetValue()); // discard history, so output is 0 - APSARA_TEST_EQUAL_FATAL(0, processorInstance.mProcOutRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseOutSizeBytes->GetValue()); - APSARA_TEST_EQUAL_FATAL(2, processor.mProcDiscardRecordsTotal->GetValue()); - APSARA_TEST_EQUAL_FATAL(0, processor.mProcParseErrorTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processorInstance.mProcOutRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcParseOutSizeBytes->GetValue()); + APSARA_TEST_EQUAL_FATAL(2UL, processor.mProcDiscardRecordsTotal->GetValue()); + APSARA_TEST_EQUAL_FATAL(0UL, processor.mProcParseErrorTotal->GetValue()); } void ProcessorParseTimestampNativeUnittest::TestProcessEventPreciseTimestampLegacy() {