From fb4e67e42294f43b4d60ba4ab43b0bf4b8a7c403 Mon Sep 17 00:00:00 2001 From: abingcbc Date: Tue, 15 Oct 2024 10:20:17 +0800 Subject: [PATCH] fix unittest --- core/file_server/ContainerInfo.cpp | 1 - core/file_server/reader/LogFileReader.cpp | 20 +- core/pipeline/GlobalConfig.h | 5 +- core/pipeline/Pipeline.cpp | 12 +- core/pipeline/serializer/SLSSerializer.cpp | 1 - core/pipeline/serializer/SLSSerializer.h | 7 +- .../event_handler/ModifyHandlerUnittest.cpp | 46 ++++- core/unittest/flusher/FlusherSLSUnittest.cpp | 20 +- .../processor/ProcessorTagNativeUnittest.cpp | 24 +-- core/unittest/reader/DeletedFileUnittest.cpp | 6 +- core/unittest/reader/ForceReadUnittest.cpp | 51 +++-- .../reader/GetLastLineDataUnittest.cpp | 45 +++- .../reader/JsonLogFileReaderUnittest.cpp | 72 +++++-- .../unittest/reader/LogFileReaderUnittest.cpp | 193 +++++++++++++----- .../RemoveLastIncompleteLogUnittest.cpp | 58 ++++-- 15 files changed, 409 insertions(+), 152 deletions(-) diff --git a/core/file_server/ContainerInfo.cpp b/core/file_server/ContainerInfo.cpp index d903bd5c47..dac3524f91 100644 --- a/core/file_server/ContainerInfo.cpp +++ b/core/file_server/ContainerInfo.cpp @@ -68,7 +68,6 @@ bool ContainerInfo::ParseAllByJSONObj(const Json::Value& paramsAll, } bool ContainerInfo::ParseByJSONObj(const Json::Value& params, ContainerInfo& containerInfo, std::string& errorMsg) { - bool isOldCheckpoint = !params.isMember("MetaDatas"); containerInfo.mJson = params; if (params.isMember("ID") && params["ID"].isString()) { if (params["ID"].empty()) { diff --git a/core/file_server/reader/LogFileReader.cpp b/core/file_server/reader/LogFileReader.cpp index 3b9be5f92e..0e27d72997 100644 --- a/core/file_server/reader/LogFileReader.cpp +++ b/core/file_server/reader/LogFileReader.cpp @@ -2463,16 +2463,18 @@ void LogFileReader::SetEventGroupMetaAndTag(PipelineEventGroup& group) { } // 3. container name tag, external k8s env/label tag auto containerExtraTags = GetContainerExtraTags(); - for (size_t i = 0; i < containerExtraTags->size(); ++i) { - auto key = ContainerInfo::GetFileTagKey((*containerExtraTags)[i].key()); - if (key != TagKey::UNKOWN) { // container name tag - auto keyName = mTagConfig.first->GetFileTagKeyName(key); - if (!keyName.empty()) { - StringBuffer b = group.GetSourceBuffer()->CopyString((*containerExtraTags)[i].value()); - group.SetTagNoCopy(keyName, StringView(b.data, b.size)); + if (containerExtraTags) { + for (size_t i = 0; i < containerExtraTags->size(); ++i) { + auto key = ContainerInfo::GetFileTagKey((*containerExtraTags)[i].key()); + if (key != TagKey::UNKOWN) { // container name tag + auto keyName = mTagConfig.first->GetFileTagKeyName(key); + if (!keyName.empty()) { + StringBuffer b = group.GetSourceBuffer()->CopyString((*containerExtraTags)[i].value()); + group.SetTagNoCopy(keyName, StringView(b.data, b.size)); + } + } else { // external k8s env/label tag + group.SetTag((*containerExtraTags)[i].key(), (*containerExtraTags)[i].value()); } - } else { // external k8s env/label tag - group.SetTag((*containerExtraTags)[i].key(), (*containerExtraTags)[i].value()); } } // 4. inode diff --git a/core/pipeline/GlobalConfig.h b/core/pipeline/GlobalConfig.h index 942e606dee..3eeeda78b4 100644 --- a/core/pipeline/GlobalConfig.h +++ b/core/pipeline/GlobalConfig.h @@ -20,6 +20,7 @@ #include #include +#include #include namespace logtail { @@ -38,8 +39,8 @@ struct GlobalConfig { uint32_t mProcessPriority = 0; bool mEnableTimestampNanosecond = false; bool mUsingOldContentTag = false; - Json::Value mPipelineMetaTagKey; - Json::Value mAgentEnvMetaTagKey; + std::unordered_map mPipelineMetaTagKey; + std::unordered_map mAgentEnvMetaTagKey; }; } // namespace logtail diff --git a/core/pipeline/Pipeline.cpp b/core/pipeline/Pipeline.cpp index 8c6fe1be1c..4250f22f86 100644 --- a/core/pipeline/Pipeline.cpp +++ b/core/pipeline/Pipeline.cpp @@ -475,8 +475,16 @@ void Pipeline::CopyNativeGlobalParamToGoPipeline(Json::Value& pipeline) { Json::Value& global = pipeline["global"]; global["EnableTimestampNanosecond"] = mContext.GetGlobalConfig().mEnableTimestampNanosecond; global["UsingOldContentTag"] = mContext.GetGlobalConfig().mUsingOldContentTag; - global["PipelineMetaTagKey"] = mContext.GetGlobalConfig().mPipelineMetaTagKey; - global["AgentEnvMetaTagKey"] = mContext.GetGlobalConfig().mAgentEnvMetaTagKey; + Json::Value pipelineMetaTagKey; + for (const auto& kv : mContext.GetGlobalConfig().mPipelineMetaTagKey) { + pipelineMetaTagKey[kv.first] = kv.second; + } + global["PipelineMetaTagKey"] = pipelineMetaTagKey; + Json::Value agentEnvMetaTagKey; + for (const auto& kv : mContext.GetGlobalConfig().mAgentEnvMetaTagKey) { + agentEnvMetaTagKey[kv.first] = kv.second; + } + global["AgentEnvMetaTagKey"] = agentEnvMetaTagKey; } } diff --git a/core/pipeline/serializer/SLSSerializer.cpp b/core/pipeline/serializer/SLSSerializer.cpp index c17d7e4d01..29f8b64e65 100644 --- a/core/pipeline/serializer/SLSSerializer.cpp +++ b/core/pipeline/serializer/SLSSerializer.cpp @@ -14,7 +14,6 @@ #include "pipeline/serializer/SLSSerializer.h" -#include "application/Application.h" #include "common/Flags.h" #include "common/TimeUtil.h" #include "common/compression/CompressType.h" diff --git a/core/pipeline/serializer/SLSSerializer.h b/core/pipeline/serializer/SLSSerializer.h index c9bdb56798..2184862e99 100644 --- a/core/pipeline/serializer/SLSSerializer.h +++ b/core/pipeline/serializer/SLSSerializer.h @@ -19,6 +19,7 @@ #include #include +#include "common/TagConstants.h" #include "pipeline/serializer/Serializer.h" namespace logtail { @@ -38,8 +39,10 @@ struct CompressedLogGroup { CompressedLogGroup(std::string&& data, size_t rawSize) : mData(std::move(data)), mRawSize(rawSize) {} }; -template<> -bool Serializer>::DoSerialize(std::vector&& p, std::string& output, std::string& errorMsg); +template <> +bool Serializer>::DoSerialize(std::vector&& p, + std::string& output, + std::string& errorMsg); class SLSEventGroupListSerializer : public Serializer> { public: diff --git a/core/unittest/event_handler/ModifyHandlerUnittest.cpp b/core/unittest/event_handler/ModifyHandlerUnittest.cpp index 490ed7890a..f16bce7c80 100644 --- a/core/unittest/event_handler/ModifyHandlerUnittest.cpp +++ b/core/unittest/event_handler/ModifyHandlerUnittest.cpp @@ -118,8 +118,12 @@ class ModifyHandlerUnittest : public ::testing::Test { ProcessQueueManager::GetInstance()->CreateOrUpdateBoundedQueue(0, 0, ctx); // build a reader - mReaderPtr = std::make_shared( - gRootDir, gLogName, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + mReaderPtr = std::make_shared(gRootDir, + gLogName, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); mReaderPtr->UpdateReaderManual(); APSARA_TEST_TRUE_FATAL(mReaderPtr->CheckFileSignatureAndOffset(true)); @@ -140,6 +144,7 @@ class ModifyHandlerUnittest : public ::testing::Test { FileDiscoveryOptions discoveryOpts; FileReaderOptions readerOpts; MultilineOptions multilineOpts; + FileTagOptions tagOpts; PipelineContext ctx; FileDiscoveryConfig mConfig; @@ -214,8 +219,12 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::string logPath1 = logPath + ".1"; writeLog(logPath1, "a sample log\n"); auto devInode1 = GetFileDevInode(logPath1); - auto reader1 = std::make_shared( - gRootDir, basicLogName, devInode1, std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + auto reader1 = std::make_shared(gRootDir, + basicLogName, + devInode1, + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader1->mRealLogPath = logPath1; reader1->mLastFileSignatureSize = sigSize; reader1->mLastFileSignatureHash = sigHash; @@ -223,8 +232,12 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::string logPath2 = logPath + ".2"; writeLog(logPath2, "a sample log\n"); auto devInode2 = GetFileDevInode(logPath2); - auto reader2 = std::make_shared( - gRootDir, basicLogName, devInode2, std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + auto reader2 = std::make_shared(gRootDir, + basicLogName, + devInode2, + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader2->mRealLogPath = logPath2; reader2->mLastFileSignatureSize = sigSize; reader2->mLastFileSignatureHash = sigHash; @@ -240,8 +253,12 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::string logPath3 = logPath + ".3"; writeLog(logPath3, "a sample log\n"); auto devInode3 = GetFileDevInode(logPath3); - auto reader3 = std::make_shared( - gRootDir, basicLogName, devInode3, std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + auto reader3 = std::make_shared(gRootDir, + basicLogName, + devInode3, + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader3->mRealLogPath = logPath3; reader3->mLastFileSignatureSize = sigSize; reader3->mLastFileSignatureHash = sigHash; @@ -249,8 +266,12 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::string logPath4 = logPath + ".4"; writeLog(logPath4, "a sample log\n"); auto devInode4 = GetFileDevInode(logPath4); - auto reader4 = std::make_shared( - gRootDir, basicLogName, devInode4, std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + auto reader4 = std::make_shared(gRootDir, + basicLogName, + devInode4, + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader4->mRealLogPath = logPath4; reader4->mLastFileSignatureSize = sigSize; reader4->mLastFileSignatureHash = sigHash; @@ -269,6 +290,7 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx), std::make_pair(&discoveryOpts, &ctx), + std::make_pair(&tagOpts, &ctx), 0, false); // recover reader from checkpoint, random order @@ -278,6 +300,7 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx), std::make_pair(&discoveryOpts, &ctx), + std::make_pair(&tagOpts, &ctx), 0, false); handlerPtr->CreateLogFileReaderPtr(gRootDir, @@ -286,6 +309,7 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx), std::make_pair(&discoveryOpts, &ctx), + std::make_pair(&tagOpts, &ctx), 0, false); handlerPtr->CreateLogFileReaderPtr(gRootDir, @@ -294,6 +318,7 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx), std::make_pair(&discoveryOpts, &ctx), + std::make_pair(&tagOpts, &ctx), 0, false); handlerPtr->CreateLogFileReaderPtr(gRootDir, @@ -302,6 +327,7 @@ void ModifyHandlerUnittest::TestRecoverReaderFromCheckpoint() { std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx), std::make_pair(&discoveryOpts, &ctx), + std::make_pair(&tagOpts, &ctx), 0, false); APSARA_TEST_EQUAL_FATAL(handlerPtr->mNameReaderMap.size(), 1); diff --git a/core/unittest/flusher/FlusherSLSUnittest.cpp b/core/unittest/flusher/FlusherSLSUnittest.cpp index a760f5b8cd..ee99705d42 100644 --- a/core/unittest/flusher/FlusherSLSUnittest.cpp +++ b/core/unittest/flusher/FlusherSLSUnittest.cpp @@ -591,7 +591,7 @@ void FlusherSLSUnittest::TestSend() { // replayed group PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); @@ -632,7 +632,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); APSARA_TEST_EQUAL(2, logGroup.logtags_size()); - APSARA_TEST_EQUAL("__hostname__", logGroup.logtags(0).key()); + APSARA_TEST_EQUAL(TagDefaultKey[TagKey::HOST_NAME], logGroup.logtags(0).key()); APSARA_TEST_EQUAL("hostname", logGroup.logtags(0).value()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(1).key()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -648,7 +648,7 @@ void FlusherSLSUnittest::TestSend() { flusher.mBatcher.GetEventFlushStrategy().SetMaxCnt(1); PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); @@ -686,7 +686,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); APSARA_TEST_EQUAL(2, logGroup.logtags_size()); - APSARA_TEST_EQUAL("__hostname__", logGroup.logtags(0).key()); + APSARA_TEST_EQUAL(TagDefaultKey[TagKey::HOST_NAME], logGroup.logtags(0).key()); APSARA_TEST_EQUAL("hostname", logGroup.logtags(0).value()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(1).key()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -733,7 +733,7 @@ void FlusherSLSUnittest::TestSend() { flusher.mBatcher.GetEventFlushStrategy().SetMaxCnt(1); PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); @@ -766,7 +766,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); APSARA_TEST_EQUAL(3, logGroup.logtags_size()); - APSARA_TEST_EQUAL("__hostname__", logGroup.logtags(0).key()); + APSARA_TEST_EQUAL(TagDefaultKey[TagKey::HOST_NAME], logGroup.logtags(0).key()); APSARA_TEST_EQUAL("hostname", logGroup.logtags(0).value()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(1).key()); APSARA_TEST_EQUAL("tag_key", logGroup.logtags(2).key()); @@ -815,7 +815,7 @@ void FlusherSLSUnittest::TestSend() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); @@ -870,7 +870,7 @@ void FlusherSLSUnittest::TestSend() { APSARA_TEST_EQUAL("uuid", logGroup.machineuuid()); APSARA_TEST_EQUAL("172.0.0.1", logGroup.source()); APSARA_TEST_EQUAL(2, logGroup.logtags_size()); - APSARA_TEST_EQUAL("__hostname__", logGroup.logtags(0).key()); + APSARA_TEST_EQUAL(TagDefaultKey[TagKey::HOST_NAME], logGroup.logtags(0).key()); APSARA_TEST_EQUAL("hostname", logGroup.logtags(0).value()); APSARA_TEST_EQUAL("__pack_id__", logGroup.logtags(1).key()); APSARA_TEST_EQUAL(1, logGroup.logs_size()); @@ -917,7 +917,7 @@ void FlusherSLSUnittest::TestFlush() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); @@ -961,7 +961,7 @@ void FlusherSLSUnittest::TestFlushAll() { PipelineEventGroup group(make_shared()); group.SetMetadata(EventGroupMetaKey::SOURCE_ID, string("source-id")); - group.SetTag(LOG_RESERVED_KEY_HOSTNAME, "hostname"); + group.SetTag(TagDefaultKey[TagKey::HOST_NAME], "hostname"); group.SetTag(LOG_RESERVED_KEY_SOURCE, "172.0.0.1"); group.SetTag(LOG_RESERVED_KEY_MACHINE_UUID, "uuid"); group.SetTag(LOG_RESERVED_KEY_TOPIC, "topic"); diff --git a/core/unittest/processor/ProcessorTagNativeUnittest.cpp b/core/unittest/processor/ProcessorTagNativeUnittest.cpp index 7855259280..e6ebea0082 100644 --- a/core/unittest/processor/ProcessorTagNativeUnittest.cpp +++ b/core/unittest/processor/ProcessorTagNativeUnittest.cpp @@ -79,14 +79,14 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE_FATAL(processor.Init(config)); processor.Process(eventGroup); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), - eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); + // APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_PATH)); + // APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), + // eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); + // APSARA_TEST_FALSE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); #ifdef __ENTERPRISE__ - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); - APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), - eventGroup.GetTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); + // APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); + // APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), + // eventGroup.GetTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); #endif } @@ -98,11 +98,11 @@ void ProcessorTagNativeUnittest::TestProcess() { APSARA_TEST_TRUE_FATAL(processor.Init(config)); processor.Process(eventGroup); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), - eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); - APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); - APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag(LOG_RESERVED_KEY_HOSTNAME)); + // APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_PATH)); + // APSARA_TEST_EQUAL_FATAL(eventGroup.GetMetadata(EventGroupMetaKey::LOG_FILE_PATH), + // eventGroup.GetTag(LOG_RESERVED_KEY_PATH)); + // APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_HOSTNAME)); + // APSARA_TEST_EQUAL_FATAL(LogFileProfiler::mHostname, eventGroup.GetTag(LOG_RESERVED_KEY_HOSTNAME)); #ifdef __ENTERPRISE__ APSARA_TEST_TRUE_FATAL(eventGroup.HasTag(LOG_RESERVED_KEY_USER_DEFINED_ID)); APSARA_TEST_EQUAL_FATAL(EnterpriseConfigProvider::GetInstance()->GetUserDefinedIdSet(), diff --git a/core/unittest/reader/DeletedFileUnittest.cpp b/core/unittest/reader/DeletedFileUnittest.cpp index 23bf0500e7..884c932ef6 100644 --- a/core/unittest/reader/DeletedFileUnittest.cpp +++ b/core/unittest/reader/DeletedFileUnittest.cpp @@ -32,8 +32,9 @@ class DeletedFileUnittest : public testing::Test { hostLogPathFile, DevInode(), make_pair(&readerOpts, &ctx), - make_pair(&multilineOpts, &ctx))); - } + make_pair(&multilineOpts, &ctx), + make_pair(&tagOpts, &ctx))); + } void TearDown() override { INT32_FLAG(force_release_deleted_file_fd_timeout) = -1; } @@ -41,6 +42,7 @@ class DeletedFileUnittest : public testing::Test { LogFileReaderPtr reader; FileReaderOptions readerOpts; MultilineOptions multilineOpts; + FileTagOptions tagOpts; PipelineContext ctx; string hostLogPathDir = "."; string hostLogPathFile = "DeletedFileUnittest.txt"; diff --git a/core/unittest/reader/ForceReadUnittest.cpp b/core/unittest/reader/ForceReadUnittest.cpp index fa99c10eec..19ce5498ad 100644 --- a/core/unittest/reader/ForceReadUnittest.cpp +++ b/core/unittest/reader/ForceReadUnittest.cpp @@ -25,10 +25,10 @@ #include "common/JsonUtil.h" #include "config/PipelineConfig.h" #include "file_server/ConfigManager.h" +#include "file_server/FileServer.h" #include "file_server/event/BlockEventManager.h" #include "file_server/event/Event.h" #include "file_server/event_handler/EventHandler.h" -#include "file_server/FileServer.h" #include "logger/Logger.h" #include "pipeline/Pipeline.h" #include "pipeline/queue/ProcessQueueManager.h" @@ -133,6 +133,7 @@ class ForceReadUnittest : public testing::Test { FileDiscoveryOptions discoveryOpts; FileReaderOptions readerOpts; MultilineOptions multilineOpts; + FileTagOptions tagOpts; PipelineContext ctx; FileDiscoveryConfig mConfig; }; @@ -144,8 +145,12 @@ void ForceReadUnittest::TestTimeoutForceRead() { { // read -> add timeout event -> handle timeout -> valid -> read empty -> not rollback Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -182,8 +187,12 @@ void ForceReadUnittest::TestTimeoutForceRead() { { // read -> write -> add timeout event -> handle timeout -> valid -> read not empty -> rollback Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -224,8 +233,12 @@ void ForceReadUnittest::TestTimeoutForceRead() { // read -> add timeout event -> write -> read -> handle timeout -> event invalid LOG_WARNING(sLogger, ("This case is difficult to test", "test")); Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -298,8 +311,12 @@ void ForceReadUnittest::TestFileCloseForceRead() { { // file close -> handle timeout -> valid -> not rollback Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -342,8 +359,12 @@ void ForceReadUnittest::TestAddTimeoutEvent() { { // read part -> not add timeout event Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -367,8 +388,12 @@ void ForceReadUnittest::TestAddTimeoutEvent() { { // read all -> add timeout event Init(); - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); diff --git a/core/unittest/reader/GetLastLineDataUnittest.cpp b/core/unittest/reader/GetLastLineDataUnittest.cpp index 761c225a57..e99ac67f42 100644 --- a/core/unittest/reader/GetLastLineDataUnittest.cpp +++ b/core/unittest/reader/GetLastLineDataUnittest.cpp @@ -13,8 +13,8 @@ // limitations under the License. #include "common/FileSystemUtil.h" -#include "file_server/reader/LogFileReader.h" #include "common/memory/SourceBuffer.h" +#include "file_server/reader/LogFileReader.h" #include "unittest/Unittest.h" namespace logtail { @@ -66,6 +66,7 @@ class LastMatchedContainerdTextLineUnittest : public ::testing::Test { std::unique_ptr expectedContent; FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; static std::string logPathDir; static std::string gbkFile; @@ -82,8 +83,12 @@ std::string LastMatchedContainerdTextLineUnittest::utf8File; void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine() { { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -511,8 +516,12 @@ void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineSingleLine void LastMatchedContainerdTextLineUnittest::TestLastContainerdTextLineMerge() { { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -978,6 +987,7 @@ class LastMatchedDockerJsonFileUnittest : public ::testing::Test { std::unique_ptr expectedContent; FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; static std::string logPathDir; static std::string gbkFile; @@ -993,8 +1003,12 @@ std::string LastMatchedDockerJsonFileUnittest::utf8File; void LastMatchedDockerJsonFileUnittest::TestLastDockerJsonFile() { { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -1233,6 +1247,7 @@ class LastMatchedContainerdTextWithDockerJsonUnittest : public ::testing::Test { std::unique_ptr expectedContent; FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; static std::string logPathDir; static std::string gbkFile; @@ -1248,8 +1263,12 @@ std::string LastMatchedContainerdTextWithDockerJsonUnittest::utf8File; void LastMatchedContainerdTextWithDockerJsonUnittest::TestContainerdTextWithDockerJson() { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(0); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); @@ -1303,8 +1322,12 @@ void LastMatchedContainerdTextWithDockerJsonUnittest::TestContainerdTextWithDock void LastMatchedContainerdTextWithDockerJsonUnittest::TestDockerJsonWithContainerdText() { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); BaseLineParse* baseLineParsePtr = nullptr; baseLineParsePtr = logFileReader.GetParser(LogFileReader::BUFFER_SIZE); logFileReader.mLineParsers.emplace_back(baseLineParsePtr); diff --git a/core/unittest/reader/JsonLogFileReaderUnittest.cpp b/core/unittest/reader/JsonLogFileReaderUnittest.cpp index cd9f16f805..1522aac796 100644 --- a/core/unittest/reader/JsonLogFileReaderUnittest.cpp +++ b/core/unittest/reader/JsonLogFileReaderUnittest.cpp @@ -89,9 +89,14 @@ void JsonLogFileReaderUnittest::TestReadGBK() { { // buffer size big enough and is json MultilineOptions multilineOpts; FileReaderOptions readerOpts; + FileTagOptions tagOpts; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - JsonLogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + JsonLogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -111,8 +116,13 @@ void JsonLogFileReaderUnittest::TestReadGBK() { multilineOpts.Init(config, ctx, ""); FileReaderOptions readerOpts; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - JsonLogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 23; size_t BUFFER_SIZE_UTF8 = 25; // "{"first":"iLogtail 为可" reader.UpdateReaderManual(); @@ -130,8 +140,13 @@ void JsonLogFileReaderUnittest::TestReadGBK() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - JsonLogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -151,8 +166,13 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { { // buffer size big enough and is json MultilineOptions multilineOpts; FileReaderOptions readerOpts; - JsonLogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -168,8 +188,13 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { // should read buffer size MultilineOptions multilineOpts; FileReaderOptions readerOpts; - JsonLogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 25; reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -185,8 +210,13 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { // should read until last json MultilineOptions multilineOpts; FileReaderOptions readerOpts; - JsonLogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -203,8 +233,13 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { { // read twice MultilineOptions multilineOpts; FileReaderOptions readerOpts; - JsonLogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions tagOpts; + JsonLogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -232,8 +267,12 @@ void JsonLogFileReaderUnittest::TestReadUTF8() { class RemoveLastIncompleteLogUnittest : public ::testing::Test { public: void SetUp() override { - mLogFileReader.reset(new JsonLogFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx))); + mLogFileReader.reset(new JsonLogFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx))); } void TestRemoveLastIncompleteLogSingleLine(); @@ -248,6 +287,7 @@ class RemoveLastIncompleteLogUnittest : public ::testing::Test { std::unique_ptr mLogFileReader; MultilineOptions multilineOpts; FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; }; diff --git a/core/unittest/reader/LogFileReaderUnittest.cpp b/core/unittest/reader/LogFileReaderUnittest.cpp index 165e4e258d..2748ad8986 100644 --- a/core/unittest/reader/LogFileReaderUnittest.cpp +++ b/core/unittest/reader/LogFileReaderUnittest.cpp @@ -21,8 +21,8 @@ #include "common/RuntimeUtil.h" #include "common/memory/SourceBuffer.h" #include "file_server/FileServer.h" -#include "protobuf/sls/sls_logs.pb.h" #include "file_server/reader/LogFileReader.h" +#include "protobuf/sls/sls_logs.pb.h" #include "unittest/Unittest.h" DECLARE_FLAG_INT32(force_release_deleted_file_fd_timeout); @@ -77,6 +77,7 @@ class LogFileReaderUnittest : public ::testing::Test { static std::string utf8File; FileDiscoveryOptions discoveryOpts; FileReaderOptions readerOpts; + FileTagOptions fileTagOpts; PipelineContext ctx; }; @@ -93,8 +94,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -109,8 +114,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -132,8 +141,12 @@ void LogFileReaderUnittest::TestReadGBK() { readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; multilineOpts.Init(config, ctx, ""); - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 14; size_t BUFFER_SIZE_UTF8 = 15; // "ilogtail 为可" reader.UpdateReaderManual(); @@ -154,8 +167,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); // reader.mDiscardUnmatch = false; reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -178,8 +195,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); // reader.mDiscardUnmatch = false; reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -207,8 +228,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); // reader.mDiscardUnmatch = false; reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -237,8 +262,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); LogBuffer logBuffer; @@ -255,8 +284,12 @@ void LogFileReaderUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -306,8 +339,12 @@ void LogFileReaderUnittest::TestReadUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -321,8 +358,12 @@ void LogFileReaderUnittest::TestReadUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); reader.CheckFileSignatureAndOffset(true); @@ -344,8 +385,12 @@ void LogFileReaderUnittest::TestReadUTF8() { multilineOpts.Init(config, ctx, ""); FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 15; reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -365,8 +410,12 @@ void LogFileReaderUnittest::TestReadUTF8() { multilineOpts.Init(config, ctx, ""); FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -387,8 +436,12 @@ void LogFileReaderUnittest::TestReadUTF8() { multilineOpts.Init(config, ctx, ""); FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -414,8 +467,12 @@ void LogFileReaderUnittest::TestReadUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -442,8 +499,12 @@ void LogFileReaderUnittest::TestReadUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); LogBuffer logBuffer; @@ -459,8 +520,12 @@ void LogFileReaderUnittest::TestReadUTF8() { multilineOpts.Init(config, ctx, ""); FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader.mLogFileOp.GetFileSize(); @@ -555,6 +620,7 @@ class LogMultiBytesUnittest : public ::testing::Test { static std::string gbkFile; static std::string utf8File; FileDiscoveryOptions discoveryOpts; + FileTagOptions fileTagOpts; PipelineContext ctx; }; @@ -572,8 +638,12 @@ void LogMultiBytesUnittest::TestAlignLastCharacterUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader logFileReader( - "", "", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("", + "", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); std::string expectedLog = "为可观测场景而"; std::string testLog = expectedLog + "生"; size_t result = logFileReader.AlignLastCharacter(const_cast(testLog.data()), expectedLog.size()); @@ -583,8 +653,13 @@ void LogMultiBytesUnittest::TestAlignLastCharacterUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader logFileReader( - "", "", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions fileTagOpts; + LogFileReader logFileReader("", + "", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); std::string expectedLog = "为可观测场景而"; std::string testLog = expectedLog + "生"; size_t result = logFileReader.AlignLastCharacter(const_cast(testLog.data()), expectedLog.size() + 1); @@ -597,8 +672,13 @@ void LogMultiBytesUnittest::TestAlignLastCharacterGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader logFileReader( - "", "", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions fileTagOpts; + LogFileReader logFileReader("", + "", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); { // case: no align std::string expectedLog = "\xce\xaa\xbf\xc9\xb9\xdb\xb2\xe2\xb3\xa1\xbe\xb0\xb6\xf8"; // equal to "为可观测场景而" @@ -619,8 +699,13 @@ void LogMultiBytesUnittest::TestReadUTF8() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions fileTagOpts; + LogFileReader reader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 13; // equal to "iLogtail 为" plus one illegal byte reader.UpdateReaderManual(); reader.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); @@ -639,8 +724,13 @@ void LogMultiBytesUnittest::TestReadGBK() { FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; readerOpts.mFileEncoding = FileReaderOptions::Encoding::GBK; - LogFileReader reader( - logPathDir, gbkFile, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + FileTagOptions fileTagOpts; + LogFileReader reader(logPathDir, + gbkFile, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); LogFileReader::BUFFER_SIZE = 12; // equal to "iLogtail 为" plus one illegal byte size_t BUFFER_SIZE_UTF8 = 12; // "ilogtail 为可" reader.UpdateReaderManual(); @@ -680,6 +770,7 @@ class LogFileReaderCheckpointUnittest : public ::testing::Test { static std::string logPathDir; static std::string utf8File; FileDiscoveryOptions discoveryOpts; + FileTagOptions fileTagOpts; PipelineContext ctx; }; @@ -693,8 +784,12 @@ void LogFileReaderCheckpointUnittest::TestDumpMetaToMem() { MultilineOptions multilineOpts; FileReaderOptions readerOpts; readerOpts.mInputType = FileReaderOptions::InputType::InputFile; - LogFileReader reader1( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader1(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader1.UpdateReaderManual(); reader1.InitReader(true, LogFileReader::BACKWARD_TO_BEGINNING); int64_t fileSize = reader1.mLogFileOp.GetFileSize(); @@ -708,8 +803,12 @@ void LogFileReaderCheckpointUnittest::TestDumpMetaToMem() { APSARA_TEST_GE_FATAL(reader1.mCache.size(), 0UL); reader1.DumpMetaToMem(false); // second read - LogFileReader reader2( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader reader2(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&fileTagOpts, &ctx)); reader2.UpdateReaderManual(); reader2.InitReader(false, LogFileReader::BACKWARD_TO_BEGINNING); reader2.CheckFileSignatureAndOffset(true); diff --git a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp index 5ce57053b3..3acdffca80 100644 --- a/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp +++ b/core/unittest/reader/RemoveLastIncompleteLogUnittest.cpp @@ -70,6 +70,7 @@ class RemoveLastIncompleteLogUnittest : public ::testing::Test { std::unique_ptr expectedContent; FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; static std::string logPathDir; static std::string gbkFile; @@ -85,8 +86,12 @@ std::string RemoveLastIncompleteLogUnittest::utf8File; void RemoveLastIncompleteLogUnittest::TestSingleline() { MultilineOptions multilineOpts; - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); { // case single line std::string line1 = "first."; std::string line2 = "second."; @@ -139,8 +144,12 @@ void RemoveLastIncompleteLogUnittest::TestMultiline() { config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - logPathDir, utf8File, DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader(logPathDir, + utf8File, + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); { // case multi line std::vector index; std::string firstLog = LOG_BEGIN_STRING + "first.\nmultiline1\nmultiline2"; @@ -206,6 +215,7 @@ class RemoveLastIncompleteLogMultilineUnittest : public ::testing::Test { private: FileReaderOptions readerOpts; + FileTagOptions tagOpts; PipelineContext ctx; }; @@ -221,8 +231,12 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBe config["ContinuePattern"] = LOG_CONTINUE_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with begin continue std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + '\n'; @@ -272,8 +286,12 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBe config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with begin end std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n'; @@ -322,8 +340,12 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithBe config["StartPattern"] = LOG_BEGIN_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with begin std::string expectMatch = LOG_BEGIN_STRING + "\n" + LOG_UNMATCH + "\n" + LOG_UNMATCH + '\n'; @@ -363,8 +385,12 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithCo config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with continue end std::string expectMatch = LOG_CONTINUE_STRING + "\n" + LOG_CONTINUE_STRING + "\n" + LOG_END_STRING + '\n'; @@ -413,8 +439,12 @@ void RemoveLastIncompleteLogMultilineUnittest::TestRemoveLastIncompleteLogWithEn config["EndPattern"] = LOG_END_REGEX; MultilineOptions multilineOpts; multilineOpts.Init(config, ctx, ""); - LogFileReader logFileReader( - "dir", "file", DevInode(), std::make_pair(&readerOpts, &ctx), std::make_pair(&multilineOpts, &ctx)); + LogFileReader logFileReader("dir", + "file", + DevInode(), + std::make_pair(&readerOpts, &ctx), + std::make_pair(&multilineOpts, &ctx), + std::make_pair(&tagOpts, &ctx)); // logFileReader.mDiscardUnmatch = true; { // case: end with end std::string expectMatch = LOG_UNMATCH + "\n" + LOG_UNMATCH + "\n" + LOG_END_STRING + '\n';