Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

change trace tag #1852

Open
wants to merge 21 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 19 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions core/constants/TagConstants.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ namespace logtail {
const std::string DEFAULT_TAG_POD_UID = "pod_uid";
const std::string DEFAULT_TAG_CONTAINER_NAME = "container_name";
const std::string DEFAULT_TAG_CONTAINER_IP = "container_ip";
const std::string DEFAULT_TAG_CONTAINER_ID = "container_id";
const std::string DEFAULT_TAG_IMAGE_NAME = "image_name";

////////////////////////// LOG ////////////////////////
Expand Down Expand Up @@ -55,6 +56,39 @@ namespace logtail {
const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID = "__user_defined_id__";
#endif

////////////////////////// Trace ////////////////////////
#ifndef __ENTERPRISE__ // 开源版
const std::string DEFAULT_TRACE_TAG_NAMESPACE = "k8s.namespace.name";
NameHaibinZhang marked this conversation as resolved.
Show resolved Hide resolved
const std::string DEFAULT_TRACE_TAG_CONTAINER_ID = "k8s.container.id";
const std::string DEFAULT_TRACE_TAG_PID = "service.instance.id";
const std::string DEFAULT_TRACE_TAG_REMOTE_IP = "remote.ip";
NameHaibinZhang marked this conversation as resolved.
Show resolved Hide resolved
const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_NAME = "peer.workload.name";
const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_KIND = "peer.workload.kind";
const std::string DEFAULT_TRACE_TAG_PEER_NAMESPACE = "peer.namespace";
const std::string DEFAULT_TRACE_TAG_APPID = DEFAULT_TRACE_TAG_PID;
const std::string DEFAULT_TRACE_TAG_IMAGES = "container.image.name";
const std::string DEFAULT_TRACE_TAG_LABELS = "k8s.pod.label";
const std::string DEFAULT_TRACE_TAG_WORKLOAD_KIND = "workload.kind";
const std::string DEFAULT_TRACE_TAG_WORKLOAD_NAME = "workload.name";
const std::string DEFAULT_TRACE_TAG_SERVICENAME = "service.name";
#else
const std::string DEFAULT_TRACE_TAG_NAMESPACE = "namespace";
const std::string DEFAULT_TRACE_TAG_CONTAINER_ID = "container.id";
const std::string DEFAULT_TRACE_TAG_PID = "pid";
const std::string DEFAULT_TRACE_TAG_REMOTE_IP = "remote_ip";
const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_NAME = "peerWorkloadName";
const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_KIND = "peerWorkloadKind";
const std::string DEFAULT_TRACE_TAG_PEER_NAMESPACE = "peerNamespace";
const std::string DEFAULT_TRACE_TAG_APPID = "armsAppId";
const std::string DEFAULT_TRACE_TAG_IMAGES = "images";
const std::string DEFAULT_TRACE_TAG_LABELS = "labels";
const std::string DEFAULT_TRACE_TAG_WORKLOAD_KIND = "workloadKind";
const std::string DEFAULT_TRACE_TAG_WORKLOAD_NAME = "workloadName";
const std::string DEFAULT_TRACE_TAG_SERVICENAME = "serviceName";

#endif


////////////////////////// METRIC ////////////////////////
const std::string DEFAULT_METRIC_TAG_NAMESPACE = DEFAULT_TAG_NAMESPACE;
const std::string DEFAULT_METRIC_TAG_POD_NAME = DEFAULT_TAG_POD_NAME;
Expand Down
15 changes: 15 additions & 0 deletions core/constants/TagConstants.h
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,21 @@ namespace logtail {
extern const std::string DEFAULT_LOG_TAG_USER_DEFINED_ID;
#endif

////////////////////////// TRACE ////////////////////////
extern const std::string DEFAULT_TRACE_TAG_NAMESPACE;
extern const std::string DEFAULT_TRACE_TAG_CONTAINER_ID;
extern const std::string DEFAULT_TRACE_TAG_PID;
extern const std::string DEFAULT_TRACE_TAG_REMOTE_IP;
extern const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_NAME;
extern const std::string DEFAULT_TRACE_TAG_PEER_WORKLOAD_KIND;
extern const std::string DEFAULT_TRACE_TAG_PEER_NAMESPACE;
extern const std::string DEFAULT_TRACE_TAG_APPID;
extern const std::string DEFAULT_TRACE_TAG_IMAGES;
extern const std::string DEFAULT_TRACE_TAG_LABELS;
extern const std::string DEFAULT_TRACE_TAG_WORKLOAD_KIND;
extern const std::string DEFAULT_TRACE_TAG_WORKLOAD_NAME;
extern const std::string DEFAULT_TRACE_TAG_SERVICENAME;

////////////////////////// METRIC ////////////////////////
extern const std::string DEFAULT_METRIC_TAG_NAMESPACE;
extern const std::string DEFAULT_METRIC_TAG_POD_NAME;
Expand Down
30 changes: 15 additions & 15 deletions core/metadata/K8sMetadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,32 +33,32 @@ size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* userp) {
}

bool K8sMetadata::FromInfoJson(const Json::Value& json, k8sContainerInfo& info) {
if (!json.isMember(imageKey) || !json.isMember(labelsKey) || !json.isMember(namespaceKey)
|| !json.isMember(workloadKindKey) || !json.isMember(workloadNameKey)) {
if (!json.isMember(TAG_IMAGES) || !json.isMember(TAG_LABELS) || !json.isMember(TAG_NAMESPACE)
|| !json.isMember(TAG_WORKLOADKIND) || !json.isMember(TAG_WORKLOADNAME)) {
return false;
}

for (const auto& key : json[imageKey].getMemberNames()) {
if (json[imageKey].isMember(key)) {
info.images[key] = json[imageKey][key].asString();
for (const auto& key : json[TAG_IMAGES].getMemberNames()) {
if (json[TAG_IMAGES].isMember(key)) {
info.images[key] = json[TAG_IMAGES][key].asString();
}
}
for (const auto& key : json[labelsKey].getMemberNames()) {
if (json[labelsKey].isMember(key)) {
info.labels[key] = json[labelsKey][key].asString();
for (const auto& key : json[TAG_LABELS].getMemberNames()) {
if (json[TAG_LABELS].isMember(key)) {
info.labels[key] = json[TAG_LABELS][key].asString();

if (key == appIdKey) {
info.appId = json[labelsKey][key].asString();
if (key == DEFAULT_TRACE_TAG_APPID) {
info.appId = json[TAG_LABELS][key].asString();
}
}
}

info.k8sNamespace = json[namespaceKey].asString();
if (json.isMember(serviceNameKey)) {
info.serviceName = json[serviceNameKey].asString();
info.k8sNamespace = json[TAG_NAMESPACE].asString();
if (json.isMember(TAG_SERVICENAME)) {
info.serviceName = json[TAG_SERVICENAME].asString();
}
info.workloadKind = json[workloadKindKey].asString();
info.workloadName = json[workloadNameKey].asString();
info.workloadKind = json[TAG_WORKLOADKIND].asString();
info.workloadName = json[TAG_WORKLOADNAME].asString();
info.timestamp = std::time(0);
return true;
}
Expand Down
14 changes: 7 additions & 7 deletions core/metadata/K8sMetadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#include <curl/curl.h>
#include "common/LRUCache.h"
#include "app_config/AppConfig.h"
#include "TagConstants.h"
#include <json/json.h>
#include "common/Flags.h"

Expand All @@ -24,13 +25,12 @@ DECLARE_FLAG_INT32(loong_collector_k8s_meta_service_port);

namespace logtail {

const static std::string appIdKey = "armsAppId";
const static std::string imageKey = "images";
const static std::string labelsKey = "labels";
const static std::string namespaceKey = "namespace";
const static std::string workloadKindKey = "workloadKind";
const static std::string workloadNameKey = "workloadName";
const static std::string serviceNameKey = "serviceName";
const std::string TAG_NAMESPACE = "namespace";
const std::string TAG_IMAGES = "images";
const std::string TAG_WORKLOADNAME = "workloadName";
const std::string TAG_WORKLOADKIND = "workloadKind";
const std::string TAG_SERVICENAME = "serviceName";
const std::string TAG_LABELS = "labels";

struct k8sContainerInfo {
std::unordered_map<std::string, std::string> images;
Expand Down
22 changes: 11 additions & 11 deletions core/metadata/LabelingK8sMetadata.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
#include "models/SpanEvent.h"
#include "monitor/metric_constants/MetricConstants.h"
#include <boost/utility/string_view.hpp>
#include "K8sMetadata.h"


using logtail::StringView;

Expand Down Expand Up @@ -75,7 +75,7 @@ bool LabelingK8sMetadata::AddLabels(Event& e, std::vector<std::string>& containe
bool res = true;

auto& k8sMetadata = K8sMetadata::GetInstance();
StringView containerIdViewKey(containerIdKey);
StringView containerIdViewKey(DEFAULT_TRACE_TAG_CONTAINER_ID);
NameHaibinZhang marked this conversation as resolved.
Show resolved Hide resolved
StringView containerIdView = e.HasTag(containerIdViewKey) ? e.GetTag(containerIdViewKey) : StringView{};
if (!containerIdView.empty()) {
std::string containerId(containerIdView);
Expand All @@ -84,14 +84,14 @@ bool LabelingK8sMetadata::AddLabels(Event& e, std::vector<std::string>& containe
containerVec.push_back(containerId);
res = false;
} else {
e.SetTag(workloadNameKey, containerInfo->workloadName);
e.SetTag(workloadKindKey, containerInfo->workloadKind);
e.SetTag(namespaceKey, containerInfo->k8sNamespace);
e.SetTag(serviceNameKey, containerInfo->serviceName);
e.SetTag(pidKey, containerInfo->appId);
e.SetTag(DEFAULT_TRACE_TAG_WORKLOAD_NAME, containerInfo->workloadName);
e.SetTag(DEFAULT_TRACE_TAG_WORKLOAD_KIND, containerInfo->workloadKind);
e.SetTag(DEFAULT_TRACE_TAG_NAMESPACE, containerInfo->k8sNamespace);
e.SetTag(DEFAULT_TRACE_TAG_SERVICENAME, containerInfo->serviceName);
e.SetTag(DEFAULT_TRACE_TAG_PID, containerInfo->appId);
}
}
StringView ipView(remoteIpKey);
StringView ipView(DEFAULT_TRACE_TAG_REMOTE_IP);
StringView remoteIpView = e.HasTag(ipView) ? e.GetTag(ipView) : StringView{};
if (!remoteIpView.empty()) {
std::string remoteIp(remoteIpView);
Expand All @@ -100,9 +100,9 @@ bool LabelingK8sMetadata::AddLabels(Event& e, std::vector<std::string>& containe
remoteIpVec.push_back(remoteIp);
res = false;
} else {
e.SetTag(peerWorkloadNameKey, ipInfo->workloadName);
e.SetTag(peerWorkloadKindKey, ipInfo->workloadKind);
e.SetTag(peerNamespaceKey, ipInfo->k8sNamespace);
e.SetTag(DEFAULT_TRACE_TAG_PEER_WORKLOAD_NAME, ipInfo->workloadName);
e.SetTag(DEFAULT_TRACE_TAG_PEER_WORKLOAD_KIND, ipInfo->workloadKind);
e.SetTag(DEFAULT_TRACE_TAG_PEER_NAMESPACE, ipInfo->k8sNamespace);
}
}
return res;
Expand Down
8 changes: 1 addition & 7 deletions core/metadata/LabelingK8sMetadata.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,10 @@

#include "models/LogEvent.h"
#include "plugin/interface/Processor.h"
#include "K8sMetadata.h"

namespace logtail {

const static std::string containerIdKey = "container.id";
const static std::string pidKey = "pid";
const static std::string remoteIpKey = "remote_ip";
const static std::string peerWorkloadNameKey = "peerWorkloadName";
const static std::string peerWorkloadKindKey = "peerWorkloadKind";
const static std::string peerNamespaceKey = "peerNamespace";

class LabelingK8sMetadata {
public:
void AddLabelToLogGroup(PipelineEventGroup& logGroup);
Expand Down
62 changes: 46 additions & 16 deletions core/unittest/metadata/K8sMetadataUnittest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,36 @@ class k8sMetadataUnittest : public ::testing::Test {
}

public:

void TestFromInfoJson() {
// 创建Json测试数据
Json::Value testJson;
testJson["images"]["image1"] = "nginx:latest";
testJson["labels"]["app_id"] = "my_app_id";
testJson["namespace"] = "default";
testJson["workloadKind"] = "Deployment";
testJson["workloadName"] = "nginx-deployment";
testJson["serviceName"] = "nginx-service";

k8sContainerInfo info;
auto& k8sMetadata = K8sMetadata::GetInstance();

// 调用函数
bool result = k8sMetadata.FromInfoJson(testJson, info);

// 验证结果
EXPECT_TRUE(result);
EXPECT_EQ(info.images["image1"], "nginx:latest");
EXPECT_EQ(info.labels["app_id"], "my_app_id");
EXPECT_EQ(info.k8sNamespace, "default");
EXPECT_EQ(info.workloadKind, "Deployment");
EXPECT_EQ(info.workloadName, "nginx-deployment");
EXPECT_EQ(info.serviceName, "nginx-service");
}

void TestGetByContainerIds() {
LOG_INFO(sLogger, ("TestGetByContainerIds() begin", time(NULL)));
const std::string jsonData = R"({"containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},"images":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}})";
const std::string jsonData = R"({"containerd://286effd2650c0689b779018e42e9ec7aa3d2cb843005e038204e85fc3d4f9144":{"namespace":"default","workloadName":"oneagent-demo-658648895b","workloadKind":"replicaset","serviceName":"","labels":{"app":"oneagent-demo","pod-template-hash":"658648895b"},"envs":{},""images"":{"oneagent-demo":"sls-opensource-registry.cn-shanghai.cr.aliyuncs.com/ilogtail-community-edition/centos7-cve-fix:1.0.0"}}})";

Json::Value root;
Json::CharReaderBuilder readerBuilder;
Expand Down Expand Up @@ -75,7 +102,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"COREDNS_NAMESPACE": "",
"COREDNS_POD_NAME": ""
},
"images": {
""images"": {
"coredns": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/coredns:v1.9.3.10-7dfca203-aliyun"
}
},
Expand All @@ -94,7 +121,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"KUBE_NODE_NAME": "",
"SERVICE_TYPE": "provisioner"
},
"images": {
""images"": {
"csi-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-plugin:v1.30.3-921e63a-aliyun",
"external-csi-snapshotter": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-snapshotter:v4.0.0-a230d5b-aliyun",
"external-disk-attacher": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-attacher:v4.5.0-4a01fda6-aliyun",
Expand All @@ -119,7 +146,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"envs": {
"NODE_NAME": ""
},
"images": {
""images"": {
"kube-proxy-worker": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/kube-proxy:v1.30.1-aliyun.1"
}
}
Expand Down Expand Up @@ -148,7 +175,7 @@ class k8sMetadataUnittest : public ::testing::Test {
{
"name": "test",
"tags": {
"remote_ip": "172.16.20.108"
"remote.ip": "172.16.20.108"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
Expand All @@ -174,7 +201,7 @@ class k8sMetadataUnittest : public ::testing::Test {
processor.AddLabelToLogGroup(eventGroup);
EventsContainer& eventsEnd = eventGroup.MutableEvents();
auto& metricEvent = eventsEnd[0].Cast<MetricEvent>();
APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peerWorkloadName").to_string());
APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peer.workload.name").to_string());
APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr);
}

Expand All @@ -195,7 +222,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"COREDNS_NAMESPACE": "",
"COREDNS_POD_NAME": ""
},
"images": {
""images"": {
"coredns": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/coredns:v1.9.3.10-7dfca203-aliyun"
}
},
Expand All @@ -214,7 +241,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"KUBE_NODE_NAME": "",
"SERVICE_TYPE": "provisioner"
},
"images": {
""images"": {
"csi-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-plugin:v1.30.3-921e63a-aliyun",
"external-csi-snapshotter": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-snapshotter:v4.0.0-a230d5b-aliyun",
"external-disk-attacher": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-attacher:v4.5.0-4a01fda6-aliyun",
Expand All @@ -239,7 +266,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"envs": {
"NODE_NAME": ""
},
"images": {
""images"": {
"kube-proxy-worker": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/kube-proxy:v1.30.1-aliyun.1"
}
}
Expand Down Expand Up @@ -274,7 +301,7 @@ class k8sMetadataUnittest : public ::testing::Test {
mSpanEvent->SetStartTimeNs(1715826723000000000);
mSpanEvent->SetEndTimeNs(1715826725000000000);
mSpanEvent->SetTag(string("key1"), string("value1"));
mSpanEvent->SetTag(string("remote_ip"), string("172.16.20.108"));
mSpanEvent->SetTag(string("remote.ip"), string("172.16.20.108"));
SpanEvent::InnerEvent* e = mSpanEvent->AddEvent();
e->SetName("test_event");
e->SetTimestampNs(1715826724000000000);
Expand All @@ -287,7 +314,9 @@ class k8sMetadataUnittest : public ::testing::Test {
mSpanEvent->SetScopeTag(string("key2"), string("value2"));
LabelingK8sMetadata& processor = *(new LabelingK8sMetadata);
processor.AddLabels(*mSpanEvent, container_vec, remote_ip_vec);
APSARA_TEST_EQUAL("kube-proxy-worker", mSpanEvent->GetTag("peerWorkloadName").to_string());
APSARA_TEST_EQUAL("kube-system", mSpanEvent->GetTag("k8s.namespace.name").to_string());
APSARA_TEST_EQUAL("daemonset", mSpanEvent->GetTag("peer.workload.kind").to_string());
APSARA_TEST_EQUAL("kube-proxy-worker", mSpanEvent->GetTag("peer.workload.name").to_string());
APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr);
}

Expand All @@ -309,7 +338,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"COREDNS_NAMESPACE": "",
"COREDNS_POD_NAME": ""
},
"images": {
""images"": {
"coredns": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/coredns:v1.9.3.10-7dfca203-aliyun"
}
},
Expand All @@ -328,7 +357,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"KUBE_NODE_NAME": "",
"SERVICE_TYPE": "provisioner"
},
"images": {
""images"": {
"csi-provisioner": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-plugin:v1.30.3-921e63a-aliyun",
"external-csi-snapshotter": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-snapshotter:v4.0.0-a230d5b-aliyun",
"external-disk-attacher": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/csi-attacher:v4.5.0-4a01fda6-aliyun",
Expand All @@ -353,7 +382,7 @@ class k8sMetadataUnittest : public ::testing::Test {
"envs": {
"NODE_NAME": ""
},
"images": {
""images"": {
"kube-proxy-worker": "registry-cn-chengdu-vpc.ack.aliyuncs.com/acs/kube-proxy:v1.30.1-aliyun.1"
}
}
Expand Down Expand Up @@ -383,7 +412,7 @@ class k8sMetadataUnittest : public ::testing::Test {
{
"name": "test",
"tags": {
"remote_ip": "172.16.20.108"
"remote.ip": "172.16.20.108"
},
"timestamp" : 12345678901,
"timestampNanosecond" : 0,
Expand Down Expand Up @@ -412,7 +441,7 @@ class k8sMetadataUnittest : public ::testing::Test {
processor.AddLabels(events[0].Cast<MetricEvent>(), container_vec, remote_ip_vec);
EventsContainer& eventsEnd = eventGroup.MutableEvents();
auto& metricEvent = eventsEnd[0].Cast<MetricEvent>();
APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peerWorkloadName").to_string());
APSARA_TEST_EQUAL("kube-proxy-worker", metricEvent.GetTag("peer.workload.name").to_string());
APSARA_TEST_TRUE_FATAL(k8sMetadata.GetInfoByIpFromCache("10.41.0.2") != nullptr);
}
};
Expand All @@ -421,6 +450,7 @@ APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestGetByContainerIds, 0);
APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestGetByLocalHost, 1);
APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToMetric, 2);
APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestAddLabelToSpan, 3);
APSARA_UNIT_TEST_CASE(k8sMetadataUnittest, TestFromInfoJson, 4);



Expand Down
Loading