[Fix](security) All sk in ms log should be encrypted

cloud/src/meta-service/meta_service_helper.h

@@ -19,7 +19,9 @@
 
 #include <brpc/controller.h>
 #include <gen_cpp/cloud.pb.h>
+#include <openssl/md5.h>
 
+#include <iomanip>
 #include <memory>
 #include <string>
 #include <string_view>
@@ -29,12 +31,26 @@
 #include "common/logging.h"
 #include "common/stopwatch.h"
 #include "common/util.h"
+#include "cpp/sync_point.h"
 #include "meta-service/keys.h"
 #include "meta-service/txn_kv.h"
 #include "meta-service/txn_kv_error.h"
 #include "resource-manager/resource_manager.h"
 
 namespace doris::cloud {
+inline std::string md5(const std::string& str) {
+    unsigned char digest[MD5_DIGEST_LENGTH];
+    MD5_CTX context;
+    MD5_Init(&context);
+    MD5_Update(&context, str.c_str(), str.length());
+    MD5_Final(digest, &context);
+
+    std::ostringstream ss;
+    for (unsigned char i : digest) {
+        ss << std::setw(2) << std::setfill('0') << std::hex << (int)i;
+    }
+    return ss.str();
+}
 
 template <class Request>
 void begin_rpc(std::string_view func_name, brpc::Controller* ctrl, const Request* req) {
@@ -101,7 +117,34 @@ void finish_rpc(std::string_view func_name, brpc::Controller* ctrl, Response* re
         LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
                   << " status=" << res->status().ShortDebugString()
                   << " delete_bitmap_size=" << res->segment_delete_bitmaps_size();
-
+    } else if constexpr (std::is_same_v<Response, GetObjStoreInfoResponse> ||
+                         std::is_same_v<Response, GetStageResponse>) {
+        std::string debug_string = res->DebugString();
+        // Start position for searching "sk" fields
+        size_t pos = 0;
+        // Iterate through the string and find all occurrences of "sk: "
+        while ((pos = debug_string.find("sk: ", pos)) != std::string::npos) {
+            // Find the start and end of the "sk" value (assumed to be within quotes)
+            // Start after the quote
+            size_t sk_value_start = debug_string.find('\"', pos) + 1;
+            // End at the next quote
+            size_t sk_value_end = debug_string.find('\"', sk_value_start);
+
+            // Extract the "sk" value
+            std::string sk_value =
+                    debug_string.substr(sk_value_start, sk_value_end - sk_value_start);
+            // Encrypt the "sk" value with MD5
+            std::string encrypted_sk = "md5: " + md5(sk_value);
+
+            // Replace the original "sk" value with the encrypted MD5 value
+            debug_string.replace(sk_value_start, sk_value_end - sk_value_start, encrypted_sk);
+
+            // Move the position to the end of the current "sk" field and continue searching
+            pos = sk_value_end;
+        }
+        TEST_SYNC_POINT_CALLBACK("sk_finish_rpc", &debug_string);
+        LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
+                  << " response=" << debug_string;
     } else {
         LOG(INFO) << "finish " << func_name << " from " << ctrl->remote_side()
                   << " response=" << res->ShortDebugString();

cloud/src/meta-service/meta_service_resource.cpp

@@ -203,6 +203,8 @@ void MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
                                          GetObjStoreInfoResponse* response,
                                          ::google::protobuf::Closure* done) {
     RPC_PREPROCESS(get_obj_store_info);
+    TEST_SYNC_POINT_CALLBACK("obj-store-info_sk_response", &response);
+    TEST_SYNC_POINT_RETURN_WITH_VOID("obj-store-info_sk_response_return");
     // Prepare data
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
@@ -2590,6 +2592,8 @@ void MetaServiceImpl::get_stage(google::protobuf::RpcController* controller,
                                 const GetStageRequest* request, GetStageResponse* response,
                                 ::google::protobuf::Closure* done) {
     RPC_PREPROCESS(get_stage);
+    TEST_SYNC_POINT_CALLBACK("stage_sk_response", &response);
+    TEST_SYNC_POINT_RETURN_WITH_VOID("stage_sk_response_return");
     std::string cloud_unique_id = request->has_cloud_unique_id() ? request->cloud_unique_id() : "";
     if (cloud_unique_id.empty()) {
         code = MetaServiceCode::INVALID_ARGUMENT;

cloud/test/meta_service_http_test.cpp

@@ -1456,4 +1456,83 @@ TEST(MetaServiceHttpTest, TxnLazyCommit) {
     }
 }
 
+TEST(MetaServiceHttpTest, get_stage_response_sk) {
+    auto sp = SyncPoint::get_instance();
+    sp->enable_processing();
+    std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+                                                          [&](...) { sp->disable_processing(); });
+
+    GetStageResponse res;
+    auto* stage = res.add_stage();
+    stage->mutable_obj_info()->set_ak("stage-ak");
+    stage->mutable_obj_info()->set_sk("stage-sk");
+    auto foo = [res](auto args) { (*(try_any_cast<GetStageResponse**>(args[0])))->CopyFrom(res); };
+    sp->set_call_back("stage_sk_response", foo);
+    sp->set_call_back("stage_sk_response_return",
+                      [](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });
+
+    auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+    auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);
+
+    auto bar = [](auto args) {
+        std::cout << *try_any_cast<std::string*>(args[0]);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("stage-sk") == std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: f497d053066fa4b7d3b1f6564597d233") != std::string::npos);
+    };
+    sp->set_call_back("sk_finish_rpc", bar);
+
+    GetStageResponse res1;
+    GetStageRequest req1;
+    brpc::Controller cntl;
+    ms->get_stage(&cntl, &req1, &res1, nullptr);
+}
+
+TEST(MetaServiceHttpTest, get_obj_store_info_response_sk) {
+    auto sp = SyncPoint::get_instance();
+    sp->enable_processing();
+    std::unique_ptr<int, std::function<void(int*)>> defer((int*)0x01,
+                                                          [&](...) { sp->disable_processing(); });
+
+    GetObjStoreInfoResponse res;
+    auto* obj_info = res.add_obj_info();
+    obj_info->set_ak("obj-store-info-ak1");
+    obj_info->set_sk("obj-store-info-sk1");
+    obj_info = res.add_storage_vault()->mutable_obj_info();
+    obj_info->set_ak("obj-store-info-ak2");
+    obj_info->set_sk("obj-store-info-sk2");
+    auto foo = [res](auto args) {
+        (*(try_any_cast<GetObjStoreInfoResponse**>(args[0])))->CopyFrom(res);
+    };
+    sp->set_call_back("obj-store-info_sk_response", foo);
+    sp->set_call_back("obj-store-info_sk_response_return",
+                      [](auto&& args) { *try_any_cast<bool*>(args.back()) = true; });
+
+    auto rate_limiter = std::make_shared<cloud::RateLimiter>();
+
+    auto ms = std::make_unique<cloud::MetaServiceImpl>(nullptr, nullptr, rate_limiter);
+
+    auto bar = [](auto args) {
+        std::cout << *try_any_cast<std::string*>(args[0]);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk1") ==
+                    std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: 35d5a637fd9d45a28207a888b751efc4") != std::string::npos);
+
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0])).find("obj-store-info-sk2") ==
+                    std::string::npos);
+        EXPECT_TRUE((*try_any_cast<std::string*>(args[0]))
+                            .find("md5: 01d7473ae201a2ecdf1f7c064eb81a95") != std::string::npos);
+    };
+    sp->set_call_back("sk_finish_rpc", bar);
+
+    GetObjStoreInfoResponse res1;
+    GetObjStoreInfoRequest req1;
+    brpc::Controller cntl;
+    ms->get_obj_store_info(&cntl, &req1, &res1, nullptr);
+}
+
 } // namespace doris::cloud