From 62a58bff4c2f640f1afcba8c754058d5f77d420f Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Sun, 8 Dec 2024 04:26:23 +0800 Subject: [PATCH] [fix](storage vault) Fix missing use_path_style when create storage vault (#45155) --- .../meta-service/meta_service_resource.cpp | 27 +++++++++++-------- .../apache/doris/catalog/StorageVault.java | 4 +++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index db232eb5314783..4fa8cc5a132296 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -765,6 +765,7 @@ struct ObjectStorageDesc { std::string& endpoint; std::string& external_endpoint; std::string& region; + bool& use_path_style; }; static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, @@ -777,7 +778,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, msg = "s3 obj info err " + proto_to_json(*request); return -1; } - auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_desc; + auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style] = obj_desc; const auto& obj = request->has_obj() ? request->obj() : request->vault().obj_info(); // Prepare data if (!obj.has_ak() || !obj.has_sk()) { @@ -803,6 +804,7 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, endpoint = obj.has_endpoint() ? obj.endpoint() : ""; external_endpoint = obj.has_external_endpoint() ? obj.external_endpoint() : ""; region = obj.has_region() ? obj.region() : ""; + use_path_style = obj.use_path_style(); // obj size > 1k, refuse if (obj.ByteSizeLong() > 1024) { code = MetaServiceCode::INVALID_ARGUMENT; @@ -812,13 +814,13 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, return 0; } -static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info, +static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_desc, const ObjectStoreInfoPB& obj, InstanceInfoPB& instance, EncryptionInfoPB& encryption_info, AkSkPair& cipher_ak_sk_pair) { ObjectStoreInfoPB last_item; - auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region] = obj_info; + auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style] = obj_desc; auto now_time = std::chrono::system_clock::now(); uint64_t time = std::chrono::duration_cast(now_time.time_since_epoch()).count(); @@ -840,6 +842,7 @@ static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_info, last_item.set_region(region); last_item.set_provider(obj.provider()); last_item.set_sse_enabled(instance.sse_enabled()); + last_item.set_use_path_style(use_path_style); return last_item; } @@ -848,14 +851,15 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr AlterObjStoreInfoResponse* response, ::google::protobuf::Closure* done) { std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region; + bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; RPC_PREPROCESS(alter_storage_vault); switch (request->op()) { case AlterObjStoreInfoRequest::ADD_S3_VAULT: case AlterObjStoreInfoRequest::DROP_S3_VAULT: { - auto tmp_desc = - ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region}; + auto tmp_desc = ObjectStorageDesc { + ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style}; if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info, cipher_ak_sk_pair)) { return; @@ -994,8 +998,8 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr } } // calc id - auto tmp_tuple = - ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region}; + auto tmp_tuple = ObjectStorageDesc { + ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style}; ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance, encryption_info, cipher_ak_sk_pair); if (instance.storage_vault_names().end() != @@ -1138,6 +1142,7 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont AlterObjStoreInfoResponse* response, ::google::protobuf::Closure* done) { std::string ak, sk, bucket, prefix, endpoint, external_endpoint, region; + bool use_path_style; EncryptionInfoPB encryption_info; AkSkPair cipher_ak_sk_pair; RPC_PREPROCESS(alter_obj_store_info); @@ -1145,8 +1150,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont case AlterObjStoreInfoRequest::ADD_OBJ_INFO: case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK: case AlterObjStoreInfoRequest::UPDATE_AK_SK: { - auto tmp_desc = - ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region}; + auto tmp_desc = ObjectStorageDesc { + ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style}; if (0 != extract_object_storage_info(request, code, msg, tmp_desc, encryption_info, cipher_ak_sk_pair)) { return; @@ -1285,8 +1290,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont } } // calc id - auto tmp_tuple = - ObjectStorageDesc {ak, sk, bucket, prefix, endpoint, external_endpoint, region}; + auto tmp_tuple = ObjectStorageDesc { + ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style}; ObjectStoreInfoPB last_item = object_info_pb_factory(tmp_tuple, obj, instance, encryption_info, cipher_ak_sk_pair); instance.add_obj_info()->CopyFrom(last_item); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java index df9310526e4757..9d45ce7bdd8f51 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVault.java @@ -23,6 +23,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.qe.ShowResultSetMetaData; import com.google.common.base.Strings; @@ -144,6 +145,9 @@ public void setId(String id) { vault.modifyProperties(stmt.getProperties()); break; case S3: + if (!stmt.getProperties().containsKey(PropertyConverter.USE_PATH_STYLE)) { + stmt.getProperties().put(PropertyConverter.USE_PATH_STYLE, "true"); + } CreateResourceStmt resourceStmt = new CreateResourceStmt(false, ifNotExists, name, stmt.getProperties()); resourceStmt.analyzeResourceType();