Skip to content

Commit

Permalink
Merge remote-tracking branch 'baidu_tera/master' into rb.m.ng
Browse files Browse the repository at this point in the history
  • Loading branch information
@lylei
lylei committed Nov 4, 2015
2 parents 1b2eddb + 2e5dec2 commit 0323460
Show file tree
Hide file tree
Showing 10 changed files with 107 additions and 81 deletions.
8 changes: 6 additions & 2 deletions src/io/tablet_io.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,12 @@ std::string TabletIO::GetTableName() const {

std::string TabletIO::GetTablePath() const {
if (!m_tablet_path.empty()) {
return m_tablet_path.substr(FLAGS_tera_tabletnode_path_prefix.size());
std::string path =
m_tablet_path.substr(FLAGS_tera_tabletnode_path_prefix.size());
if (path.at(0) == '/') {
path = path.substr(1);
}
return path;
} else {
return m_tablet_path;
}
Expand Down Expand Up @@ -1482,7 +1487,6 @@ void TabletIO::SetupOptionsForLG() {
<< ", buffer_size:" << lg_info->memtable_ldb_write_buffer_size
<< ", block_size:" << lg_info->memtable_ldb_block_size;
}
LOG(INFO) << ", sst_size: " << lg_schema.sst_size() << " Bytes.";
lg_info->sst_size = lg_schema.sst_size();
m_ldb_options.sst_size = lg_schema.sst_size();
exist_lg_list->insert(lg_i);
Expand Down
4 changes: 2 additions & 2 deletions src/leveldb/db/db_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1697,10 +1697,10 @@ void DBImpl::GetApproximateSizes(uint64_t* size, std::vector<uint64_t>* lgsize)
// add mem&imm size
if (size) {
if (mem_) {
size += mem_->ApproximateMemoryUsage();
*size += mem_->ApproximateMemoryUsage();
}
if (imm_) {
size += imm_->ApproximateMemoryUsage();
*size += imm_->ApproximateMemoryUsage();
}
}
}
Expand Down
1 change: 0 additions & 1 deletion src/leveldb/include/leveldb/env.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,6 @@ class EnvWrapper : public Env {
return target_->NewLogger(fname, result);
}
virtual void SetLogger(Logger* logger) {
Logger::SetDefaultLogger(logger);
return target_->SetLogger(logger);
}
uint64_t NowMicros() {
Expand Down
1 change: 1 addition & 0 deletions src/leveldb/util/env_posix.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,7 @@ class PosixEnv : public Env {
}
}
virtual void SetLogger(Logger* logger) {
Logger::SetDefaultLogger(logger);
info_log_ = logger;
thread_pool_.SetLogger(logger);
}
Expand Down
71 changes: 54 additions & 17 deletions src/master/master_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ DECLARE_bool(tera_ins_enabled);
DECLARE_int64(tera_sdk_perf_counter_log_interval);

DECLARE_bool(tera_acl_enabled);
DECLARE_bool(tera_only_root_create_table);
DECLARE_string(tera_master_gc_strategy);

namespace tera {
Expand Down Expand Up @@ -155,6 +156,7 @@ bool MasterImpl::Init() {
m_zk_adapter.reset(new FakeMasterZkAdapter(this, m_local_addr));
}

LOG(INFO) << "[acl] " << (FLAGS_tera_acl_enabled ? "enabled" : "disabled");
SetMasterStatus(kIsSecondary);
m_thread_pool->AddTask(boost::bind(&MasterImpl::InitAsync, this));
return true;
Expand Down Expand Up @@ -462,16 +464,37 @@ bool MasterImpl::IsRootUser(const std::string& token) {
return m_user_manager->UserNameToToken("root") == token;
}

// user is admin or user is in admin_group
bool MasterImpl::CheckUserPermissionOnTable(const std::string& token, TablePtr table) {
std::string group_name = table->GetSchema().admin_group();
std::string user_name = m_user_manager->TokenToUserName(token);
return (m_user_manager->IsUserInGroup(user_name, group_name)
|| (table->GetSchema().admin() == m_user_manager->TokenToUserName(token)));
}

template <typename Request>
bool MasterImpl::HasPermissionOnTable(const Request* request, TablePtr table) {
if (!FLAGS_tera_acl_enabled
|| IsRootUser(request->user_token())
|| ((table->GetSchema().admin_group() == "") && (table->GetSchema().admin() == ""))
|| (request->has_user_token()
&& CheckUserPermissionOnTable(request->user_token(), table))) {
return true;

}
return false;
}

template <typename Request, typename Response, typename Callback>
bool MasterImpl::HasTablePermission(const Request* request, Response* response,
Callback* done, TablePtr table, const char* operate) {
bool MasterImpl::HasPermissionOrReturn(const Request* request, Response* response,
Callback* done, TablePtr table, const char* operate) {
// check permission
if (!FLAGS_tera_acl_enabled
|| IsRootUser(request->user_token())) {
LOG(INFO) << "[acl] is acl enabled: " << FLAGS_tera_acl_enabled;
if (HasPermissionOnTable(request, table)) {
return true;
} else {
LOG(INFO) << "[acl] fail to " << operate;
std::string token = request->has_user_token() ? request->user_token() : "";
LOG(INFO) << "[acl] " << m_user_manager->TokenToUserName(token)
<< ":" << token << " fail to " << operate;
response->set_sequence_id(request->sequence_id());
response->set_status(kNotPermission);
done->Run();
Expand Down Expand Up @@ -661,11 +684,13 @@ void MasterImpl::CreateTable(const CreateTableRequest* request,
done->Run();
return;
}
if (FLAGS_tera_acl_enabled && !IsRootUser(request->user_token())) {
response->set_sequence_id(request->sequence_id());
response->set_status(kNotPermission);
done->Run();
return;
if (FLAGS_tera_acl_enabled
&& !IsRootUser(request->user_token())
&& FLAGS_tera_only_root_create_table) {
response->set_sequence_id(request->sequence_id());
response->set_status(kNotPermission);
done->Run();
return;
}
if (!request->schema().alias().empty()) {
bool alias_exist = false;
Expand Down Expand Up @@ -786,7 +811,7 @@ void MasterImpl::DeleteTable(const DeleteTableRequest* request,
done->Run();
return;
}
if (!HasTablePermission(request, response, done, table, "delete table")) {
if (!HasPermissionOrReturn(request, response, done, table, "delete table")) {
return;
}

Expand Down Expand Up @@ -842,7 +867,7 @@ void MasterImpl::DisableTable(const DisableTableRequest* request,
done->Run();
return;
}
if (!HasTablePermission(request, response, done, table, "disable table")) {
if (!HasPermissionOrReturn(request, response, done, table, "disable table")) {
return;
}

Expand Down Expand Up @@ -901,7 +926,7 @@ void MasterImpl::EnableTable(const EnableTableRequest* request,
done->Run();
return;
}
if (!HasTablePermission(request, response, done, table, "enable table")) {
if (!HasPermissionOrReturn(request, response, done, table, "enable table")) {
return;
}

Expand Down Expand Up @@ -952,7 +977,7 @@ void MasterImpl::UpdateTable(const UpdateTableRequest* request,
done->Run();
return;
}
if (!HasTablePermission(request, response, done, table, "update table")) {
if (!HasPermissionOrReturn(request, response, done, table, "update table")) {
return;
}

Expand Down Expand Up @@ -1074,11 +1099,19 @@ void MasterImpl::ShowTables(const ShowTablesRequest* request,
TableMetaList* table_meta_list = response->mutable_table_meta_list();
for (uint32_t i = 0; i < table_list.size(); ++i) {
TablePtr table = table_list[i];
// if a user has NO permission on a table,
// he/she should not notice this table
if (!HasPermissionOnTable(request, table)) {
continue;
}
table->ToMeta(table_meta_list->add_meta());
}
TabletMetaList* tablet_meta_list = response->mutable_tablet_meta_list();
for (uint32_t i = 0; i < tablet_list.size(); ++i) {
TabletPtr tablet = tablet_list[i];
if (!HasPermissionOnTable(request, tablet->GetTable())) {
continue;
}
TabletMeta meta;
tablet->ToMeta(&meta);
tablet_meta_list->add_meta()->CopyFrom(meta);
Expand Down Expand Up @@ -1128,6 +1161,9 @@ void MasterImpl::ShowTabletNodes(const ShowTabletNodesRequest* request,
std::vector<TabletPtr> tablet_list;
m_tablet_manager->FindTablet(request->addr(), &tablet_list);
for (size_t i = 0; i < tablet_list.size(); ++i) {
if (!HasPermissionOnTable(request, tablet_list[i]->GetTable())) {
continue;
}
TabletMeta* meta = response->mutable_tabletmeta_list()->add_meta();
TabletCounter* counter = response->mutable_tabletmeta_list()->add_counter();
tablet_list[i]->ToMeta(meta);
Expand Down Expand Up @@ -3460,12 +3496,13 @@ void MasterImpl::QueryTabletNodeCallback(std::string addr, QueryRequest* request
ClearUnusedSnapshots(tablet, meta);
VLOG(30) << "[query] " << tablet;
} else {
VLOG(30) << "fail to match tablet: " << meta.table_name()
LOG(WARNING) << "fail to match tablet: " << meta.table_name()
<< ", path: " << meta.path()
<< ", range: [" << DebugString(key_start)
<< ", " << DebugString(key_end)
<< "], size: " << meta.size()
<< ", addr: " << meta.server_addr();
<< ", addr: " << meta.server_addr()
<< ", tablet: " << tablet;
}
}

Expand Down
10 changes: 8 additions & 2 deletions src/master/master_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -528,9 +528,15 @@ class MasterImpl {

bool IsRootUser(const std::string& token);

bool CheckUserPermissionOnTable(const std::string& token, TablePtr table);

template <typename Request>
bool HasPermissionOnTable(const Request* request, TablePtr table);

template <typename Request, typename Response, typename Callback>
bool HasTablePermission(const Request* request, Response* response,
Callback* done, TablePtr table, const char* operate);
bool HasPermissionOrReturn(const Request* request, Response* response,
Callback* done, TablePtr table, const char* operate);

void FillAlias(const std::string& key, const std::string& value);
private:
mutable Mutex m_status_mutex;
Expand Down
61 changes: 23 additions & 38 deletions src/sdk/client_impl.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,43 +511,24 @@ bool ClientImpl::ShowTablesInfo(const string& name,
TableMeta* meta,
TabletMetaList* tablet_list,
ErrorCode* err) {
if (meta == NULL || tablet_list == NULL) {
return false;
}
tablet_list->Clear();
std::string internal_table_name;
if (!GetInternalTableName(name, err, &internal_table_name)) {
LOG(ERROR) << "faild to scan meta schema";
return false;
TableMetaList table_list;
bool result = DoShowTablesInfo(&table_list, tablet_list, name, err);
if (result) {
meta->CopyFrom(table_list.meta(0));
}
master::MasterClient master_client(_cluster->MasterAddr());

ShowTablesRequest request;
ShowTablesResponse response;
request.set_sequence_id(0);
request.set_start_table_name(internal_table_name);
request.set_max_table_num(1);
request.set_user_token(GetUserToken(_user_identity, _user_passcode));

if (master_client.ShowTables(&request, &response) &&
response.status() == kMasterOk) {
if (response.table_meta_list().meta_size() == 0) {
return false;
} else if (response.table_meta_list().meta(0).table_name() != internal_table_name) {
return false;
}
meta->CopyFrom(response.table_meta_list().meta(0));
tablet_list->CopyFrom(response.tablet_meta_list());
return true;
}
LOG(ERROR) << "fail to show table info: " << name;
err->SetFailed(ErrorCode::kSystem, StatusCodeToString(response.status()));
return false;
return result;
}

bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,
TabletMetaList* tablet_list,
ErrorCode* err) {
return DoShowTablesInfo(table_list, tablet_list, "", err);
}

bool ClientImpl::DoShowTablesInfo(TableMetaList* table_list,
TabletMetaList* tablet_list,
const string& table_name,
ErrorCode* err) {
if (table_list == NULL || tablet_list == NULL) {
return false;
}
Expand All @@ -556,14 +537,17 @@ bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,

master::MasterClient master_client(_cluster->MasterAddr());
std::string start_tablet_key;
std::string start_table_name;
std::string start_table_name = table_name;
bool has_more = true;
bool has_error = false;
bool table_meta_copied = false;
std::string err_msg;
while(has_more && !has_error) {
ShowTablesRequest request;
ShowTablesResponse response;
if (!table_name.empty()) {
request.set_max_table_num(1);
}
request.set_start_table_name(start_table_name);
request.set_start_tablet_key(start_tablet_key);
request.set_max_tablet_num(FLAGS_tera_sdk_show_max_num); //tablets be fetched at most in one RPC
Expand Down Expand Up @@ -614,17 +598,18 @@ bool ClientImpl::ShowTablesInfo(TableMetaList* table_list,

if (has_error) {
LOG(ERROR) << "fail to show table info.";
err->SetFailed(ErrorCode::kSystem, err_msg);
if (err != NULL) {
err->SetFailed(ErrorCode::kSystem, err_msg);
}
return false;
}
return true;
}


bool ClientImpl::ShowTabletNodesInfo(const string& addr,
TabletNodeInfo* info,
TabletMetaList* tablet_list,
ErrorCode* err) {
TabletNodeInfo* info,
TabletMetaList* tablet_list,
ErrorCode* err) {
if (info == NULL || tablet_list == NULL) {
return false;
}
Expand Down Expand Up @@ -655,7 +640,7 @@ bool ClientImpl::ShowTabletNodesInfo(const string& addr,
}

bool ClientImpl::ShowTabletNodesInfo(std::vector<TabletNodeInfo>* infos,
ErrorCode* err) {
ErrorCode* err) {
if (infos == NULL) {
return false;
}
Expand Down
7 changes: 7 additions & 0 deletions src/sdk/client_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,13 @@ class ClientImpl : public Client {
bool CheckReturnValue(StatusCode status, std::string& reason, ErrorCode* err);
bool GetInternalTableName(const std::string& table_name, ErrorCode* err,
std::string* internal_table_name);

/// show all tables info: `table_name' should be an empty string
/// show a single table info: `table_name' should be the table name
bool DoShowTablesInfo(TableMetaList* table_list,
TabletMetaList* tablet_list,
const string& table_name,
ErrorCode* err);
private:
ClientImpl(const ClientImpl&);
void operator=(const ClientImpl&);
Expand Down
1 change: 0 additions & 1 deletion src/tabletnode/tabletnode_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
#include "proto/master_rpc.pb.h"
#include "proto/tabletnode.pb.h"
#include "proto/tabletnode_rpc.pb.h"
#include "proto/table_meta.pb.h"
#include "tabletnode/rpc_compactor.h"
#include "tabletnode/tabletnode_sysinfo.h"
#include "utils/rpc_timer_list.h"
Expand Down
Loading

0 comments on commit 0323460

Please sign in to comment.