issue=#1244 tera_main split to tera_master and tabletserver (#1245)

00k · taocp · commit 5cd780ad0494 · 2017-05-02T09:48:29.000+08:00
diff --git a/Makefile b/Makefile
@@ -36,7 +36,8 @@ OTHER_SRC := $(wildcard src/zk/*.cc) $(wildcard src/utils/*.cc) $(VERSION_SRC) \
              src/tera_flags.cc
 COMMON_SRC := $(wildcard src/common/base/*.cc) $(wildcard src/common/net/*.cc) \
               $(wildcard src/common/file/*.cc) $(wildcard src/common/file/recordio/*.cc) \
-              $(wildcard src/common/console/*.cc) 
+              $(wildcard src/common/console/*.cc)
+SERVER_WRAPPER_SRC := src/tera_main_wrapper.cc
 SERVER_SRC := src/tera_main.cc src/tera_entry.cc
 CLIENT_SRC := src/teracli_main.cc
 TEST_CLIENT_SRC := src/tera_test_main.cc
@@ -45,7 +46,7 @@ MONITOR_SRC := src/monitor/teramo_main.cc
 MARK_SRC := src/benchmark/mark.cc src/benchmark/mark_main.cc
 TEST_SRC := src/utils/test/prop_tree_test.cc src/utils/test/tprinter_test.cc \
             src/io/test/tablet_io_test.cc src/io/test/tablet_scanner_test.cc \
-	     src/master/test/master_impl_test.cc src/io/test/load_test.cc
+            src/master/test/master_impl_test.cc src/io/test/load_test.cc
 
 TEST_OUTPUT := test_output
 UNITTEST_OUTPUT := $(TEST_OUTPUT)/unittest
@@ -58,6 +59,7 @@ PROTO_OBJ := $(PROTO_SRC:.cc=.o)
 JNI_TERA_OBJ := $(JNI_TERA_SRC:.cc=.o)
 OTHER_OBJ := $(OTHER_SRC:.cc=.o)
 COMMON_OBJ := $(COMMON_SRC:.cc=.o)
+SERVER_WRAPPER_OBJ := $(SERVER_WRAPPER_SRC:.cc=.o)
 SERVER_OBJ := $(SERVER_SRC:.cc=.o)
 CLIENT_OBJ := $(CLIENT_SRC:.cc=.o)
 TEST_CLIENT_OBJ := $(TEST_CLIENT_SRC:.cc=.o)
@@ -68,11 +70,12 @@ HTTP_OBJ := $(HTTP_SRC:.cc=.o)
 TEST_OBJ := $(TEST_SRC:.cc=.o)
 ALL_OBJ := $(MASTER_OBJ) $(TABLETNODE_OBJ) $(IO_OBJ) $(SDK_OBJ) $(PROTO_OBJ) \
            $(JNI_TERA_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(SERVER_OBJ) $(CLIENT_OBJ) \
-           $(TEST_CLIENT_OBJ) $(TERA_C_OBJ) $(MONITOR_OBJ) $(MARK_OBJ) $(TEST_OBJ)
+           $(TEST_CLIENT_OBJ) $(TERA_C_OBJ) $(MONITOR_OBJ) $(MARK_OBJ) $(TEST_OBJ) \
+           $(SERVER_WRAPPER_OBJ)
 LEVELDB_LIB := src/leveldb/libleveldb.a
 LEVELDB_UTIL := src/leveldb/util/histogram.o src/leveldb/port/port_posix.o
 
-PROGRAM = tera_main teracli teramo tera_test
+PROGRAM = tera_main tera_master tabletserver teracli teramo tera_test
 LIBRARY = libtera.a
 SOLIBRARY = libtera.so
 TERA_C_SO = libtera_c.so
@@ -114,8 +117,15 @@ cleanall:
 	$(MAKE) clean
 	rm -rf build
 
-tera_main: $(SERVER_OBJ) $(MASTER_OBJ) $(TABLETNODE_OBJ) $(IO_OBJ) $(SDK_OBJ) \
-           $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
+tera_main: src/tera_main_wrapper.o src/version.o src/tera_flags.o
+	$(CXX) -o $@ $^ $(LDFLAGS)
+
+tera_master: $(SERVER_OBJ) $(MASTER_OBJ) $(IO_OBJ) $(SDK_OBJ) \
+             $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
+	$(CXX) -o $@ $^ $(LDFLAGS)
+
+tabletserver: $(SERVER_OBJ) $(TABLETNODE_OBJ) $(IO_OBJ) $(SDK_OBJ) \
+              $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
 libtera.a: $(SDK_OBJ) $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_UTIL)
@@ -178,8 +188,8 @@ tablet_scanner_test: src/io/test/tablet_scanner_test.o src/tabletnode/tabletnode
                      $(IO_OBJ) $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
-master_impl_test: src/master/test/master_impl_test.o src/tera_entry.o $(MASTER_OBJ) $(TABLETNODE_OBJ) $(IO_OBJ) $(SDK_OBJ) \
-           $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
+master_impl_test: src/master/test/master_impl_test.o src/tera_entry.cc $(MASTER_OBJ) $(IO_OBJ) $(SDK_OBJ) \
+                  $(PROTO_OBJ) $(OTHER_OBJ) $(COMMON_OBJ) $(LEVELDB_LIB)
 	$(CXX) -o $@ $^ $(LDFLAGS)
 
 $(ALL_OBJ): %.o: %.cc $(PROTO_OUT_H)
diff --git a/doc/cn/cluster_setup.md b/doc/cn/cluster_setup.md
@@ -47,12 +47,12 @@ Tera集群搭建
   * 在master节点上，执行以下命令
   ```
   cd ${tera_prefix}/bin
-  nohup ./tera_main --flagfile=../conf/tera.flag --tera_role=master &> ../log/master.stderr &
+  nohup ./tera_master --flagfile=../conf/tera.flag &> ../log/master.stderr &
   ```
   * 在TabletServer节点上，执行以下命令
   ```
   cd ${tera_prefix}/bin
-  nohup ./tera_main --flagfile=../conf/tera.flag --tera_role=tabletnode &> ../log/tabletserver.stderr &
+  nohup ./tabletserver --flagfile=../conf/tera.flag &> ../log/tabletserver.stderr &
   ```
 5. 停止tera
-  * 用kill命令杀掉tera_main对应的进程即可
+  * 用kill命令杀掉tera_master和tabletserver对应的进程即可
diff --git a/doc/cn/onebox.md b/doc/cn/onebox.md
@@ -8,7 +8,7 @@
 
 ## 准备工作
 1. 完成Tera的编译，请参考：https://github.com/baidu/tera/wiki/Build-Manual
-2. 将编译生成的tera_main, teracli两个二进制文件放入example/onebox/bin
+2. 将编译生成的tera_master, tabletserver, teracli三个二进制文件放入example/onebox/bin
 3. 如有需要，通过修改example/onebox/bin/config中的选项配置tabletnode个数
 4. 进入example/onebox/bin/目录
 
diff --git a/doc/en/onebox.md b/doc/en/onebox.md
@@ -9,7 +9,7 @@ Your can run Tera in pseudo-distributed mode which means that Tera runs on a sin
 1. git clone https://github.com/baidu/tera.git
 1. cd tera
 1. ./build.sh
-1. cp {tera_main,teracli} example/onebox/bin
+1. cp {tera_master,tabletserver,teracli} example/onebox/bin
 1. cd example/onebox/bin
 
 
diff --git a/doc/ubuntu_install_onebox_and_cluster.md b/doc/ubuntu_install_onebox_and_cluster.md
@@ -49,7 +49,7 @@ ubuntu安装tera(单机和集群)
 
 3. 等待编译结束 & 单机体验
 
-  将编译生成的tera_main和teracli文件copy到example/onebox/bin目录下，进入目录执行：sh launch_tera.sh。然后执行./teracli进终端交互。Have fun！
+  将编译生成的tera_master，tabletserver和teracli文件copy到example/onebox/bin目录下，进入目录执行：sh launch_tera.sh。然后执行./teracli进终端交互。Have fun！
   详见：https://github.com/baidu/tera/blob/master/doc/onebox-cn.md
 
 
@@ -262,10 +262,10 @@ ubuntu安装tera(单机和集群)
   > 在zookeeper目录中执行：./bin/zkServer.sh status，找到leader那台机器，即master
   > 进入tera_root的bin目录，
     在master上执行：
-    nohup ./tera_main --flagfile=../conf/tera.flag --tera_role=master &> ../log/master.stderr &
+    nohup ./tera_master --flagfile=../conf/tera.flag &> ../log/master.stderr &
     
     在其他两台slave机器上执行：
-    nohup ./tera_main --flagfile=../conf/tera.flag --tera_role=tabletnode &> ../log/tabletserver.stderr &
+    nohup ./tabletserver --flagfile=../conf/tera.flag &> ../log/tabletserver.stderr &
 
   > 在任意一台机器上的tera_root/bin目录中执行：./teracli
     Have Fun！
diff --git a/example/docker/master b/example/docker/master
@@ -3,4 +3,4 @@
 export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/amd64/:/home/ubuntu/leiliyuan/hadoop-1.2.1/lib:$(hadoop classpath)
 cd /opt/tera/bin
 
-nohup ./tera_main --flagfile=../conf/tera.flag --tera_local_addr $1 --tera_role=master &> ../../share/log/master.stderr &
+nohup ./tera_master --flagfile=../conf/tera.flag --tera_local_addr $1 &> ../../share/log/master.stderr &
diff --git a/example/docker/tabletnode b/example/docker/tabletnode
@@ -2,4 +2,4 @@
 
 export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:$JAVA_HOME/jre/lib/amd64/:/home/ubuntu/leiliyuan/hadoop-1.2.1/lib:$(hadoop classpath)
 cd /opt/tera/bin
-nohup ./tera_main --flagfile=../conf/tera.flag --tera_local_addr $1 --tera_role=tabletnode &> ../../share/log/tabletserver.stderr &
+nohup ./tabletserver --flagfile=../conf/tera.flag --tera_local_addr $1 &> ../../share/log/tabletserver.stderr &
diff --git a/example/onebox/bin/kill_tera.sh b/example/onebox/bin/kill_tera.sh
@@ -1,8 +1,13 @@
 #!/bin/bash
 source ./config
 
-for ((i=0; i<=${TABLETNODE_NUM}; i++)); do
-    PID=`ps x | grep tera_main | grep $((PORT+i)) | awk '{print $1}'`;
+PID=`ps x | grep tera_master | grep $PORT | awk '{print $1}'`;
+if [ ${PID}"x" != "x" ]; then
+    kill -9 $PID;
+fi
+
+for ((i=1; i<=${TABLETNODE_NUM}; i++)); do
+    PID=`ps x | grep tabletserver | grep $((PORT+i)) | awk '{print $1}'`;
     if [ ${PID}"x" != "x" ]; then
         kill -9 $PID;
     fi
diff --git a/example/onebox/bin/launch_tera.sh b/example/onebox/bin/launch_tera.sh
@@ -33,9 +33,8 @@ for ((i=1; i<=$TABLETNODE_NUM; i++)); do
     if [ ! -x $CACHE_PATH ];then
         mkdir -p $CACHE_PATH
     fi
-    ${CURRENT_DIR}/tera_main \
+    ${CURRENT_DIR}/tabletserver \
         --flagfile=${CURRENT_DIR}/../conf/tera.flag \
-        --tera_role=tabletnode \
         --tera_tabletnode_port=$((PORT+i)) \
         --tera_leveldb_log_path=${LEVELDB_LOG_FILE} \
         --tera_tabletnode_cache_paths=${CACHE_PATH} \
@@ -53,9 +52,8 @@ MASTER_LOG_FILE=${CURRENT_DIR}/../log/master.stderr
 if [ -f ${MASTER_LOG_FILE} ];then
     mv ${MASTER_LOG_FILE} ${MASTER_LOG_FILE}.${TIME}
 fi
-${CURRENT_DIR}/tera_main \
+${CURRENT_DIR}/tera_master \
     --flagfile=${CURRENT_DIR}/../conf/tera.flag \
-    --tera_role=master \
     --tera_master_port=${PORT} \
     --tera_fake_zk_path_prefix=${FAKE_ZK_PATH_PREFIX} \
     --tera_log_prefix=master &> ${MASTER_LOG_FILE} </dev/null &
diff --git a/ft_test.sh b/ft_test.sh
@@ -51,7 +51,8 @@ if ! $run_without_fetch; then
     cp example/onebox/conf/tera.flag $test_dir/conf
 
     cp build/bin/teracli $test_dir/bin
-    cp build/bin/tera_main $test_dir/bin
+    cp build/bin/tera_master $test_dir/bin
+    cp build/bin/tabletserver $test_dir/bin
     cp build/bin/tera_test $test_dir/bin
     cp build/benchmark/tera_bench $test_dir/bin
     cp build/benchmark/tera_mark $test_dir/bin
diff --git a/src/io/tablet_io.cc b/src/io/tablet_io.cc
@@ -69,7 +69,7 @@ DECLARE_bool(tera_tablet_use_memtable_on_leveldb);
 DECLARE_int64(tera_tablet_memtable_ldb_write_buffer_size);
 DECLARE_int64(tera_tablet_memtable_ldb_block_size);
 
-extern tera::Counter row_read_delay;
+tera::Counter row_read_delay;
 
 namespace tera {
 namespace io {
diff --git a/src/master/master_entry.cc b/src/master/master_entry.cc
@@ -16,6 +16,14 @@ DECLARE_string(tera_master_port);
 DECLARE_int32(tera_master_rpc_server_max_inflow);
 DECLARE_int32(tera_master_rpc_server_max_outflow);
 
+std::string GetTeraEntryName() {
+    return "master";
+}
+
+tera::TeraEntry* GetTeraEntry() {
+    return new tera::master::MasterEntry();
+}
+
 namespace tera {
 namespace master {
 
diff --git a/src/tabletnode/tabletnode_entry.cc b/src/tabletnode/tabletnode_entry.cc
@@ -34,6 +34,14 @@ DECLARE_int32(tera_tabletnode_hang_detect_threshold);
 DECLARE_int32(tera_tabletnode_rpc_server_max_inflow);
 DECLARE_int32(tera_tabletnode_rpc_server_max_outflow);
 
+std::string GetTeraEntryName() {
+    return "tabletnode";
+}
+
+tera::TeraEntry* GetTeraEntry() {
+    return new tera::tabletnode::TabletNodeEntry();
+}
+
 namespace tera {
 namespace tabletnode {
 
diff --git a/src/tabletnode/tabletnode_sysinfo.cc b/src/tabletnode/tabletnode_sysinfo.cc
@@ -85,7 +85,7 @@ extern tera::Counter ssd_write_size_counter;
 }
 
 tera::Counter rand_read_delay;
-tera::Counter row_read_delay;
+extern tera::Counter row_read_delay;
 tera::Counter range_error_counter;
 tera::Counter read_pending_counter;
 tera::Counter write_pending_counter;
diff --git a/src/tera_main.cc b/src/tera_main.cc
@@ -8,43 +8,29 @@
 #include <glog/logging.h>
 
 #include "common/base/scoped_ptr.h"
-#include "master/master_entry.h"
-#include "tabletnode/tabletnode_entry.h"
 #include "tera_entry.h"
 #include "utils/utils_cmd.h"
 #include "version.h"
 
-DECLARE_string(tera_role);
 DECLARE_string(tera_log_prefix);
 DECLARE_string(tera_local_addr);
 
+extern std::string GetTeraEntryName();
+extern tera::TeraEntry* GetTeraEntry();
+
 volatile sig_atomic_t g_quit = 0;
 
 static void SignalIntHandler(int sig) {
     g_quit = 1;
 }
 
-tera::TeraEntry* SwitchTeraEntry() {
-    const std::string& server_name = FLAGS_tera_role;
-
-    if (server_name == "master") {
-        return new tera::master::MasterEntry();
-    } else if (server_name == "tabletnode") {
-        return new tera::tabletnode::TabletNodeEntry();
-    }
-    LOG(ERROR) << "FLAGS_tera_role should be one of ("
-        << "master | tabletnode"
-        << "), not : " << FLAGS_tera_role;
-    return NULL;
-}
-
 int main(int argc, char** argv) {
     ::google::ParseCommandLineFlags(&argc, &argv, true);
     ::google::InitGoogleLogging(argv[0]);
     if (!FLAGS_tera_log_prefix.empty()) {
         tera::utils::SetupLog(FLAGS_tera_log_prefix);
     } else {
-        tera::utils::SetupLog(FLAGS_tera_role);
+        tera::utils::SetupLog(GetTeraEntryName());
     }
 
     if (argc > 1) {
@@ -58,7 +44,7 @@ int main(int argc, char** argv) {
     signal(SIGINT, SignalIntHandler);
     signal(SIGTERM, SignalIntHandler);
 
-    scoped_ptr<tera::TeraEntry> entry(SwitchTeraEntry());
+    scoped_ptr<tera::TeraEntry> entry(GetTeraEntry());
     if (entry.get() == NULL) {
         return -1;
     }
@@ -72,10 +58,6 @@ int main(int argc, char** argv) {
             LOG(ERROR) << "Server run error ,and then exit now ";
             break;
         }
-        // jvm会抢注这个, 时刻准备着抢回来
-        signal(SIGINT, SignalIntHandler);
-        signal(SIGTERM, SignalIntHandler);
-        // signal(SIGSEGV, SIG_DFL); // 如果这个被改回SIG_DFL, jvm会core, 不知道为啥...
     }
     if (g_quit) {
         LOG(INFO) << "received interrupt signal from user, will stop";
diff --git a/src/tera_main_wrapper.cc b/src/tera_main_wrapper.cc
@@ -0,0 +1,48 @@
+// Copyright (c) 2017, Baidu.com, Inc. All Rights Reserved
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <iostream>
+#include <signal.h>
+#include <string.h>
+#include <vector>
+#include <unistd.h>
+
+#include <gflags/gflags.h>
+
+#include "version.h"
+
+DECLARE_string(tera_role);
+
+int main(int argc, char** argv) {
+    if (argc > 1 && strcmp(argv[1], "version") == 0) {
+        PrintSystemVersion();
+        return 0;
+    }
+
+    google::AllowCommandLineReparsing();
+    google::ParseCommandLineFlags(&argc, &argv, false);
+
+    const char* program = NULL;
+    if (FLAGS_tera_role == "master") {
+        program = "./tera_master";
+    } else if (FLAGS_tera_role == "tabletnode") {
+        program = "./tabletserver";
+    } else {
+        std::cerr << "FLAGS_tera_role should be one of (master | tabletnode)" << std::endl;
+        return -1;
+    }
+
+    std::vector<char*> myargv;
+    myargv.resize(argc + 1);
+    myargv[0] = (char*)"tera_main";
+    for (int i = 1; i < argc; i++) {
+        myargv[i] = argv[i];
+    }
+    myargv[argc] = NULL;
+    if (-1 == execv(program, &myargv[0])) {
+        std::cerr << "execv " << program << " error: " << errno << std::endl;
+        return -1;
+    }
+    return 0;
+}
diff --git a/test/testcase/shell_script/launch_ts_first.sh b/test/testcase/shell_script/launch_ts_first.sh
@@ -25,9 +25,8 @@ MASTER_LOG_FILE=${CURRENT_DIR}/../log/master.stderr
 if [ -f ${MASTER_LOG_FILE} ];then
     mv ${MASTER_LOG_FILE} ${MASTER_LOG_FILE}.${TIME}
 fi
-${CURRENT_DIR}/tera_main \
+${CURRENT_DIR}/tera_master \
     --flagfile=${CURRENT_DIR}/../conf/tera.flag \
-    --tera_role=master \
     --tera_master_port=${PORT} \
     --tera_fake_zk_path_prefix=${FAKE_ZK_PATH_PREFIX} \
     --tera_log_prefix=master &> ${MASTER_LOG_FILE} </dev/null &
@@ -47,9 +46,8 @@ for ((i=1; i<=$TABLETNODE_NUM; i++)); do
     if [ ! -x $CACHE_PATH ];then
         mkdir -p $CACHE_PATH
     fi
-    ${CURRENT_DIR}/tera_main \
+    ${CURRENT_DIR}/tabletserver \
         --flagfile=${CURRENT_DIR}/../conf/tera.flag \
-        --tera_role=tabletnode \
         --tera_tabletnode_port=$((PORT+i)) \
         --tera_leveldb_log_path=${LEVELDB_LOG_FILE} \
         --tera_tabletnode_cache_paths=${CACHE_PATH} \
