Merge remote-tracking branch 'baidu_tera/master'

Author: lylei9

.travis.yml

@@ -1,7 +1,7 @@
 language: cpp
 compiler: gcc
 install:
-    - sudo apt-get install libboost-dev
+    - sudo apt-get install libboost-dev uuid-dev
 
     - wget https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.gz
     - tar xf protobuf-2.6.1.tar.gz
@@ -32,6 +32,9 @@ install:
     - tar zxvf gperftools-2.2.1.tar.gz
     - ( cd gperftools-2.2.1 && ./configure --with-pic && make -j4 && sudo make install )
 
+    - git clone https://github.com/fxsjy/ins
+    - (cd ins && PBRPC_PATH=../sofa-pbrpc/output/ make sdk )
+
 before_script:
     - sed -i 's/^SOFA_PBRPC_PREFIX=.*/SOFA_PBRPC_PREFIX=.\/sofa-pbrpc\/output/' depends.mk
     - sed -i 's/^PROTOBUF_PREFIX=.*/PROTOBUF_PREFIX=\/usr\/local/' depends.mk
@@ -41,6 +44,7 @@ before_script:
     - sed -i 's/^GLOG_PREFIX=.*/GLOG_PREFIX=\/usr\/local/' depends.mk
     - sed -i 's/^GPERFTOOLS_PREFIX=.*/GPERFTOOLS_PREFIX=\/usr\/local/' depends.mk
     - sed -i 's/^BOOST_INCDIR=.*/BOOST_INCDIR=\/usr\/local\/include/' depends.mk
+    - sed -i 's/^INS_PREFIX=.*/INS_PREFIX=.\/ins\/output/' depends.mk
 
 script:
     - make -j4

build.sh

@@ -118,6 +118,21 @@ make -j4
 make install
 cd -
 
+wget --no-check-certificate https://raw.githubusercontent.com/fxsjy/thirdparty/master/uuid-1.6.2.tar.gz
+tar -xzvf uuid-1.6.2.tar.gz
+cd uuid-1.6.2
+./configure ${DEPS_CONFIG} 
+make -j4
+make install
+cd -
+
+git clone https://github.com/fxsjy/ins
+cd ins
+sed -i "s|^PREFIX=.*|PREFIX=${DEPS_PREFIX}|" Makefile
+sed -i "s|^PROTOC=.*|PROTOC=${DEPS_PREFIX}/bin/protoc|" Makefile
+make install_sdk
+cd -
+
 cd ${WORK_DIR}
 
 ########################################
@@ -132,6 +147,7 @@ sed -i 's/^GFLAGS_PREFIX=.*/GFLAGS_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^GLOG_PREFIX=.*/GLOG_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^GPERFTOOLS_PREFIX=.*/GPERFTOOLS_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^BOOST_INCDIR=.*/BOOST_INCDIR=.\/thirdparty\/boost_1_57_0/' depends.mk
+sed -i 's/^INS_PREFIX=.*/INS_PREFIX=.\/thirdparty/' depends.mk
 
 ########################################
 # build tera

depends.mk

@@ -12,6 +12,7 @@ ZOOKEEPER_PREFIX=
 GFLAGS_PREFIX=
 GLOG_PREFIX=
 GPERFTOOLS_PREFIX=
+INS_PREFIX=
 BOOST_INCDIR=
 
 SOFA_PBRPC_INCDIR = $(SOFA_PBRPC_PREFIX)/include
@@ -21,6 +22,7 @@ ZOOKEEPER_INCDIR = $(ZOOKEEPER_PREFIX)/include
 GFLAGS_INCDIR = $(GFLAGS_PREFIX)/include
 GLOG_INCDIR = $(GLOG_PREFIX)/include
 GPERFTOOLS_INCDIR = $(GPERFTOOLS_PREFIX)/include
+INS_INCDIR = $(INS_PREFIX)/include
 
 SOFA_PBRPC_LIBDIR = $(SOFA_PBRPC_PREFIX)/lib
 PROTOBUF_LIBDIR = $(PROTOBUF_PREFIX)/lib
@@ -29,6 +31,7 @@ ZOOKEEPER_LIBDIR = $(ZOOKEEPER_PREFIX)/lib
 GFLAGS_LIBDIR = $(GFLAGS_PREFIX)/lib
 GLOG_LIBDIR = $(GLOG_PREFIX)/lib
 GPERFTOOLS_LIBDIR = $(GPERFTOOLS_PREFIX)/lib
+INS_LIBDIR = $(INS_PREFIX)/lib
 
 PROTOC = $(PROTOBUF_PREFIX)/bin/protoc
 
@@ -39,10 +42,10 @@ PROTOC = $(PROTOBUF_PREFIX)/bin/protoc
 DEPS_INCPATH = -I$(SOFA_PBRPC_INCDIR) -I$(PROTOBUF_INCDIR) \
                -I$(SNAPPY_INCDIR) -I$(ZOOKEEPER_INCDIR) \
                -I$(GFLAGS_INCDIR) -I$(GLOG_INCDIR) \
-               -I$(GPERFTOOLS_INCDIR) -I$(BOOST_INCDIR)
+               -I$(GPERFTOOLS_INCDIR) -I$(BOOST_INCDIR) -I$(INS_INCDIR)
 DEPS_LDPATH = -L$(SOFA_PBRPC_LIBDIR) -L$(PROTOBUF_LIBDIR) \
               -L$(SNAPPY_LIBDIR) -L$(ZOOKEEPER_LIBDIR) \
               -L$(GFLAGS_LIBDIR) -L$(GLOG_LIBDIR) \
-              -L$(GPERFTOOLS_LIBDIR)
-DEPS_LDFLAGS = -lsofa-pbrpc -lprotobuf -lsnappy -lzookeeper_mt \
-               -lgflags -lglog -ltcmalloc_minimal
+              -L$(GPERFTOOLS_LIBDIR) -L$(INS_LIBDIR)
+DEPS_LDFLAGS = -lins_sdk -lsofa-pbrpc -lprotobuf -lsnappy -lzookeeper_mt \
+               -lgflags -lglog -ltcmalloc_minimal -luuid

internal_build.sh

@@ -101,6 +101,13 @@ make -j4
 make install
 cd -
 
+git clone [email protected]:fxsjy/ins.git
+cd ins
+sed -i "s|^PREFIX=.*|PREFIX=${DEPS_PREFIX}|" Makefile
+sed -i "s|^PROTOC=.*|PROTOC=${DEPS_PREFIX}/bin/protoc|" Makefile
+make install_sdk
+cd -
+
 cd ${WORK_DIR}
 
 ########################################
@@ -115,6 +122,7 @@ sed -i 's/^GFLAGS_PREFIX=.*/GFLAGS_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^GLOG_PREFIX=.*/GLOG_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^GPERFTOOLS_PREFIX=.*/GPERFTOOLS_PREFIX=.\/thirdparty/' depends.mk
 sed -i 's/^BOOST_INCDIR=.*/BOOST_INCDIR=.\/thirdparty\/boost_1_57_0/' depends.mk
+sed -i 's/^INS_PREFIX=.*/INS_PREFIX=.\/thirdparty/' depends.mk
 
 ########################################
 # build tera

src/benchmark/mark_main.cc

@@ -12,8 +12,6 @@
 #include <sys/time.h>
 #include <time.h>
 
-#include <openssl/md5.h>
-
 #include "benchmark/mark.h"
 
 DECLARE_string(flagfile);
@@ -32,6 +30,7 @@ DEFINE_int64(max_outflow, -1, "max_outflow");
 DEFINE_int64(max_rate, -1, "max_rate");
 DEFINE_bool(scan_streaming, false, "enable streaming scan");
 DEFINE_int64(batch_count, 1, "batch_count(sync)");
+DEFINE_bool(seq_write, false, "enable sequential write");
 
 int mode = 0;
 int type = 0;
@@ -472,13 +471,15 @@ int main(int argc, char** argv) {
     ::google::ParseCommandLineFlags(&argc, &argv, true);
 
     tera::ErrorCode err;
-    tera::Client* client = tera::Client::NewClient("./tera.flag", "sample");
+    tera::Client* client = tera::Client::NewClient("./tera.flag", "tera_mark");
     if (NULL == client) {
         std::cerr << "fail to create client: " << tera::strerr(err) << std::endl;
         return -1;
     }
 
-    tera::Table* table = client->OpenTable(FLAGS_tablename, &err);
+    tera::TableOptions options;
+    options.sequential_write = FLAGS_seq_write;
+    tera::Table* table = client->OpenTable(FLAGS_tablename, options, &err);
     if (NULL == table) {
         std::cerr << "fail to open table: " << tera::strerr(err) << std::endl;
         return -1;

src/common/thread_pool.h

@@ -97,8 +97,9 @@ class ThreadPool {
         work_cv_.Signal();
         return bg_item.id;
     }
-    /// Cancel a delayed task ,if running, wait
-    bool CancelTask(int64_t task_id) {
+    /// Cancel a delayed task
+    /// if running, wait if non_block==false; return immediately if non_block==true
+    bool CancelTask(int64_t task_id, bool non_block = false, bool* is_running = NULL) {
         if (task_id == 0) {
             return false;
         }
@@ -108,10 +109,18 @@ class ThreadPool {
                 if (running_task_id_ != task_id) {
                     BGMap::iterator it = latest_.find(task_id);
                     if (it == latest_.end()) {
+                        if (is_running != NULL) {
+                            *is_running = false;
+                        }
                         return false;
                     }
                     latest_.erase(it);
                     return true;
+                } else if (non_block) {
+                    if (is_running != NULL) {
+                        *is_running = true;
+                    }
+                    return false;
                 }
             }
             timespec ts = {0, 100000};

src/io/tablet_io.cc

@@ -15,6 +15,7 @@
 #include "io/tablet_writer.h"
 #include "io/timekey_comparator.h"
 #include "io/ttlkv_compact_strategy.h"
+#include "io/utils_leveldb.h"
 #include "leveldb/cache.h"
 #include "leveldb/compact_strategy.h"
 #include "leveldb/env.h"
@@ -66,27 +67,6 @@ extern tera::Counter row_read_delay;
 namespace tera {
 namespace io {
 
-#define OPERATE_WITH_RETRY(func, tips, sleep_time, max_retry) \
-do { \
-    int32_t retry = 0; \
-    bool ret = false; \
-    while ( !(ret = func)  && retry < max_retry ) { \
-        ThisThread::Sleep(sleep_time); \
-        if (max_retry < 10 || retry % (max_retry / 10 + 1) == 0) { \
-            LOG(WARNING) << "try again to " << tips << ": " << retry; \
-        } \
-        retry++; \
-    } \
-    if (!ret) { \
-        LOG(ERROR) << "can not " << tips << " status: " << db_status.ToString(); \
-    } \
-} while (0)
-
-#define OPERATE_FILE_WITH_RETRY(func, tips) \
-    OPERATE_WITH_RETRY(func, tips, \
-                       FLAGS_tera_io_retry_period, \
-                       0 /*FLAGS_tera_io_retry_max_times*/);
-
 TabletIO::TabletIO()
     : m_async_writer(NULL),
       m_compact_status(kTableNotCompact),
@@ -1258,30 +1238,25 @@ void TabletIO::SetupOptionsForLG() {
 
         leveldb::LG_info* lg_info = new leveldb::LG_info(lg_schema.id());
 
-        if (FLAGS_tera_leveldb_env_type == "local") {
-            m_ldb_options.env = lg_info->env = leveldb::NewPosixEnv();
-            m_ldb_options.seek_latency = FLAGS_tera_leveldb_env_local_seek_latency;
-        } else {
-            if (store == MemoryStore) {
-                m_ldb_options.env = lg_info->env = leveldb::EnvInMemory();
-                m_ldb_options.seek_latency = 0;
-                m_ldb_options.block_cache =
-                    leveldb::NewLRUCache(FLAGS_tera_memenv_block_cache_size * 1024 * 1024);
-                m_mem_store_activated = true;
-            } else if (store == FlashStore) {
-                if (!FLAGS_tera_tabletnode_cache_enabled) {
-                    m_ldb_options.env = lg_info->env =
-                        leveldb::EnvFlash(m_ldb_options.info_log);
-                } else {
-                    LOG(INFO) << "activate block-level Cache store";
-                    m_ldb_options.env = lg_info->env = leveldb::EnvThreeLevelCache();
-                }
-                m_ldb_options.seek_latency = FLAGS_tera_leveldb_env_local_seek_latency;
+        if (store == MemoryStore) {
+            m_ldb_options.env = lg_info->env = LeveldbMemEnv();
+            m_ldb_options.seek_latency = 0;
+            m_ldb_options.block_cache =
+                leveldb::NewLRUCache(FLAGS_tera_memenv_block_cache_size * 1024 * 1024);
+            m_mem_store_activated = true;
+        } else if (store == FlashStore) {
+            if (!FLAGS_tera_tabletnode_cache_enabled) {
+                m_ldb_options.env = lg_info->env = LeveldbFlashEnv(m_ldb_options.info_log);
             } else {
-                m_ldb_options.env = lg_info->env = leveldb::EnvDfs();
-                m_ldb_options.seek_latency = FLAGS_tera_leveldb_env_dfs_seek_latency;
+                LOG(INFO) << "activate block-level Cache store";
+                m_ldb_options.env = lg_info->env = leveldb::EnvThreeLevelCache();
             }
+            m_ldb_options.seek_latency = FLAGS_tera_leveldb_env_local_seek_latency;
+        } else {
+            m_ldb_options.env = lg_info->env = LeveldbBaseEnv();
+            m_ldb_options.seek_latency = FLAGS_tera_leveldb_env_dfs_seek_latency;
         }
+
         if (compress) {
             lg_info->compression = leveldb::kSnappyCompression;
         }