issue=1258, Tcache support block-level cache evict

baidu · May 18, 2017 · 72a31b8 · 72a31b8
1 parent f15ac00
commit 72a31b8
Show file tree

Hide file tree

Showing 10 changed files with 1,372 additions and 38 deletions.
diff --git a/src/io/tablet_io.cc b/src/io/tablet_io.cc
@@ -1668,18 +1668,25 @@ void TabletIO::SetupOptionsForLG() {
             lg_info->env = LeveldbMockEnv();
         } else if (store == MemoryStore) {
             if (FLAGS_tera_use_flash_for_memenv) {
-                lg_info->env = LeveldbFlashEnv();
+                if (FLAGS_tera_tabletnode_block_cache_enabled) {
+                    LOG(INFO) << "MemLG[" << lg_i << "] activate TCache";
+                    lg_info->env = io::DefaultBlockCacheEnv();
+                } else {
+                    lg_info->env = LeveldbFlashEnv();
+                }
             } else {
                 lg_info->env = LeveldbMemEnv();
             }
             lg_info->seek_latency = 0;
             lg_info->block_cache = m_memory_cache;
         } else if (store == FlashStore) {
-            if (!FLAGS_tera_tabletnode_cache_enabled) {
-                lg_info->env = LeveldbFlashEnv();
+            if (FLAGS_tera_tabletnode_block_cache_enabled) {
+                //LOG(INFO) << "activate block-level Cache store";
+                //lg_info->env = leveldb::EnvThreeLevelCache();
+                LOG(INFO) << "FlashLG[" << lg_i << "] activate TCache";
+                lg_info->env = io::DefaultBlockCacheEnv();
             } else {
-                LOG(INFO) << "activate block-level Cache store";
-                lg_info->env = leveldb::EnvThreeLevelCache();
+                lg_info->env = LeveldbFlashEnv();
             }
             lg_info->seek_latency = FLAGS_tera_leveldb_env_local_seek_latency;
         } else {

diff --git a/src/io/utils_leveldb.cc b/src/io/utils_leveldb.cc
@@ -18,6 +18,7 @@
 #include "leveldb/comparator.h"
 #include "leveldb/env_dfs.h"
 #include "leveldb/env_flash.h"
+#include "leveldb/block_cache.h"
 #include "leveldb/env_inmem.h"
 #include "leveldb/env_mock.h"
 #include "leveldb/table_utils.h"
@@ -31,6 +32,7 @@ DECLARE_string(tera_leveldb_env_hdfs2_nameservice_list);
 DECLARE_string(tera_tabletnode_path_prefix);
 DECLARE_string(tera_dfs_so_path);
 DECLARE_string(tera_dfs_conf);
+DECLARE_int32(tera_leveldb_block_cache_env_num_thread);
 
 namespace tera {
 namespace io {
@@ -66,6 +68,20 @@ leveldb::Env* LeveldbBaseEnv() {
     }
 }
 
+// Tcache: default env
+static pthread_once_t block_cache_once = PTHREAD_ONCE_INIT;
+static Env* default_block_cache_env;
+static void InitDefaultBlockCacheEnv() {
+    default_block_cache_env = new BlockCacheEnv(leveldbBaseEnv);
+    default_block_cache_env->SetBackgroundThreads(FLAGS_tera_leveldb_block_cache_env_num_thread);
+}
+
+leveldb::Env* DefaultBlockCacheEnv() {
+    pthread_once(&block_cache_once, InitDefaultBlockCacheEnv);
+    return default_block_cache_env;
+}
+
+// mem env
 leveldb::Env* LeveldbMemEnv() {
     static Mutex mutex;
     static leveldb::Env* mem_env = NULL;
@@ -78,6 +94,7 @@ leveldb::Env* LeveldbMemEnv() {
     return mem_env;
 }
 
+// flash env
 leveldb::Env* LeveldbFlashEnv() {
     static Mutex mutex;
     static leveldb::Env* flash_env = NULL;

diff --git a/src/io/utils_leveldb.h b/src/io/utils_leveldb.h
@@ -18,6 +18,8 @@ void InitDfsEnv();
 // return the base env leveldb used (dfs/local), singleton
 leveldb::Env* LeveldbBaseEnv();
 
+leveldb::Env* DefaultBlockCacheEnv(); // ssd + base
+
 // return the mem env leveldb used, singleton
 leveldb::Env* LeveldbMemEnv();
 

diff --git a/src/leveldb/include/leveldb/block_cache.h b/src/leveldb/include/leveldb/block_cache.h
@@ -0,0 +1,99 @@
+// Copyright (c) 2015, Baidu.com, Inc. All Rights Reserved
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef  STOREAGE_LEVELDB_UTIL_BLOCK_CACHE_H
+#define  STOREAGE_LEVELDB_UTIL_BLOCK_CACHE_H
+#include "leveldb/status.h"
+#include "leveldb/options.h"
+
+namespace leveldb {
+/////////////////////////////////////////////
+// Tcache
+/////////////////////////////////////////////
+uint64_t kBlockSize = 8192UL;
+uint64_t kDataSetSize = 134217728UL;
+uint64_t kFidBatchNum = 200000UL;
+uint64_t kCacheSize = 350000000000UL;
+uint64_t kMetaBlockSize = 2000UL;
+uint64_t kMetaTableSize = 500UL;
+uint64_t kWriteBufferSize = 1048576UL;
+
+struct BlockCacheOptions {
+    Options opts;
+    std::string cache_dir;
+    uint64_t block_size;
+    uint64_t dataset_size;
+    uint64_t fid_batch_num;
+    uint64_t cache_size;
+    uint64_t dataset_num;
+    uint64_t meta_block_cache_size;
+    uint64_t meta_table_cache_size;
+    uint64_t write_buffer_size;
+    Env* env;
+
+    BlockCacheOptions()
+    : block_size(kBlockSize),
+      dataset_size(kDataSetSize),
+      fid_batch_num(kFidBatchNum),
+      cache_size(kCacheSize),
+      meta_block_cache_size(kMetaBlockSize),
+      meta_table_cache_size(kMetaTableSize),
+      write_buffer_size(kWriteBufferSize),
+      env(NULL) {
+          dataset_num = cache_size / dataset_size + 1;
+    }
+};
+
+class BlockCacheWritableFile;
+class BlockCacheRandomAccessFile;
+class BlockCacheImpl;
+
+class BlockCacheEnv : public EnvWrapper {
+public:
+    BlockCacheEnv(Env* base);
+
+    ~BlockCacheEnv();
+
+    virtual Status FileExists(const std::string& fname);
+
+    virtual Status GetChildren(const std::string& path,
+                               std::vector<std::string>* result);
+
+    virtual Status DeleteFile(const std::string& fname);
+
+    virtual Status CreateDir(const std::string& name);
+
+    virtual Status DeleteDir(const std::string& name);
+
+    virtual Status CopyFile(const std::string& from,
+                            const std::string& to);
+
+    virtual Status GetFileSize(const std::string& fname, uint64_t* size);
+
+    virtual Status RenameFile(const std::string& src, const std::string& target);
+
+    virtual Status LockFile(const std::string& fname, FileLock** lock);
+
+    virtual Status UnlockFile(FileLock* lock);
+
+    virtual Status NewSequentialFile(const std::string& fname,
+                                     SequentialFile** result); // never cache log
+
+    // cache relatively
+    virtual Status NewRandomAccessFile(const std::string& fname,
+                                       RandomAccessFile** result); // cache Pread
+
+    virtual Status NewWritableFile(const std::string& fname,
+                                   WritableFile** result); // cache Append
+    Status LoadCache(const BlockCacheOptions& opts, const std::string& cache_dir);
+
+private:
+    std::vector<BlockCacheImpl*> cache_vec_;
+    Env* dfs_env_;
+};
+
+Env* NewBlockCacheEnv(Env* base);
+
+} // leveldb
+
diff --git a/src/leveldb/include/leveldb/cache.h b/src/leveldb/include/leveldb/cache.h
@@ -31,7 +31,11 @@ class Cache;
 
 // Create a new cache with a fixed size capacity.  This implementation
 // of Cache uses a least-recently-used eviction policy.
-extern Cache* NewLRUCache(size_t capacity);
+enum LRUCacheType {
+    kLRUCacheDefault = 0,
+    kLRUCache2Q = 1,
+};
+extern Cache* NewLRUCache(size_t capacityint, LRUCacheType type = kLRUCacheDefault);
 
 class Cache {
  public: