Add scan-perf #2

scan-perf/CMakeLists.txt

@@ -0,0 +1,24 @@
+cmake_minimum_required(VERSION 3.5)
+project(scan_perf)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_VERBOSE_MAKEFILE on)
+set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb ")
+set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall")
+
+add_executable(scan_perf main.cpp
+                         rocksdb.h
+                         rocksdb.cpp)
+
+include_directories(third/rocksdb/include
+                    third/benchmark/include)
+
+target_link_libraries(scan_perf
+        third/rocksdb/librocksdb.a
+        third/benchmark/build/src/libbenchmark.a
+        pthread
+        zstd
+        lz4
+        bz2
+        snappy
+        z)

scan-perf/main.cpp

@@ -0,0 +1,63 @@
+#include <iostream>
+#include <benchmark/benchmark.h>
+#include <unistd.h>
+
+#include "rocksdb.h"
+
+using namespace rocksdb;
+
+/*
+static void test(benchmark::State& state, RocksDB* db)
+{
+  for (auto _ : state) {
+    printf("%s\n", "once");
+    db->run();
+  }
+}
+*/
+
+auto test = [](benchmark::State& state, RocksDB* db) {
+  for (auto _ : state) {
+    db->run();
+  }
+};
+
+int main(int argc, char* argv[])
+{
+  RocksDB* db = new RocksDB("testdb");
+
+  if (argc < 2) {
+    goto usage;
+  }
+  if (!strcmp(argv[1], "load")) {
+    std::string file_path = "data.txt";
+    size_t nums = 1000;
+
+    if (argc >= 3) {
+      file_path = argv[2];
+    }
+    if (argc >= 4) {
+      nums = (size_t)atoi(argv[3]);
+    }
+
+    db->load(file_path, nums);
+    db->compact();
+  } else if (!strcmp(argv[1], "compact")) {
+    db->compact();
+  } else if (!strcmp(argv[1], "run")) {
+    printf("%s\n", "benchmark begin!");
+
+    benchmark::RegisterBenchmark("test", test, db);
+    benchmark::RunSpecifiedBenchmarks();
+  } else {
+    goto usage;
+  }
+
+  delete db;
+
+  return 0;
+
+  usage:
+  fprintf(stderr, "%s\n", "Usage: ./scan_perf {load [file path] [nums]}/compact/run");
+  return 1;
+}

scan-perf/rocksdb.cpp

@@ -0,0 +1,161 @@
+//
+// Created by frost on 18-9-20.
+//
+
+#include <benchmark/benchmark.h>
+#include "rocksdb.h"
+
+RocksDB::RocksDB(std::string&& db_name_) : db_name(std::move(db_name_))
+{
+  // rocksdb options
+  db_options.create_if_missing = true;
+
+  fprintf(stderr, "Open db: %s\n", db_name.c_str());
+  rocksdb::Status s = rocksdb::DB::Open(db_options, db_name, &db);
+  if (!s.ok()) {
+    fprintf(stderr, "FATAL: db(%s) open error: %s\n",
+            db_name.c_str(), s.ToString().c_str());
+    exit(1);
+  }
+
+  const auto& opt = db->GetOptions();
+
+  fprintf(stderr,
+  R"EOS(rocksdb options:
+opt.write_buffer_size              = %lld
+opt.target_file_size_base          = %lld
+opt.target_file_size_multiplier    = %d
+)EOS"
+  , (long long)opt.write_buffer_size
+  , (long long)opt.target_file_size_base
+  , opt.target_file_size_multiplier);
+}
+
+RocksDB::~RocksDB()
+{
+  assert(db != nullptr);
+  delete db;
+}
+
+bool RocksDB::load(std::string fpath, size_t limit)
+{
+  FILE* file = fopen(fpath.c_str(), "r");
+  if (file == nullptr) {
+    fprintf(stderr, "FATAL: invalid filename: %s\n", fpath.c_str());
+    exit(1);
+  }
+
+  printf("Begin load %ld lines data from %s\n", limit, fpath.c_str());
+  size_t nums = 0;
+  char* buf = (char *)malloc(2048000 * sizeof(char));
+  size_t len;
+  auto key = (char *)malloc(10240 * sizeof(char));
+  auto value = (char *)malloc(2048000 * sizeof(char));
+  rocksdb::Status s;
+  while (nums < limit) {
+    ssize_t read = getline(&buf, &len, file);
+    if (read != -1) {
+      ssize_t pos = 0;
+      while (pos < read) {
+        if (buf[pos++] == '\t') {
+          memcpy(key, buf, (size_t)(pos - 1));
+          key[pos - 1] = 0;
+          memcpy(value, buf + pos, (size_t)(read - pos -1));
+          value[read - pos -1] = 0;
+
+          s = db->Put(write_options, key, value);
+          // printf("key: %s value: %s\n", key, value);
+          if (!s.ok()) {
+            fprintf(stderr, "WARN: error insert: %s\n", buf);
+          }
+
+          break;
+        }
+      }
+    } else {
+      fprintf(stderr, "ERROR: file EOF, lines < limit(%ld).\n", limit);
+      break;
+    }
+    nums++;
+    if (!(nums % 1000)) {
+      printf("\rinsert: %ld", nums);
+    }
+  }
+
+  printf("\n%s\n", "Load data done!");
+
+  free(buf);
+  free(key);
+  free(value);
+
+  return true;
+}
+
+bool RocksDB::compact()
+{
+  rocksdb::CompactRangeOptions cro;
+  cro.exclusive_manual_compaction = false;
+  cro.target_level = 6;
+  printf("%s\n", "Begin compact...");
+  rocksdb::Status s = db->CompactRange(cro, nullptr, nullptr);
+  if (!s.ok()) {
+    fprintf(stderr, "Compact db error: %s\n", s.ToString().c_str());
+    return false;
+  }
+  printf("%s\n", "Compact done!");
+  return true;
+}
+
+void RocksDB::load_key(std::string fpath, size_t limit)
+{
+  FILE* file = fopen(fpath.c_str(), "r");
+  if (file == nullptr) {
+    fprintf(stderr, "FATAL: invalid filename: %s\n", fpath.c_str());
+    exit(1);
+  }
+
+  printf("Begin load %ld keys from %s\n", limit, fpath.c_str());
+  size_t nums = 0;
+  char* buf = (char *)malloc(2048000 * sizeof(char));
+  size_t len;
+  auto key = (char *)malloc(10240 * sizeof(char));
+
+  while (nums < limit) {
+    ssize_t read = getline(&buf, &len, file);
+    if (read != -1) {
+      ssize_t pos = 0;
+      while (pos < read) {
+        if (buf[pos++] == '\t') {
+          memcpy(key, buf, (size_t) (pos - 1));
+          key[pos - 1] = 0;
+          keys.emplace_back(std::string(key));
+
+          break;
+        }
+      }
+    } else {
+      fprintf(stderr, "ERROR: file EOF, lines < limit(%ld).\n", limit);
+      break;
+    }
+  }
+
+  printf("%s\n", "Load keys done");
+}
+
+void RocksDB::run()
+{
+  load_key("data.txt", 10000);
+  rocksdb::Iterator* iter = db->NewIterator(read_options);
+  for(auto& key : keys) {
+    iter->Seek(key);
+    if (!iter->Valid()) {
+      fprintf(stderr, "Seek key=%s error\n", key.c_str());
+    }
+    for (int i = 0; i < 10; i++) {
+      iter->Prev();
+      if (!iter->Valid()) {
+        break;
+      }
+    }
+  }
+}

scan-perf/rocksdb.h

@@ -0,0 +1,33 @@
+//
+// Created by frost on 18-9-20.
+//
+
+#ifndef SCAN_PERF_ROCKSDB_H
+#define SCAN_PERF_ROCKSDB_H
+
+#include <string>
+#include <vector>
+#include <rocksdb/db.h>
+#include <rocksdb/options.h>
+
+class RocksDB {
+public:
+  explicit RocksDB(std::string&& db_name);
+  ~RocksDB();
+  bool load(std::string fpath, size_t limit);
+  bool compact();
+  void run();
+
+private:
+  std::string db_name;
+  rocksdb::DB* db;
+  rocksdb::WriteOptions write_options;
+  rocksdb::ReadOptions read_options;
+  rocksdb::Options db_options;
+  std::vector<std::string> keys;
+
+  void load_key(std::string fpath, size_t limit);
+};
+
+
+#endif //SCAN_PERF_ROCKSDB_H