From ef55fc686bc10f0a3bedc687f76fb04efb1f1b49 Mon Sep 17 00:00:00 2001 From: vrofze Date: Wed, 26 Sep 2018 19:42:23 +0800 Subject: [PATCH] Add scan-perf #2 --- scan-perf/CMakeLists.txt | 24 ++++++ scan-perf/main.cpp | 63 +++++++++++++++ scan-perf/rocksdb.cpp | 161 +++++++++++++++++++++++++++++++++++++++ scan-perf/rocksdb.h | 33 ++++++++ 4 files changed, 281 insertions(+) create mode 100644 scan-perf/CMakeLists.txt create mode 100644 scan-perf/main.cpp create mode 100644 scan-perf/rocksdb.cpp create mode 100644 scan-perf/rocksdb.h diff --git a/scan-perf/CMakeLists.txt b/scan-perf/CMakeLists.txt new file mode 100644 index 0000000..41fd5c8 --- /dev/null +++ b/scan-perf/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.5) +project(scan_perf) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_VERBOSE_MAKEFILE on) +set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g -ggdb ") +set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall") + +add_executable(scan_perf main.cpp + rocksdb.h + rocksdb.cpp) + +include_directories(third/rocksdb/include + third/benchmark/include) + +target_link_libraries(scan_perf + third/rocksdb/librocksdb.a + third/benchmark/build/src/libbenchmark.a + pthread + zstd + lz4 + bz2 + snappy + z) \ No newline at end of file diff --git a/scan-perf/main.cpp b/scan-perf/main.cpp new file mode 100644 index 0000000..0b9151e --- /dev/null +++ b/scan-perf/main.cpp @@ -0,0 +1,63 @@ +#include +#include +#include + +#include "rocksdb.h" + +using namespace rocksdb; + +/* +static void test(benchmark::State& state, RocksDB* db) +{ + for (auto _ : state) { + printf("%s\n", "once"); + db->run(); + } +} +*/ + +auto test = [](benchmark::State& state, RocksDB* db) { + for (auto _ : state) { + db->run(); + } +}; + +int main(int argc, char* argv[]) +{ + RocksDB* db = new RocksDB("testdb"); + + if (argc < 2) { + goto usage; + } + if (!strcmp(argv[1], "load")) { + std::string file_path = "data.txt"; + size_t nums = 1000; + + if (argc >= 3) { + file_path = argv[2]; + } + if (argc >= 4) { + nums = (size_t)atoi(argv[3]); + } + + db->load(file_path, nums); + db->compact(); + } else if (!strcmp(argv[1], "compact")) { + db->compact(); + } else if (!strcmp(argv[1], "run")) { + printf("%s\n", "benchmark begin!"); + + benchmark::RegisterBenchmark("test", test, db); + benchmark::RunSpecifiedBenchmarks(); + } else { + goto usage; + } + + delete db; + + return 0; + + usage: + fprintf(stderr, "%s\n", "Usage: ./scan_perf {load [file path] [nums]}/compact/run"); + return 1; +} \ No newline at end of file diff --git a/scan-perf/rocksdb.cpp b/scan-perf/rocksdb.cpp new file mode 100644 index 0000000..41356b6 --- /dev/null +++ b/scan-perf/rocksdb.cpp @@ -0,0 +1,161 @@ +// +// Created by frost on 18-9-20. +// + +#include +#include "rocksdb.h" + +RocksDB::RocksDB(std::string&& db_name_) : db_name(std::move(db_name_)) +{ + // rocksdb options + db_options.create_if_missing = true; + + fprintf(stderr, "Open db: %s\n", db_name.c_str()); + rocksdb::Status s = rocksdb::DB::Open(db_options, db_name, &db); + if (!s.ok()) { + fprintf(stderr, "FATAL: db(%s) open error: %s\n", + db_name.c_str(), s.ToString().c_str()); + exit(1); + } + + const auto& opt = db->GetOptions(); + + fprintf(stderr, + R"EOS(rocksdb options: +opt.write_buffer_size = %lld +opt.target_file_size_base = %lld +opt.target_file_size_multiplier = %d +)EOS" + , (long long)opt.write_buffer_size + , (long long)opt.target_file_size_base + , opt.target_file_size_multiplier); +} + +RocksDB::~RocksDB() +{ + assert(db != nullptr); + delete db; +} + +bool RocksDB::load(std::string fpath, size_t limit) +{ + FILE* file = fopen(fpath.c_str(), "r"); + if (file == nullptr) { + fprintf(stderr, "FATAL: invalid filename: %s\n", fpath.c_str()); + exit(1); + } + + printf("Begin load %ld lines data from %s\n", limit, fpath.c_str()); + size_t nums = 0; + char* buf = (char *)malloc(2048000 * sizeof(char)); + size_t len; + auto key = (char *)malloc(10240 * sizeof(char)); + auto value = (char *)malloc(2048000 * sizeof(char)); + rocksdb::Status s; + while (nums < limit) { + ssize_t read = getline(&buf, &len, file); + if (read != -1) { + ssize_t pos = 0; + while (pos < read) { + if (buf[pos++] == '\t') { + memcpy(key, buf, (size_t)(pos - 1)); + key[pos - 1] = 0; + memcpy(value, buf + pos, (size_t)(read - pos -1)); + value[read - pos -1] = 0; + + s = db->Put(write_options, key, value); + // printf("key: %s value: %s\n", key, value); + if (!s.ok()) { + fprintf(stderr, "WARN: error insert: %s\n", buf); + } + + break; + } + } + } else { + fprintf(stderr, "ERROR: file EOF, lines < limit(%ld).\n", limit); + break; + } + nums++; + if (!(nums % 1000)) { + printf("\rinsert: %ld", nums); + } + } + + printf("\n%s\n", "Load data done!"); + + free(buf); + free(key); + free(value); + + return true; +} + +bool RocksDB::compact() +{ + rocksdb::CompactRangeOptions cro; + cro.exclusive_manual_compaction = false; + cro.target_level = 6; + printf("%s\n", "Begin compact..."); + rocksdb::Status s = db->CompactRange(cro, nullptr, nullptr); + if (!s.ok()) { + fprintf(stderr, "Compact db error: %s\n", s.ToString().c_str()); + return false; + } + printf("%s\n", "Compact done!"); + return true; +} + +void RocksDB::load_key(std::string fpath, size_t limit) +{ + FILE* file = fopen(fpath.c_str(), "r"); + if (file == nullptr) { + fprintf(stderr, "FATAL: invalid filename: %s\n", fpath.c_str()); + exit(1); + } + + printf("Begin load %ld keys from %s\n", limit, fpath.c_str()); + size_t nums = 0; + char* buf = (char *)malloc(2048000 * sizeof(char)); + size_t len; + auto key = (char *)malloc(10240 * sizeof(char)); + + while (nums < limit) { + ssize_t read = getline(&buf, &len, file); + if (read != -1) { + ssize_t pos = 0; + while (pos < read) { + if (buf[pos++] == '\t') { + memcpy(key, buf, (size_t) (pos - 1)); + key[pos - 1] = 0; + keys.emplace_back(std::string(key)); + + break; + } + } + } else { + fprintf(stderr, "ERROR: file EOF, lines < limit(%ld).\n", limit); + break; + } + } + + printf("%s\n", "Load keys done"); +} + +void RocksDB::run() +{ + load_key("data.txt", 10000); + rocksdb::Iterator* iter = db->NewIterator(read_options); + for(auto& key : keys) { + iter->Seek(key); + if (!iter->Valid()) { + fprintf(stderr, "Seek key=%s error\n", key.c_str()); + } + for (int i = 0; i < 10; i++) { + iter->Prev(); + if (!iter->Valid()) { + break; + } + } + } +} \ No newline at end of file diff --git a/scan-perf/rocksdb.h b/scan-perf/rocksdb.h new file mode 100644 index 0000000..065fcf1 --- /dev/null +++ b/scan-perf/rocksdb.h @@ -0,0 +1,33 @@ +// +// Created by frost on 18-9-20. +// + +#ifndef SCAN_PERF_ROCKSDB_H +#define SCAN_PERF_ROCKSDB_H + +#include +#include +#include +#include + +class RocksDB { +public: + explicit RocksDB(std::string&& db_name); + ~RocksDB(); + bool load(std::string fpath, size_t limit); + bool compact(); + void run(); + +private: + std::string db_name; + rocksdb::DB* db; + rocksdb::WriteOptions write_options; + rocksdb::ReadOptions read_options; + rocksdb::Options db_options; + std::vector keys; + + void load_key(std::string fpath, size_t limit); +}; + + +#endif //SCAN_PERF_ROCKSDB_H