Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Start to vendor lmdb-js-lite #18

Merged
merged 16 commits into from
Aug 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
258 changes: 241 additions & 17 deletions Cargo.lock

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions crates/lmdb-js-lite/.cargo/config.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[target.aarch64-unknown-linux-musl]
linker = "aarch64-linux-musl-gcc"
rustflags = ["-C", "target-feature=-crt-static"]
[target.x86_64-pc-windows-msvc]
rustflags = ["-C", "target-feature=+crt-static"]
3 changes: 3 additions & 0 deletions crates/lmdb-js-lite/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
/typescript/*.js
/*.node
/databases
9 changes: 9 additions & 0 deletions crates/lmdb-js-lite/.mocharc.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"$schema": "https://json.schemastore.org/mocharc",
"spec": "__test__/*.test.ts",
"timeout": 20000,
"require": [
"ts-node/register",
"@atlaspack/test-utils/src/mochaSetup.js"
]
}
14 changes: 14 additions & 0 deletions crates/lmdb-js-lite/.npmignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
target
yamadapc marked this conversation as resolved.
Show resolved Hide resolved
Cargo.lock
.cargo
.github
npm
.eslintrc
.prettierignore
rustfmt.toml
yarn.lock
*.node
.yarn
__test__
__benchmarks__
renovate.json
36 changes: 36 additions & 0 deletions crates/lmdb-js-lite/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
[package]
edition = "2021"
name = "lmdb-js-lite"
version = "0.1.5"
repository = "https://github.com/atlassian-labs/atlaspack"

[[bench]]
name = "lmdb_js_safe_benchmark"
harness = false

[lib]
crate-type = ["cdylib", "lib"]

[dependencies]
anyhow = "1.0.86"
crossbeam = "0.8.4"
heed = "0.20.3"
lazy_static = "1.5.0"
napi = { version = "3.0.0-alpha.8", default-features = false, features = ["napi4", "tokio"] }
napi-derive = "3.0.0-alpha.7"
rayon = "1.10.0"
thiserror = "1.0.63"
tracing = "0.1.40"
tracing-subscriber = "0.3.18"
lz4_flex = "0.11.3"

[dev-dependencies]
criterion = "0.5.1"
rand = "0.9.0-alpha.2"

[build-dependencies]
napi-build = "2.1.3"

[profile.release]
lto = true
strip = "symbols"
9 changes: 9 additions & 0 deletions crates/lmdb-js-lite/__benchmarks__/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# lmdb-js-lite benchmarks

Simple benchmarks of the NAPI interface against lmdb-js.

## Usage

```
for i in __benchmarks__/*.ts ; do printf "$i\n=========================\n"; yarn ts-node $i; done
```
yamadapc marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import { randomBytes } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import * as v8 from "node:v8";
import { open as openLMDBUnsafe } from "lmdb";

const ENTRY_SIZE = 64 * 1024; // 64KB
const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB

let key = 0;

function generateEntry() {
return {
key: String(key++),
value: randomBytes(ENTRY_SIZE),
};
}

async function main() {
rmSync("./databases/unsafe", {
recursive: true,
force: true,
});
mkdirSync("./databases/unsafe", {
recursive: true,
});

const unsafeDB = openLMDBUnsafe({
path: "./databases/unsafe/read",
encoding: "binary",
compression: true,
eventTurnBatching: true,
});

console.log("Generating entries for testing");
const entries = [...Array(NUM_ENTRIES)].map(() => {
yamadapc marked this conversation as resolved.
Show resolved Hide resolved
return generateEntry();
});
await unsafeDB.transaction(() => {
console.log("Writing entries");
for (let entry of entries) {
unsafeDB.put(entry.key, entry.value);
}
unsafeDB.put(
"benchmarkInfo",
v8.serialize({
NUM_ENTRIES,
}),
);
});
}

main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
57 changes: 57 additions & 0 deletions crates/lmdb-js-lite/__benchmarks__/prepare-read-benchmark.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import { randomBytes } from "node:crypto";
import { mkdirSync, rmSync } from "node:fs";
import * as v8 from "node:v8";
import { Lmdb } from "../index";

const ENTRY_SIZE = 64 * 1024; // 64KB
const ASYNC_WRITES = true;
const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB
const MAP_SIZE = 1024 * 1024 * 1024 * 10;

let key = 0;

function generateEntry() {
return {
key: String(key++),
value: randomBytes(ENTRY_SIZE),
};
}

async function main() {
rmSync("./databases/safe", {
recursive: true,
force: true,
});
mkdirSync("./databases/safe", {
recursive: true,
});

const safeDB = new Lmdb({
path: "./databases/safe/read",
asyncWrites: ASYNC_WRITES,
mapSize: MAP_SIZE,
});

console.log("Generating entries for testing");
const entries = [...Array(NUM_ENTRIES)].map(() => {
return generateEntry();
});
console.log("Writing entries");
await safeDB.startWriteTransaction();
for (let entry of entries) {
await safeDB.put(entry.key, entry.value);
}
await safeDB.put(
"benchmarkInfo",
v8.serialize({
NUM_ENTRIES,
}),
);
await safeDB.commitWriteTransaction();
safeDB.close();
}

main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
63 changes: 63 additions & 0 deletions crates/lmdb-js-lite/__benchmarks__/read-throughput-safe.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import { Lmdb } from "../index";
import * as v8 from "node:v8";

const MAX_TIME = 10000;
const ASYNC_WRITES = true;
const MAP_SIZE = 1024 * 1024 * 1024 * 10;

async function main() {
const safeDB = new Lmdb({
path: "./databases/safe/read",
asyncWrites: ASYNC_WRITES,
mapSize: MAP_SIZE,
});

const value = safeDB.getSync("benchmarkInfo");
if (!value) throw new Error("Run prepare-read-benchmark.ts");
const benchmarkInfo = v8.deserialize(value);
console.log(benchmarkInfo);

const { NUM_ENTRIES } = benchmarkInfo;
console.log("(transaction) Reading all entries out");
safeDB.startReadTransaction();
{
const start = Date.now();
const readEntries = [];
let i = 0;
while (Date.now() - start < MAX_TIME) {
readEntries.push(safeDB.getSync(String(i % NUM_ENTRIES)));
i += 1;
}
const duration = Date.now() - start;
const throughput = readEntries.length / duration;
console.log(
"(transaction) Safe Throughput:",
throughput,
"entries / second",
);
}
safeDB.commitReadTransaction();

console.log("(no-transaction) Reading all entries out");
{
const start = Date.now();
const readEntries = [];
let i = 0;
while (Date.now() - start < MAX_TIME) {
readEntries.push(safeDB.getSync(String(i % NUM_ENTRIES)));
i += 1;
}
const duration = Date.now() - start;
const throughput = readEntries.length / duration;
console.log(
"(no-transaction) Safe Throughput:",
throughput,
"entries / second",
);
}
}

main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
38 changes: 38 additions & 0 deletions crates/lmdb-js-lite/__benchmarks__/read-throughput-unsafe.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
import { open as openLMDBUnsafe } from "lmdb";
import v8 from "node:v8";

const MAX_TIME = 10000;

async function main() {
const unsafeDB = openLMDBUnsafe({
path: "./databases/unsafe/read",
compression: true,
encoding: "binary",
eventTurnBatching: true,
});

const value = unsafeDB.get("benchmarkInfo");
if (!value) throw new Error("Run prepare-read-benchmark.ts");
const benchmarkInfo = v8.deserialize(value);
console.log(benchmarkInfo);
const { NUM_ENTRIES } = benchmarkInfo;

console.log("Reading all entries out");
{
const start = Date.now();
const readEntries = [];
let i = 0;
while (Date.now() - start < MAX_TIME) {
readEntries.push(unsafeDB.get(String(i % NUM_ENTRIES)));
i += 1;
}
const duration = Date.now() - start;
const throughput = readEntries.length / duration;
console.log("Unsafe Throughput:", throughput, "entries / second");
}
}

main().catch((err) => {
console.error(err);
process.exitCode = 1;
});
Loading