Start to vendor lmdb-js-lite (#18)

crates/lmdb-js-lite/.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.aarch64-unknown-linux-musl]
+linker = "aarch64-linux-musl-gcc"
+rustflags = ["-C", "target-feature=-crt-static"]
+[target.x86_64-pc-windows-msvc]
+rustflags = ["-C", "target-feature=+crt-static"]

crates/lmdb-js-lite/.gitignore

@@ -0,0 +1,3 @@
+/typescript/*.js
+/*.node
+/databases

crates/lmdb-js-lite/.mocharc.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://json.schemastore.org/mocharc",
+  "spec": "__test__/*.test.ts",
+  "timeout": 20000,
+  "require": [
+    "ts-node/register",
+    "@atlaspack/test-utils/src/mochaSetup.js"
+  ]
+}

crates/lmdb-js-lite/.npmignore

@@ -0,0 +1,14 @@
+target
+Cargo.lock
+.cargo
+.github
+npm
+.eslintrc
+.prettierignore
+rustfmt.toml
+yarn.lock
+*.node
+.yarn
+__test__
+__benchmarks__
+renovate.json

crates/lmdb-js-lite/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+edition = "2021"
+name = "lmdb-js-lite"
+version = "0.1.5"
+repository = "https://github.com/atlassian-labs/atlaspack"
+
+[[bench]]
+name = "lmdb_js_safe_benchmark"
+harness = false
+
+[lib]
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+anyhow = "1.0.86"
+crossbeam = "0.8.4"
+heed = "0.20.3"
+lazy_static = "1.5.0"
+napi = { version = "3.0.0-alpha.8", default-features = false, features = ["napi4", "tokio"] }
+napi-derive = "3.0.0-alpha.7"
+rayon = "1.10.0"
+thiserror = "1.0.63"
+tracing = "0.1.40"
+tracing-subscriber = "0.3.18"
+lz4_flex = "0.11.3"
+
+[dev-dependencies]
+criterion = "0.5.1"
+rand = "0.9.0-alpha.2"
+
+[build-dependencies]
+napi-build = "2.1.3"
+
+[profile.release]
+lto = true
+strip = "symbols"

crates/lmdb-js-lite/__benchmarks__/README.md

@@ -0,0 +1,9 @@
+# lmdb-js-lite benchmarks
+
+Simple benchmarks of the NAPI interface against lmdb-js.
+
+## Usage
+
+```
+for i in __benchmarks__/*.ts ; do printf "$i\n=========================\n"; yarn ts-node $i; done
+```

crates/lmdb-js-lite/__benchmarks__/prepare-read-benchmark-unsafe.ts

@@ -0,0 +1,55 @@
+import { randomBytes } from "node:crypto";
+import { mkdirSync, rmSync } from "node:fs";
+import * as v8 from "node:v8";
+import { open as openLMDBUnsafe } from "lmdb";
+
+const ENTRY_SIZE = 64 * 1024; // 64KB
+const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB
+
+let key = 0;
+
+function generateEntry() {
+  return {
+    key: String(key++),
+    value: randomBytes(ENTRY_SIZE),
+  };
+}
+
+async function main() {
+  rmSync("./databases/unsafe", {
+    recursive: true,
+    force: true,
+  });
+  mkdirSync("./databases/unsafe", {
+    recursive: true,
+  });
+
+  const unsafeDB = openLMDBUnsafe({
+    path: "./databases/unsafe/read",
+    encoding: "binary",
+    compression: true,
+    eventTurnBatching: true,
+  });
+
+  console.log("Generating entries for testing");
+  const entries = [...Array(NUM_ENTRIES)].map(() => {
+    return generateEntry();
+  });
+  await unsafeDB.transaction(() => {
+    console.log("Writing entries");
+    for (let entry of entries) {
+      unsafeDB.put(entry.key, entry.value);
+    }
+    unsafeDB.put(
+      "benchmarkInfo",
+      v8.serialize({
+        NUM_ENTRIES,
+      }),
+    );
+  });
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});

crates/lmdb-js-lite/__benchmarks__/prepare-read-benchmark.ts

@@ -0,0 +1,57 @@
+import { randomBytes } from "node:crypto";
+import { mkdirSync, rmSync } from "node:fs";
+import * as v8 from "node:v8";
+import { Lmdb } from "../index";
+
+const ENTRY_SIZE = 64 * 1024; // 64KB
+const ASYNC_WRITES = true;
+const NUM_ENTRIES = Math.floor((1024 * 1024 * 1024 * 5) / ENTRY_SIZE); // Total memory used 1GB
+const MAP_SIZE = 1024 * 1024 * 1024 * 10;
+
+let key = 0;
+
+function generateEntry() {
+  return {
+    key: String(key++),
+    value: randomBytes(ENTRY_SIZE),
+  };
+}
+
+async function main() {
+  rmSync("./databases/safe", {
+    recursive: true,
+    force: true,
+  });
+  mkdirSync("./databases/safe", {
+    recursive: true,
+  });
+
+  const safeDB = new Lmdb({
+    path: "./databases/safe/read",
+    asyncWrites: ASYNC_WRITES,
+    mapSize: MAP_SIZE,
+  });
+
+  console.log("Generating entries for testing");
+  const entries = [...Array(NUM_ENTRIES)].map(() => {
+    return generateEntry();
+  });
+  console.log("Writing entries");
+  await safeDB.startWriteTransaction();
+  for (let entry of entries) {
+    await safeDB.put(entry.key, entry.value);
+  }
+  await safeDB.put(
+    "benchmarkInfo",
+    v8.serialize({
+      NUM_ENTRIES,
+    }),
+  );
+  await safeDB.commitWriteTransaction();
+  safeDB.close();
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});

crates/lmdb-js-lite/__benchmarks__/read-throughput-safe.ts

@@ -0,0 +1,63 @@
+import { Lmdb } from "../index";
+import * as v8 from "node:v8";
+
+const MAX_TIME = 10000;
+const ASYNC_WRITES = true;
+const MAP_SIZE = 1024 * 1024 * 1024 * 10;
+
+async function main() {
+  const safeDB = new Lmdb({
+    path: "./databases/safe/read",
+    asyncWrites: ASYNC_WRITES,
+    mapSize: MAP_SIZE,
+  });
+
+  const value = safeDB.getSync("benchmarkInfo");
+  if (!value) throw new Error("Run prepare-read-benchmark.ts");
+  const benchmarkInfo = v8.deserialize(value);
+  console.log(benchmarkInfo);
+
+  const { NUM_ENTRIES } = benchmarkInfo;
+  console.log("(transaction) Reading all entries out");
+  safeDB.startReadTransaction();
+  {
+    const start = Date.now();
+    const readEntries = [];
+    let i = 0;
+    while (Date.now() - start < MAX_TIME) {
+      readEntries.push(safeDB.getSync(String(i % NUM_ENTRIES)));
+      i += 1;
+    }
+    const duration = Date.now() - start;
+    const throughput = readEntries.length / duration;
+    console.log(
+      "(transaction) Safe Throughput:",
+      throughput,
+      "entries / second",
+    );
+  }
+  safeDB.commitReadTransaction();
+
+  console.log("(no-transaction) Reading all entries out");
+  {
+    const start = Date.now();
+    const readEntries = [];
+    let i = 0;
+    while (Date.now() - start < MAX_TIME) {
+      readEntries.push(safeDB.getSync(String(i % NUM_ENTRIES)));
+      i += 1;
+    }
+    const duration = Date.now() - start;
+    const throughput = readEntries.length / duration;
+    console.log(
+      "(no-transaction) Safe Throughput:",
+      throughput,
+      "entries / second",
+    );
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});

crates/lmdb-js-lite/__benchmarks__/read-throughput-unsafe.ts

@@ -0,0 +1,38 @@
+import { open as openLMDBUnsafe } from "lmdb";
+import v8 from "node:v8";
+
+const MAX_TIME = 10000;
+
+async function main() {
+  const unsafeDB = openLMDBUnsafe({
+    path: "./databases/unsafe/read",
+    compression: true,
+    encoding: "binary",
+    eventTurnBatching: true,
+  });
+
+  const value = unsafeDB.get("benchmarkInfo");
+  if (!value) throw new Error("Run prepare-read-benchmark.ts");
+  const benchmarkInfo = v8.deserialize(value);
+  console.log(benchmarkInfo);
+  const { NUM_ENTRIES } = benchmarkInfo;
+
+  console.log("Reading all entries out");
+  {
+    const start = Date.now();
+    const readEntries = [];
+    let i = 0;
+    while (Date.now() - start < MAX_TIME) {
+      readEntries.push(unsafeDB.get(String(i % NUM_ENTRIES)));
+      i += 1;
+    }
+    const duration = Date.now() - start;
+    const throughput = readEntries.length / duration;
+    console.log("Unsafe Throughput:", throughput, "entries / second");
+  }
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exitCode = 1;
+});