From 1f05215ed1da3ad188d4723ca365b36600650393 Mon Sep 17 00:00:00 2001 From: WANG YEFU Date: Sun, 4 Aug 2024 16:36:26 +0800 Subject: [PATCH] Add nodejs binding (#9) --- .github/workflows/ci.yml | 66 ++++++++++++++++--- .gitignore | 5 +- benchmark/benchmark.py | 2 +- bindings/nodejs/package.json.tpl | 10 +++ bindings/nodejs/packages/README.md | 1 + .../vectorlite-darwin-arm64/package.json | 13 ++++ .../vectorlite-darwin-arm64/src/index.js | 7 ++ .../vectorlite-darwin-x64/package.json | 13 ++++ .../vectorlite-darwin-x64/src/index.js | 7 ++ .../vectorlite-linux-x64/package.json | 13 ++++ .../vectorlite-linux-x64/src/index.js | 7 ++ .../vectorlite-win32-x64/package.json | 13 ++++ .../vectorlite-win32-x64/src/index.js | 7 ++ bindings/nodejs/packages/vectorlite/README.md | 34 ++++++++++ .../nodejs/packages/vectorlite/package.json | 23 +++++++ .../nodejs/packages/vectorlite/src/index.js | 28 ++++++++ .../nodejs/packages/vectorlite/test/test.js | 27 ++++++++ extract_wheels.sh | 26 ++++++++ 18 files changed, 290 insertions(+), 12 deletions(-) create mode 100644 bindings/nodejs/package.json.tpl create mode 100644 bindings/nodejs/packages/README.md create mode 100644 bindings/nodejs/packages/vectorlite-darwin-arm64/package.json create mode 100644 bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js create mode 100644 bindings/nodejs/packages/vectorlite-darwin-x64/package.json create mode 100644 bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js create mode 100644 bindings/nodejs/packages/vectorlite-linux-x64/package.json create mode 100644 bindings/nodejs/packages/vectorlite-linux-x64/src/index.js create mode 100644 bindings/nodejs/packages/vectorlite-win32-x64/package.json create mode 100644 bindings/nodejs/packages/vectorlite-win32-x64/src/index.js create mode 100644 bindings/nodejs/packages/vectorlite/README.md create mode 100644 bindings/nodejs/packages/vectorlite/package.json create mode 100644 bindings/nodejs/packages/vectorlite/src/index.js create mode 100644 bindings/nodejs/packages/vectorlite/test/test.js create mode 100644 extract_wheels.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2d21f4..23aeb63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,13 @@ on: type: 'choice' options: ['no', 'pypi', 'testpypi'] + publish_npm: + description: 'whether to publish the npm packages' + required: false + default: 'no' + type: 'choice' + options: ['no', 'yes'] + pull_request: branches: - main @@ -88,9 +95,6 @@ jobs: if: ${{ github.event.inputs.upload_wheel != 'no' && github.event_name != 'pull_request' }} needs: build_wheels runs-on: ubuntu-latest - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-13, macos-14] steps: - uses: actions/checkout@v4 @@ -98,10 +102,9 @@ jobs: - uses: benjlevesque/short-sha@v3.0 id: short_sha - # Download artifact + # Download all artifacts - uses: actions/download-artifact@v4 with: - name: vectorlite-wheel-${{ matrix.os }}-${{ steps.short_sha.outputs.sha }} path: ./wheelhouse - name: Upload to test.pypi.org @@ -109,17 +112,60 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }} - run: pipx run twine upload --repository testpypi wheelhouse/*.whl + run: pipx run twine upload --repository testpypi wheelhouse/**/*.whl - name: Upload to pypi.org - if: ${{ github.event.inputs.upload_wheel == 'pypi' && startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.event.inputs.upload_wheel == 'pypi' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} - run: pipx run twine upload wheelhouse/*.whl + run: pipx run twine upload wheelhouse/**/*.whl - name: Fail if uploading to pypi.org without a tag - if: ${{ github.event.inputs.upload_wheel == 'pypi' && !startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.event.inputs.upload_wheel == 'pypi' && !(startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} run: | echo "Error: Uploading to pypi.org requires a tag" - exit 1 \ No newline at end of file + exit 1 + + publish_npm_pkgs: + name: Upload npm packages + if: ${{ github.event.inputs.publish_npm == 'yes' && github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }} + needs: build_wheels + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - uses: benjlevesque/short-sha@v3.0 + id: short_sha + + # Download all artifacts + - uses: actions/download-artifact@v4 + with: + path: ./wheelhouse + + # extract vectorlite from wheels and copy to nodejs bindings directory + - name: unzip wheels + run: | + sh extract_wheels.sh + + - uses: actions/setup-node@v4 + with: + node-version: lts/* + registry-url: 'https://registry.npmjs.org' + + - name: Test locally + working-directory: bindings/nodejs/packages/vectorlite + run: | + npm i -D + npm link ../vectorlite-linux-x64 + npm run test + + - name: Publish to npm + working-directory: bindings/nodejs + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }} + run: | + mv package.json.tpl package.json + npm publish --workspaces --access public + diff --git a/.gitignore b/.gitignore index 9727cf4..5281674 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,7 @@ vcpkg_installed/* *.pyc dist/* -*egg-info \ No newline at end of file +*egg-info + +node_modules +bindings/nodejs/vectorlite/package-lock.json \ No newline at end of file diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index e37cfc5..deb407c 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -34,7 +34,7 @@ def timeit(func): cursor = conn.cursor() -NUM_ELEMENTS = 10000 # number of vectors +NUM_ELEMENTS = 1000 # number of vectors NUM_QUERIES = 100 # number of queries DIMS = [256, 1024] diff --git a/bindings/nodejs/package.json.tpl b/bindings/nodejs/package.json.tpl new file mode 100644 index 0000000..5c764f0 --- /dev/null +++ b/bindings/nodejs/package.json.tpl @@ -0,0 +1,10 @@ +{ + "name": "vectorlite-workspaces", + "workspaces": [ + "packages/vectorlite-linux-x64", + "packages/vectorlite-win32-x64", + "packages/vectorlite-darwin-x64", + "packages/vectorlite-darwin-arm64", + "packages/vectorlite" + ] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/README.md b/bindings/nodejs/packages/README.md new file mode 100644 index 0000000..36f97c3 --- /dev/null +++ b/bindings/nodejs/packages/README.md @@ -0,0 +1 @@ +This folder hosts nodejs bindings for vectorlite. `vectorlite.[so|dll|dylib]` is copied to their own platform dependent package folder. Please check ci.yml for details. \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json b/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json new file mode 100644 index 0000000..0a698b6 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-arm64/package.json @@ -0,0 +1,13 @@ +{ + "name": "@1yefuwang1/vectorlite-darwin-arm64", + "version": "0.1.0", + "homepage": "https://github.com/1yefuwang1/vectorlite", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["darwin"], + "cpu": ["arm64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js b/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-x64/package.json b/bindings/nodejs/packages/vectorlite-darwin-x64/package.json new file mode 100644 index 0000000..c2c1c31 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-x64/package.json @@ -0,0 +1,13 @@ +{ + "name": "@1yefuwang1/vectorlite-darwin-x64", + "version": "0.1.0", + "homepage": "https://github.com/1yefuwang1/vectorlite", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["darwin"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js b/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-linux-x64/package.json b/bindings/nodejs/packages/vectorlite-linux-x64/package.json new file mode 100644 index 0000000..33e348a --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-linux-x64/package.json @@ -0,0 +1,13 @@ +{ + "name": "@1yefuwang1/vectorlite-linux-x64", + "version": "0.1.0", + "homepage": "https://github.com/1yefuwang1/vectorlite", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["linux"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js b/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-linux-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-win32-x64/package.json b/bindings/nodejs/packages/vectorlite-win32-x64/package.json new file mode 100644 index 0000000..be21078 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-win32-x64/package.json @@ -0,0 +1,13 @@ +{ + "name": "@1yefuwang1/vectorlite-win32-x64", + "version": "0.1.0", + "homepage": "https://github.com/1yefuwang1/vectorlite", + "main": "src/index.js", + "files": ["src"], + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "os": ["win32"], + "cpu": ["x64"] +} \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js b/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js new file mode 100644 index 0000000..f1cc5d2 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite-win32-x64/src/index.js @@ -0,0 +1,7 @@ +const path = require('path'); + +function vectorlitePath() { + return path.join(__dirname, 'vectorlite'); +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/README.md b/bindings/nodejs/packages/vectorlite/README.md new file mode 100644 index 0000000..c467b35 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/README.md @@ -0,0 +1,34 @@ +# `vectorlite` for nodejs +Vectorlite is a fast and tunable vector search extension for SQLite. +For more info, please check https://github.com/1yefuwang1/vectorlite. +# Example +Below is an example of using it with `better-sqlite3`. +```javascript +const sqlite3 = require('better-sqlite3'); +const vectorlite = require('vectorlite'); + +const db = new sqlite3(':memory:'); +db.loadExtension(vectorlite.vectorlitePath()); + +console.log(db.prepare('select vectorlite_info()').all()); + +// Create a vectorlite virtual table hosting 10-dimensional float32 vectors with hnsw index +db.exec('create virtual table test using vectorlite(vec float32[10], hnsw(max_elements=100));') + +// insert a json vector +db.prepare('insert into test(rowid, vec) values (?, vector_from_json(?))').run([0, JSON.stringify(Array.from({length: 10}, () => Math.random()))]); +// insert a raw vector +db.prepare('insert into test(rowid, vec) values (?, ?)').run([1, Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +// a normal vector query +let result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2))') + .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +console.log(result); + +// a vector query with rowid filter +result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2)) and rowid in (1,2,3)') + .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +console.log(result); +``` \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/package.json b/bindings/nodejs/packages/vectorlite/package.json new file mode 100644 index 0000000..7292930 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/package.json @@ -0,0 +1,23 @@ +{ + "name": "vectorlite", + "version": "0.1.0", + "homepage": "https://github.com/1yefuwang1/vectorlite", + "main": "src/index.js", + "files": ["src"], + "scripts": { + "test": "node test/test.js" + }, + "author": "1yefuwang1@gmail.com", + "license": "Apache-2.0", + "description": "A fast and tunable vector search extension for SQLite", + "keywords": ["sqlite3", "vector database", "vectordb"], + "devDependencies": { + "better-sqlite3": "^11.1.2" + }, + "optionalDependencies": { + "@1yefuwang1/vectorlite-darwin-x64": "0.1.0", + "@1yefuwang1/vectorlite-darwin-arm64": "0.1.0", + "@1yefuwang1/vectorlite-linux-x64": "0.1.0", + "@1yefuwang1/vectorlite-win32-x64": "0.1.0" + } +} diff --git a/bindings/nodejs/packages/vectorlite/src/index.js b/bindings/nodejs/packages/vectorlite/src/index.js new file mode 100644 index 0000000..c59abc5 --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/src/index.js @@ -0,0 +1,28 @@ +const os = require('os'); + +const supportedPlatformsAndArchs = { + 'darwin-x64': '@1yefuwang1/vectorlite-darwin-x64', + 'darwin-arm64': '@1yefuwang1/vectorlite-darwin-arm64', + 'linux-x64': '@1yefuwang1/vectorlite-linux-x64', + 'win32-x64': '@1yefuwang1/vectorlite-win32-x64', +}; + +const platformAndArch = `${os.platform()}-${os.arch()}`; + +let vectorlitePathCache = undefined; + +// Returns path to the vectorlite shared library +function vectorlitePath() { + if (vectorlitePathCache) { + return vectorlitePathCache; + } + const packageName = supportedPlatformsAndArchs[platformAndArch]; + if (!packageName) { + throw new Error(`Platform ${platformAndArch} is not supported`); + } + const package = require(packageName); + vectorlitePathCache = package.vectorlitePath(); + return vectorlitePathCache; +} + +exports.vectorlitePath = vectorlitePath; \ No newline at end of file diff --git a/bindings/nodejs/packages/vectorlite/test/test.js b/bindings/nodejs/packages/vectorlite/test/test.js new file mode 100644 index 0000000..f00a55d --- /dev/null +++ b/bindings/nodejs/packages/vectorlite/test/test.js @@ -0,0 +1,27 @@ +const sqlite3 = require('better-sqlite3'); +const vectorlite = require('../src/index.js'); + +const db = new sqlite3(':memory:'); +db.loadExtension(vectorlite.vectorlitePath()); + +console.log(db.prepare('select vectorlite_info()').all()); + +// Create a vectorlite virtual table hosting 10-dimensional float32 vectors with hnsw index +db.exec('create virtual table test using vectorlite(vec float32[10], hnsw(max_elements=100));') + +// insert a json vector +db.prepare('insert into test(rowid, vec) values (?, vector_from_json(?))').run([0, JSON.stringify(Array.from({length: 10}, () => Math.random()))]); +// insert a raw vector +db.prepare('insert into test(rowid, vec) values (?, ?)').run([1, Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +// a normal vector query +let result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2))') + .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +console.log(result); + +// a vector query with rowid filter +result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2)) and rowid in (1,2,3)') + .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]); + +console.log(result); \ No newline at end of file diff --git a/extract_wheels.sh b/extract_wheels.sh new file mode 100644 index 0000000..b03d0c7 --- /dev/null +++ b/extract_wheels.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +for wheel in wheelhouse/vectorlite-wheel*/*.whl; do + unziped_dir=$wheel.unzipped + unzip $wheel -d $unziped_dir + + case "$wheel" in + *linux*x86_64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.so bindings/nodejs/packages/vectorlite-linux-x64/src + ;; + *win*amd64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dll bindings/nodejs/packages/vectorlite-win32-x64/src + ;; + *macosx*arm64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-arm64/src + ;; + *macosx*x86_64.whl) + cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-x64/src + ;; + *) + echo "Unknown wheel type: $wheel" + exit 1 + ;; + esac +done +