Add nodejs binding (1yefuwang1#9)

.github/workflows/ci.yml

@@ -10,6 +10,13 @@ on:
         type: 'choice'
         options: ['no', 'pypi', 'testpypi']
 
+      publish_npm:
+        description: 'whether to publish the npm packages'
+        required: false
+        default: 'no'
+        type: 'choice'
+        options: ['no', 'yes']
+
   pull_request:
     branches:
       - main
@@ -88,38 +95,77 @@ jobs:
     if: ${{ github.event.inputs.upload_wheel != 'no' && github.event_name != 'pull_request' }}
     needs: build_wheels
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macos-13, macos-14]
 
     steps:
       - uses: actions/checkout@v4
 
       - uses: benjlevesque/[email protected]
         id: short_sha
 
-      # Download artifact
+      # Download all artifacts
       - uses: actions/download-artifact@v4
         with:
-          name: vectorlite-wheel-${{ matrix.os }}-${{ steps.short_sha.outputs.sha }}
           path: ./wheelhouse
 
       - name: Upload to test.pypi.org
         if: ${{ github.event.inputs.upload_wheel == 'testpypi' }}
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.TEST_PYPI_TOKEN }}
-        run: pipx run twine upload --repository testpypi wheelhouse/*.whl
+        run: pipx run twine upload --repository testpypi wheelhouse/**/*.whl
 
       - name: Upload to pypi.org
-        if: ${{ github.event.inputs.upload_wheel == 'pypi' && startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ github.event.inputs.upload_wheel == 'pypi' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }}
         env:
           TWINE_USERNAME: __token__
           TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-        run: pipx run twine upload wheelhouse/*.whl
+        run: pipx run twine upload wheelhouse/**/*.whl
 
       - name: Fail if uploading to pypi.org without a tag
-        if: ${{ github.event.inputs.upload_wheel == 'pypi' && !startsWith(github.ref, 'refs/tags/v') }}
+        if: ${{ github.event.inputs.upload_wheel == 'pypi' && !(startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }}
         run: | 
           echo "Error: Uploading to pypi.org requires a tag"
-          exit 1
+          exit 1
+
+  publish_npm_pkgs:
+    name: Upload npm packages
+    if: ${{ github.event.inputs.publish_npm == 'yes' && github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || startsWith(github.ref, 'refs/heads/release/')) }}
+    needs: build_wheels
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: benjlevesque/[email protected]
+        id: short_sha
+
+      # Download all artifacts
+      - uses: actions/download-artifact@v4
+        with:
+          path: ./wheelhouse
+
+      # extract vectorlite from wheels and copy to nodejs bindings directory
+      - name: unzip wheels
+        run: |
+          sh extract_wheels.sh
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Test locally
+        working-directory: bindings/nodejs/packages/vectorlite
+        run: |
+          npm i -D
+          npm link ../vectorlite-linux-x64
+          npm run test
+
+      - name: Publish to npm
+        working-directory: bindings/nodejs
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_PUBLISH_TOKEN }}
+        run: |
+          mv package.json.tpl package.json
+          npm publish --workspaces --access public
+

.gitignore

@@ -41,4 +41,7 @@ vcpkg_installed/*
 *.pyc
 dist/*
 
-*egg-info
+*egg-info
+
+node_modules
+bindings/nodejs/vectorlite/package-lock.json

benchmark/benchmark.py

@@ -34,7 +34,7 @@ def timeit(func):
 
 cursor = conn.cursor()
 
-NUM_ELEMENTS = 10000  # number of vectors
+NUM_ELEMENTS = 1000  # number of vectors
 NUM_QUERIES = 100  # number of queries
 
 DIMS = [256, 1024]

bindings/nodejs/package.json.tpl

@@ -0,0 +1,10 @@
+{
+  "name": "vectorlite-workspaces",
+  "workspaces": [
+    "packages/vectorlite-linux-x64",
+    "packages/vectorlite-win32-x64",
+    "packages/vectorlite-darwin-x64",
+    "packages/vectorlite-darwin-arm64",
+    "packages/vectorlite"
+  ]
+}

bindings/nodejs/packages/README.md

@@ -0,0 +1 @@
+This folder hosts nodejs bindings for vectorlite. `vectorlite.[so|dll|dylib]` is copied to their own platform dependent package folder. Please check ci.yml for details.

bindings/nodejs/packages/vectorlite-darwin-arm64/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@1yefuwang1/vectorlite-darwin-arm64",
+  "version": "0.1.0",
+  "homepage": "https://github.com/1yefuwang1/vectorlite",
+  "main": "src/index.js",
+  "files": ["src"],
+  "author": "[email protected]",
+  "license": "Apache-2.0",
+  "description": "A fast and tunable vector search extension for SQLite",
+  "keywords": ["sqlite3", "vector database", "vectordb"],
+  "os": ["darwin"],
+  "cpu": ["arm64"]
+}

bindings/nodejs/packages/vectorlite-darwin-arm64/src/index.js

@@ -0,0 +1,7 @@
+const path = require('path');
+
+function vectorlitePath() {
+    return path.join(__dirname, 'vectorlite');
+}
+
+exports.vectorlitePath = vectorlitePath;

bindings/nodejs/packages/vectorlite-darwin-x64/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@1yefuwang1/vectorlite-darwin-x64",
+  "version": "0.1.0",
+  "homepage": "https://github.com/1yefuwang1/vectorlite",
+  "main": "src/index.js",
+  "files": ["src"],
+  "author": "[email protected]",
+  "license": "Apache-2.0",
+  "description": "A fast and tunable vector search extension for SQLite",
+  "keywords": ["sqlite3", "vector database", "vectordb"],
+  "os": ["darwin"],
+  "cpu": ["x64"]
+}

bindings/nodejs/packages/vectorlite-darwin-x64/src/index.js

@@ -0,0 +1,7 @@
+const path = require('path');
+
+function vectorlitePath() {
+    return path.join(__dirname, 'vectorlite');
+}
+
+exports.vectorlitePath = vectorlitePath;

bindings/nodejs/packages/vectorlite-linux-x64/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@1yefuwang1/vectorlite-linux-x64",
+  "version": "0.1.0",
+  "homepage": "https://github.com/1yefuwang1/vectorlite",
+  "main": "src/index.js",
+  "files": ["src"],
+  "author": "[email protected]",
+  "license": "Apache-2.0",
+  "description": "A fast and tunable vector search extension for SQLite",
+  "keywords": ["sqlite3", "vector database", "vectordb"],
+  "os": ["linux"],
+  "cpu": ["x64"]
+}

bindings/nodejs/packages/vectorlite-linux-x64/src/index.js

@@ -0,0 +1,7 @@
+const path = require('path');
+
+function vectorlitePath() {
+    return path.join(__dirname, 'vectorlite');
+}
+
+exports.vectorlitePath = vectorlitePath;

bindings/nodejs/packages/vectorlite-win32-x64/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "@1yefuwang1/vectorlite-win32-x64",
+  "version": "0.1.0",
+  "homepage": "https://github.com/1yefuwang1/vectorlite",
+  "main": "src/index.js",
+  "files": ["src"],
+  "author": "[email protected]",
+  "license": "Apache-2.0",
+  "description": "A fast and tunable vector search extension for SQLite",
+  "keywords": ["sqlite3", "vector database", "vectordb"],
+  "os": ["win32"],
+  "cpu": ["x64"]
+}

bindings/nodejs/packages/vectorlite-win32-x64/src/index.js

@@ -0,0 +1,7 @@
+const path = require('path');
+
+function vectorlitePath() {
+    return path.join(__dirname, 'vectorlite');
+}
+
+exports.vectorlitePath = vectorlitePath;

bindings/nodejs/packages/vectorlite/README.md

@@ -0,0 +1,34 @@
+# `vectorlite` for nodejs
+Vectorlite is a fast and tunable vector search extension for SQLite.
+For more info, please check https://github.com/1yefuwang1/vectorlite.
+# Example
+Below is an example of using it with `better-sqlite3`.
+```javascript
+const sqlite3 = require('better-sqlite3');
+const vectorlite = require('vectorlite');
+
+const db = new sqlite3(':memory:');
+db.loadExtension(vectorlite.vectorlitePath());
+
+console.log(db.prepare('select vectorlite_info()').all());
+
+// Create a vectorlite virtual table hosting 10-dimensional float32 vectors with hnsw index
+db.exec('create virtual table test using vectorlite(vec float32[10], hnsw(max_elements=100));')
+
+// insert a json vector
+db.prepare('insert into test(rowid, vec) values (?, vector_from_json(?))').run([0, JSON.stringify(Array.from({length: 10}, () => Math.random()))]);
+// insert a raw vector
+db.prepare('insert into test(rowid, vec) values (?, ?)').run([1, Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+// a normal vector query
+let result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2))')
+    .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+console.log(result);
+
+// a vector query with rowid filter
+result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2)) and rowid in (1,2,3)')
+    .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+console.log(result);
+```

bindings/nodejs/packages/vectorlite/package.json

@@ -0,0 +1,23 @@
+{
+  "name": "vectorlite",
+  "version": "0.1.0",
+  "homepage": "https://github.com/1yefuwang1/vectorlite",
+  "main": "src/index.js",
+  "files": ["src"],
+  "scripts": {
+    "test": "node test/test.js"
+  },
+  "author": "[email protected]",
+  "license": "Apache-2.0",
+  "description": "A fast and tunable vector search extension for SQLite",
+  "keywords": ["sqlite3", "vector database", "vectordb"],
+  "devDependencies": {
+    "better-sqlite3": "^11.1.2"
+  },
+  "optionalDependencies": {
+    "@1yefuwang1/vectorlite-darwin-x64": "0.1.0",
+    "@1yefuwang1/vectorlite-darwin-arm64": "0.1.0",
+    "@1yefuwang1/vectorlite-linux-x64": "0.1.0",
+    "@1yefuwang1/vectorlite-win32-x64": "0.1.0"
+  }
+}

bindings/nodejs/packages/vectorlite/src/index.js

@@ -0,0 +1,28 @@
+const os = require('os');
+
+const supportedPlatformsAndArchs = {
+    'darwin-x64': '@1yefuwang1/vectorlite-darwin-x64',
+    'darwin-arm64': '@1yefuwang1/vectorlite-darwin-arm64',
+    'linux-x64': '@1yefuwang1/vectorlite-linux-x64',
+    'win32-x64': '@1yefuwang1/vectorlite-win32-x64',
+};
+
+const platformAndArch = `${os.platform()}-${os.arch()}`;
+
+let vectorlitePathCache = undefined;
+
+// Returns path to the vectorlite shared library
+function vectorlitePath() {
+    if (vectorlitePathCache) {
+        return vectorlitePathCache;
+    }
+    const packageName = supportedPlatformsAndArchs[platformAndArch];
+    if (!packageName) {
+        throw new Error(`Platform ${platformAndArch} is not supported`);
+    }
+    const package = require(packageName);
+    vectorlitePathCache = package.vectorlitePath();
+    return vectorlitePathCache;
+}
+
+exports.vectorlitePath = vectorlitePath;

bindings/nodejs/packages/vectorlite/test/test.js

@@ -0,0 +1,27 @@
+const sqlite3 = require('better-sqlite3');
+const vectorlite = require('../src/index.js');
+
+const db = new sqlite3(':memory:');
+db.loadExtension(vectorlite.vectorlitePath());
+
+console.log(db.prepare('select vectorlite_info()').all());
+
+// Create a vectorlite virtual table hosting 10-dimensional float32 vectors with hnsw index
+db.exec('create virtual table test using vectorlite(vec float32[10], hnsw(max_elements=100));')
+
+// insert a json vector
+db.prepare('insert into test(rowid, vec) values (?, vector_from_json(?))').run([0, JSON.stringify(Array.from({length: 10}, () => Math.random()))]);
+// insert a raw vector
+db.prepare('insert into test(rowid, vec) values (?, ?)').run([1, Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+// a normal vector query
+let result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2))')
+    .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+console.log(result);
+
+// a vector query with rowid filter
+result = db.prepare('select rowid from test where knn_search(vec, knn_param(?, 2)) and rowid in (1,2,3)')
+    .all([Buffer.from(Float32Array.from(Array.from({length: 10}, () => Math.random())).buffer)]);
+
+console.log(result);

extract_wheels.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+for wheel in wheelhouse/vectorlite-wheel*/*.whl; do
+    unziped_dir=$wheel.unzipped
+    unzip $wheel -d $unziped_dir
+
+    case "$wheel" in
+        *linux*x86_64.whl)
+            cp $unziped_dir/vectorlite_py/vectorlite.so bindings/nodejs/packages/vectorlite-linux-x64/src
+            ;;
+        *win*amd64.whl)
+            cp $unziped_dir/vectorlite_py/vectorlite.dll bindings/nodejs/packages/vectorlite-win32-x64/src
+            ;;
+        *macosx*arm64.whl)
+            cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-arm64/src
+            ;;
+        *macosx*x86_64.whl)
+            cp $unziped_dir/vectorlite_py/vectorlite.dylib bindings/nodejs/packages/vectorlite-darwin-x64/src
+            ;;
+        *)
+            echo "Unknown wheel type: $wheel"
+            exit 1
+            ;;
+    esac    
+done
+