Add a feature to link custom duckdb extension

.gitignore

@@ -28,3 +28,6 @@ compile_commands.json
 data/tpch/**/*.parquet
 .idea
 lib/.idea
+
+check_duckdb
+wasm_setup

Makefile

@@ -28,6 +28,7 @@ JS_FILTER=
 EXTENSION_CACHE_DIR="${ROOT_DIR}/.ccache/extension"
 EXCEL_EXTENSION_CACHE_FILE="${EXTENSION_CACHE_DIR}/excel"
 JSON_EXTENSION_CACHE_FILE="${EXTENSION_CACHE_DIR}/json"
+CUSTOM_EXTENSION_ENVIRONMENT=DUCKDB_WEB_CUSTOM_EXTENSION_DIRS="${CUSTOM_EXTENSION_DIRS}"
 
 cpp_lib: lib_tests
 
@@ -256,27 +257,43 @@ wasm_setup: set_environment check_duckdb wrapped_wasm_caches
 
 .PHONY: wasm_dev
 wasm_dev: wasm_setup
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev mvp
+ifneq (${DUCKDB_SKIP_BUILD_EH}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev eh
+endif
+ifneq (${DUCKDB_SKIP_BUILD_COI}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh dev coi
+endif
 
 .PHONY: wasm_relperf
 wasm_relperf: wasm_setup
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf mvp
+ifneq (${DUCKDB_SKIP_BUILD_EH}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf eh
+endif
+ifneq (${DUCKDB_SKIP_BUILD_COI}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relperf coi
+endif
 
 .PHONY: wasm_relsize
 wasm_relsize: wasm_setup
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize mvp
+ifneq (${DUCKDB_SKIP_BUILD_EH}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize eh
+endif
+ifneq (${DUCKDB_SKIP_BUILD_COI}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh relsize coi
+endif
 
 .PHONY: wasm_debug
 wasm_debug: wasm_setup
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh
-	${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug mvp
+ifneq (${DUCKDB_SKIP_BUILD_EH}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug eh
+endif
+ifneq (${DUCKDB_SKIP_BUILD_COI}, 1)
+	${CUSTOM_EXTENSION_ENVIRONMENT} ${EXEC_ENVIRONMENT} ${ROOT_DIR}/scripts/wasm_build_lib.sh debug coi
+endif
 
 wasm: wasm_relperf
 
@@ -397,7 +414,6 @@ eslint:
 # Install all yarn packages
 .PHONY: yarn_install
 yarn_install:
-	yarn
 	yarn install
 
 .PHONY: examples

examples/bare-browser/package.json

@@ -5,7 +5,7 @@
     "main": "index.js",
     "license": "MIT",
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm"
+        "@duckdb/duckdb-wasm": "1.11.0"
     },
     "devDependencies": {
         "http-server": "^14.1.1"

examples/bare-node/package.json

@@ -5,7 +5,7 @@
     "main": "index.js",
     "license": "MIT",
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm"
+        "@duckdb/duckdb-wasm": "1.11.0"
     },
     "scripts": {
         "test": "node ./index.cjs"

examples/esbuild-browser/package.json

@@ -5,7 +5,7 @@
     "main": "index.js",
     "license": "MIT",
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm"
+        "@duckdb/duckdb-wasm": "1.11.0"
     },
     "devDependencies": {
         "esbuild": "^0.19.5",

examples/esbuild-node/package.json

@@ -5,7 +5,7 @@
     "main": "index.js",
     "license": "MIT",
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../../packages/duckdb-wasm"
+        "@duckdb/duckdb-wasm": "1.11.0"
     },
     "devDependencies": {
         "esbuild": "^0.19.5",

lib/CMakeLists.txt

@@ -10,6 +10,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(ignoreMe "${DUCKDB_WASM_VERSION}")
+set(DUCKDB_WEB_CUSTOM_EXTENSION_DIRS $ENV{DUCKDB_WEB_CUSTOM_EXTENSION_DIRS})
 
 option(DUCKDB_WASM_LOADABLE_EXTENSIONS "Build with loadable extensions" OFF)
 option(DUCKDB_WASM_LOADABLE_EXTENSIONS_UNSIGNED "Build with loadable extensions" OFF)
@@ -287,6 +288,40 @@ target_link_libraries(duckdb_web_parquet duckdb duckdb_parquet)
 target_link_libraries(duckdb_web_excel duckdb duckdb_excel)
 target_link_libraries(duckdb_web_json duckdb duckdb_json)
 
+# ---------------------------------------------------------------------------
+# Custom extensions
+
+macro(register_custom_extension name lib)
+  set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES "${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES};${name}" PARENT_SCOPE)
+  set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS "${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS};${lib}" PARENT_SCOPE)
+endmacro()
+
+set(DUCKDB_WEB_CUSTOM_EXTENSION_NAMES "")
+set(DUCKDB_WEB_CUSTOM_EXTENSION_LIBS "")
+
+if(DUCKDB_WEB_CUSTOM_EXTENSION_DIRS)
+  foreach(dir ${DUCKDB_WEB_CUSTOM_EXTENSION_DIRS})
+    message(STATUS "Adding custom extension dir ${dir}")
+    add_subdirectory(${dir} ${CMAKE_BINARY_DIR}/${dir})
+  endforeach()
+endif()
+
+foreach(lib ${DUCKDB_WEB_CUSTOM_EXTENSION_LIBS})
+  target_link_libraries(duckdb_web ${lib})
+endforeach()
+
+set(GENERATED_HEADER_FILE ${PROJECT_BINARY_DIR}/codegen/include/duckdb/generated_extensions_header.hpp)
+set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS "")
+set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS "")
+
+foreach(name ${DUCKDB_WEB_CUSTOM_EXTENSION_NAMES})
+  set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS "${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS}extern void ${name}_init(duckdb::DuckDB *db);")
+  set(DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS "${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS}${name}_init(db);")
+endforeach()
+
+configure_file(${CMAKE_SOURCE_DIR}/extension/generated_custom_extensions_header.hpp.in ${GENERATED_HEADER_FILE})
+include_directories(${PROJECT_BINARY_DIR}/codegen/include)
+
 # ---------------------------------------------------------------------------
 # Emscripten
 
@@ -305,6 +340,8 @@ if(EMSCRIPTEN)
       -s WARN_ON_UNDEFINED_SYMBOLS=0 \
       -s ALLOW_MEMORY_GROWTH=1 \
       -s MAXIMUM_MEMORY=4GB \
+      -s INITIAL_MEMORY=32MB \
+      -s STACK_SIZE=16MB \
       -s MODULARIZE=1 \
       -s EXPORT_NAME='DuckDB' \
       -s EXPORTED_FUNCTIONS='[ \

lib/extension/CMakeLists.template.txt

@@ -0,0 +1,9 @@
+# This is a CMake file template for a custom extension.
+# This file is expected to be copied to the extension directory, and specified via
+# `CUSTOM_EXTENSION_DIRS` variable for duckdb-wasm Makefile.
+
+cmake_minimum_required(VERSION 3.10)
+
+# This CMake file is expected to call `register_custom_extension` to tell
+# duckdb-wasm CMake file to link the custom extension library and call the init-function at the startup.
+register_custom_extension("libhello_ext" "/path/to/libhello_ext.a")

lib/extension/generated_custom_extensions_header.hpp.in

@@ -0,0 +1,14 @@
+#ifndef DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP
+#define DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP
+
+#include "duckdb/main/database.hpp"
+
+extern "C" {
+${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_FUNCTIONS}
+}
+
+void LoadCustomExtensions(duckdb::DuckDB *db) {
+    ${DUCKDB_WEB_CUSTOM_EXTENSION_INIT_CALLS}
+}
+
+#endif  // DUCKDB_WASM_GENERATED_CUSTOM_EXTENSIONS_HEADER_HPP

lib/src/webdb.cc

@@ -62,6 +62,7 @@
 #include "duckdb/web/udf.h"
 #include "duckdb/web/utils/debug.h"
 #include "duckdb/web/utils/wasm_response.h"
+#include "duckdb/generated_extensions_header.hpp"
 #include "rapidjson/document.h"
 #include "rapidjson/error/en.h"
 #include "rapidjson/rapidjson.h"
@@ -829,6 +830,7 @@ arrow::Status WebDB::Open(std::string_view args_json) {
         duckdb_web_json_init(db.get());
 #endif
 #endif  // WASM_LOADABLE_EXTENSIONS
+        LoadCustomExtensions(db.get());
         RegisterCustomExtensionOptions(db);
 
         // Reset state that is specific to the old database

packages/benchmarks/package.json

@@ -4,7 +4,7 @@
     "license": "MIT",
     "private": true,
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../duckdb-wasm",
+        "@duckdb/duckdb-wasm": "1.11.0",
         "apache-arrow": "^13.0.0",
         "apache-arrow-3": "npm:apache-arrow@^3.0.0",
         "arquero": "^5.2.0",

packages/duckdb-wasm-app/package.json

@@ -18,8 +18,8 @@
         "node": ">=14.13"
     },
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../duckdb-wasm",
-        "@duckdb/duckdb-wasm-shell": "file:../duckdb-wasm-shell",
+        "@duckdb/duckdb-wasm": "1.11.0",
+        "@duckdb/duckdb-wasm-shell": "1.11.0",
         "apache-arrow": "^13.0.0",
         "bootstrap": "^5.3.2",
         "classnames": "^2.3.2",
@@ -54,6 +54,7 @@
         "file-loader": "^6.2.0",
         "html-webpack-plugin": "^5.5.3",
         "jasmine-core": "^5.1.1",
+        "ifdef-loader": "^2.3.2",
         "jasmine-spec-reporter": "^7.0.0",
         "karma": "^6.4.2",
         "karma-chrome-launcher": "^3.2.0",

packages/duckdb-wasm-app/src/app.tsx

@@ -14,18 +14,27 @@ import 'react-popper-tooltip/dist/styles.css';
 
 import * as duckdb from '@duckdb/duckdb-wasm';
 import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm';
+/// #if EH_BUILD_ENABLED
 import duckdb_wasm_eh from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm';
+/// #endif
+/// #if COI_BUILD_ENABLED
 import duckdb_wasm_coi from '@duckdb/duckdb-wasm/dist/duckdb-coi.wasm';
+/// #endif
 
 const DUCKDB_BUNDLES: duckdb.DuckDBBundles = {
     mvp: {
         mainModule: duckdb_wasm,
         mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString(),
     },
+    /// #if EH_BUILD_ENABLED
     eh: {
         mainModule: duckdb_wasm_eh,
         mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString(),
     },
+    /// #else
+    eh: undefined,
+    /// #endif
+    /// #if COI_BUILD_ENABLED
     coi: {
         mainModule: duckdb_wasm_coi,
         mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-coi.worker.js', import.meta.url).toString(),
@@ -34,6 +43,9 @@ const DUCKDB_BUNDLES: duckdb.DuckDBBundles = {
             import.meta.url,
         ).toString(),
     },
+    /// #else
+    coi: undefined,
+    /// #endif
 };
 const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING);
 

packages/duckdb-wasm-app/webpack.common.js

@@ -33,9 +33,20 @@ export function configure(params) {
                 },
                 {
                     test: /\.tsx?$/,
-                    loader: 'ts-loader',
                     exclude: /node_modules/,
-                    options: params.tsLoaderOptions,
+                    use: [
+                        {
+                            loader: 'ts-loader',
+                            options: params.tsLoaderOptions,
+                        },
+                        {
+                            loader: 'ifdef-loader',
+                            options: {
+                                EH_BUILD_ENABLED: process.env['DUCKDB_SKIP_BUILD_EH'] !== '1',
+                                COI_BUILD_ENABLED: process.env['DUCKDB_SKIP_BUILD_COI'] !== '1',
+                            },
+                        },
+                    ],
                 },
                 {
                     test: /\.css$/,

packages/duckdb-wasm-shell/package.json

@@ -24,7 +24,7 @@
         "csv"
     ],
     "dependencies": {
-        "@duckdb/duckdb-wasm": "file:../duckdb-wasm",
+        "@duckdb/duckdb-wasm": "1.11.0",
         "xterm": "^5.1.0",
         "xterm-addon-fit": "^0.7.0",
         "xterm-addon-web-links": "^0.9.0",