Implement force-install and update extension (#5368)

acquamarin · web-flow · commit 2c15d16bc8ea · 2025-05-09T17:52:08.000-04:00
diff --git a/extension/algo/test/test_files/basic.test b/extension/algo/test/test_files/basic.test
@@ -4,8 +4,7 @@
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL algo FROM 'http://localhost/extension/repo/'
----- 1
-Extension: algo has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load algo
 ---- ok
 -STATEMENT load algo
diff --git a/extension/delta/src/installer/delta_install_func.cpp b/extension/delta/src/installer/delta_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"delta", repo};
+    kuzu::extension::InstallExtensionInfo info{"delta", repo, false /* forceInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/delta/test/test_files/delta.test b/extension/delta/test/test_files/delta.test
@@ -91,8 +91,7 @@ Ethan|28|1800.600000|1996-03-10 00:00:00
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL DELTA FROM 'http://localhost/extension/repo/'
----- 1
-Extension: delta has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension DELTA
 ---- ok
 -LOG LoadFromLocalDeltaTable
diff --git a/extension/duckdb/src/include/installer/duckdb_installer.h b/extension/duckdb/src/include/installer/duckdb_installer.h
@@ -10,7 +10,7 @@ class KUZU_API DuckDBInstaller final : public extension::ExtensionInstaller {
     DuckDBInstaller(const extension::InstallExtensionInfo& info, main::ClientContext& context)
         : ExtensionInstaller{info, context} {}
 
-    void install() override;
+    bool install() override;
 };
 
 } // namespace duckdb_extension
diff --git a/extension/duckdb/src/installer/duckdb_install_func.cpp b/extension/duckdb/src/installer/duckdb_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"duckdb", repo};
+    kuzu::extension::InstallExtensionInfo info{"duckdb", repo, false /* forceInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/duckdb/src/installer/duckdb_installer.cpp b/extension/duckdb/src/installer/duckdb_installer.cpp
@@ -6,7 +6,7 @@
 namespace kuzu {
 namespace duckdb_extension {
 
-void DuckDBInstaller::install() {
+bool DuckDBInstaller::install() {
     auto loaderFileRepoInfo =
         extension::ExtensionUtils::getExtensionLoaderRepoInfo(info.name, info.repo);
     auto localLoaderFilePath =
@@ -23,6 +23,7 @@ void DuckDBInstaller::install() {
             tryDownloadExtensionFile(dependencyLibRepoInfo, localDependencyLibPath);
         }
     }
+    return true;
 }
 
 } // namespace duckdb_extension
diff --git a/extension/duckdb/test/test_files/duckdb.test b/extension/duckdb/test/test_files/duckdb.test
@@ -307,8 +307,7 @@ Runtime exception: No loaded extension can handle database type: sqlite.
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL duckdb FROM 'http://localhost/extension/repo/'
----- 1
-Extension: duckdb has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension duckdb
 ---- ok
 -STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/dataset/databases/duckdb_database/tinysnb.db' as tinysnb (dbtype duckdb, skip_unsupported_table = true);
diff --git a/extension/fts/test/test_files/basic.test b/extension/fts/test/test_files/basic.test
@@ -308,8 +308,7 @@ title3
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL fts FROM 'http://localhost/extension/repo/'
----- 1
-Extension: fts has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load fts
 ---- ok
 -STATEMENT CALL CREATE_FTS_INDEX('Book', 'book_index', ['abstract', 'author', 'title'], stemmer := 'porter');
diff --git a/extension/httpfs/test/test_files/http.test b/extension/httpfs/test/test_files/http.test
@@ -164,8 +164,7 @@ Binder exception: Attaching a kuzu database without an alias is not allowed.
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL httpfs FROM 'http://localhost/extension/repo/'
----- 1
-Extension: httpfs has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension httpfs
 ---- ok
 -STATEMENT load from "http://localhost/dataset/tinysnb/vPerson.csv" return * limit 5;
diff --git a/extension/iceberg/src/installer/iceberg_install_func.cpp b/extension/iceberg/src/installer/iceberg_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"iceberg", repo};
+    kuzu::extension::InstallExtensionInfo info{"iceberg", repo, false /* forceInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/iceberg/test/test_files/iceberg.test b/extension/iceberg/test/test_files/iceberg.test
@@ -148,8 +148,7 @@ lineitem_iceberg/metadata/cf3d0be5-cf70-453d-ad8f-48fdc412e608-m0.avro|1|DATA|AD
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL iceberg FROM 'http://localhost/extension/repo/'
----- 1
-Extension: iceberg has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension iceberg
 ---- ok
 -STATEMENT LOAD FROM '${KUZU_ROOT_DIRECTORY}/extension/iceberg/test/iceberg_tables/lineitem_iceberg' (file_format='iceberg', allow_moved_paths=true) RETURN l_linenumber, l_quantity, l_shipdate, l_shipinstruct ORDER BY l_quantity, l_shipdate LIMIT 5;
diff --git a/extension/json/test/scan_json.test b/extension/json/test/scan_json.test
@@ -166,8 +166,7 @@ Unable to find primary key value 6.|${KUZU_ROOT_DIRECTORY}/dataset/json-error/in
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL json FROM 'http://localhost/extension/repo/'
----- 1
-Extension: json has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension json
 ---- ok
 -STATEMENT LOAD FROM "${KUZU_ROOT_DIRECTORY}/dataset/tinysnb_json/vMovies.json" RETURN *;
diff --git a/extension/neo4j/test/test_files/install_neo4j.test b/extension/neo4j/test/test_files/install_neo4j.test
@@ -4,8 +4,7 @@
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL neo4j FROM 'http://localhost/extension/repo/'
----- 1
-Extension: neo4j has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension neo4j
 ---- ok
 
diff --git a/extension/postgres/src/installer/postgres_install_func.cpp b/extension/postgres/src/installer/postgres_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"postgres", repo};
+    kuzu::extension::InstallExtensionInfo info{"postgres", repo, false /* forceInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/postgres/test/test_files/install_postgres.test b/extension/postgres/test/test_files/install_postgres.test
@@ -4,7 +4,6 @@
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL postgres FROM 'http://localhost/extension/repo/'
----- 1
-Extension: postgres has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension postgres
 ---- ok
diff --git a/extension/sqlite/src/installer/sqlite_install_func.cpp b/extension/sqlite/src/installer/sqlite_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"sqlite", repo};
+    kuzu::extension::InstallExtensionInfo info{"sqlite", repo, false /* forceInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/sqlite/test/test_files/sqlite.test b/extension/sqlite/test/test_files/sqlite.test
@@ -110,8 +110,7 @@ Conversion Error: timestamp field value out of range: "08:30:00", expected forma
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL sqlite FROM 'http://localhost/extension/repo/'
----- 1
-Extension: sqlite has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT load extension sqlite
 ---- ok
 -STATEMENT ATTACH '${KUZU_ROOT_DIRECTORY}/extension/sqlite/test/sqlite_database/tinysnb' as tinysnb (dbtype sqlite);
diff --git a/extension/unity_catalog/src/installer/unity_catalog_install_func.cpp b/extension/unity_catalog/src/installer/unity_catalog_install_func.cpp
@@ -9,7 +9,7 @@ extern "C" {
 #define INIT_EXPORT __attribute__((visibility("default")))
 #endif
 INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {
-    kuzu::extension::InstallExtensionInfo info{"unity_catalog", repo};
+    kuzu::extension::InstallExtensionInfo info{"unity_catalog", repo, false /* falseInstall */};
     kuzu::duckdb_extension::DuckDBInstaller installer{info, context};
     installer.install();
 }
diff --git a/extension/vector/test/test_files/small.test b/extension/vector/test/test_files/small.test
@@ -362,8 +362,7 @@ embeddings|e_hnsw_index|HNSW|[vec]|True|CALL CREATE_VECTOR_INDEX('embeddings', '
 
 -CASE InstallOfficialExtension
 -STATEMENT INSTALL vector FROM 'http://localhost/extension/repo/'
----- 1
-Extension: vector has been installed from repo: http://localhost/extension/repo/.
+---- ok
 -STATEMENT LOAD vector
 ---- ok
 -STATEMENT CREATE NODE TABLE embeddings (id int64, vec FLOAT[8], PRIMARY KEY (id));
diff --git a/scripts/antlr4/Cypher.g4 b/scripts/antlr4/Cypher.g4
@@ -100,6 +100,8 @@ FALSE : ( 'F' | 'f' ) ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'S' | 's' ) ( 'E' | 'e' ) ;
 
 FROM : ( 'F' | 'f' ) ( 'R' | 'r' ) ( 'O' | 'o' ) ( 'M' | 'm' ) ;
 
+FORCE : ( 'F' | 'f' ) ( 'O' | 'o' ) ( 'R' | 'r' ) ( 'C' | 'c' ) ( 'E' | 'e' ) ;
+
 GLOB : ( 'G' | 'g' ) ( 'L' | 'l' ) ( 'O' | 'o' ) ( 'B' | 'b' ) ;
 
 GRAPH : ( 'G' | 'g' ) ( 'R' | 'r' ) ( 'A' | 'a' ) ( 'P' | 'p' ) ( 'H' | 'h' ) ;
@@ -212,6 +214,8 @@ UNWIND : ( 'U' | 'u' ) ( 'N' | 'n' ) ( 'W' | 'w' ) ( 'I' | 'i' ) ( 'N' | 'n' ) (
 
 UNINSTALL : ( 'U' | 'u' ) ( 'N' | 'n' ) ( 'I' | 'i' ) ( 'N' | 'n' ) ( 'S' | 's' ) ( 'T' | 't' ) ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' ) ;
 
+UPDATE : ( 'U' | 'u' ) ( 'P' | 'p' ) ( 'D' | 'd' ) ( 'A' | 'a' ) ( 'T' | 't' ) ( 'E' | 'e' ) ;
+
 USE : ( 'U' | 'u' ) ( 'S' | 's' ) ( 'E' | 'e' ) ;
 
 WHEN : ( 'W' | 'w' ) ( 'H' | 'h' ) ( 'E' | 'e' ) ( 'N' | 'n' ) ;
@@ -439,17 +443,21 @@ kU_Transaction
 kU_Extension
     : kU_LoadExtension
         | kU_InstallExtension
-        | kU_UninstallExtension;
+        | kU_UninstallExtension
+        | kU_UpdateExtension ;
 
 kU_LoadExtension
     : LOAD SP (EXTENSION SP)? ( StringLiteral | oC_Variable ) ;
 
 kU_InstallExtension
-    : INSTALL SP oC_Variable (SP FROM SP StringLiteral)?;
+    : (FORCE SP)? INSTALL SP oC_Variable (SP FROM SP StringLiteral)?;
 
 kU_UninstallExtension
     : UNINSTALL SP oC_Variable;
 
+kU_UpdateExtension
+    : UPDATE SP oC_Variable;
+
 oC_Query
     : oC_RegularQuery ;
 
@@ -940,6 +948,7 @@ kU_NonReservedKeywords
         | EXPLAIN
         | EXPORT
         | EXTENSION
+        | FORCE
         | GRAPH
         | IF
         | IS
@@ -969,6 +978,7 @@ kU_NonReservedKeywords
         | TYPE
         | USE
         | UNINSTALL
+        | UPDATE
         | WRITE
         | FROM
         | TO
diff --git a/scripts/antlr4/hash.md5 b/scripts/antlr4/hash.md5
@@ -1 +1 @@
-4cbc063c8233c423a3fef323b0a395cc
+878e8588d090a233c250fde729a6fe88
diff --git a/src/antlr4/Cypher.g4 b/src/antlr4/Cypher.g4
@@ -206,17 +206,21 @@ kU_Transaction
 kU_Extension
     : kU_LoadExtension
         | kU_InstallExtension
-        | kU_UninstallExtension;
+        | kU_UninstallExtension
+        | kU_UpdateExtension ;
 
 kU_LoadExtension
     : LOAD SP (EXTENSION SP)? ( StringLiteral | oC_Variable ) ;
 
 kU_InstallExtension
-    : INSTALL SP oC_Variable (SP FROM SP StringLiteral)?;
+    : (FORCE SP)? INSTALL SP oC_Variable (SP FROM SP StringLiteral)?;
 
 kU_UninstallExtension
     : UNINSTALL SP oC_Variable;
 
+kU_UpdateExtension
+    : UPDATE SP oC_Variable;
+
 oC_Query
     : oC_RegularQuery ;
 
@@ -707,6 +711,7 @@ kU_NonReservedKeywords
         | EXPLAIN
         | EXPORT
         | EXTENSION
+        | FORCE
         | GRAPH
         | IF
         | IS
@@ -736,6 +741,7 @@ kU_NonReservedKeywords
         | TYPE
         | USE
         | UNINSTALL
+        | UPDATE
         | WRITE
         | FROM
         | TO
diff --git a/src/antlr4/keywords.txt b/src/antlr4/keywords.txt
@@ -41,6 +41,7 @@ EXPORT
 EXTENSION
 FALSE
 FROM
+FORCE
 GLOB
 GRAPH
 GROUP
@@ -97,6 +98,7 @@ TYPE
 UNION
 UNWIND
 UNINSTALL
+UPDATE
 USE
 WHEN
 WHERE
diff --git a/src/extension/extension_installer.cpp b/src/extension/extension_installer.cpp
@@ -37,12 +37,15 @@ void ExtensionInstaller::tryDownloadExtensionFile(const ExtensionRepoInfo& repoI
     fileInfo->syncFile();
 }
 
-void ExtensionInstaller::install() {
-    installExtension();
-    installDependencies();
+bool ExtensionInstaller::install() {
+    auto install = installExtension();
+    if (install) {
+        installDependencies();
+    }
+    return install;
 }
 
-void ExtensionInstaller::installExtension() {
+bool ExtensionInstaller::installExtension() {
     auto vfs = context.getVFSUnsafe();
     auto localExtensionDir = context.getExtensionDir();
     if (!vfs->fileOrPathExists(localExtensionDir, &context)) {
@@ -53,14 +56,20 @@ void ExtensionInstaller::installExtension() {
     if (!vfs->fileOrPathExists(localDirForExtension)) {
         vfs->createDir(localDirForExtension);
     }
+    auto localLibFilePath =
+        extension::ExtensionUtils::getLocalPathForExtensionLib(&context, info.name);
+    if (vfs->fileOrPathExists(localLibFilePath) && !info.forceInstall) {
+        // The extension has been installed, skip downloading from the repo.
+        return false;
+    }
     auto localDirForSharedLib = extension::ExtensionUtils::getLocalPathForSharedLib(&context);
     if (!vfs->fileOrPathExists(localDirForSharedLib)) {
         vfs->createDir(localDirForSharedLib);
     }
     auto libFileRepoInfo = extension::ExtensionUtils::getExtensionLibRepoInfo(info.name, info.repo);
-    auto localLibFilePath =
-        extension::ExtensionUtils::getLocalPathForExtensionLib(&context, info.name);
+
     tryDownloadExtensionFile(libFileRepoInfo, localLibFilePath);
+    return true;
 }
 
 void ExtensionInstaller::installDependencies() {
diff --git a/src/include/extension/extension_action.h b/src/include/extension/extension_action.h
@@ -33,10 +33,11 @@ struct ExtensionAuxInfo {
 
 struct InstallExtensionAuxInfo : public ExtensionAuxInfo {
     std::string extensionRepo;
+    bool forceInstall;
 
-    explicit InstallExtensionAuxInfo(std::string extensionRepo, std::string path)
+    explicit InstallExtensionAuxInfo(std::string extensionRepo, std::string path, bool forceInstall)
         : ExtensionAuxInfo{ExtensionAction::INSTALL, std::move(path)},
-          extensionRepo{std::move(extensionRepo)} {}
+          extensionRepo{std::move(extensionRepo)}, forceInstall{forceInstall} {}
 
     std::unique_ptr<ExtensionAuxInfo> copy() override {
         return std::make_unique<InstallExtensionAuxInfo>(*this);
diff --git a/src/include/extension/extension_installer.h b/src/include/extension/extension_installer.h
@@ -14,6 +14,10 @@ namespace extension {
 struct InstallExtensionInfo {
     std::string name;
     std::string repo;
+    bool forceInstall;
+
+    InstallExtensionInfo(std::string name, std::string repo, bool forceInstall)
+        : name{std::move(name)}, repo{std::move(repo)}, forceInstall{forceInstall} {}
 };
 
 class KUZU_API ExtensionInstaller {
@@ -23,13 +27,13 @@ class KUZU_API ExtensionInstaller {
 
     virtual ~ExtensionInstaller() = default;
 
-    virtual void install();
+    virtual bool install();
 
 protected:
     void tryDownloadExtensionFile(const ExtensionRepoInfo& info, const std::string& localFilePath);
 
 private:
-    void installExtension();
+    bool installExtension();
     void installDependencies();
 
 protected:
diff --git a/src/include/processor/operator/simple/install_extension.h b/src/include/processor/operator/simple/install_extension.h
diff --git a/src/parser/transform/transform_extension.cpp b/src/parser/transform/transform_extension.cpp
diff --git a/src/processor/map/map_simple.cpp b/src/processor/map/map_simple.cpp
diff --git a/src/processor/operator/simple/install_extension.cpp b/src/processor/operator/simple/install_extension.cpp
diff --git a/test/test_files/extension/extension.test b/test/test_files/extension/extension.test
diff --git a/third_party/antlr4_cypher/cypher_lexer.cpp b/third_party/antlr4_cypher/cypher_lexer.cpp
diff --git a/third_party/antlr4_cypher/cypher_parser.cpp b/third_party/antlr4_cypher/cypher_parser.cpp
diff --git a/third_party/antlr4_cypher/include/cypher_lexer.h b/third_party/antlr4_cypher/include/cypher_lexer.h
diff --git a/third_party/antlr4_cypher/include/cypher_parser.h b/third_party/antlr4_cypher/include/cypher_parser.h
diff --git a/tools/shell/include/keywords.h b/tools/shell/include/keywords.h

Original file line number	Diff line number	Diff line change
`@@ -9,7 +9,7 @@ extern "C" {`
`9`	`9`	`#define INIT_EXPORT __attribute__((visibility("default")))`
`10`	`10`	`#endif`
`11`	`11`	`INIT_EXPORT void install(const std::string& repo, kuzu::main::ClientContext& context) {`
`12`		`- kuzu::extension::InstallExtensionInfo info{"delta", repo};`
	`12`	`+ kuzu::extension::InstallExtensionInfo info{"delta", repo, false /* forceInstall */};`
`13`	`13`	`kuzu::duckdb_extension::DuckDBInstaller installer{info, context};`
`14`	`14`	`installer.install();`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`namespace kuzu {`
`7`	`7`	`namespace duckdb_extension {`
`8`	`8`
`9`		`-void DuckDBInstaller::install() {`
	`9`	`+bool DuckDBInstaller::install() {`
`10`	`10`	`auto loaderFileRepoInfo =`
`11`	`11`	`extension::ExtensionUtils::getExtensionLoaderRepoInfo(info.name, info.repo);`
`12`	`12`	`auto localLoaderFilePath =`
`@@ -23,6 +23,7 @@ void DuckDBInstaller::install() {`
`23`	`23`	`tryDownloadExtensionFile(dependencyLibRepoInfo, localDependencyLibPath);`
`24`	`24`	`}`
`25`	`25`	`}`
	`26`	`+ return true;`
`26`	`27`	`}`
`27`	`28`
`28`	`29`	`} // namespace duckdb_extension`