diff --git a/core/src/fs.ts b/core/src/fs.ts index 359254e2ee..8b556ec053 100644 --- a/core/src/fs.ts +++ b/core/src/fs.ts @@ -24,11 +24,11 @@ const readFile: (path: string) => Promise = (path) => window.core.api?.readFile(path); /** * Check whether the file exists - * @param {string} path + * @param {string} path * @returns {boolean} A boolean indicating whether the path is a file. */ -const checkFileExists = (path: string): Promise => - window.core.api?.checkFileExists(path); +const exists = (path: string): Promise => + window.core.api?.exists(path); /** * List the directory files * @param {string} path - The path of the directory to list files. @@ -83,7 +83,7 @@ export const fs = { isDirectory, writeFile, readFile, - checkFileExists, + exists, listFiles, mkdir, rmdir, diff --git a/core/src/types/index.ts b/core/src/types/index.ts index c6de436cf3..969d3de5ca 100644 --- a/core/src/types/index.ts +++ b/core/src/types/index.ts @@ -41,11 +41,8 @@ export type MessageRequest = { /** Messages for constructing a chat completion request **/ messages?: ChatCompletionMessage[]; - /** Runtime parameters for constructing a chat completion request **/ - parameters?: ModelRuntimeParams; - /** Settings for constructing a chat completion request **/ - model?: ModelInfo + model?: ModelInfo; }; /** diff --git a/electron/handlers/fs.ts b/electron/handlers/fs.ts index 1e2df5c599..acc0ed2dab 100644 --- a/electron/handlers/fs.ts +++ b/electron/handlers/fs.ts @@ -56,7 +56,7 @@ export function handleFsIPCs() { * @param path - The path of the file to check. * @returns A promise that resolves with a boolean indicating whether the file exists. */ - ipcMain.handle('checkFileExists', async (_event, path: string) => { + ipcMain.handle('exists', async (_event, path: string) => { return new Promise((resolve, reject) => { const fullPath = join(userSpacePath, path) fs.existsSync(fullPath) ? resolve(true) : resolve(false) diff --git a/electron/invokers/fs.ts b/electron/invokers/fs.ts index e59eb4c86d..e1aa67cca3 100644 --- a/electron/invokers/fs.ts +++ b/electron/invokers/fs.ts @@ -31,7 +31,7 @@ export function fsInvokers() { * Reads a file at the specified path. * @param {string} path - The path of the file to read. */ - checkFileExists: (path: string) => ipcRenderer.invoke('checkFileExists', path), + exists: (path: string) => ipcRenderer.invoke('exists', path), /** * Writes data to a file at the specified path. diff --git a/extensions/inference-nitro-extension/src/@types/global.d.ts b/extensions/inference-nitro-extension/src/@types/global.d.ts index 5776cca202..642f109090 100644 --- a/extensions/inference-nitro-extension/src/@types/global.d.ts +++ b/extensions/inference-nitro-extension/src/@types/global.d.ts @@ -10,10 +10,10 @@ declare const INFERENCE_URL: string; * @property settings.embedding - Whether to use embedding. */ interface EngineSettings { - ctx_len: number; - ngl: number; - cont_batching: boolean; - embedding: boolean; + ctx_len: number; + ngl: number; + cont_batching: boolean; + embedding: boolean; } /** @@ -21,7 +21,6 @@ interface EngineSettings { * @property error - An error message if the model fails to load. */ interface ModelOperationResponse { - error?: any; - modelFile?: string; - } - \ No newline at end of file + error?: any; + modelFile?: string; +} diff --git a/extensions/inference-nitro-extension/src/index.ts b/extensions/inference-nitro-extension/src/index.ts index f8bbd2a092..a1c125ce55 100644 --- a/extensions/inference-nitro-extension/src/index.ts +++ b/extensions/inference-nitro-extension/src/index.ts @@ -33,17 +33,17 @@ import { join } from "path"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceNitroExtension implements InferenceExtension { - private static readonly _homeDir = 'engines' - private static readonly _engineMetadataFileName = 'nitro.json' + private static readonly _homeDir = "engines"; + private static readonly _engineMetadataFileName = "nitro.json"; - static _currentModel: Model; + private static _currentModel: Model; - static _engineSettings: EngineSettings = { - "ctx_len": 2048, - "ngl": 100, - "cont_batching": false, - "embedding": false - } + private static _engineSettings: EngineSettings = { + ctx_len: 2048, + ngl: 100, + cont_batching: false, + embedding: false, + }; controller = new AbortController(); isCancelled = false; @@ -59,12 +59,12 @@ export default class JanInferenceNitroExtension implements InferenceExtension { * Subscribes to events emitted by the @janhq/core package. */ onLoad(): void { - fs.mkdir(JanInferenceNitroExtension._homeDir) - this.writeDefaultEngineSettings() + fs.mkdir(JanInferenceNitroExtension._homeDir); + this.writeDefaultEngineSettings(); // Events subscription events.on(EventName.OnMessageSent, (data) => - JanInferenceNitroExtension.handleMessageRequest(data, this) + JanInferenceNitroExtension.handleMessageRequest(data, this) ); events.on(EventName.OnModelInit, (model: Model) => { @@ -112,42 +112,51 @@ export default class JanInferenceNitroExtension implements InferenceExtension { private async writeDefaultEngineSettings() { try { - const engine_json = join(JanInferenceNitroExtension._homeDir, JanInferenceNitroExtension._engineMetadataFileName) - if (await fs.checkFileExists(engine_json)) { - JanInferenceNitroExtension._engineSettings = JSON.parse(await fs.readFile(engine_json)) - } - else { - await fs.writeFile(engine_json, JSON.stringify(JanInferenceNitroExtension._engineSettings, null, 2)) + const engineFile = join( + JanInferenceNitroExtension._homeDir, + JanInferenceNitroExtension._engineMetadataFileName + ); + if (await fs.exists(engineFile)) { + JanInferenceNitroExtension._engineSettings = JSON.parse( + await fs.readFile(engineFile) + ); + } else { + await fs.writeFile( + engineFile, + JSON.stringify(JanInferenceNitroExtension._engineSettings, null, 2) + ); } } catch (err) { - console.error(err) + console.error(err); } } private static async handleModelInit(model: Model) { - if (model.engine !== "nitro") { return } + if (model.engine !== "nitro") { + return; + } const userSpacePath = await getUserSpace(); const modelFullPath = join(userSpacePath, "models", model.id, model.id); - const nitro_init_result = await executeOnMain(MODULE, "initModel", { + const nitroInitResult = await executeOnMain(MODULE, "initModel", { modelFullPath: modelFullPath, - model: model + model: model, }); - if (nitro_init_result.error === null) { - events.emit(EventName.OnModelFail, model) - } - else{ + if (nitroInitResult.error === null) { + events.emit(EventName.OnModelFail, model); + } else { JanInferenceNitroExtension._currentModel = model; events.emit(EventName.OnModelReady, model); } } private static async handleModelStop(model: Model) { - if (model.engine !== 'nitro') { return } - else { - await executeOnMain(MODULE, "stopModel") - events.emit(EventName.OnModelStopped, model) + if (model.engine !== "nitro") { + return; + } else { + await executeOnMain(MODULE, "stopModel"); + events.emit(EventName.OnModelStopped, model); } } @@ -171,18 +180,17 @@ export default class JanInferenceNitroExtension implements InferenceExtension { return new Promise(async (resolve, reject) => { requestInference( - data.messages ?? [], - JanInferenceNitroExtension._engineSettings, - JanInferenceNitroExtension._currentModel - ) - .subscribe({ + data.messages ?? [], + JanInferenceNitroExtension._engineSettings, + JanInferenceNitroExtension._currentModel + ).subscribe({ next: (_content) => {}, - complete: async () => { - resolve(message); - }, - error: async (err) => { - reject(err); - }, + complete: async () => { + resolve(message); + }, + error: async (err) => { + reject(err); + }, }); }); } @@ -197,7 +205,9 @@ export default class JanInferenceNitroExtension implements InferenceExtension { data: MessageRequest, instance: JanInferenceNitroExtension ) { - if (data.model.engine !== 'nitro') { return } + if (data.model.engine !== "nitro") { + return; + } const timestamp = Date.now(); const message: ThreadMessage = { id: ulid(), @@ -216,11 +226,11 @@ export default class JanInferenceNitroExtension implements InferenceExtension { instance.controller = new AbortController(); requestInference( - data.messages ?? [], - JanInferenceNitroExtension._engineSettings, - JanInferenceNitroExtension._currentModel, - instance.controller - ).subscribe({ + data.messages ?? [], + JanInferenceNitroExtension._engineSettings, + JanInferenceNitroExtension._currentModel, + instance.controller + ).subscribe({ next: (content) => { const messageContent: ThreadContent = { type: ContentType.Text, diff --git a/extensions/inference-nitro-extension/src/module.ts b/extensions/inference-nitro-extension/src/module.ts index 1323dca6d4..d36553f409 100644 --- a/extensions/inference-nitro-extension/src/module.ts +++ b/extensions/inference-nitro-extension/src/module.ts @@ -26,9 +26,9 @@ let currentModelFile = null; */ function stopModel(): Promise { return new Promise((resolve, reject) => { - checkAndUnloadNitro() - resolve({ error: undefined}) - }) + checkAndUnloadNitro(); + resolve({ error: undefined }); + }); } /** @@ -39,33 +39,32 @@ function stopModel(): Promise { * TODO: Should it be startModel instead? */ function initModel(wrapper: any): Promise { - currentModelFile = wrapper.modelFullPath; - if (wrapper.model.engine !== "nitro") { - return Promise.resolve({ error: "Not a nitro model" }) - } - else { - log.info("Started to load model " + wrapper.model.modelFullPath); - const settings = { - llama_model_path: currentModelFile, - ...wrapper.model.settings, - }; - log.info(`Load model settings: ${JSON.stringify(settings, null, 2)}`); - return ( - // 1. Check if the port is used, if used, attempt to unload model / kill nitro process - validateModelVersion() - .then(checkAndUnloadNitro) - // 2. Spawn the Nitro subprocess - .then(spawnNitroProcess) - // 4. Load the model into the Nitro subprocess (HTTP POST request) - .then(() => loadLLMModel(settings)) - // 5. Check if the model is loaded successfully - .then(validateModelStatus) - .catch((err) => { - log.error("error: " + JSON.stringify(err)); - return { error: err, currentModelFile }; - }) - ); - } + currentModelFile = wrapper.modelFullPath; + if (wrapper.model.engine !== "nitro") { + return Promise.resolve({ error: "Not a nitro model" }); + } else { + log.info("Started to load model " + wrapper.model.modelFullPath); + const settings = { + llama_model_path: currentModelFile, + ...wrapper.model.settings, + }; + log.info(`Load model settings: ${JSON.stringify(settings, null, 2)}`); + return ( + // 1. Check if the port is used, if used, attempt to unload model / kill nitro process + validateModelVersion() + .then(checkAndUnloadNitro) + // 2. Spawn the Nitro subprocess + .then(spawnNitroProcess) + // 4. Load the model into the Nitro subprocess (HTTP POST request) + .then(() => loadLLMModel(settings)) + // 5. Check if the model is loaded successfully + .then(validateModelStatus) + .catch((err) => { + log.error("error: " + JSON.stringify(err)); + return { error: err, currentModelFile }; + }) + ); + } } /** @@ -148,13 +147,12 @@ async function checkAndUnloadNitro() { // If inUse - try unload or kill process, otherwise do nothing if (inUse) { // Attempt to unload model - return await fetch(NITRO_HTTP_UNLOAD_MODEL_URL, { + return fetch(NITRO_HTTP_UNLOAD_MODEL_URL, { method: "GET", headers: { "Content-Type": "application/json", }, - }) - .catch((err) => { + }).catch((err) => { console.error(err); // Fallback to kill the port return killSubprocess(); diff --git a/extensions/inference-openai-extension/src/@types/global.d.ts b/extensions/inference-openai-extension/src/@types/global.d.ts index 988c3c7db0..5e9fd4d8a9 100644 --- a/extensions/inference-openai-extension/src/@types/global.d.ts +++ b/extensions/inference-openai-extension/src/@types/global.d.ts @@ -3,25 +3,25 @@ import { Model } from "@janhq/core"; declare const MODULE: string; declare interface EngineSettings { - base_url?: string; - api_key?: string; + base_url?: string; + api_key?: string; } enum OpenAIChatCompletionModelName { - 'gpt-3.5-turbo-instruct' = 'gpt-3.5-turbo-instruct', - 'gpt-3.5-turbo-instruct-0914' = 'gpt-3.5-turbo-instruct-0914', - 'gpt-4-1106-preview' = 'gpt-4-1106-preview', - 'gpt-3.5-turbo-0613' = 'gpt-3.5-turbo-0613', - 'gpt-3.5-turbo-0301' = 'gpt-3.5-turbo-0301', - 'gpt-3.5-turbo' = 'gpt-3.5-turbo', - 'gpt-3.5-turbo-16k-0613' = 'gpt-3.5-turbo-16k-0613', - 'gpt-3.5-turbo-1106' = 'gpt-3.5-turbo-1106', - 'gpt-4-vision-preview' = 'gpt-4-vision-preview', - 'gpt-4' = 'gpt-4', - 'gpt-4-0314' = 'gpt-4-0314', - 'gpt-4-0613' = 'gpt-4-0613', + "gpt-3.5-turbo-instruct" = "gpt-3.5-turbo-instruct", + "gpt-3.5-turbo-instruct-0914" = "gpt-3.5-turbo-instruct-0914", + "gpt-4-1106-preview" = "gpt-4-1106-preview", + "gpt-3.5-turbo-0613" = "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-0301" = "gpt-3.5-turbo-0301", + "gpt-3.5-turbo" = "gpt-3.5-turbo", + "gpt-3.5-turbo-16k-0613" = "gpt-3.5-turbo-16k-0613", + "gpt-3.5-turbo-1106" = "gpt-3.5-turbo-1106", + "gpt-4-vision-preview" = "gpt-4-vision-preview", + "gpt-4" = "gpt-4", + "gpt-4-0314" = "gpt-4-0314", + "gpt-4-0613" = "gpt-4-0613", } declare type OpenAIModel = Omit & { - id: OpenAIChatCompletionModelName; -}; \ No newline at end of file + id: OpenAIChatCompletionModelName; +}; diff --git a/extensions/inference-openai-extension/src/index.ts b/extensions/inference-openai-extension/src/index.ts index 1fecf17b5e..4eb179da8c 100644 --- a/extensions/inference-openai-extension/src/index.ts +++ b/extensions/inference-openai-extension/src/index.ts @@ -31,14 +31,14 @@ import { EngineSettings, OpenAIModel } from "./@types/global"; * It also subscribes to events emitted by the @janhq/core package and handles new message requests. */ export default class JanInferenceOpenAIExtension implements InferenceExtension { - private static readonly _homeDir = 'engines' - private static readonly _engineMetadataFileName = 'openai.json' - - static _currentModel: OpenAIModel; - - static _engineSettings: EngineSettings = { - "base_url": "https://api.openai.com/v1", - "api_key": "sk-" + private static readonly _homeDir = "engines"; + private static readonly _engineMetadataFileName = "openai.json"; + + private static _currentModel: OpenAIModel; + + private static _engineSettings: EngineSettings = { + base_url: "https://api.openai.com/v1", + api_key: "sk-", }; controller = new AbortController(); @@ -56,8 +56,8 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension { * Subscribes to events emitted by the @janhq/core package. */ onLoad(): void { - fs.mkdir(JanInferenceOpenAIExtension._homeDir) - JanInferenceOpenAIExtension.writeDefaultEngineSettings() + fs.mkdir(JanInferenceOpenAIExtension._homeDir); + JanInferenceOpenAIExtension.writeDefaultEngineSettings(); // Events subscription events.on(EventName.OnMessageSent, (data) => @@ -87,20 +87,27 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension { modelId: string, settings?: ModelSettingParams ): Promise { - return + return; } static async writeDefaultEngineSettings() { try { - const engine_json = join(JanInferenceOpenAIExtension._homeDir, JanInferenceOpenAIExtension._engineMetadataFileName) - if (await fs.checkFileExists(engine_json)) { - JanInferenceOpenAIExtension._engineSettings = JSON.parse(await fs.readFile(engine_json)) - } - else { - await fs.writeFile(engine_json, JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2)) + const engineFile = join( + JanInferenceOpenAIExtension._homeDir, + JanInferenceOpenAIExtension._engineMetadataFileName + ); + if (await fs.exists(engineFile)) { + JanInferenceOpenAIExtension._engineSettings = JSON.parse( + await fs.readFile(engineFile) + ); + } else { + await fs.writeFile( + engineFile, + JSON.stringify(JanInferenceOpenAIExtension._engineSettings, null, 2) + ); } } catch (err) { - console.error(err) + console.error(err); } } /** @@ -137,35 +144,39 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension { }; return new Promise(async (resolve, reject) => { - requestInference(data.messages ?? [], - JanInferenceOpenAIExtension._engineSettings, - JanInferenceOpenAIExtension._currentModel) - .subscribe({ - next: (_content) => {}, - complete: async () => { - resolve(message); - }, - error: async (err) => { - reject(err); - }, + requestInference( + data.messages ?? [], + JanInferenceOpenAIExtension._engineSettings, + JanInferenceOpenAIExtension._currentModel + ).subscribe({ + next: (_content) => {}, + complete: async () => { + resolve(message); + }, + error: async (err) => { + reject(err); + }, }); }); } private static async handleModelInit(model: OpenAIModel) { - if (model.engine !== 'openai') { return } - else { - JanInferenceOpenAIExtension._currentModel = model - JanInferenceOpenAIExtension.writeDefaultEngineSettings() + if (model.engine !== "openai") { + return; + } else { + JanInferenceOpenAIExtension._currentModel = model; + JanInferenceOpenAIExtension.writeDefaultEngineSettings(); // Todo: Check model list with API key - events.emit(EventName.OnModelReady, model) + events.emit(EventName.OnModelReady, model); // events.emit(EventName.OnModelFail, model) } } private static async handleModelStop(model: OpenAIModel) { - if (model.engine !== 'openai') { return } - events.emit(EventName.OnModelStopped, model) + if (model.engine !== "openai") { + return; + } + events.emit(EventName.OnModelStopped, model); } /** @@ -178,8 +189,10 @@ export default class JanInferenceOpenAIExtension implements InferenceExtension { data: MessageRequest, instance: JanInferenceOpenAIExtension ) { - if (data.model.engine !== 'openai') { return } - + if (data.model.engine !== "openai") { + return; + } + const timestamp = Date.now(); const message: ThreadMessage = { id: ulid(), diff --git a/web/hooks/useSendChatMessage.ts b/web/hooks/useSendChatMessage.ts index 2a9e7a102b..970aedbecd 100644 --- a/web/hooks/useSendChatMessage.ts +++ b/web/hooks/useSendChatMessage.ts @@ -231,7 +231,6 @@ export default function useSendChatMessage() { await WaitForModelStarting(modelId) setQueuedMessage(false) } - console.log('messageRequest', messageRequest) events.emit(EventName.OnMessageSent, messageRequest) }