From b68950d9f32cc48e3b8adf64830513c7f85ae730 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 26 Dec 2024 15:12:00 +0700 Subject: [PATCH 1/5] chore: select default engine variant base on user device specs --- .../engine-management-extension/package.json | 12 +- .../rolldown.config.js | 39 ++++ .../src/@types/global.d.ts | 1 + .../engine-management-extension/src/error.ts | 10 + .../engine-management-extension/src/index.ts | 40 +++- .../src/node/cpuInfo.ts | 0 .../src/node/index.test.ts} | 181 ++---------------- .../src/node/index.ts} | 43 +---- .../webpack.config.js | 35 ---- .../inference-cortex-extension/package.json | 4 +- .../src/@types/global.d.ts | 10 + .../src/node/index.test.ts | 11 -- .../src/node/index.ts | 42 ++-- 13 files changed, 153 insertions(+), 275 deletions(-) create mode 100644 extensions/engine-management-extension/rolldown.config.js create mode 100644 extensions/engine-management-extension/src/error.ts rename extensions/{inference-cortex-extension => engine-management-extension}/src/node/cpuInfo.ts (100%) rename extensions/{inference-cortex-extension/src/node/execute.test.ts => engine-management-extension/src/node/index.test.ts} (63%) rename extensions/{inference-cortex-extension/src/node/execute.ts => engine-management-extension/src/node/index.ts} (70%) delete mode 100644 extensions/engine-management-extension/webpack.config.js diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index 8dca02513d..c9af0ea646 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -4,11 +4,12 @@ "version": "1.0.0", "description": "This extension enables manage engines", "main": "dist/index.js", + "node": "dist/node/index.cjs.js", "author": "Jan ", "license": "MIT", "scripts": { "test": "jest", - "build": "tsc -b . && webpack --config webpack.config.js", + "build": "rolldown -c", "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { @@ -16,17 +17,23 @@ "./main": "./dist/module.js" }, "devDependencies": { + "@rollup/plugin-replace": "^6.0.2", "cpx": "^1.5.0", "rimraf": "^3.0.2", + "rolldown": "^1.0.0-beta.1", "ts-loader": "^9.5.0", "webpack": "^5.88.2", "webpack-cli": "^5.1.4" }, "dependencies": { "@janhq/core": "file:../../core", + "cpu-instructions": "^0.0.13", "ky": "^1.7.2", "p-queue": "^8.0.1" }, + "bundledDependencies": [ + "cpu-instructions" + ], "engines": { "node": ">=18.0.0" }, @@ -34,6 +41,5 @@ "dist/*", "package.json", "README.md" - ], - "bundleDependencies": [] + ] } diff --git a/extensions/engine-management-extension/rolldown.config.js b/extensions/engine-management-extension/rolldown.config.js new file mode 100644 index 0000000000..3214c2fc09 --- /dev/null +++ b/extensions/engine-management-extension/rolldown.config.js @@ -0,0 +1,39 @@ +import { defineConfig } from 'rolldown' +import replace from '@rollup/plugin-replace' +import pkgJson from './package.json' with { type: 'json' } + +export default defineConfig([ + { + input: 'src/index.ts', + output: { + format: 'esm', + file: 'dist/index.js', + }, + plugins: [ + replace({ + NODE: JSON.stringify(`${pkgJson.name}/${pkgJson.node}`), + API_URL: JSON.stringify('http://127.0.0.1:39291'), + SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), + }), + ], + }, + { + input: 'src/node/index.ts', + output: { + format: 'cjs', + file: 'dist/node/index.cjs.js', + }, + }, + { + input: 'src/node/cpuInfo.ts', + output: { + format: 'cjs', + file: 'dist/node/cpuInfo.js', + }, + external: ['cpu-instructions'], + resolve: { + extensions: ['.ts', '.js', '.svg'], + }, + }, +]) diff --git a/extensions/engine-management-extension/src/@types/global.d.ts b/extensions/engine-management-extension/src/@types/global.d.ts index 0a8fa23bb6..a8a5ee4515 100644 --- a/extensions/engine-management-extension/src/@types/global.d.ts +++ b/extensions/engine-management-extension/src/@types/global.d.ts @@ -3,6 +3,7 @@ declare global { declare const API_URL: string declare const CORTEX_ENGINE_VERSION: string declare const SOCKET_URL: string + declare const NODE: string interface Core { api: APIFunctions diff --git a/extensions/engine-management-extension/src/error.ts b/extensions/engine-management-extension/src/error.ts new file mode 100644 index 0000000000..50c75f22fd --- /dev/null +++ b/extensions/engine-management-extension/src/error.ts @@ -0,0 +1,10 @@ +/** + * Custom Engine Error + */ +export class EngineError extends Error { + message: string + constructor(message: string) { + super() + this.message = message + } +} diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index 3d8de47457..c589b6a85b 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -5,9 +5,12 @@ import { Engines, EngineVariant, EngineReleased, + executeOnMain, + systemInformation, } from '@janhq/core' import ky, { HTTPError } from 'ky' import PQueue from 'p-queue' +import { EngineError } from './error' /** * JSONEngineManagementExtension is a EngineManagementExtension implementation that provides @@ -22,11 +25,29 @@ export default class JSONEngineManagementExtension extends EngineManagementExten async onLoad() { this.queue.add(() => this.healthz()) try { - await this.getDefaultEngineVariant(InferenceEngine.cortex_llamacpp) + const variant = await this.getDefaultEngineVariant( + InferenceEngine.cortex_llamacpp + ) + // Check whether should use bundled version or installed version + // Only use larger version + if (this.compareVersions(CORTEX_ENGINE_VERSION, variant.version) > 0) { + throw new EngineError( + 'Default engine version is smaller than bundled version' + ) + } } catch (error) { - if (error instanceof HTTPError && error.response.status === 400) { + if ( + (error instanceof HTTPError && error.response.status === 400) || + error instanceof EngineError + ) { + const systemInfo = await systemInformation() + const variant = await executeOnMain( + NODE, + 'engineVariant', + systemInfo.gpuSetting + ) await this.setDefaultEngineVariant(InferenceEngine.cortex_llamacpp, { - variant: 'mac-arm64', + variant: variant, version: `${CORTEX_ENGINE_VERSION}`, }) } else { @@ -174,11 +195,22 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * Do health check on cortex.cpp * @returns */ - healthz(): Promise { + async healthz(): Promise { return ky .get(`${API_URL}/healthz`, { retry: { limit: 20, delay: () => 500, methods: ['get'] }, }) .then(() => {}) } + + private compareVersions(version1: string, version2: string): number { + const parseVersion = (version: string) => version.split('.').map(Number) + + const [major1, minor1, patch1] = parseVersion(version1.replace(/^v/, '')) + const [major2, minor2, patch2] = parseVersion(version2.replace(/^v/, '')) + + if (major1 !== major2) return major1 - major2 + if (minor1 !== minor2) return minor1 - minor2 + return patch1 - patch2 + } } diff --git a/extensions/inference-cortex-extension/src/node/cpuInfo.ts b/extensions/engine-management-extension/src/node/cpuInfo.ts similarity index 100% rename from extensions/inference-cortex-extension/src/node/cpuInfo.ts rename to extensions/engine-management-extension/src/node/cpuInfo.ts diff --git a/extensions/inference-cortex-extension/src/node/execute.test.ts b/extensions/engine-management-extension/src/node/index.test.ts similarity index 63% rename from extensions/inference-cortex-extension/src/node/execute.test.ts rename to extensions/engine-management-extension/src/node/index.test.ts index 1bcefce9d5..c73feb9c66 100644 --- a/extensions/inference-cortex-extension/src/node/execute.test.ts +++ b/extensions/engine-management-extension/src/node/index.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from '@jest/globals' -import { engineVariant, executableCortexFile } from './execute' +import engine from './index' import { GpuSetting } from '@janhq/core/node' import { cpuInfo } from 'cpu-instructions' import { fork } from 'child_process' @@ -62,20 +62,9 @@ describe('test executable cortex file', () => { Object.defineProperty(process, 'arch', { value: 'arm64', }) - expect(executableCortexFile(testSettings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: - originalPlatform === 'darwin' - ? expect.stringContaining(`cortex-server`) - : expect.anything(), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) mockFork.mockReturnValue(mockProcess) - expect(engineVariant(testSettings)).resolves.toEqual('mac-arm64') + expect(engine.engineVariant(testSettings)).resolves.toEqual('mac-arm64') }) it('executes on MacOS', () => { @@ -99,18 +88,7 @@ describe('test executable cortex file', () => { value: 'x64', }) - expect(executableCortexFile(testSettings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: - originalPlatform === 'darwin' - ? expect.stringContaining(`cortex-server`) - : expect.anything(), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - expect(engineVariant(testSettings)).resolves.toEqual('mac-amd64') + expect(engine.engineVariant(testSettings)).resolves.toEqual('mac-amd64') }) it('executes on Windows CPU', () => { @@ -131,15 +109,7 @@ describe('test executable cortex file', () => { } mockFork.mockReturnValue(mockProcess) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server.exe`), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - expect(engineVariant()).resolves.toEqual('windows-amd64-avx') + expect(engine.engineVariant()).resolves.toEqual('windows-amd64-avx') }) it('executes on Windows Cuda 11', () => { @@ -176,15 +146,8 @@ describe('test executable cortex file', () => { send: jest.fn(), } mockFork.mockReturnValue(mockProcess) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server.exe`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + + expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-avx2-cuda-11-7' ) }) @@ -221,15 +184,8 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server.exe`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + + expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-noavx-cuda-12-0' ) mockFork.mockReturnValue({ @@ -240,7 +196,7 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(engineVariant(settings)).resolves.toEqual( + expect(engine.engineVariant(settings)).resolves.toEqual( 'windows-amd64-avx2-cuda-12-0' ) }) @@ -261,15 +217,8 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - expect(engineVariant()).resolves.toEqual('linux-amd64-noavx') + + expect(engine.engineVariant()).resolves.toEqual('linux-amd64-noavx') }) it('executes on Linux Cuda 11', () => { @@ -306,15 +255,9 @@ describe('test executable cortex file', () => { send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) + expect(engine.engineVariant(settings)).resolves.toBe( + 'linux-amd64-avx2-cuda-11-7' ) - expect(engineVariant(settings)).resolves.toBe('linux-amd64-avx2-cuda-11-7') }) it('executes on Linux Cuda 12', () => { @@ -349,15 +292,8 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + + expect(engine.engineVariant(settings)).resolves.toEqual( 'linux-amd64-avx2-cuda-12-0' ) }) @@ -383,16 +319,7 @@ describe('test executable cortex file', () => { send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + expect(engine.engineVariant(settings)).resolves.toEqual( `linux-amd64-${instruction}` ) }) @@ -416,15 +343,7 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server.exe`), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + expect(engine.engineVariant(settings)).resolves.toEqual( `windows-amd64-${instruction}` ) }) @@ -465,15 +384,7 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server.exe`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + expect(engine.engineVariant(settings)).resolves.toEqual( `windows-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) }) @@ -514,15 +425,7 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual( + expect(engine.engineVariant(settings)).resolves.toEqual( `linux-amd64-${instruction === 'avx512' || instruction === 'avx2' ? 'avx2' : 'noavx'}-cuda-12-0` ) }) @@ -564,50 +467,8 @@ describe('test executable cortex file', () => { }), send: jest.fn(), }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: expect.stringContaining(`cortex-server`), - cudaVisibleDevices: '0', - vkVisibleDevices: '0', - }) - ) - expect(engineVariant(settings)).resolves.toEqual(`linux-amd64-vulkan`) - }) - }) - - // Generate test for different cpu instructions on MacOS - it(`executes on MacOS with different instructions`, () => { - Object.defineProperty(process, 'platform', { - value: 'darwin', - }) - const cpuInstructions = ['avx512', 'avx2', 'avx', 'noavx'] - cpuInstructions.forEach(() => { - Object.defineProperty(process, 'platform', { - value: 'darwin', - }) - const settings: GpuSetting = { - ...testSettings, - run_mode: 'cpu', - } - mockFork.mockReturnValue({ - on: jest.fn((event, callback) => { - if (event === 'message') { - callback('noavx') - } - }), - send: jest.fn(), - }) - expect(executableCortexFile(settings)).toEqual( - expect.objectContaining({ - enginePath: expect.stringContaining('shared'), - executablePath: - originalPlatform === 'darwin' - ? expect.stringContaining(`cortex-server`) - : expect.anything(), - cudaVisibleDevices: '', - vkVisibleDevices: '', - }) + expect(engine.engineVariant(settings)).resolves.toEqual( + `linux-amd64-vulkan` ) }) }) diff --git a/extensions/inference-cortex-extension/src/node/execute.ts b/extensions/engine-management-extension/src/node/index.ts similarity index 70% rename from extensions/inference-cortex-extension/src/node/execute.ts rename to extensions/engine-management-extension/src/node/index.ts index 0b091d464d..c55fef7204 100644 --- a/extensions/inference-cortex-extension/src/node/execute.ts +++ b/extensions/engine-management-extension/src/node/index.ts @@ -1,13 +1,6 @@ import * as path from 'path' -import { GpuSetting, appResourcePath, log } from '@janhq/core/node' +import { GpuSetting, log } from '@janhq/core/node' import { fork } from 'child_process' - -export interface CortexExecutableOptions { - enginePath: string - executablePath: string - cudaVisibleDevices: string - vkVisibleDevices: string -} /** * The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu. * @param settings @@ -37,14 +30,6 @@ const os = (): string => { : 'linux-amd64' } -/** - * The cortex.cpp extension based on the current platform. - * @returns .exe if on Windows, otherwise an empty string. - */ -const extension = (): '.exe' | '' => { - return process.platform === 'win32' ? '.exe' : '' -} - /** * The CUDA version that will be set - either '11-7' or '12-0'. * @param settings @@ -89,30 +74,10 @@ const cpuInstructions = async (): Promise => { }) } -/** - * The executable options for the cortex.cpp extension. - */ -export const executableCortexFile = ( - gpuSetting?: GpuSetting -): CortexExecutableOptions => { - let cudaVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? '' - let vkVisibleDevices = gpuSetting?.gpus_in_use.join(',') ?? '' - let binaryName = `cortex-server${extension()}` - const binPath = path.join(__dirname, '..', 'bin') - return { - enginePath: path.join(appResourcePath(), 'shared'), - executablePath: path.join(binPath, binaryName), - cudaVisibleDevices, - vkVisibleDevices, - } -} - /** * Find which variant to run based on the current platform. */ -export const engineVariant = async ( - gpuSetting?: GpuSetting -): Promise => { +const engineVariant = async (gpuSetting?: GpuSetting): Promise => { const cpuInstruction = await cpuInstructions() log(`[CORTEX]: CPU instruction: ${cpuInstruction}`) let engineVariant = [ @@ -135,3 +100,7 @@ export const engineVariant = async ( log(`[CORTEX]: Engine variant: ${engineVariant}`) return engineVariant } + +export default { + engineVariant, +} diff --git a/extensions/engine-management-extension/webpack.config.js b/extensions/engine-management-extension/webpack.config.js deleted file mode 100644 index 82499fb4bc..0000000000 --- a/extensions/engine-management-extension/webpack.config.js +++ /dev/null @@ -1,35 +0,0 @@ -const webpack = require('webpack') - -module.exports = { - experiments: { outputModule: true }, - entry: './src/index.ts', // Adjust the entry point to match your project's main file - mode: 'production', - module: { - rules: [ - { - test: /\.tsx?$/, - use: 'ts-loader', - exclude: /node_modules/, - }, - ], - }, - output: { - filename: 'index.js', // Adjust the output file name as needed - library: { type: 'module' }, // Specify ESM output format - }, - plugins: [ - new webpack.DefinePlugin({ - API_URL: JSON.stringify('http://127.0.0.1:39291'), - SOCKET_URL: JSON.stringify('ws://127.0.0.1:39291'), - CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), - }), - ], - resolve: { - extensions: ['.ts', '.js'], - }, - // Do not minify the output, otherwise it breaks the class registration - optimization: { - minimize: false, - }, - // Add loaders and other configuration as needed for your project -} diff --git a/extensions/inference-cortex-extension/package.json b/extensions/inference-cortex-extension/package.json index 507ab4ebc8..1a5ad11694 100644 --- a/extensions/inference-cortex-extension/package.json +++ b/extensions/inference-cortex-extension/package.json @@ -47,7 +47,6 @@ }, "dependencies": { "@janhq/core": "file:../../core", - "cpu-instructions": "^0.0.13", "decompress": "^4.2.1", "fetch-retry": "^5.0.6", "ky": "^1.7.2", @@ -69,7 +68,6 @@ "tcp-port-used", "fetch-retry", "@janhq/core", - "decompress", - "cpu-instructions" + "decompress" ] } diff --git a/extensions/inference-cortex-extension/src/@types/global.d.ts b/extensions/inference-cortex-extension/src/@types/global.d.ts index 139d836a56..3c310477d7 100644 --- a/extensions/inference-cortex-extension/src/@types/global.d.ts +++ b/extensions/inference-cortex-extension/src/@types/global.d.ts @@ -13,3 +13,13 @@ interface ModelOperationResponse { error?: any modelFile?: string } + +/** + * Cortex Executable Options Interface + */ +interface CortexExecutableOptions { + enginePath: string + executablePath: string + cudaVisibleDevices: string + vkVisibleDevices: string +} diff --git a/extensions/inference-cortex-extension/src/node/index.test.ts b/extensions/inference-cortex-extension/src/node/index.test.ts index ff9d7c2fc3..bdfd99d035 100644 --- a/extensions/inference-cortex-extension/src/node/index.test.ts +++ b/extensions/inference-cortex-extension/src/node/index.test.ts @@ -54,17 +54,6 @@ jest.mock('child_process', () => ({ }, })) -jest.mock('./execute', () => ({ - executableCortexFile: () => { - return { - enginePath: 'enginePath', - executablePath: 'executablePath', - cudaVisibleDevices: 'cudaVisibleDevices', - vkVisibleDevices: 'vkVisibleDevices', - } - }, -})) - import index from './index' describe('dispose', () => { diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index a13bf60284..9f71bc0026 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -1,6 +1,10 @@ import path from 'path' -import { getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' -import { engineVariant, executableCortexFile } from './execute' +import { + appResourcePath, + getJanDataFolderPath, + log, + SystemInformation, +} from '@janhq/core/node' import { ProcessWatchdog } from './watchdog' // The HOST address to use for the Nitro subprocess @@ -15,21 +19,16 @@ function run(systemInfo?: SystemInformation): Promise { log(`[CORTEX]:: Spawning cortex subprocess...`) return new Promise(async (resolve, reject) => { - let executableOptions = executableCortexFile( - // If ngl is not set or equal to 0, run on CPU with correct instructions - systemInfo?.gpuSetting - ? { - ...systemInfo.gpuSetting, - run_mode: systemInfo.gpuSetting.run_mode, - } - : undefined - ) - + let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' + let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` + const binPath = path.join(__dirname, '..', 'bin') + const enginePath = path.join(appResourcePath(), 'shared') + const executablePath = path.join(binPath, binaryName) // Execute the binary - log(`[CORTEX]:: Spawn cortex at path: ${executableOptions.executablePath}`) - log(`[CORTEX]:: Cortex engine path: ${executableOptions.enginePath}`) + log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`) + log(`[CORTEX]:: Cortex engine path: ${enginePath}`) - addEnvPaths(executableOptions.enginePath) + addEnvPaths(enginePath) const dataFolderPath = getJanDataFolderPath() if (watchdog) { @@ -37,7 +36,7 @@ function run(systemInfo?: SystemInformation): Promise { } watchdog = new ProcessWatchdog( - executableOptions.executablePath, + executablePath, [ '--start-server', '--port', @@ -48,14 +47,14 @@ function run(systemInfo?: SystemInformation): Promise { dataFolderPath, ], { - cwd: executableOptions.enginePath, + cwd: enginePath, env: { ...process.env, - ENGINE_PATH: executableOptions.enginePath, - CUDA_VISIBLE_DEVICES: executableOptions.cudaVisibleDevices, + ENGINE_PATH: enginePath, + CUDA_VISIBLE_DEVICES: gpuVisibleDevices, // Vulkan - Support 1 device at a time for now - ...(executableOptions.vkVisibleDevices?.length > 0 && { - GGML_VULKAN_DEVICE: executableOptions.vkVisibleDevices[0], + ...(gpuVisibleDevices?.length > 0 && { + GGML_VK_VISIBLE_DEVICES: gpuVisibleDevices, }), }, } @@ -96,5 +95,4 @@ export interface CortexProcessInfo { export default { run, dispose, - engineVariant, } From 274c940e01539ef7e6a88ec81d3433bc6d840604 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 26 Dec 2024 22:34:08 +0700 Subject: [PATCH 2/5] chore: symlink engine variants --- .../engine-management-extension/package.json | 3 +- .../rolldown.config.js | 6 +++ .../engine-management-extension/src/index.ts | 3 ++ .../src/node/index.ts | 47 ++++++++++++++++++- .../inference-cortex-extension/download.sh | 2 - .../src/node/index.ts | 13 +---- 6 files changed, 58 insertions(+), 16 deletions(-) diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index c9af0ea646..b4ba074ad5 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -32,7 +32,8 @@ "p-queue": "^8.0.1" }, "bundledDependencies": [ - "cpu-instructions" + "cpu-instructions", + "@janhq/core" ], "engines": { "node": ">=18.0.0" diff --git a/extensions/engine-management-extension/rolldown.config.js b/extensions/engine-management-extension/rolldown.config.js index 3214c2fc09..edebf5efaf 100644 --- a/extensions/engine-management-extension/rolldown.config.js +++ b/extensions/engine-management-extension/rolldown.config.js @@ -20,10 +20,16 @@ export default defineConfig([ }, { input: 'src/node/index.ts', + external: ['@janhq/core/node'], output: { format: 'cjs', file: 'dist/node/index.cjs.js', }, + plugins: [ + replace({ + CORTEX_ENGINE_VERSION: JSON.stringify('v0.1.42'), + }), + ], }, { input: 'src/node/cpuInfo.ts', diff --git a/extensions/engine-management-extension/src/index.ts b/extensions/engine-management-extension/src/index.ts index c589b6a85b..334df8cc1e 100644 --- a/extensions/engine-management-extension/src/index.ts +++ b/extensions/engine-management-extension/src/index.ts @@ -23,6 +23,9 @@ export default class JSONEngineManagementExtension extends EngineManagementExten * Called when the extension is loaded. */ async onLoad() { + // Symlink Engines Directory + await executeOnMain(NODE, 'symlinkEngines') + // Run Healthcheck this.queue.add(() => this.healthz()) try { const variant = await this.getDefaultEngineVariant( diff --git a/extensions/engine-management-extension/src/node/index.ts b/extensions/engine-management-extension/src/node/index.ts index c55fef7204..3053fe7a39 100644 --- a/extensions/engine-management-extension/src/node/index.ts +++ b/extensions/engine-management-extension/src/node/index.ts @@ -1,6 +1,13 @@ import * as path from 'path' -import { GpuSetting, log } from '@janhq/core/node' +import { + appResourcePath, + getJanDataFolderPath, + GpuSetting, + log, +} from '@janhq/core/node' import { fork } from 'child_process' +import { mkdir, readdir, symlink } from 'fs/promises' + /** * The GPU runMode that will be set - either 'vulkan', 'cuda', or empty for cpu. * @param settings @@ -101,6 +108,44 @@ const engineVariant = async (gpuSetting?: GpuSetting): Promise => { return engineVariant } +/** + * Create symlink to each variant for the default bundled version + */ +const symlinkEngines = async () => { + const sourceEnginePath = path.join( + appResourcePath(), + 'shared', + 'engines', + 'cortex.llamacpp' + ) + const symlinkEnginePath = path.join( + getJanDataFolderPath(), + 'engines', + 'cortex.llamacpp' + ) + const variantFolders = await readdir(sourceEnginePath) + for (const variant of variantFolders) { + const targetVariantPath = path.join( + sourceEnginePath, + variant, + CORTEX_ENGINE_VERSION + ) + const symlinkVariantPath = path.join( + symlinkEnginePath, + variant, + CORTEX_ENGINE_VERSION + ) + + await mkdir(path.join(sourceEnginePath, variant), { + recursive: true, + }).catch(console.error) + + await symlink(targetVariantPath, symlinkVariantPath).catch(console.error) + console.log(`Symlink created: ${targetVariantPath} -> ${symlinkEnginePath}`) + } +} + export default { engineVariant, + symlinkEngines, } diff --git a/extensions/inference-cortex-extension/download.sh b/extensions/inference-cortex-extension/download.sh index b0f3b36e35..0ccbdcb1ad 100755 --- a/extensions/inference-cortex-extension/download.sh +++ b/extensions/inference-cortex-extension/download.sh @@ -30,8 +30,6 @@ if [ "$OS_TYPE" == "Linux" ]; then download "${ENGINE_DOWNLOAD_URL}-linux-amd64-vulkan.tar.gz" -e --strip 1 -o "${SHARED_PATH}/engines/cortex.llamacpp/linux-amd64-vulkan/v${ENGINE_VERSION}" 1 download "${CUDA_DOWNLOAD_URL}/cuda-12-0-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1 download "${CUDA_DOWNLOAD_URL}/cuda-11-7-linux-amd64.tar.gz" -e --strip 1 -o "${SHARED_PATH}" 1 - mkdir -p "${SHARED_PATH}/engines/cortex.llamacpp/deps" - touch "${SHARED_PATH}/engines/cortex.llamacpp/deps/keep" elif [ "$OS_TYPE" == "Darwin" ]; then # macOS downloads diff --git a/extensions/inference-cortex-extension/src/node/index.ts b/extensions/inference-cortex-extension/src/node/index.ts index 9f71bc0026..4ce35c83da 100644 --- a/extensions/inference-cortex-extension/src/node/index.ts +++ b/extensions/inference-cortex-extension/src/node/index.ts @@ -1,10 +1,5 @@ import path from 'path' -import { - appResourcePath, - getJanDataFolderPath, - log, - SystemInformation, -} from '@janhq/core/node' +import { getJanDataFolderPath, log, SystemInformation } from '@janhq/core/node' import { ProcessWatchdog } from './watchdog' // The HOST address to use for the Nitro subprocess @@ -22,13 +17,9 @@ function run(systemInfo?: SystemInformation): Promise { let gpuVisibleDevices = systemInfo?.gpuSetting?.gpus_in_use.join(',') ?? '' let binaryName = `cortex-server${process.platform === 'win32' ? '.exe' : ''}` const binPath = path.join(__dirname, '..', 'bin') - const enginePath = path.join(appResourcePath(), 'shared') const executablePath = path.join(binPath, binaryName) // Execute the binary log(`[CORTEX]:: Spawn cortex at path: ${executablePath}`) - log(`[CORTEX]:: Cortex engine path: ${enginePath}`) - - addEnvPaths(enginePath) const dataFolderPath = getJanDataFolderPath() if (watchdog) { @@ -47,10 +38,8 @@ function run(systemInfo?: SystemInformation): Promise { dataFolderPath, ], { - cwd: enginePath, env: { ...process.env, - ENGINE_PATH: enginePath, CUDA_VISIBLE_DEVICES: gpuVisibleDevices, // Vulkan - Support 1 device at a time for now ...(gpuVisibleDevices?.length > 0 && { From 157d3b507473da2d119f2d13176fed9a1950550a Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 27 Dec 2024 10:28:19 +0700 Subject: [PATCH 3/5] chore: rolldown.config in mjs format --- extensions/engine-management-extension/package.json | 2 +- .../{rolldown.config.js => rolldown.config.mjs} | 0 extensions/engine-management-extension/tsconfig.json | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename extensions/engine-management-extension/{rolldown.config.js => rolldown.config.mjs} (100%) diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index b4ba074ad5..0dfa9b02e5 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -9,7 +9,7 @@ "license": "MIT", "scripts": { "test": "jest", - "build": "rolldown -c", + "build": "rolldown -c rolldown.config.mjs", "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" }, "exports": { diff --git a/extensions/engine-management-extension/rolldown.config.js b/extensions/engine-management-extension/rolldown.config.mjs similarity index 100% rename from extensions/engine-management-extension/rolldown.config.js rename to extensions/engine-management-extension/rolldown.config.mjs diff --git a/extensions/engine-management-extension/tsconfig.json b/extensions/engine-management-extension/tsconfig.json index 8427123e73..891d28a605 100644 --- a/extensions/engine-management-extension/tsconfig.json +++ b/extensions/engine-management-extension/tsconfig.json @@ -11,5 +11,5 @@ "rootDir": "./src" }, "include": ["./src"], - "exclude": ["src/**/*.test.ts"] + "exclude": ["src/**/*.test.ts", "rolldown.config.mjs"] } From 0b619ac94604eb128c9ea1dd8c9ee41a0f4182c7 Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 27 Dec 2024 11:27:52 +0700 Subject: [PATCH 4/5] chore: binary codesign --- extensions/engine-management-extension/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/engine-management-extension/package.json b/extensions/engine-management-extension/package.json index 0dfa9b02e5..de690c1ab6 100644 --- a/extensions/engine-management-extension/package.json +++ b/extensions/engine-management-extension/package.json @@ -10,7 +10,7 @@ "scripts": { "test": "jest", "build": "rolldown -c rolldown.config.mjs", - "build:publish": "rimraf *.tgz --glob && yarn build && npm pack && cpx *.tgz ../../pre-install" + "build:publish": "rimraf *.tgz --glob && yarn build && ../../.github/scripts/auto-sign.sh && npm pack && cpx *.tgz ../../pre-install" }, "exports": { ".": "./dist/index.js", From 2be92c75d9d5648ab84277595b611cf5f5eeb32a Mon Sep 17 00:00:00 2001 From: Louis Date: Fri, 27 Dec 2024 16:08:36 +0700 Subject: [PATCH 5/5] fix: download state in footer bar and variant status --- extensions/inference-cortex-extension/src/index.ts | 5 ++++- web/containers/Providers/EventListener.tsx | 11 +++++++---- web/hooks/useDownloadState.ts | 5 ----- web/screens/Settings/Engines/Settings.tsx | 12 ++++++++++-- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/extensions/inference-cortex-extension/src/index.ts b/extensions/inference-cortex-extension/src/index.ts index b631a048f9..ba9cb3b369 100644 --- a/extensions/inference-cortex-extension/src/index.ts +++ b/extensions/inference-cortex-extension/src/index.ts @@ -289,7 +289,10 @@ export default class JanInferenceCortexExtension extends LocalOAIEngine { ) if (data.task.type === 'Engine') { - events.emit(EngineEvent.OnEngineUpdate, {}) + events.emit(EngineEvent.OnEngineUpdate, { + type: data.type, + percent: percent, + }) } else { if (data.type === DownloadTypes.DownloadSuccess) { // Delay for the state update from cortex.cpp diff --git a/web/containers/Providers/EventListener.tsx b/web/containers/Providers/EventListener.tsx index f28057626f..55c172beb8 100644 --- a/web/containers/Providers/EventListener.tsx +++ b/web/containers/Providers/EventListener.tsx @@ -96,10 +96,7 @@ const EventListener = () => { const onFileDownloadSuccess = useCallback( async (state: DownloadState) => { console.debug('onFileDownloadSuccess', state) - if ( - state.downloadType !== 'extension' && - state.downloadType !== 'Engine' - ) { + if (state.downloadType !== 'extension') { // Update model metadata accordingly const model = ModelManager.instance().models.get(state.modelId) if (model) { @@ -111,6 +108,12 @@ const EventListener = () => { ...model.parameters, } as Partial) .catch((e) => console.debug(e)) + + toaster({ + title: 'Download Completed', + description: `Download ${state.modelId} completed`, + type: 'success', + }) } state.downloadState = 'end' setDownloadState(state) diff --git a/web/hooks/useDownloadState.ts b/web/hooks/useDownloadState.ts index 32a9d32556..632ca31a8f 100644 --- a/web/hooks/useDownloadState.ts +++ b/web/hooks/useDownloadState.ts @@ -54,11 +54,6 @@ export const setDownloadStateAtom = atom( (e) => e.id === state.modelId ) if (model) set(downloadedModelsAtom, (prev) => [...prev, model]) - toaster({ - title: 'Download Completed', - description: `Download ${state.modelId} completed`, - type: 'success', - }) } } else if (state.downloadState === 'error') { // download error diff --git a/web/screens/Settings/Engines/Settings.tsx b/web/screens/Settings/Engines/Settings.tsx index 4fd04d85a9..572e8f1ef5 100644 --- a/web/screens/Settings/Engines/Settings.tsx +++ b/web/screens/Settings/Engines/Settings.tsx @@ -1,5 +1,5 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ -import React, { useCallback, useEffect, useState } from 'react' +import React, { useCallback, useEffect, useMemo, useState } from 'react' import { EngineEvent, events, InferenceEngine } from '@janhq/core' import { Button, ScrollArea, Badge, Select } from '@janhq/joi' @@ -67,6 +67,14 @@ const EngineSettings = ({ engine }: { engine: InferenceEngine }) => { defaultEngineVariant?.variant ) + const selectedVariant = useMemo( + () => + options?.map((e) => e.value).includes(selectedVariants) + ? selectedVariants + : undefined, + [selectedVariants, options] + ) + useEffect(() => { if (defaultEngineVariant?.variant) { setSelectedVariants(defaultEngineVariant.variant || '') @@ -157,7 +165,7 @@ const EngineSettings = ({ engine }: { engine: InferenceEngine }) => {