feat: remote engine management

core/src/browser/extensions/engines/helpers/sse.ts

@@ -22,7 +22,9 @@
       headers: {
         'Content-Type': 'application/json',
         'Access-Control-Allow-Origin': '*',
-        'Accept': model.parameters?.stream ? 'text/event-stream' : 'application/json',
+        'Accept': model.parameters?.stream
+          ? 'text/event-stream'
+          : 'application/json',
         ...headers,
       },
       body: JSON.stringify(requestBody),
@@ -34,8 +36,8 @@
           let errorCode = ErrorCode.Unknown
           if (data.error) {
             errorCode = data.error.code ?? data.error.type ?? ErrorCode.Unknown
           } else if (response.status === 401) {
             errorCode = ErrorCode.InvalidApiKey
           }
           const error = {
             message: data.error?.message ?? data.message ?? 'Error occurred.',
@@ -47,12 +49,24 @@
         }
         // There could be overriden stream parameter in the model
         // that is set in request body (transformed payload)
-        if (requestBody?.stream === false || model.parameters?.stream === false) {
+        if (
+          requestBody?.stream === false ||
+          model.parameters?.stream === false
+        ) {
           const data = await response.json()
+          if (data.error || data.message) {
+            subscriber.error(data.error ?? data)
+            subscriber.complete()
+            return
+          }
           if (transformResponse) {
             subscriber.next(transformResponse(data))
           } else {
-            subscriber.next(data.choices[0]?.message?.content ?? '')
+            subscriber.next(
+              data.choices
+                ? data.choices[0]?.message?.content
+                : (data.content[0]?.text ?? '')
+            )
           }
         } else {
           const stream = response.body
@@ -71,32 +85,32 @@
             for (const line of lines) {
               try {
                 if (transformResponse) {
                   content += transformResponse(line)
                   subscriber.next(content ?? '')
                 } else {
                   const toParse = cachedLines + line
                   if (!line.includes('data: [DONE]')) {
                     const data = JSON.parse(toParse.replace('data: ', ''))
                     if ('error' in data) {
                       subscriber.error(data.error)
                       subscriber.complete()
                       return
                     }
                     content += data.choices[0]?.delta?.content ?? ''
                     if (content.startsWith('assistant: ')) {
                       content = content.replace('assistant: ', '')
                     }
                     if (content !== '') subscriber.next(content)
                   }
                 }
               } catch {
                 cachedLines = line
               }
             }
           }
         }
         subscriber.complete()
       })
       .catch((err) => subscriber.error(err))
   })
 }

core/src/browser/extensions/enginesManagement.ts

@@ -3,6 +3,7 @@
   Engines,
   EngineVariant,
   EngineReleased,
+  EngineConfig,
   DefaultEngineVariant,
 } from '../../types'
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
@@ -14,7 +15,7 @@
  */
 export abstract class EngineManagementExtension extends BaseExtension {
   type(): ExtensionTypeEnum | undefined {
     return ExtensionTypeEnum.Engine
   }
 
   /**
@@ -55,8 +56,16 @@
    * @returns A Promise that resolves to intall of engine.
    */
   abstract installEngine(
-    name: InferenceEngine,
-    engineConfig: { variant: string; version?: string }
+    name: string,
+    engineConfig: EngineConfig
+  ): Promise<{ messages: string }>
+
+  /**
+   * Add a new remote engine
+   * @returns A Promise that resolves to intall of engine.
+   */
+  abstract addRemoteEngine(
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
@@ -65,14 +74,16 @@
    */
   abstract uninstallEngine(
     name: InferenceEngine,
-    engineConfig: { variant: string; version: string }
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
    * @param name - Inference engine name.
    * @returns A Promise that resolves to an object of default engine.
    */
-  abstract getDefaultEngineVariant(name: InferenceEngine): Promise<DefaultEngineVariant>
+  abstract getDefaultEngineVariant(
+    name: InferenceEngine
+  ): Promise<DefaultEngineVariant>
 
   /**
    * @body variant - string
@@ -81,11 +92,19 @@
    */
   abstract setDefaultEngineVariant(
     name: InferenceEngine,
-    engineConfig: { variant: string; version: string }
+    engineConfig: EngineConfig
   ): Promise<{ messages: string }>
 
   /**
    * @returns A Promise that resolves to update engine.
    */
-  abstract updateEngine(name: InferenceEngine): Promise<{ messages: string }>
+  abstract updateEngine(
+    name: InferenceEngine,
+    engineConfig?: EngineConfig
+  ): Promise<{ messages: string }>
+
+  /**
+   * @returns A Promise that resolves to an object of remote models list .
+   */
+  abstract getRemoteModels(name: InferenceEngine | string): Promise<any>
 }

core/src/node/helper/config.test.ts

@@ -1,28 +1,19 @@
-import { getEngineConfiguration } from './config';
-import { getAppConfigurations, defaultAppConfig } from './config';
-
-import { getJanExtensionsPath } from './config';
-import { getJanDataFolderPath } from './config';
-it('should return undefined for invalid engine ID', async () => {
-  const config = await getEngineConfiguration('invalid_engine');
-  expect(config).toBeUndefined();
-});
+import { getAppConfigurations, defaultAppConfig } from './config'
 
+import { getJanExtensionsPath, getJanDataFolderPath } from './config'
 
 it('should return default config when CI is e2e', () => {
-  process.env.CI = 'e2e';
-  const config = getAppConfigurations();
-  expect(config).toEqual(defaultAppConfig());
-});
-
+  process.env.CI = 'e2e'
+  const config = getAppConfigurations()
+  expect(config).toEqual(defaultAppConfig())
+})
 
 it('should return extensions path when retrieved successfully', () => {
-  const extensionsPath = getJanExtensionsPath();
-  expect(extensionsPath).not.toBeUndefined();
-});
-
+  const extensionsPath = getJanExtensionsPath()
+  expect(extensionsPath).not.toBeUndefined()
+})
 
 it('should return data folder path when retrieved successfully', () => {
-  const dataFolderPath = getJanDataFolderPath();
-  expect(dataFolderPath).not.toBeUndefined();
-});
+  const dataFolderPath = getJanDataFolderPath()
+  expect(dataFolderPath).not.toBeUndefined()
+})

core/src/node/helper/config.ts

@@ -1,8 +1,7 @@
-import { AppConfiguration, SettingComponentProps } from '../../types'
+import { AppConfiguration } from '../../types'
 import { join, resolve } from 'path'
 import fs from 'fs'
 import os from 'os'
-import childProcess from 'child_process'
 const configurationFileName = 'settings.json'
 
 /**
@@ -19,9 +18,11 @@
 
   if (!fs.existsSync(configurationFile)) {
     // create default app config if we don't have one
-    console.debug(`App config not found, creating default config at ${configurationFile}`)
+    console.debug(
+      `App config not found, creating default config at ${configurationFile}`
+    )
     fs.writeFileSync(configurationFile, JSON.stringify(appDefaultConfiguration))
     return appDefaultConfiguration
   }
 
   try {
@@ -30,20 +31,28 @@
     )
     return appConfigurations
   } catch (err) {
-    console.error(`Failed to read app config, return default config instead! Err: ${err}`)
+    console.error(
+      `Failed to read app config, return default config instead! Err: ${err}`
+    )
     return defaultAppConfig()
   }
 }
 
 const getConfigurationFilePath = () =>
   join(
-    global.core?.appPath() || process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
+    global.core?.appPath() ||
+      process.env[process.platform == 'win32' ? 'USERPROFILE' : 'HOME'],
     configurationFileName
   )
 
-export const updateAppConfiguration = (configuration: AppConfiguration): Promise<void> => {
+export const updateAppConfiguration = (
+  configuration: AppConfiguration
+): Promise<void> => {
   const configurationFile = getConfigurationFilePath()
-  console.debug('updateAppConfiguration, configurationFile: ', configurationFile)
+  console.debug(
+    'updateAppConfiguration, configurationFile: ',
+    configurationFile
+  )
 
   fs.writeFileSync(configurationFile, JSON.stringify(configuration))
   return Promise.resolve()
@@ -69,86 +78,6 @@
   return join(appConfigurations.data_folder, 'extensions')
 }
 
-/**
- * Utility function to physical cpu count
- *
- * @returns {number} The physical cpu count.
- */
-export const physicalCpuCount = async (): Promise<number> => {
-  const platform = os.platform()
-  try {
-    if (platform === 'linux') {
-      const output = await exec('lscpu -p | egrep -v "^#" | sort -u -t, -k 2,4 | wc -l')
-      return parseInt(output.trim(), 10)
-    } else if (platform === 'darwin') {
-      const output = await exec('sysctl -n hw.physicalcpu_max')
-      return parseInt(output.trim(), 10)
-    } else if (platform === 'win32') {
-      const output = await exec('WMIC CPU Get NumberOfCores')
-      return output
-        .split(os.EOL)
-        .map((line: string) => parseInt(line))
-        .filter((value: number) => !isNaN(value))
-        .reduce((sum: number, number: number) => sum + number, 1)
-    } else {
-      const cores = os.cpus().filter((cpu: any, index: number) => {
-        const hasHyperthreading = cpu.model.includes('Intel')
-        const isOdd = index % 2 === 1
-        return !hasHyperthreading || isOdd
-      })
-      return cores.length
-    }
-  } catch (err) {
-    console.warn('Failed to get physical CPU count', err)
-    // Divide by 2 to get rid of hyper threading
-    const coreCount = Math.ceil(os.cpus().length / 2)
-    console.debug('Using node API to get physical CPU count:', coreCount)
-    return coreCount
-  }
-}
-
-const exec = async (command: string): Promise<string> => {
-  return new Promise((resolve, reject) => {
-    childProcess.exec(command, { encoding: 'utf8' }, (error, stdout) => {
-      if (error) {
-        reject(error)
-      } else {
-        resolve(stdout)
-      }
-    })
-  })
-}
-
-// a hacky way to get the api key. we should comes up with a better
-// way to handle this
-export const getEngineConfiguration = async (engineId: string) => {
-  if (engineId !== 'openai' && engineId !== 'groq') return undefined
-
-  const settingDirectoryPath = join(
-    getJanDataFolderPath(),
-    'settings',
-    '@janhq',
-    engineId === 'openai' ? 'inference-openai-extension' : 'inference-groq-extension',
-    'settings.json'
-  )
-
-  const content = fs.readFileSync(settingDirectoryPath, 'utf-8')
-  const settings: SettingComponentProps[] = JSON.parse(content)
-  const apiKeyId = engineId === 'openai' ? 'openai-api-key' : 'groq-api-key'
-  const keySetting = settings.find((setting) => setting.key === apiKeyId)
-  let fullUrl = settings.find((setting) => setting.key === 'chat-completions-endpoint')
-    ?.controllerProps.value
-
-  let apiKey = keySetting?.controllerProps.value
-  if (typeof apiKey !== 'string') apiKey = ''
-  if (typeof fullUrl !== 'string') fullUrl = ''
-
-  return {
-    api_key: apiKey,
-    full_url: fullUrl,
-  }
-}
-
 /**
  * Default app configurations
  * App Data Folder default to Electron's userData
@@ -158,7 +87,10 @@
  */
 export const defaultAppConfig = (): AppConfiguration => {
   const { app } = require('electron')
-  const defaultJanDataFolder = join(app?.getPath('userData') ?? os?.homedir() ?? '', 'data')
+  const defaultJanDataFolder = join(
+    app?.getPath('userData') ?? os?.homedir() ?? '',
+    'data'
+  )
   return {
     data_folder:
       process.env.CI === 'e2e'

core/src/node/helper/resource.test.ts

@@ -1,15 +1,9 @@
-import { getSystemResourceInfo } from './resource';
+import { getSystemResourceInfo } from './resource'
 
 it('should return the correct system resource information with a valid CPU count', async () => {
-  const mockCpuCount = 4;
-  jest.spyOn(require('./config'), 'physicalCpuCount').mockResolvedValue(mockCpuCount);
-  const logSpy = jest.spyOn(require('./logger'), 'log').mockImplementation(() => {});
-
-  const result = await getSystemResourceInfo();
+  const result = await getSystemResourceInfo()
 
   expect(result).toEqual({
-    numCpuPhysicalCore: mockCpuCount,
     memAvailable: 0,
-  });
-  expect(logSpy).toHaveBeenCalledWith(`[CORTEX]::CPU information - ${mockCpuCount}`);
-});
+  })
+})

core/src/node/helper/resource.ts

@@ -1,13 +1,7 @@
 import { SystemResourceInfo } from '../../types'
-import { physicalCpuCount } from './config'
-import { log } from './logger'
 
 export const getSystemResourceInfo = async (): Promise<SystemResourceInfo> => {
-  const cpu = await physicalCpuCount()
-  log(`[CORTEX]::CPU information - ${cpu}`)
-
   return {
-    numCpuPhysicalCore: cpu,
     memAvailable: 0, // TODO: this should not be 0
   }
 }

core/src/types/engine/index.ts

@@ -1,7 +1,23 @@
 import { InferenceEngine } from '../../types'
 
 export type Engines = {
-  [key in InferenceEngine]: EngineVariant[]
+  [key in InferenceEngine]: (EngineVariant & EngineConfig)[]
+}
+
+export type EngineMetadata = {
+  get_models_url?: string
+  header_template?: string
+  transform_req?: {
+    chat_completions?: {
+      url?: string
+      template?: string
+    }
+  }
+  transform_resp?: {
+    chat_completions?: {
+      template?: string
+    }
+  }
 }
 
 export type EngineVariant = {
@@ -23,6 +39,16 @@ export type EngineReleased = {
   size: number
 }
 
+export type EngineConfig = {
+  engine?: string
+  version?: string
+  variant?: string
+  type?: string
+  url?: string
+  api_key?: string
+  metadata?: EngineMetadata
+}
+
 export enum EngineEvent {
   OnEngineUpdate = 'OnEngineUpdate',
 }