Merge pull request #4249 from janhq/feat/threads-messages-requests-to…

…-backend

feat: reroute threads and messages requests to cortex.cpp backend

core/src/browser/extensions/conversational.ts

@@ -1,4 +1,10 @@
-import { Thread, ThreadInterface, ThreadMessage, MessageInterface } from '../../types'
+import {
+  Thread,
+  ThreadInterface,
+  ThreadMessage,
+  MessageInterface,
+  ThreadAssistantInfo,
+} from '../../types'
 import { BaseExtension, ExtensionTypeEnum } from '../extension'
 
 /**
@@ -17,10 +23,21 @@ export abstract class ConversationalExtension
     return ExtensionTypeEnum.Conversational
   }
 
-  abstract getThreads(): Promise<Thread[]>
-  abstract saveThread(thread: Thread): Promise<void>
+  abstract listThreads(): Promise<Thread[]>
+  abstract createThread(thread: Partial<Thread>): Promise<Thread>
+  abstract modifyThread(thread: Thread): Promise<void>
   abstract deleteThread(threadId: string): Promise<void>
-  abstract addNewMessage(message: ThreadMessage): Promise<void>
-  abstract writeMessages(threadId: string, messages: ThreadMessage[]): Promise<void>
-  abstract getAllMessages(threadId: string): Promise<ThreadMessage[]>
+  abstract createMessage(message: Partial<ThreadMessage>): Promise<ThreadMessage>
+  abstract deleteMessage(threadId: string, messageId: string): Promise<void>
+  abstract listMessages(threadId: string): Promise<ThreadMessage[]>
+  abstract getThreadAssistant(threadId: string): Promise<ThreadAssistantInfo>
+  abstract createThreadAssistant(
+    threadId: string,
+    assistant: ThreadAssistantInfo
+  ): Promise<ThreadAssistantInfo>
+  abstract modifyThreadAssistant(
+    threadId: string,
+    assistant: ThreadAssistantInfo
+  ): Promise<ThreadAssistantInfo>
+  abstract modifyMessage(message: ThreadMessage): Promise<ThreadMessage>
 }

core/src/browser/extensions/engines/AIEngine.ts

@@ -2,7 +2,6 @@ import { events } from '../../events'
 import { BaseExtension } from '../../extension'
 import { MessageRequest, Model, ModelEvent } from '../../../types'
 import { EngineManager } from './EngineManager'
-import { ModelManager } from '../../models/manager'
 
 /**
  * Base AIEngine

core/src/node/api/restful/helper/builder.ts

@@ -6,7 +6,6 @@ import {
   mkdirSync,
   appendFileSync,
   createWriteStream,
-  rmdirSync,
 } from 'fs'
 import { JanApiRouteConfiguration, RouteConfiguration } from './configuration'
 import { join } from 'path'
@@ -126,7 +125,7 @@ export const createThread = async (thread: any) => {
     }
   }
 
-  const threadId = generateThreadId(thread.assistants[0].assistant_id)
+  const threadId = generateThreadId(thread.assistants[0]?.assistant_id)
   try {
     const updatedThread = {
       ...thread,
@@ -280,7 +279,7 @@ export const models = async (request: any, reply: any) => {
     'Content-Type': 'application/json',
   }
 
-  const response = await fetch(`${CORTEX_API_URL}/models${request.url.split('/models')[1] ?? ""}`, {
+  const response = await fetch(`${CORTEX_API_URL}/models${request.url.split('/models')[1] ?? ''}`, {
     method: request.method,
     headers: headers,
     body: JSON.stringify(request.body),

core/src/types/assistant/assistantEntity.ts

@@ -36,3 +36,10 @@ export type Assistant = {
   /** Represents the metadata of the object. */
   metadata?: Record<string, unknown>
 }
+
+export interface CodeInterpreterTool {
+  /**
+   * The type of tool being defined: `code_interpreter`
+   */
+  type: 'code_interpreter'
+}

core/src/types/message/messageEntity.ts

@@ -1,3 +1,4 @@
+import { CodeInterpreterTool } from '../assistant'
 import { ChatCompletionMessage, ChatCompletionRole } from '../inference'
 import { ModelInfo } from '../model'
 import { Thread } from '../thread'
@@ -15,6 +16,10 @@ export type ThreadMessage = {
   thread_id: string
   /** The assistant id of this thread. **/
   assistant_id?: string
+  /**
+   * A list of files attached to the message, and the tools they were added to.
+   */
+  attachments?: Array<Attachment> | null
   /** The role of the author of this message. **/
   role: ChatCompletionRole
   /** The content of this message. **/
@@ -52,6 +57,11 @@ export type MessageRequest = {
    */
   assistantId?: string
 
+  /**
+   * A list of files attached to the message, and the tools they were added to.
+   */
+  attachments: Array<Attachment> | null
+
   /** Messages for constructing a chat completion request **/
   messages?: ChatCompletionMessage[]
 
@@ -97,8 +107,7 @@ export enum ErrorCode {
  */
 export enum ContentType {
   Text = 'text',
-  Image = 'image',
-  Pdf = 'pdf',
+  Image = 'image_url',
 }
 
 /**
@@ -108,8 +117,15 @@ export enum ContentType {
 export type ContentValue = {
   value: string
   annotations: string[]
-  name?: string
-  size?: number
+}
+
+/**
+ * The `ImageContentValue` type defines the shape of a content value object of image type
+ * @data_transfer_object
+ */
+export type ImageContentValue = {
+  detail?: string
+  url?: string
 }
 
 /**
@@ -118,5 +134,37 @@ export type ContentValue = {
  */
 export type ThreadContent = {
   type: ContentType
-  text: ContentValue
+  text?: ContentValue
+  image_url?: ImageContentValue
+}
+
+export interface Attachment {
+  /**
+   * The ID of the file to attach to the message.
+   */
+  file_id?: string
+
+  /**
+   * The tools to add this file to.
+   */
+  tools?: Array<CodeInterpreterTool | Attachment.AssistantToolsFileSearchTypeOnly>
+}
+
+export namespace Attachment {
+  export interface AssistantToolsFileSearchTypeOnly {
+    /**
+     * The type of tool being defined: `file_search`
+     */
+    type: 'file_search'
+  }
+}
+
+/**
+ * On an incomplete message, details about why the message is incomplete.
+ */
+export interface IncompleteDetails {
+  /**
+   * The reason the message is incomplete.
+   */
+  reason: 'content_filter' | 'max_tokens' | 'run_cancelled' | 'run_expired' | 'run_failed'
 }

core/src/types/message/messageInterface.ts

@@ -11,20 +11,20 @@ export interface MessageInterface {
    * @param {ThreadMessage} message - The message to be added.
    * @returns {Promise<void>} A promise that resolves when the message has been added.
    */
-  addNewMessage(message: ThreadMessage): Promise<void>
-
-  /**
-   * Writes an array of messages to a specific thread.
-   * @param {string} threadId - The ID of the thread to write the messages to.
-   * @param {ThreadMessage[]} messages - The array of messages to be written.
-   * @returns {Promise<void>} A promise that resolves when the messages have been written.
-   */
-  writeMessages(threadId: string, messages: ThreadMessage[]): Promise<void>
+  createMessage(message: ThreadMessage): Promise<ThreadMessage>
 
   /**
    * Retrieves all messages from a specific thread.
    * @param {string} threadId - The ID of the thread to retrieve the messages from.
    * @returns {Promise<ThreadMessage[]>} A promise that resolves to an array of messages from the thread.
    */
-  getAllMessages(threadId: string): Promise<ThreadMessage[]>
+  listMessages(threadId: string): Promise<ThreadMessage[]>
+
+  /**
+   * Deletes a specific message from a thread.
+   * @param {string} threadId - The ID of the thread from which the message will be deleted.
+   * @param {string} messageId - The ID of the message to be deleted.
+   * @returns {Promise<void>} A promise that resolves when the message has been successfully deleted.
+   */
+  deleteMessage(threadId: string, messageId: string): Promise<void>
 }

core/src/types/thread/threadInterface.ts

@@ -11,15 +11,23 @@ export interface ThreadInterface {
    * @abstract
    * @returns {Promise<Thread[]>} A promise that resolves to an array of threads.
    */
-  getThreads(): Promise<Thread[]>
+  listThreads(): Promise<Thread[]>
 
   /**
-   * Saves a thread.
+   * Create a thread.
    * @abstract
    * @param {Thread} thread - The thread to save.
    * @returns {Promise<void>} A promise that resolves when the thread is saved.
    */
-  saveThread(thread: Thread): Promise<void>
+  createThread(thread: Thread): Promise<Thread>
+
+  /**
+   * modify a thread.
+   * @abstract
+   * @param {Thread} thread - The thread to save.
+   * @returns {Promise<void>} A promise that resolves when the thread is saved.
+   */
+  modifyThread(thread: Thread): Promise<void>
 
   /**
    * Deletes a thread.

electron/tests/config/fixtures.ts

@@ -108,7 +108,7 @@ export const test = base.extend<
 })
 
 test.beforeAll(async () => {
-  await rmSync(path.join(__dirname, '../../test-data'), {
+  rmSync(path.join(__dirname, '../../test-data'), {
     recursive: true,
     force: true,
   })
@@ -122,6 +122,5 @@ test.beforeAll(async () => {
 })
 
 test.afterAll(async () => {
-  // temporally disabling this due to the config for parallel testing WIP
   // teardownElectron()
 })

electron/tests/e2e/navigation.e2e.spec.ts

@@ -2,11 +2,8 @@ import { expect } from '@playwright/test'
 import { page, test, TIMEOUT } from '../config/fixtures'
 
 test('renders left navigation panel', async () => {
-  const settingsBtn = await page
-    .getByTestId('Thread')
-    .first()
-    .isEnabled({ timeout: TIMEOUT })
-  expect([settingsBtn].filter((e) => !e).length).toBe(0)
+  const threadBtn = page.getByTestId('Thread').first()
+  await expect(threadBtn).toBeVisible({ timeout: TIMEOUT })
   // Chat section should be there
   await page.getByTestId('Local API Server').first().click({
     timeout: TIMEOUT,

extensions/assistant-extension/src/index.ts

@@ -141,7 +141,7 @@ export default class JanAssistantExtension extends AssistantExtension {
           top_k: 2,
           chunk_size: 1024,
           chunk_overlap: 64,
-          retrieval_template: `Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer.
+          retrieval_template: `Use the following pieces of context to answer the question at the end.
 ----------------
 CONTEXT: {CONTEXT}
 ----------------

extensions/assistant-extension/src/node/index.ts

@@ -9,13 +9,14 @@ export function toolRetrievalUpdateTextSplitter(
   retrieval.updateTextSplitter(chunkSize, chunkOverlap)
 }
 export async function toolRetrievalIngestNewDocument(
+  thread: string,
   file: string,
   model: string,
   engine: string,
   useTimeWeighted: boolean
 ) {
-  const filePath = path.join(getJanDataFolderPath(), normalizeFilePath(file))
-  const threadPath = path.dirname(filePath.replace('files', ''))
+  const threadPath = path.join(getJanDataFolderPath(), 'threads', thread)
+  const filePath = path.join(getJanDataFolderPath(), 'files', file)
   retrieval.updateEmbeddingEngine(model, engine)
   return retrieval
     .ingestAgentKnowledge(filePath, `${threadPath}/memory`, useTimeWeighted)

extensions/assistant-extension/src/tools/retrieval.ts

@@ -35,6 +35,7 @@ export class RetrievalTool extends InferenceTool {
         await executeOnMain(
           NODE,
           'toolRetrievalIngestNewDocument',
+          data.thread?.id,
           docFile,
           data.model?.id,
           data.model?.engine,

extensions/conversational-extension/package.json

@@ -18,12 +18,14 @@
   "devDependencies": {
     "cpx": "^1.5.0",
     "rimraf": "^3.0.2",
+    "ts-loader": "^9.5.0",
     "webpack": "^5.88.2",
-    "webpack-cli": "^5.1.4",
-    "ts-loader": "^9.5.0"
+    "webpack-cli": "^5.1.4"
   },
   "dependencies": {
-    "@janhq/core": "file:../../core"
+    "@janhq/core": "file:../../core",
+    "ky": "^1.7.2",
+    "p-queue": "^8.0.1"
   },
   "engines": {
     "node": ">=18.0.0"

extensions/conversational-extension/src/@types/global.d.ts

@@ -0,0 +1,14 @@
+export {}
+declare global {
+  declare const API_URL: string
+  declare const SOCKET_URL: string
+
+  interface Core {
+    api: APIFunctions
+    events: EventEmitter
+  }
+  interface Window {
+    core?: Core | undefined
+    electronAPI?: any | undefined
+  }
+}