From 8e937d4c408d809fa67c124c7fb339e4a084c00b Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Mon, 10 Jun 2024 12:40:19 +0200 Subject: [PATCH 1/5] feat(assistant): changed default model to 4o && vision support (images) --- apps/api/src/app/chat/chat.config.ts | 2 +- .../chat/chat-audio/chat-audio.component.html | 16 ++--- .../chat/chat-audio/chat-audio.component.ts | 24 +++++-- .../chat-footer/chat-footer.component.html | 5 ++ .../chat/chat-footer/chat-footer.component.ts | 3 + .../chat-message/chat-message.component.html | 29 ++++++--- .../chat-message/chat-message.component.scss | 7 +++ .../chat-message/chat-message.component.ts | 13 ++++ .../chat-messages.component.html | 6 +- .../controls/files/files.component.ts | 7 ++- .../controls/files/files.directive.ts | 3 +- .../message-content.component.html | 25 ++++++++ .../message-content.component.scss | 37 +++++++++++ .../message-content.component.spec.ts | 23 +++++++ .../message-content.component.ts | 46 ++++++++++++++ .../message-content.helpers.ts | 37 +++++++++++ .../message-content.service.ts | 43 +++++++++++++ .../chat-cloud/chat-cloud.component.ts | 2 +- .../chat-iframe/chat-iframe.component.html | 1 + .../chat-iframe/chat-iframe.component.scss | 5 +- .../chat-iframe/chat-iframe.component.ts | 4 +- .../+chat/containers/chat/chat.component.ts | 2 +- .../app/modules/+chat/shared/chat.model.ts | 4 +- .../app/modules/+chat/shared/chat.service.ts | 62 ++++++++++++++++--- .../environments/environment.development.ts | 1 + apps/spa/src/environments/environment.ts | 1 + apps/spa/src/styles/_extends/_material.scss | 4 ++ .../src/environments/environment.prod.ts | 2 +- .../src/lib/chat/chat.model.ts | 3 +- .../src/lib/chat/chat.service.ts | 8 ++- 30 files changed, 373 insertions(+), 52 deletions(-) create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.html create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.scss create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.component.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.helpers.ts create mode 100644 apps/spa/src/app/components/controls/message-content/message-content.service.ts diff --git a/apps/api/src/app/chat/chat.config.ts b/apps/api/src/app/chat/chat.config.ts index ed41c1e..36ff0ef 100644 --- a/apps/api/src/app/chat/chat.config.ts +++ b/apps/api/src/app/chat/chat.config.ts @@ -6,7 +6,7 @@ export const assistantParams: AssistantCreateParams = { name: '@boldare/openai-assistant', instructions: `You are a chatbot assistant. Use the general knowledge to answer questions. Speak briefly and clearly.`, tools: [{ type: 'code_interpreter' }, { type: 'file_search' }], - model: 'gpt-4-turbo', + model: 'gpt-4o', temperature: 0.05, }; diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html index d3e56c9..fb97189 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html @@ -1,9 +1,9 @@ -@if (isAudioEnabled && message) { - - @if(!isStarted) { - play_circle - } @else { - pause_circle - } - +@if (getMessageText) { + + @if (!isStarted) { + play_circle + } @else { + pause_circle + } + } diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts index 3d1dec7..2338119 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts @@ -1,14 +1,15 @@ import { Component, Input, OnInit } from '@angular/core'; +import { MatIconModule } from '@angular/material/icon'; +import { delay } from 'rxjs'; +import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant'; +import { NgClass } from '@angular/common'; +import { getMessageText } from '../../controls/message-content/message-content.helpers'; import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; import { ChatMessage, SpeechVoice, } from '../../../modules/+chat/shared/chat.model'; import { environment } from '../../../../environments/environment'; -import { MatIconModule } from '@angular/material/icon'; -import { delay } from 'rxjs'; -import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant'; -import { NgClass } from '@angular/common'; @Component({ selector: 'ai-chat-audio', @@ -19,10 +20,17 @@ import { NgClass } from '@angular/common'; }) export class ChatAudioComponent implements OnInit { @Input() message!: ChatMessage; - isAudioEnabled = environment.isAudioEnabled; isStarted = false; audio = new Audio(); + get getMessageText(): string { + if (!environment.isAudioEnabled || !this.message) { + return ''; + } + + return getMessageText(this.message); + } + constructor(private readonly chatService: ChatClientService) {} ngOnInit(): void { @@ -42,6 +50,10 @@ export class ChatAudioComponent implements OnInit { } speech(): void { + if (!this.getMessageText) { + return; + } + this.isStarted = true; if (this.audio.src) { @@ -50,7 +62,7 @@ export class ChatAudioComponent implements OnInit { } const payload: PostSpeechDto = { - content: this.message.content, + content: getMessageText(this.message), voice: SpeechVoice.Onyx, }; diff --git a/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html b/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html index d1c6fe1..1bba187 100644 --- a/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html +++ b/apps/spa/src/app/components/chat/chat-footer/chat-footer.component.html @@ -11,6 +11,11 @@ [matTooltip]="!isDisabled ? 'Add files' : ''" [isDisabled]="isDisabled" /> } + @if (isImageContentEnabled) { + + } -} +@if (message) { + @if (message.role === 'assistant') { + + } -
- +
+ @if (messageText) { + + } - @if (message.role !== chatRole.System) { - - } -
+ @if (message.role !== chatRole.System) { + + } + + @if (messageImage.length) { +
+ @for (image of messageImage; track messageImage) { +
File ID: {{ image.image_file.file_id }}
+ } +
+ } +
} diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss index 14478b0..0be069c 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss @@ -46,3 +46,10 @@ max-width: 80%; z-index: 1; } + +.chat-message__file { + border-top: 1px dashed rgba(0, 0, 0, 0.4); + margin-top: $size-2; + padding-top: $size-2; + font-size: 11px; +} diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts index cdb0d9c..1a25913 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts @@ -7,6 +7,11 @@ import { MarkdownComponent } from 'ngx-markdown'; import { ChatAudioComponent } from '../chat-audio/chat-audio.component'; import { NgClass } from '@angular/common'; import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component'; +import { + getMessageImage, + getMessageText, +} from '../../controls/message-content/message-content.helpers'; +import { ImageFileContentBlock } from 'openai/resources/beta/threads'; @Component({ selector: 'ai-chat-message', @@ -25,6 +30,14 @@ export class ChatMessageComponent { @Input() class = ''; chatRole = ChatRole; + get messageText(): string { + return getMessageText(this.message); + } + + get messageImage(): ImageFileContentBlock[] { + return getMessageImage(this.message); + } + @HostBinding('class') get getClasses(): string { return `${this.class} is-${this.message?.role || 'none'}`; } diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html index a27b0da..950882a 100644 --- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html +++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html @@ -1,10 +1,10 @@
@for (message of initialMessages.concat(messages); track message) { - - + + } @empty { - + }
diff --git a/apps/spa/src/app/components/controls/files/files.component.ts b/apps/spa/src/app/components/controls/files/files.component.ts index 41c696d..2166b43 100644 --- a/apps/spa/src/app/components/controls/files/files.component.ts +++ b/apps/spa/src/app/components/controls/files/files.component.ts @@ -1,4 +1,4 @@ -import { Component, Input, ViewChild } from '@angular/core'; +import { Component, ElementRef, Input, ViewChild } from '@angular/core'; import { MatIcon } from '@angular/material/icon'; import { AiFilesDirective } from './files.directive'; import { toSignal } from '@angular/core/rxjs-interop'; @@ -19,7 +19,7 @@ import { ControlIconComponent } from '../control-icon/control-icon.component'; styleUrl: './files.component.scss', }) export class FilesComponent { - @ViewChild('input') input!: HTMLInputElement; + @ViewChild('input') input!: ElementRef; @Input() isDisabled = false; files = toSignal(this.fileService.files$, { initialValue: [] }); @@ -37,7 +37,8 @@ export class FilesComponent { clear(event: Event): void { event.preventDefault(); event.stopPropagation(); - this.input.files = null; + this.input.nativeElement.files = null; + this.input.nativeElement.value = ''; this.fileService.clear(); } } diff --git a/apps/spa/src/app/components/controls/files/files.directive.ts b/apps/spa/src/app/components/controls/files/files.directive.ts index 80bb359..311cfa2 100644 --- a/apps/spa/src/app/components/controls/files/files.directive.ts +++ b/apps/spa/src/app/components/controls/files/files.directive.ts @@ -6,6 +6,7 @@ import { EventEmitter, Input, } from '@angular/core'; +import { MessageContent } from 'openai/src/resources/beta/threads/messages'; @Directive({ standalone: true, @@ -13,7 +14,7 @@ import { }) export class AiFilesDirective { @Output() drop$: EventEmitter = new EventEmitter(); - @Input() files: File[] = []; + @Input() files: Array = []; event = 'init'; @HostBinding('class') get getClasses(): string { diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.html b/apps/spa/src/app/components/controls/message-content/message-content.component.html new file mode 100644 index 0000000..84cd918 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.html @@ -0,0 +1,25 @@ + + image + + @if (imageContentList$().length) { + + + {{ imageContentList$().length }} + + close + + } + + + diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.scss b/apps/spa/src/app/components/controls/message-content/message-content.component.scss new file mode 100644 index 0000000..503412f --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.scss @@ -0,0 +1,37 @@ +.files__input { + display: none; +} + +.files__counter { + display: flex; + justify-content: center; + align-items: center; + background-color: var(--color-red-500); + border-radius: 50%; + min-width: 20px; + min-height: 20px; + text-align: center; + font-size: 10px; + color: var(--color-white); + position: absolute; + right: 2px; + top: 2px; + z-index: 1; + + &:hover { + .files__number { + display: none; + } + + .files__clear { + display: block; + } + } +} + +.files__clear { + display: none; + font-size: 12px; + height: 12px; + width: 12px; +} diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts new file mode 100644 index 0000000..1b8bd43 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { MessageContentComponent } from './message-content.component'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; + +describe('MessageContentComponent', () => { + let component: MessageContentComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [MessageContentComponent, HttpClientTestingModule], + }).compileComponents(); + + fixture = TestBed.createComponent(MessageContentComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.ts new file mode 100644 index 0000000..c1a5a67 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.ts @@ -0,0 +1,46 @@ +import { Component, ElementRef, Input, ViewChild } from '@angular/core'; +import { MatIcon } from '@angular/material/icon'; +import { toSignal } from '@angular/core/rxjs-interop'; +import { MessageContentService } from './message-content.service'; +import { ControlItemComponent } from '../control-item/control-item.component'; +import { ControlIconComponent } from '../control-icon/control-icon.component'; +import { AiFilesDirective } from '../files/files.directive'; + +@Component({ + selector: 'ai-message-content', + standalone: true, + imports: [ + MatIcon, + AiFilesDirective, + ControlItemComponent, + ControlIconComponent, + ], + templateUrl: './message-content.component.html', + styleUrl: './message-content.component.scss', +}) +export class MessageContentComponent { + @ViewChild('input') input!: ElementRef; + @Input() isDisabled = false; + imageContentList$ = toSignal(this.messageContentService.data$, { + initialValue: [], + }); + + constructor(private readonly messageContentService: MessageContentService) {} + + addFiles(files: FileList) { + this.messageContentService.add(files); + } + + onFileChange(event: Event) { + const input = event.target as HTMLInputElement; + this.addFiles(input.files as FileList); + } + + clear(event: Event): void { + event.preventDefault(); + event.stopPropagation(); + this.input.nativeElement.files = null; + this.input.nativeElement.value = ''; + this.messageContentService.clear(); + } +} diff --git a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts new file mode 100644 index 0000000..4505e22 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts @@ -0,0 +1,37 @@ +import { TextContentBlock } from 'openai/resources/beta/threads/messages'; +import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages'; +import { ChatMessage } from '../../../modules/+chat/shared/chat.model'; + +export function isTextContentBlock(item: { + type: string; +}): item is TextContentBlock { + return item.type === 'text'; +} + +export function isImageFileContentBlock(item: { + type: string; +}): item is ImageFileContentBlock { + return item.type === 'image_file'; +} + +export const getMessageText = (message: ChatMessage): string => { + if (typeof message.content === 'string') { + return message.content; + } + + // @TODO: handle all types of message content + return message.content + .filter(isTextContentBlock) + .map(block => block.text.value) + .join(' '); +}; + +export const getMessageImage = ( + message: ChatMessage, +): ImageFileContentBlock[] => { + if (typeof message.content === 'string') { + return []; + } + + return message.content.filter(isImageFileContentBlock); +}; diff --git a/apps/spa/src/app/components/controls/message-content/message-content.service.ts b/apps/spa/src/app/components/controls/message-content/message-content.service.ts new file mode 100644 index 0000000..f943545 --- /dev/null +++ b/apps/spa/src/app/components/controls/message-content/message-content.service.ts @@ -0,0 +1,43 @@ +import { Injectable } from '@angular/core'; +import { BehaviorSubject } from 'rxjs'; +import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; +import { OpenAiFile } from '@boldare/openai-assistant'; + +@Injectable({ providedIn: 'root' }) +export class MessageContentService { + data$ = new BehaviorSubject([]); + + constructor(private readonly chatClientService: ChatClientService) {} + + add(files: FileList) { + const updatedFiles = [ + ...this.data$.value, + ...Object.keys(files).map(key => files[key as unknown as number]), + ]; + this.data$.next(updatedFiles); + } + + delete(index: number): void { + const updatedFiles = this.data$.value.splice(index, 1); + this.data$.next(updatedFiles); + } + + clear(): void { + this.data$.next([]); + } + + async sendFiles(): Promise { + const files = this.data$.value; + + if (!files.length) { + return []; + } + + const uploadedFilesResponse = await this.chatClientService.uploadFiles({ + files, + }); + this.clear(); + + return uploadedFilesResponse.files || []; + } +} diff --git a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts index e25496f..971cec3 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.ts @@ -1,7 +1,7 @@ import { Component } from '@angular/core'; +import { AssistantIframe } from '@boldare/ai-embedded'; import { ChatService } from '../../shared/chat.service'; import { environment } from '../../../../../environments/environment'; -import { AssistantIframe } from '@boldare/ai-embedded'; @Component({ selector: 'ai-chat-home', diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html index 1ddffde..002000b 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.html @@ -23,6 +23,7 @@ [isDisabled]="isResponding()" [isTranscriptionEnabled]="isTranscriptionEnabled" [isAttachmentEnabled]="isAttachmentEnabled" + [isImageContentEnabled]="isImageContentEnabled" (sendMessage$)="chatService.sendMessage($event)" (sendAudio$)="chatService.sendAudio($event)" /> } diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss index 1057801..65ec27b 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.scss @@ -3,12 +3,13 @@ height: 100vh; max-height: 600px; background-color: var(--color-white); - border-radius: var(--border-radius-medium); + border-radius: 0 0 var(--border-radius-medium) var(--border-radius-medium); &.chat-home__iframe .chat__content, &.chat-home__iframe ::ng-deep .messages { min-height: 280px; - height: calc(100vh - 350px); + height: calc(100vh - 300px); + max-height: 500px; overflow: auto; } } diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts index be36b2f..cf95793 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.ts @@ -38,14 +38,14 @@ export class ChatIframeComponent implements OnInit { isAttachmentEnabled = environment.isAttachmentEnabled; isRefreshEnabled = environment.isRefreshEnabled; isConfigEnabled = environment.isConfigEnabled; - initialMessages: ChatMessage[] = []; + isImageContentEnabled = environment.isImageContentEnabled; tips = [ 'Hello! 👋 How can you help me?', 'What’s the weather like in Warsaw?', 'What is the exchange rate for USD?', - 'Show me list of Pokémon', 'Show me the stats for Pikachu (Pokémon)?', ]; + initialMessages: ChatMessage[] = []; constructor( private readonly threadService: ThreadService, diff --git a/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts b/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts index ae0800a..c53cdae 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat/chat.component.ts @@ -5,8 +5,8 @@ import { RouterModule, RouterOutlet, } from '@angular/router'; -import { ChatIframeComponent } from '../chat-iframe/chat-iframe.component'; import { ChatIframeWrapperComponent } from '../../../../components/chat/chat-iframe-wrapper/chat-iframe-wrapper.component'; +import { ChatIframeComponent } from '../chat-iframe/chat-iframe.component'; @Component({ selector: 'ai-chat', diff --git a/apps/spa/src/app/modules/+chat/shared/chat.model.ts b/apps/spa/src/app/modules/+chat/shared/chat.model.ts index 5029725..6ed369f 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.model.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.model.ts @@ -1,3 +1,5 @@ +import { MessageContent } from 'openai/resources/beta/threads'; + export interface AudioResponse { content: string; } @@ -10,7 +12,7 @@ export enum ChatRole { export interface ChatMessage { metadata?: Record; - content: string; + content: string | Array; role: ChatRole; } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts index f18306e..67a3d5a 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts @@ -9,17 +9,20 @@ import { take, tap, } from 'rxjs'; +import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model'; import { ChatGatewayService } from './chat-gateway.service'; import { ChatClientService } from './chat-client.service'; import { ThreadService } from './thread.service'; import { ChatFilesService } from './chat-files.service'; import { environment } from '../../../../environments/environment'; -import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; import { + ImageFileContentBlock, Message, - TextContentBlock, + MessageContent, + Text, } from 'openai/resources/beta/threads/messages'; +import { MessageContentService } from '../../../components/controls/message-content/message-content.service'; @Injectable({ providedIn: 'root' }) export class ChatService { @@ -34,6 +37,7 @@ export class ChatService { private readonly chatClientService: ChatClientService, private readonly threadService: ThreadService, private readonly chatFilesService: ChatFilesService, + private readonly messageContentService: MessageContentService, ) { document.body.classList.add('ai-chat'); @@ -61,7 +65,7 @@ export class ChatService { return message.content?.[0]?.type === 'text'; } - parseMessages(thread: GetThreadResponseDto): ChatMessage[] { + parseMessages(thread: GetThreadResponseDto): Message[] { if (!thread.messages) { return []; } @@ -71,11 +75,7 @@ export class ChatService { .filter( message => this.isTextMessage(message) && !this.isMessageInvisible(message), - ) - .map(message => ({ - content: (message.content[0] as TextContentBlock).text.value, - role: message.role as ChatRole, - })); + ); } setInitialValues(): void { @@ -85,7 +85,10 @@ export class ChatService { filter(threadId => !!threadId), tap(() => this.isLoading$.next(true)), mergeMap(threadId => this.threadService.getThread(threadId)), - map((response: GetThreadResponseDto) => this.parseMessages(response)), + map( + (response: GetThreadResponseDto) => + this.parseMessages(response) as ChatMessage[], + ), ) .subscribe(data => { this.messages$.next(data); @@ -142,7 +145,7 @@ export class ChatService { this.addFileMessage(files); this.chatGatewayService.callStart({ - content, + content: await this.getMessageContent(content), threadId: this.threadService.threadId$.value, attachments: files.map( file => @@ -154,6 +157,45 @@ export class ChatService { }); } + async getMessageContent(content: string): Promise { + const images = (await this.messageContentService.sendFiles()) || []; + const imageFileContentList = + images?.map( + file => + ({ + type: 'image_file', + image_file: { + file_id: file.id, + }, + }) as ImageFileContentBlock, + ) || []; + + this.messages$.next([ + ...this.messages$.value.slice(0, -1), + { + content: [ + { + type: 'text', + text: { + value: content, + annotations: [], + }, + }, + ...imageFileContentList, + ], + role: ChatRole.User, + }, + ]); + + return [ + { + type: 'text', + text: content as unknown as Text, + }, + ...imageFileContentList, + ]; + } + watchTextCreated(): Subscription { return this.chatGatewayService.textCreated().subscribe(data => { this.isTyping$.next(false); diff --git a/apps/spa/src/environments/environment.development.ts b/apps/spa/src/environments/environment.development.ts index 408b7b8..80faaea 100644 --- a/apps/spa/src/environments/environment.development.ts +++ b/apps/spa/src/environments/environment.development.ts @@ -13,4 +13,5 @@ export const environment = { isConfigEnabled: false, isAutoOpen: true, isStreamingEnabled: true, + isImageContentEnabled: true, }; diff --git a/apps/spa/src/environments/environment.ts b/apps/spa/src/environments/environment.ts index 301110c..00948a5 100644 --- a/apps/spa/src/environments/environment.ts +++ b/apps/spa/src/environments/environment.ts @@ -13,4 +13,5 @@ export const environment = { isConfigEnabled: false, isAutoOpen: true, isStreamingEnabled: true, + isImageContentEnabled: true, }; diff --git a/apps/spa/src/styles/_extends/_material.scss b/apps/spa/src/styles/_extends/_material.scss index 8b3215f..bd5de81 100644 --- a/apps/spa/src/styles/_extends/_material.scss +++ b/apps/spa/src/styles/_extends/_material.scss @@ -22,6 +22,10 @@ ); } + .mat-mdc-form-field-infix { + width: 100%; + } + mat-form-field { width: 100%; font-size: 14px; diff --git a/libs/ai-embedded/src/environments/environment.prod.ts b/libs/ai-embedded/src/environments/environment.prod.ts index 0cfe45b..7efd79e 100644 --- a/libs/ai-embedded/src/environments/environment.prod.ts +++ b/libs/ai-embedded/src/environments/environment.prod.ts @@ -1,4 +1,4 @@ export const environment = { env: 'prod', - appUrl: 'https://assistant.ai.boldare.dev', + appUrl: '', }; diff --git a/libs/openai-assistant/src/lib/chat/chat.model.ts b/libs/openai-assistant/src/lib/chat/chat.model.ts index b028f5b..d38c1cc 100644 --- a/libs/openai-assistant/src/lib/chat/chat.model.ts +++ b/libs/openai-assistant/src/lib/chat/chat.model.ts @@ -2,6 +2,7 @@ import { ApiProperty } from '@nestjs/swagger'; import { ImageFile, Message, + MessageContent, MessageCreateParams, MessageDelta, Text, @@ -58,7 +59,7 @@ export class ChatCallDto { threadId!: string; @ApiProperty() - content!: string; + content!: string | Array; @ApiProperty({ required: false }) assistantId?: string; diff --git a/libs/openai-assistant/src/lib/chat/chat.service.ts b/libs/openai-assistant/src/lib/chat/chat.service.ts index f82a3a9..36e76bb 100644 --- a/libs/openai-assistant/src/lib/chat/chat.service.ts +++ b/libs/openai-assistant/src/lib/chat/chat.service.ts @@ -7,7 +7,11 @@ import { ChatCallResponseDto, } from './chat.model'; import { ChatHelpers } from './chat.helpers'; -import { Message, MessageCreateParams } from 'openai/resources/beta/threads'; +import { + Message, + MessageContentPartParam, + MessageCreateParams, +} from 'openai/resources/beta/threads'; import { AssistantStream } from 'openai/lib/AssistantStream'; import { assistantStreamEventHandler } from '../stream/stream.utils'; @@ -44,7 +48,7 @@ export class ChatService { const { threadId, content, attachments, metadata } = payload; const message: MessageCreateParams = { role: 'user', - content, + content: content as Array, attachments, metadata, }; From 7bc9544a455b557fae2ae65752c43c0da5d22dd3 Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Tue, 11 Jun 2024 11:07:18 +0200 Subject: [PATCH 2/5] feat(assistant): displaying annotations --- apps/spa/src/app/app.config.ts | 2 + .../chat/chat-audio/chat-audio.component.html | 2 +- .../chat/chat-audio/chat-audio.component.ts | 27 +++++++------ .../chat-message/chat-message.component.html | 8 ++-- .../chat-message/chat-message.component.ts | 18 +++------ .../message-content.component.ts | 2 +- .../message-content.helpers.ts | 23 ----------- .../+chat/shared/chat-gateway.service.ts | 3 +- .../app/modules/+chat/shared/chat.service.ts | 22 ++++++----- apps/spa/src/app/pipes/annotation.pipe.ts | 29 ++++++++++++++ apps/spa/src/app/pipes/message-file.pipe.ts | 18 +++++++++ apps/spa/src/app/pipes/message-text.pipe.ts | 21 ++++++++++ .../src/lib/annotations/annotations.model.ts | 13 +++++++ .../src/lib/annotations/annotations.utils.ts | 39 +++++++++++++++++++ .../src/lib/chat/chat.gateway.ts | 19 ++++++++- .../src/lib/chat/chat.model.ts | 6 +++ 16 files changed, 183 insertions(+), 69 deletions(-) create mode 100644 apps/spa/src/app/pipes/annotation.pipe.ts create mode 100644 apps/spa/src/app/pipes/message-file.pipe.ts create mode 100644 apps/spa/src/app/pipes/message-text.pipe.ts create mode 100644 libs/openai-assistant/src/lib/annotations/annotations.model.ts create mode 100644 libs/openai-assistant/src/lib/annotations/annotations.utils.ts diff --git a/apps/spa/src/app/app.config.ts b/apps/spa/src/app/app.config.ts index ec061dd..d4831a4 100644 --- a/apps/spa/src/app/app.config.ts +++ b/apps/spa/src/app/app.config.ts @@ -5,6 +5,7 @@ import { routes } from './app.routes'; import { provideAnimations } from '@angular/platform-browser/animations'; import { provideHttpClient } from '@angular/common/http'; import { provideMarkdown } from 'ngx-markdown'; +import { AnnotationPipe } from './pipes/annotation.pipe'; export const appConfig: ApplicationConfig = { providers: [ @@ -12,5 +13,6 @@ export const appConfig: ApplicationConfig = { provideAnimations(), provideHttpClient(), provideMarkdown(), + AnnotationPipe, ], }; diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html index fb97189..f7a0f1d 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html @@ -1,4 +1,4 @@ -@if (getMessageText) { +@if (isAudioEnabled && message && (message | messageText)) { @if (!isStarted) { play_circle diff --git a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts index 2338119..cd8e9e4 100644 --- a/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts +++ b/apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts @@ -2,19 +2,20 @@ import { Component, Input, OnInit } from '@angular/core'; import { MatIconModule } from '@angular/material/icon'; import { delay } from 'rxjs'; import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant'; -import { NgClass } from '@angular/common'; -import { getMessageText } from '../../controls/message-content/message-content.helpers'; +import { CommonModule } from '@angular/common'; import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; import { ChatMessage, SpeechVoice, } from '../../../modules/+chat/shared/chat.model'; +import { MessageTextPipe } from '../../../pipes/message-text.pipe'; import { environment } from '../../../../environments/environment'; @Component({ selector: 'ai-chat-audio', standalone: true, - imports: [MatIconModule, NgClass], + imports: [MatIconModule, CommonModule, MessageTextPipe], + providers: [MessageTextPipe], templateUrl: './chat-audio.component.html', styleUrl: './chat-audio.component.scss', }) @@ -22,16 +23,12 @@ export class ChatAudioComponent implements OnInit { @Input() message!: ChatMessage; isStarted = false; audio = new Audio(); + isAudioEnabled = environment.isAudioEnabled; - get getMessageText(): string { - if (!environment.isAudioEnabled || !this.message) { - return ''; - } - - return getMessageText(this.message); - } - - constructor(private readonly chatService: ChatClientService) {} + constructor( + private readonly chatService: ChatClientService, + private readonly messageTextPipe: MessageTextPipe, + ) {} ngOnInit(): void { this.audio.onended = this.onEnded.bind(this); @@ -50,7 +47,9 @@ export class ChatAudioComponent implements OnInit { } speech(): void { - if (!this.getMessageText) { + const content = this.messageTextPipe.transform(this.message); + + if (!content) { return; } @@ -62,7 +61,7 @@ export class ChatAudioComponent implements OnInit { } const payload: PostSpeechDto = { - content: getMessageText(this.message), + content, voice: SpeechVoice.Onyx, }; diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html index 1ce0d42..ce11feb 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html @@ -4,17 +4,17 @@ }
- @if (messageText) { - + @if (message | messageText) { + } @if (message.role !== chatRole.System) { } - @if (messageImage.length) { + @if ((message | messageImageFile).length) {
- @for (image of messageImage; track messageImage) { + @for (image of message | messageImageFile; track image.image_file.file_id) {
File ID: {{ image.image_file.file_id }}
}
diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts index 1a25913..3193aa4 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts @@ -7,11 +7,9 @@ import { MarkdownComponent } from 'ngx-markdown'; import { ChatAudioComponent } from '../chat-audio/chat-audio.component'; import { NgClass } from '@angular/common'; import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component'; -import { - getMessageImage, - getMessageText, -} from '../../controls/message-content/message-content.helpers'; -import { ImageFileContentBlock } from 'openai/resources/beta/threads'; +import { MessageTextPipe } from '../../../pipes/message-text.pipe'; +import { MessageImageFilePipe } from '../../../pipes/message-file.pipe'; +import { AnnotationPipe } from '../../../pipes/annotation.pipe'; @Component({ selector: 'ai-chat-message', @@ -23,6 +21,8 @@ import { ImageFileContentBlock } from 'openai/resources/beta/threads'; MarkdownComponent, ChatAudioComponent, ChatAvatarComponent, + MessageTextPipe, + MessageImageFilePipe, ], }) export class ChatMessageComponent { @@ -30,14 +30,6 @@ export class ChatMessageComponent { @Input() class = ''; chatRole = ChatRole; - get messageText(): string { - return getMessageText(this.message); - } - - get messageImage(): ImageFileContentBlock[] { - return getMessageImage(this.message); - } - @HostBinding('class') get getClasses(): string { return `${this.class} is-${this.message?.role || 'none'}`; } diff --git a/apps/spa/src/app/components/controls/message-content/message-content.component.ts b/apps/spa/src/app/components/controls/message-content/message-content.component.ts index c1a5a67..8d068e1 100644 --- a/apps/spa/src/app/components/controls/message-content/message-content.component.ts +++ b/apps/spa/src/app/components/controls/message-content/message-content.component.ts @@ -1,10 +1,10 @@ import { Component, ElementRef, Input, ViewChild } from '@angular/core'; import { MatIcon } from '@angular/material/icon'; import { toSignal } from '@angular/core/rxjs-interop'; -import { MessageContentService } from './message-content.service'; import { ControlItemComponent } from '../control-item/control-item.component'; import { ControlIconComponent } from '../control-icon/control-icon.component'; import { AiFilesDirective } from '../files/files.directive'; +import { MessageContentService } from './message-content.service'; @Component({ selector: 'ai-message-content', diff --git a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts index 4505e22..691f1c7 100644 --- a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts +++ b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts @@ -1,6 +1,5 @@ import { TextContentBlock } from 'openai/resources/beta/threads/messages'; import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages'; -import { ChatMessage } from '../../../modules/+chat/shared/chat.model'; export function isTextContentBlock(item: { type: string; @@ -13,25 +12,3 @@ export function isImageFileContentBlock(item: { }): item is ImageFileContentBlock { return item.type === 'image_file'; } - -export const getMessageText = (message: ChatMessage): string => { - if (typeof message.content === 'string') { - return message.content; - } - - // @TODO: handle all types of message content - return message.content - .filter(isTextContentBlock) - .map(block => block.text.value) - .join(' '); -}; - -export const getMessageImage = ( - message: ChatMessage, -): ImageFileContentBlock[] => { - if (typeof message.content === 'string') { - return []; - } - - return message.content.filter(isImageFileContentBlock); -}; diff --git a/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts b/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts index e6b3728..b9569af 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts @@ -3,6 +3,7 @@ import { ChatEvents } from './chat.model'; import io from 'socket.io-client'; import { ChatCallDto, + MessageWithAnnotations, TextCreatedPayload, TextDeltaPayload, TextDonePayload, @@ -37,7 +38,7 @@ export class ChatGatewayService { return this.watchEvent(ChatEvents.TextDelta); } - textDone(): Observable { + textDone(): Observable> { return this.watchEvent(ChatEvents.TextDone); } } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts index 67a3d5a..d6ad814 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@angular/core'; +import { Injectable, Injector } from '@angular/core'; import { BehaviorSubject, distinctUntilChanged, @@ -9,20 +9,21 @@ import { take, tap, } from 'rxjs'; -import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; -import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model'; -import { ChatGatewayService } from './chat-gateway.service'; -import { ChatClientService } from './chat-client.service'; -import { ThreadService } from './thread.service'; -import { ChatFilesService } from './chat-files.service'; -import { environment } from '../../../../environments/environment'; import { ImageFileContentBlock, Message, MessageContent, Text, } from 'openai/resources/beta/threads/messages'; +import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; +import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model'; +import { ChatGatewayService } from './chat-gateway.service'; +import { ChatClientService } from './chat-client.service'; +import { ThreadService } from './thread.service'; +import { ChatFilesService } from './chat-files.service'; import { MessageContentService } from '../../../components/controls/message-content/message-content.service'; +import { environment } from '../../../../environments/environment'; +import { AnnotationPipe } from '../../../pipes/annotation.pipe'; @Injectable({ providedIn: 'root' }) export class ChatService { @@ -38,6 +39,7 @@ export class ChatService { private readonly threadService: ThreadService, private readonly chatFilesService: ChatFilesService, private readonly messageContentService: MessageContentService, + private readonly annotationPipe: AnnotationPipe, ) { document.body.classList.add('ai-chat'); @@ -213,13 +215,13 @@ export class ChatService { } watchTextDone(): Subscription { - return this.chatGatewayService.textDone().subscribe(data => { + return this.chatGatewayService.textDone().subscribe(event => { this.isTyping$.next(false); this.isResponding$.next(false); this.messages$.next([ ...this.messages$.value.slice(0, -1), { - content: data.text.value, + content: this.annotationPipe.transform(event), role: ChatRole.Assistant, }, ]); diff --git a/apps/spa/src/app/pipes/annotation.pipe.ts b/apps/spa/src/app/pipes/annotation.pipe.ts new file mode 100644 index 0000000..ef35074 --- /dev/null +++ b/apps/spa/src/app/pipes/annotation.pipe.ts @@ -0,0 +1,29 @@ +import { Pipe, PipeTransform } from '@angular/core'; +import { MessageWithAnnotations } from '@boldare/openai-assistant'; +import { Message } from 'openai/resources/beta/threads'; +import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers'; + +@Pipe({ + standalone: true, + name: 'annotation' +}) +export class AnnotationPipe implements PipeTransform { + transform(event: MessageWithAnnotations<{ message: Message }>): string { + if (!isTextContentBlock(event.data.message.content[0])) { + return ''; + } + + const { text } = event.data.message.content[0]; + + if (!event.annotations?.length) { + return text.value; + } + + for (const item of event.annotations) { + const { index, annotation } = item; + text.value = text.value.replace(annotation.text, `[${index}]`); + } + + return text.value; + } +} \ No newline at end of file diff --git a/apps/spa/src/app/pipes/message-file.pipe.ts b/apps/spa/src/app/pipes/message-file.pipe.ts new file mode 100644 index 0000000..e4abf2b --- /dev/null +++ b/apps/spa/src/app/pipes/message-file.pipe.ts @@ -0,0 +1,18 @@ +import { Pipe, PipeTransform } from '@angular/core'; +import { isImageFileContentBlock } from '../components/controls/message-content/message-content.helpers'; +import { ChatMessage } from '../modules/+chat/shared/chat.model'; +import { ImageFileContentBlock } from 'openai/resources/beta/threads'; + +@Pipe({ + standalone: true, + name: 'messageImageFile' +}) +export class MessageImageFilePipe implements PipeTransform { + transform(message: ChatMessage): ImageFileContentBlock[] { + if (typeof message.content === 'string') { + return []; + } + + return message.content.filter(isImageFileContentBlock); + } +} \ No newline at end of file diff --git a/apps/spa/src/app/pipes/message-text.pipe.ts b/apps/spa/src/app/pipes/message-text.pipe.ts new file mode 100644 index 0000000..de7df80 --- /dev/null +++ b/apps/spa/src/app/pipes/message-text.pipe.ts @@ -0,0 +1,21 @@ +import { Pipe, PipeTransform } from '@angular/core'; +import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers'; +import { ChatMessage } from '../modules/+chat/shared/chat.model'; + +@Pipe({ + standalone: true, + name: 'messageText' +}) +export class MessageTextPipe implements PipeTransform { + transform(message: ChatMessage): string { + if (typeof message.content === 'string') { + return message.content; + } + + // @TODO: handle all types of message content + return message.content + .filter(isTextContentBlock) + .map(block => block.text.value) + .join(' '); + } +} \ No newline at end of file diff --git a/libs/openai-assistant/src/lib/annotations/annotations.model.ts b/libs/openai-assistant/src/lib/annotations/annotations.model.ts new file mode 100644 index 0000000..fc5dae9 --- /dev/null +++ b/libs/openai-assistant/src/lib/annotations/annotations.model.ts @@ -0,0 +1,13 @@ +import { FileObject } from "openai/resources"; +import { Annotation } from "openai/resources/beta/threads/messages"; + +export interface AnnotationData { + annotation: Annotation; + index: number; + file: FileObject; +} + +export enum AnnotationType { + file_citation = 'file_citation', + file_path = 'file_path', +} diff --git a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts new file mode 100644 index 0000000..3315653 --- /dev/null +++ b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts @@ -0,0 +1,39 @@ +import OpenAI from "openai"; +import { FileCitationAnnotation, FilePathAnnotation, Message } from "openai/resources/beta/threads"; +import { AnnotationData, AnnotationType } from "./annotations.model"; + +export const isFileCitation = (item: { type: string }): item is FileCitationAnnotation => item.type === 'file_citation'; +export const isFilePath = (item: { type: string }): item is FilePathAnnotation => item.type === 'file_path'; + +export const getAnnotations = async (message: Message, provider: OpenAI): Promise => { + if (message.content[0].type !== 'text') { + return []; + } + + const { text } = message.content[0]; + const { annotations } = text; + const annotationsData: AnnotationData[] = []; + + let index = 1; + + for (const annotation of annotations) { + let data = null; + + if (isFileCitation(annotation)) { + data = annotation[AnnotationType.file_citation]; + } + + if (isFilePath(annotation)) { + data = annotation[AnnotationType.file_path]; + } + + if (data) { + const file = await provider.files.retrieve(data.file_id); + annotationsData.push({ annotation, index, file }); + } + + index++; + } + + return annotationsData; +} diff --git a/libs/openai-assistant/src/lib/chat/chat.gateway.ts b/libs/openai-assistant/src/lib/chat/chat.gateway.ts index cb7bc96..f3666e5 100644 --- a/libs/openai-assistant/src/lib/chat/chat.gateway.ts +++ b/libs/openai-assistant/src/lib/chat/chat.gateway.ts @@ -24,12 +24,14 @@ import { RunStepCreatedPayload, RunStepDeltaPayload, RunStepDonePayload, + MessageWithAnnotations, } from './chat.model'; import { ChatService } from './chat.service'; import { CodeInterpreterToolCallDelta, FunctionToolCallDelta, } from 'openai/resources/beta/threads/runs'; +import { getAnnotations } from '../annotations/annotations.utils'; export class ChatGateway implements OnGatewayConnection { @WebSocketServer() server!: Server; @@ -132,10 +134,17 @@ export class ChatGateway implements OnGatewayConnection { socketId: string, @MessageBody() data: MessageDonePayload, ) { - this.server.to(socketId).emit(ChatEvents.MessageDone, data); + const annotations = await getAnnotations(data.message, this.chatsService.provider); + const messageWithAnnotations: MessageWithAnnotations = { + data, + annotations, + }; + + this.server.to(socketId).emit(ChatEvents.MessageDone, messageWithAnnotations); this.log( `Socket "${ChatEvents.MessageDone}" | threadId: ${data.message.thread_id}`, ); + } async emitTextCreated( @@ -152,7 +161,13 @@ export class ChatGateway implements OnGatewayConnection { } async emitTextDone(socketId: string, @MessageBody() data: TextDonePayload) { - this.server.to(socketId).emit(ChatEvents.TextDone, data); + const annotations = await getAnnotations(data.message, this.chatsService.provider); + const messageWithAnnotations: MessageWithAnnotations = { + data, + annotations, + }; + + this.server.to(socketId).emit(ChatEvents.TextDone, messageWithAnnotations); this.log( `Socket "${ChatEvents.TextDone}" | threadId: ${data.message?.thread_id} | ${data.text?.value}`, ); diff --git a/libs/openai-assistant/src/lib/chat/chat.model.ts b/libs/openai-assistant/src/lib/chat/chat.model.ts index d38c1cc..f46a2d5 100644 --- a/libs/openai-assistant/src/lib/chat/chat.model.ts +++ b/libs/openai-assistant/src/lib/chat/chat.model.ts @@ -14,6 +14,7 @@ import { ToolCallDelta, } from 'openai/resources/beta/threads/runs'; import { RunStep } from 'openai/resources/beta/threads/runs/steps'; +import { AnnotationData } from '../annotations/annotations.model'; export interface ChatAudio { file: File; @@ -146,3 +147,8 @@ export interface ChatCallCallbacks { [ChatEvents.RunStepDelta]?: (data: RunStepDeltaPayload) => Promise; [ChatEvents.RunStepDone]?: (data: RunStepDonePayload) => Promise; } + +export interface MessageWithAnnotations { + data: T; + annotations: AnnotationData[]; +} \ No newline at end of file From cc8f0d047c8faf2557835e754d013e003761fcbe Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Tue, 11 Jun 2024 13:07:13 +0200 Subject: [PATCH 3/5] feat(assistant): type improvements for messages --- .../chat-message/chat-message.component.html | 9 +- .../chat-message/chat-message.component.ts | 3 +- .../chat-messages/chat-messages.component.ts | 4 +- .../message-content.helpers.ts | 12 +-- .../app/modules/+chat/shared/chat.helpers.ts | 43 +++++++++ .../app/modules/+chat/shared/chat.model.ts | 9 +- .../app/modules/+chat/shared/chat.service.ts | 89 ++++++++----------- apps/spa/src/app/pipes/annotation.pipe.ts | 6 +- apps/spa/src/app/pipes/message-file.pipe.ts | 10 +-- apps/spa/src/app/pipes/message-text.pipe.ts | 19 ++-- .../environments/environment.development.ts | 2 +- libs/openai-assistant/src/index.ts | 1 + .../src/lib/annotations/annotations.model.ts | 4 +- .../src/lib/annotations/annotations.utils.ts | 31 ++++--- .../src/lib/annotations/index.ts | 2 + .../src/lib/chat/chat.controller.spec.ts | 10 ++- .../src/lib/chat/chat.gateway.spec.ts | 23 ++++- .../src/lib/chat/chat.gateway.ts | 15 +++- .../src/lib/chat/chat.helpers.spec.ts | 34 +++---- .../src/lib/chat/chat.helpers.ts | 15 ++-- .../src/lib/chat/chat.model.ts | 6 +- .../src/lib/chat/chat.service.spec.ts | 55 ++++++++++-- package.json | 2 + 23 files changed, 254 insertions(+), 150 deletions(-) create mode 100644 apps/spa/src/app/modules/+chat/shared/chat.helpers.ts create mode 100644 libs/openai-assistant/src/lib/annotations/index.ts diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html index ce11feb..f6ca99e 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html @@ -8,13 +8,12 @@ } - @if (message.role !== chatRole.System) { - - } - @if ((message | messageImageFile).length) {
- @for (image of message | messageImageFile; track image.image_file.file_id) { + @for ( + image of message | messageImageFile; + track image.image_file.file_id + ) {
File ID: {{ image.image_file.file_id }}
}
diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts index 3193aa4..1f86e9e 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts @@ -9,7 +9,6 @@ import { NgClass } from '@angular/common'; import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component'; import { MessageTextPipe } from '../../../pipes/message-text.pipe'; import { MessageImageFilePipe } from '../../../pipes/message-file.pipe'; -import { AnnotationPipe } from '../../../pipes/annotation.pipe'; @Component({ selector: 'ai-chat-message', @@ -26,7 +25,7 @@ import { AnnotationPipe } from '../../../pipes/annotation.pipe'; ], }) export class ChatMessageComponent { - @Input() message!: ChatMessage; + @Input() message!: Partial; @Input() class = ''; chatRole = ChatRole; diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts index 0345aff..8096c3a 100644 --- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts +++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.ts @@ -29,8 +29,8 @@ import { ChatTipsComponent } from '../chat-tips/chat-tips.component'; ], }) export class ChatMessagesComponent implements AfterViewInit, OnChanges { - @Input() initialMessages: ChatMessage[] = []; - @Input() messages: ChatMessage[] = []; + @Input() initialMessages: Partial[] = []; + @Input() messages: Partial[] = []; @Input() isTyping = false; @Input() tips: string[] = []; @Output() tipSelected$ = new EventEmitter(); diff --git a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts index 691f1c7..a559979 100644 --- a/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts +++ b/apps/spa/src/app/components/controls/message-content/message-content.helpers.ts @@ -1,14 +1,14 @@ import { TextContentBlock } from 'openai/resources/beta/threads/messages'; import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages'; -export function isTextContentBlock(item: { - type: string; +export function isTextContentBlock(item?: { + type?: string; }): item is TextContentBlock { - return item.type === 'text'; + return item?.type === 'text'; } -export function isImageFileContentBlock(item: { - type: string; +export function isImageFileContentBlock(item?: { + type?: string; }): item is ImageFileContentBlock { - return item.type === 'image_file'; + return item?.type === 'image_file'; } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts b/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts new file mode 100644 index 0000000..f56fcb5 --- /dev/null +++ b/apps/spa/src/app/modules/+chat/shared/chat.helpers.ts @@ -0,0 +1,43 @@ +import { + ImageFileContentBlock, + MessageContent, + MessageCreateParams, +} from 'openai/resources/beta/threads'; +import { TextContentBlock } from 'openai/src/resources/beta/threads/messages'; +import { ChatMessage, ChatRole } from './chat.model'; +import { CodeInterpreterTool, FileSearchTool } from 'openai/resources/beta'; + +export const textContentBlock = (content: string): TextContentBlock => ({ + type: 'text', + text: { + value: content, + annotations: [], + }, +}); + +export const imageFileContentBlock = ( + fileId: string, +): ImageFileContentBlock => ({ + type: 'image_file', + image_file: { + file_id: fileId, + }, +}); + +export const messageAttachment = ( + fileId: string, + tools: Array = [ + { type: 'code_interpreter' }, + ], +): MessageCreateParams.Attachment => ({ + file_id: fileId, + tools, +}); + +export const messageContentBlock = ( + content: MessageContent[], + role: ChatRole, +): Partial => ({ + content, + role, +}); diff --git a/apps/spa/src/app/modules/+chat/shared/chat.model.ts b/apps/spa/src/app/modules/+chat/shared/chat.model.ts index 6ed369f..c6ff091 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.model.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.model.ts @@ -1,4 +1,5 @@ -import { MessageContent } from 'openai/resources/beta/threads'; +import { AnnotationData } from '@boldare/openai-assistant'; +import { Message } from 'openai/resources/beta/threads'; export interface AudioResponse { content: string; @@ -7,12 +8,10 @@ export interface AudioResponse { export enum ChatRole { User = 'user', Assistant = 'assistant', - System = 'system', } -export interface ChatMessage { - metadata?: Record; - content: string | Array; +export interface ChatMessage extends Message { + annotations?: AnnotationData[]; role: ChatRole; } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts index d6ad814..e1c3567 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Injector } from '@angular/core'; +import { Injectable } from '@angular/core'; import { BehaviorSubject, distinctUntilChanged, @@ -10,10 +10,10 @@ import { tap, } from 'rxjs'; import { - ImageFileContentBlock, Message, MessageContent, Text, + TextContentBlock, } from 'openai/resources/beta/threads/messages'; import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant'; import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model'; @@ -24,6 +24,13 @@ import { ChatFilesService } from './chat-files.service'; import { MessageContentService } from '../../../components/controls/message-content/message-content.service'; import { environment } from '../../../../environments/environment'; import { AnnotationPipe } from '../../../pipes/annotation.pipe'; +import { + imageFileContentBlock, + messageAttachment, + messageContentBlock, + textContentBlock, +} from './chat.helpers'; +import { isTextContentBlock } from '../../../components/controls/message-content/message-content.helpers'; @Injectable({ providedIn: 'root' }) export class ChatService { @@ -31,7 +38,7 @@ export class ChatService { isVisible$ = new BehaviorSubject(environment.isAutoOpen); isTyping$ = new BehaviorSubject(false); isResponding$ = new BehaviorSubject(false); - messages$ = new BehaviorSubject([]); + messages$ = new BehaviorSubject[]>([]); constructor( private readonly chatGatewayService: ChatGatewayService, @@ -121,7 +128,7 @@ export class ChatService { window?.top?.postMessage('changeView', '*'); } - addMessage(message: ChatMessage): void { + addMessage(message: Partial): void { this.messages$.next([...this.messages$.value, message]); } @@ -129,19 +136,14 @@ export class ChatService { if (!files?.length) { return; } - - this.addMessage({ - content: `The user has attached files to the message: ${files - .map(file => file.filename) - .join(', ')}`, - role: ChatRole.System, - }); } async sendMessage(content: string, role = ChatRole.User): Promise { this.isTyping$.next(true); this.isResponding$.next(true); - this.addMessage({ content, role }); + + const message = messageContentBlock([textContentBlock(content)], role); + this.addMessage(message); const files = await this.chatFilesService.sendFiles(); this.addFileMessage(files); @@ -149,44 +151,18 @@ export class ChatService { this.chatGatewayService.callStart({ content: await this.getMessageContent(content), threadId: this.threadService.threadId$.value, - attachments: files.map( - file => - ({ - file_id: file.id, - tools: [{ type: 'code_interpreter' }], - }) || [], - ), + attachments: files.map(file => messageAttachment(file.id) || []), }); } async getMessageContent(content: string): Promise { const images = (await this.messageContentService.sendFiles()) || []; const imageFileContentList = - images?.map( - file => - ({ - type: 'image_file', - image_file: { - file_id: file.id, - }, - }) as ImageFileContentBlock, - ) || []; + images?.map(file => imageFileContentBlock(file.id)) || []; this.messages$.next([ ...this.messages$.value.slice(0, -1), - { - content: [ - { - type: 'text', - text: { - value: content, - annotations: [], - }, - }, - ...imageFileContentList, - ], - role: ChatRole.User, - }, + messageContentBlock([textContentBlock(content)], ChatRole.User), ]); return [ @@ -202,7 +178,12 @@ export class ChatService { return this.chatGatewayService.textCreated().subscribe(data => { this.isTyping$.next(false); this.isResponding$.next(true); - this.addMessage({ content: data.text.value, role: ChatRole.Assistant }); + + const message = messageContentBlock( + [textContentBlock(data.text.value)], + ChatRole.Assistant, + ); + this.addMessage(message); }); } @@ -210,7 +191,14 @@ export class ChatService { return this.chatGatewayService.textDelta().subscribe(data => { const length = this.messages$.value.length; this.isResponding$.next(true); - this.messages$.value[length - 1].content = data.text.value; + + const lastMessageContent = this.messages$.value[length - 1]?.content?.[0]; + + if (isTextContentBlock(lastMessageContent)) { + ( + this.messages$.value[length - 1].content?.[0] as TextContentBlock + ).text = data.text; + } }); } @@ -218,13 +206,14 @@ export class ChatService { return this.chatGatewayService.textDone().subscribe(event => { this.isTyping$.next(false); this.isResponding$.next(false); - this.messages$.next([ - ...this.messages$.value.slice(0, -1), - { - content: this.annotationPipe.transform(event), - role: ChatRole.Assistant, - }, - ]); + + const annotationContent = this.annotationPipe.transform(event); + const message = messageContentBlock( + [textContentBlock(annotationContent)], + ChatRole.Assistant, + ); + + this.messages$.next([...this.messages$.value.slice(0, -1), message]); }); } diff --git a/apps/spa/src/app/pipes/annotation.pipe.ts b/apps/spa/src/app/pipes/annotation.pipe.ts index ef35074..975f7c9 100644 --- a/apps/spa/src/app/pipes/annotation.pipe.ts +++ b/apps/spa/src/app/pipes/annotation.pipe.ts @@ -5,7 +5,7 @@ import { isTextContentBlock } from '../components/controls/message-content/messa @Pipe({ standalone: true, - name: 'annotation' + name: 'annotation', }) export class AnnotationPipe implements PipeTransform { transform(event: MessageWithAnnotations<{ message: Message }>): string { @@ -14,7 +14,7 @@ export class AnnotationPipe implements PipeTransform { } const { text } = event.data.message.content[0]; - + if (!event.annotations?.length) { return text.value; } @@ -26,4 +26,4 @@ export class AnnotationPipe implements PipeTransform { return text.value; } -} \ No newline at end of file +} diff --git a/apps/spa/src/app/pipes/message-file.pipe.ts b/apps/spa/src/app/pipes/message-file.pipe.ts index e4abf2b..e2b75ff 100644 --- a/apps/spa/src/app/pipes/message-file.pipe.ts +++ b/apps/spa/src/app/pipes/message-file.pipe.ts @@ -5,14 +5,14 @@ import { ImageFileContentBlock } from 'openai/resources/beta/threads'; @Pipe({ standalone: true, - name: 'messageImageFile' + name: 'messageImageFile', }) export class MessageImageFilePipe implements PipeTransform { - transform(message: ChatMessage): ImageFileContentBlock[] { + transform(message: Partial): ImageFileContentBlock[] { if (typeof message.content === 'string') { return []; } - - return message.content.filter(isImageFileContentBlock); + + return message?.content?.filter(isImageFileContentBlock) || []; } -} \ No newline at end of file +} diff --git a/apps/spa/src/app/pipes/message-text.pipe.ts b/apps/spa/src/app/pipes/message-text.pipe.ts index de7df80..6dd3463 100644 --- a/apps/spa/src/app/pipes/message-text.pipe.ts +++ b/apps/spa/src/app/pipes/message-text.pipe.ts @@ -4,18 +4,21 @@ import { ChatMessage } from '../modules/+chat/shared/chat.model'; @Pipe({ standalone: true, - name: 'messageText' + name: 'messageText', + pure: false, }) export class MessageTextPipe implements PipeTransform { - transform(message: ChatMessage): string { + transform(message: Partial): string { if (typeof message.content === 'string') { return message.content; } - + // @TODO: handle all types of message content - return message.content - .filter(isTextContentBlock) - .map(block => block.text.value) - .join(' '); + return ( + message.content + ?.filter(isTextContentBlock) + ?.map(block => block.text.value) + ?.join(' ') || '' + ); } -} \ No newline at end of file +} diff --git a/apps/spa/src/environments/environment.development.ts b/apps/spa/src/environments/environment.development.ts index 80faaea..4b76138 100644 --- a/apps/spa/src/environments/environment.development.ts +++ b/apps/spa/src/environments/environment.development.ts @@ -12,6 +12,6 @@ export const environment = { isRefreshEnabled: true, isConfigEnabled: false, isAutoOpen: true, - isStreamingEnabled: true, + isStreamingEnabled: false, isImageContentEnabled: true, }; diff --git a/libs/openai-assistant/src/index.ts b/libs/openai-assistant/src/index.ts index c7b0f65..cd8c72c 100644 --- a/libs/openai-assistant/src/index.ts +++ b/libs/openai-assistant/src/index.ts @@ -5,3 +5,4 @@ export * from './lib/chat'; export * from './lib/run'; export * from './lib/files'; export * from './lib/threads'; +export * from './lib/annotations'; diff --git a/libs/openai-assistant/src/lib/annotations/annotations.model.ts b/libs/openai-assistant/src/lib/annotations/annotations.model.ts index fc5dae9..4956421 100644 --- a/libs/openai-assistant/src/lib/annotations/annotations.model.ts +++ b/libs/openai-assistant/src/lib/annotations/annotations.model.ts @@ -1,5 +1,5 @@ -import { FileObject } from "openai/resources"; -import { Annotation } from "openai/resources/beta/threads/messages"; +import { FileObject } from 'openai/resources'; +import { Annotation } from 'openai/resources/beta/threads/messages'; export interface AnnotationData { annotation: Annotation; diff --git a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts index 3315653..815b3db 100644 --- a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts +++ b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts @@ -1,11 +1,22 @@ -import OpenAI from "openai"; -import { FileCitationAnnotation, FilePathAnnotation, Message } from "openai/resources/beta/threads"; -import { AnnotationData, AnnotationType } from "./annotations.model"; - -export const isFileCitation = (item: { type: string }): item is FileCitationAnnotation => item.type === 'file_citation'; -export const isFilePath = (item: { type: string }): item is FilePathAnnotation => item.type === 'file_path'; - -export const getAnnotations = async (message: Message, provider: OpenAI): Promise => { +import OpenAI from 'openai'; +import { + FileCitationAnnotation, + FilePathAnnotation, + Message, +} from 'openai/resources/beta/threads'; +import { AnnotationData, AnnotationType } from './annotations.model'; + +export const isFileCitation = (item: { + type: string; +}): item is FileCitationAnnotation => item.type === 'file_citation'; +export const isFilePath = (item: { + type: string; +}): item is FilePathAnnotation => item.type === 'file_path'; + +export const getAnnotations = async ( + message: Message, + provider: OpenAI, +): Promise => { if (message.content[0].type !== 'text') { return []; } @@ -31,9 +42,9 @@ export const getAnnotations = async (message: Message, provider: OpenAI): Promis const file = await provider.files.retrieve(data.file_id); annotationsData.push({ annotation, index, file }); } - + index++; } return annotationsData; -} +}; diff --git a/libs/openai-assistant/src/lib/annotations/index.ts b/libs/openai-assistant/src/lib/annotations/index.ts new file mode 100644 index 0000000..12056f9 --- /dev/null +++ b/libs/openai-assistant/src/lib/annotations/index.ts @@ -0,0 +1,2 @@ +export * from './annotations.model'; +export * from './annotations.utils'; diff --git a/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts b/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts index 25466bc..656b443 100644 --- a/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts +++ b/libs/openai-assistant/src/lib/chat/chat.controller.spec.ts @@ -28,7 +28,15 @@ describe('ChatController', () => { jest .spyOn(chatService, 'call') .mockResolvedValue({} as ChatCallResponseDto); - const payload = { content: 'Hello' } as ChatCallDto; + const payload = { + threadId: '123', + content: [ + { + text: { value: 'Hello', annotations: [] }, + type: 'text', + }, + ], + } as ChatCallDto; await chatController.call(payload); diff --git a/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts b/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts index ed00cb1..1785ba8 100644 --- a/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts +++ b/libs/openai-assistant/src/lib/chat/chat.gateway.spec.ts @@ -3,6 +3,7 @@ import { Socket } from 'socket.io'; import { ChatGateway } from './chat.gateway'; import { ChatModule } from './chat.module'; import { ChatService } from './chat.service'; +import { ChatCallDto } from './chat.model'; describe('ChatGateway', () => { let chatGateway: ChatGateway; @@ -17,9 +18,15 @@ describe('ChatGateway', () => { chatService = moduleRef.get(ChatService); chatGateway = new ChatGateway(chatService); - jest - .spyOn(chatService, 'call') - .mockResolvedValue({ threadId: '123', content: 'Hello' }); + jest.spyOn(chatService, 'call').mockResolvedValue({ + threadId: '123', + content: [ + { + text: { value: 'Hello', annotations: [] }, + type: 'text', + }, + ], + }); }); it('should be defined', () => { @@ -28,7 +35,15 @@ describe('ChatGateway', () => { describe('listenForMessages', () => { it('should call chatService.call', async () => { - const request = { threadId: '123', content: 'Hello' }; + const request = { + threadId: '123', + content: [ + { + text: { value: 'Hello', annotations: [] }, + type: 'text', + }, + ], + } as ChatCallDto; await chatGateway.listenForMessages(request, {} as Socket); diff --git a/libs/openai-assistant/src/lib/chat/chat.gateway.ts b/libs/openai-assistant/src/lib/chat/chat.gateway.ts index f3666e5..a27e790 100644 --- a/libs/openai-assistant/src/lib/chat/chat.gateway.ts +++ b/libs/openai-assistant/src/lib/chat/chat.gateway.ts @@ -134,17 +134,21 @@ export class ChatGateway implements OnGatewayConnection { socketId: string, @MessageBody() data: MessageDonePayload, ) { - const annotations = await getAnnotations(data.message, this.chatsService.provider); + const annotations = await getAnnotations( + data.message, + this.chatsService.provider, + ); const messageWithAnnotations: MessageWithAnnotations = { data, annotations, }; - this.server.to(socketId).emit(ChatEvents.MessageDone, messageWithAnnotations); + this.server + .to(socketId) + .emit(ChatEvents.MessageDone, messageWithAnnotations); this.log( `Socket "${ChatEvents.MessageDone}" | threadId: ${data.message.thread_id}`, ); - } async emitTextCreated( @@ -161,7 +165,10 @@ export class ChatGateway implements OnGatewayConnection { } async emitTextDone(socketId: string, @MessageBody() data: TextDonePayload) { - const annotations = await getAnnotations(data.message, this.chatsService.provider); + const annotations = await getAnnotations( + data.message, + this.chatsService.provider, + ); const messageWithAnnotations: MessageWithAnnotations = { data, annotations, diff --git a/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts b/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts index 6630e10..3ccaeca 100644 --- a/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts +++ b/libs/openai-assistant/src/lib/chat/chat.helpers.spec.ts @@ -4,6 +4,7 @@ import { PagePromise } from 'openai/core'; import { ChatModule } from './chat.module'; import { ChatHelpers } from './chat.helpers'; import { AiService } from '../ai'; +import { error } from 'console'; describe('ChatService', () => { let chatbotHelpers: ChatHelpers; @@ -23,7 +24,7 @@ describe('ChatService', () => { }); describe('getAnswer', () => { - it('should return a string', async () => { + it('should return array of MessageContent', async () => { const threadMessage: Message = { content: [ { @@ -44,17 +45,17 @@ describe('ChatService', () => { } as unknown as Message; jest - .spyOn(chatbotHelpers, 'getLastMessage') + .spyOn(chatbotHelpers, 'geRunMessage') .mockReturnValue(Promise.resolve(threadMessage)); const result = await chatbotHelpers.getAnswer({} as Run); - expect(result).toBe('Hello'); + expect(result).toBe(threadMessage.content); }); }); describe('parseThreadMessage', () => { - it('should return a string', () => { + it('should return a array of MessageContent', () => { const threadMessage: Message = { content: [ { @@ -64,36 +65,21 @@ describe('ChatService', () => { annotations: [], }, }, - { - type: 'text', - text: { - value: 'Hello 2', - annotations: [], - }, - }, ], } as unknown as Message; const result = chatbotHelpers.parseThreadMessage(threadMessage); - expect(result).toBe('Hello'); - }); - - it('should return a default message', () => { - const result = chatbotHelpers.parseThreadMessage(); - - expect(result).toBe( - `Seems I'm lost, would you mind reformulating your question`, - ); + expect(result).toBe(threadMessage.content); }); }); - describe('getLastMessage', () => { + describe('geRunMessage', () => { it('should return a ThreadMessage', async () => { const threadMessagesPage = { data: [ { run_id: '1', role: 'assistant', id: '1' }, - { run_id: '1', role: 'user', id: '2' }, + { run_id: '2', role: 'user', id: '2' }, { run_id: '1', role: 'assistant', id: '3' }, ], } as unknown as MessagesPage; @@ -104,7 +90,7 @@ describe('ChatService', () => { threadMessagesPage as unknown as PagePromise, ); - const result = await chatbotHelpers.getLastMessage({ id: '1' } as Run); + const result = await chatbotHelpers.geRunMessage({ id: '1' } as Run); expect(result).toBe(threadMessagesPage.data[2]); }); @@ -123,7 +109,7 @@ describe('ChatService', () => { threadMessagesPage as unknown as PagePromise, ); - const result = await chatbotHelpers.getLastMessage({ id: '1' } as Run); + const result = await chatbotHelpers.geRunMessage({ id: '1' } as Run); expect(result).toBe(undefined); }); diff --git a/libs/openai-assistant/src/lib/chat/chat.helpers.ts b/libs/openai-assistant/src/lib/chat/chat.helpers.ts index a23520e..d4e00fe 100644 --- a/libs/openai-assistant/src/lib/chat/chat.helpers.ts +++ b/libs/openai-assistant/src/lib/chat/chat.helpers.ts @@ -1,5 +1,5 @@ import { Injectable } from '@nestjs/common'; -import { Message, Run, TextContentBlock } from 'openai/resources/beta/threads'; +import { Message, MessageContent, Run } from 'openai/resources/beta/threads'; import { AiService } from '../ai'; @Injectable() @@ -9,21 +9,20 @@ export class ChatHelpers { constructor(private readonly aiService: AiService) {} - async getAnswer(run: Run): Promise { - const lastThreadMessage = await this.getLastMessage(run); + async getAnswer(run: Run): Promise { + const lastThreadMessage = await this.geRunMessage(run); return this.parseThreadMessage(lastThreadMessage); } - parseThreadMessage(message?: Message): string { + parseThreadMessage(message?: Message): MessageContent[] { if (!message) { - return `Seems I'm lost, would you mind reformulating your question`; + throw `Seems I'm lost, would you mind reformulating your question`; } - const content = message.content[0] as TextContentBlock; - return content.text.value; + return message.content; } - async getLastMessage( + async geRunMessage( run: Run, role = 'assistant', ): Promise { diff --git a/libs/openai-assistant/src/lib/chat/chat.model.ts b/libs/openai-assistant/src/lib/chat/chat.model.ts index f46a2d5..1646932 100644 --- a/libs/openai-assistant/src/lib/chat/chat.model.ts +++ b/libs/openai-assistant/src/lib/chat/chat.model.ts @@ -52,7 +52,7 @@ export class ChatCallResponseDto { threadId!: string; @ApiProperty() - content!: string; + content!: Array; } export class ChatCallDto { @@ -60,7 +60,7 @@ export class ChatCallDto { threadId!: string; @ApiProperty() - content!: string | Array; + content!: Array; @ApiProperty({ required: false }) assistantId?: string; @@ -151,4 +151,4 @@ export interface ChatCallCallbacks { export interface MessageWithAnnotations { data: T; annotations: AnnotationData[]; -} \ No newline at end of file +} diff --git a/libs/openai-assistant/src/lib/chat/chat.service.spec.ts b/libs/openai-assistant/src/lib/chat/chat.service.spec.ts index a64162c..f1b9715 100644 --- a/libs/openai-assistant/src/lib/chat/chat.service.spec.ts +++ b/libs/openai-assistant/src/lib/chat/chat.service.spec.ts @@ -1,6 +1,6 @@ import { Test } from '@nestjs/testing'; import { APIPromise } from 'openai/core'; -import { Message, Run } from 'openai/resources/beta/threads'; +import { Message, MessageContent, Run } from 'openai/resources/beta/threads'; import { AssistantStream } from 'openai/lib/AssistantStream'; import { AiModule } from './../ai/ai.module'; import { ChatModule } from './chat.module'; @@ -29,9 +29,17 @@ describe('ChatService', () => { jest.spyOn(runService, 'resolve').mockReturnThis(); - jest - .spyOn(chatbotHelpers, 'getAnswer') - .mockReturnValue(Promise.resolve('Hello response') as Promise); + jest.spyOn(chatbotHelpers, 'getAnswer').mockReturnValue( + Promise.resolve([ + { + type: 'text', + text: { + value: 'Hello response', + annotations: [], + }, + }, + ]) as Promise, + ); jest .spyOn(chatService.threads.messages, 'create') @@ -49,7 +57,18 @@ describe('ChatService', () => { describe('call', () => { it('should create "thread run"', async () => { - const payload = { content: 'Hello', threadId: '1' } as ChatCallDto; + const payload = { + content: [ + { + type: 'text', + text: { + value: 'Hello', + annotations: [], + }, + }, + ], + threadId: '1', + } as ChatCallDto; const spyOnThreadRunsCreate = jest .spyOn(chatService.threads.messages, 'create') .mockResolvedValue({} as Message); @@ -60,14 +79,36 @@ describe('ChatService', () => { }); it('should return ChatCallResponse', async () => { - const payload = { content: 'Hello', threadId: '1' } as ChatCallDto; + const payload = { + content: [ + { + type: 'text', + text: { + value: 'Hello response', + annotations: [], + }, + }, + ], + threadId: '1', + } as ChatCallDto; jest .spyOn(chatService.threads.runs, 'create') .mockResolvedValue({} as Run); const result = await chatService.call(payload); - expect(result).toEqual({ content: 'Hello response', threadId: '1' }); + expect(result).toEqual({ + content: [ + { + type: 'text', + text: { + value: 'Hello response', + annotations: [], + }, + }, + ], + threadId: '1', + }); }); }); diff --git a/package.json b/package.json index f6d134b..b6fed07 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "cp:readme": "cp ./README.md ./dist/libs/openai-assistant/README.md", "lint": "nx run-many --parallel --target=lint --all", "test:openai-assistant": "nx test openai-assistant", + "test:watch:openai-assistant": "nx test openai-assistant --watch", "test:spa": "nx test spa", + "test:watch:spa": "nx test spa --watch", "test": "nx run-many --parallel --target=test --all", "format": "nx format:write", "prepare": "husky" From 7fb65e45c7705f3f80092a9e6f5769d38d5c2b55 Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Wed, 12 Jun 2024 10:08:57 +0200 Subject: [PATCH 4/5] feat(assistant): displaying annotations in the conversation --- apps/spa/src/app/app.config.ts | 2 - .../chat-annotation.component.html | 7 +++ .../chat-annotation.component.scss | 9 +++ .../chat-annotation.component.spec.ts | 25 ++++++++ .../chat-annotation.component.ts | 39 ++++++++++++ .../chat-annotations.component.html | 10 ++++ .../chat-annotations.component.scss | 33 +++++++++++ .../chat-annotations.component.spec.ts | 22 +++++++ .../chat-annotations.component.ts | 15 +++++ .../chat-message/chat-message.component.html | 7 ++- .../chat-message/chat-message.component.scss | 1 - .../chat-message.component.spec.ts | 2 +- .../chat-message/chat-message.component.ts | 6 +- .../chat-cloud/chat-cloud.component.spec.ts | 6 +- .../chat-home/chat-home.component.spec.ts | 2 + .../chat-iframe/chat-iframe.component.spec.ts | 2 + .../+chat/shared/chat-client.service.ts | 7 +++ .../app/modules/+chat/shared/chat.service.ts | 37 ++++++------ apps/spa/src/app/pipes/annotation.pipe.ts | 59 +++++++++++++++---- apps/spa/src/app/pipes/message-file.pipe.ts | 1 + .../environments/environment.development.ts | 2 +- apps/spa/src/styles/_extends/_markdown.scss | 15 +++++ apps/spa/src/styles/_extends/_material.scss | 22 +++++++ apps/spa/src/styles/styles.scss | 1 + .../src/lib/annotations/annotations.utils.ts | 1 + .../src/lib/files/files.controller.ts | 8 +++ .../src/lib/files/files.service.ts | 4 ++ 27 files changed, 302 insertions(+), 43 deletions(-) create mode 100644 apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.html create mode 100644 apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.scss create mode 100644 apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts create mode 100644 apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.ts create mode 100644 apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html create mode 100644 apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.scss create mode 100644 apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.spec.ts create mode 100644 apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.ts create mode 100644 apps/spa/src/styles/_extends/_markdown.scss diff --git a/apps/spa/src/app/app.config.ts b/apps/spa/src/app/app.config.ts index d4831a4..ec061dd 100644 --- a/apps/spa/src/app/app.config.ts +++ b/apps/spa/src/app/app.config.ts @@ -5,7 +5,6 @@ import { routes } from './app.routes'; import { provideAnimations } from '@angular/platform-browser/animations'; import { provideHttpClient } from '@angular/common/http'; import { provideMarkdown } from 'ngx-markdown'; -import { AnnotationPipe } from './pipes/annotation.pipe'; export const appConfig: ApplicationConfig = { providers: [ @@ -13,6 +12,5 @@ export const appConfig: ApplicationConfig = { provideAnimations(), provideHttpClient(), provideMarkdown(), - AnnotationPipe, ], }; diff --git a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.html b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.html new file mode 100644 index 0000000..7813f13 --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.html @@ -0,0 +1,7 @@ +
+ [{{ index }}] +
diff --git a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.scss b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.scss new file mode 100644 index 0000000..4e289f2 --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.scss @@ -0,0 +1,9 @@ +@import 'settings'; + +.annotation { + cursor: pointer; + + &:hover { + background-color: rgba(0, 0, 0, 0.15); + } +} diff --git a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts new file mode 100644 index 0000000..53f589b --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { ChatAnnotationComponent } from './chat-annotation.component'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; + +describe('ChatAnnotationComponent', () => { + let component: ChatAnnotationComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ + ChatAnnotationComponent, + HttpClientTestingModule, + ], + }).compileComponents(); + + fixture = TestBed.createComponent(ChatAnnotationComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.ts b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.ts new file mode 100644 index 0000000..b5ade9b --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.ts @@ -0,0 +1,39 @@ +import { Component, Input } from '@angular/core'; +import { MatTooltip } from '@angular/material/tooltip'; +import { Annotation } from 'openai/resources/beta/threads'; +import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service'; +import { FileObject } from 'openai/resources'; +import { isFileCitation } from '../../../pipes/annotation.pipe'; +import { take } from 'rxjs'; + +@Component({ + selector: 'ai-chat-annotation', + standalone: true, + templateUrl: './chat-annotation.component.html', + styleUrl: './chat-annotation.component.scss', + imports: [MatTooltip], +}) +export class ChatAnnotationComponent { + @Input() annotation!: Annotation; + @Input() index = 1; + fileDetails!: FileObject; + + get fileId(): string { + return isFileCitation(this.annotation) + ? this.annotation.file_citation.file_id + : this.annotation.file_path.file_id; + } + + constructor(private chatClientService: ChatClientService) {} + + showDetails() { + if (!this.fileId || !!this.fileDetails) { + return; + } + + this.chatClientService + .retriveFile(this.fileId) + .pipe(take(1)) + .subscribe(details => (this.fileDetails = details)); + } +} diff --git a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html new file mode 100644 index 0000000..62994ff --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html @@ -0,0 +1,10 @@ +@if (message && message.type === 'text' && message.text.annotations.length) { +
Annotations:
+
+ @for (annotation of message.text.annotations; track annotation.text) { + + [{{ $index + 1 }}] + + } +
+} diff --git a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.scss b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.scss new file mode 100644 index 0000000..df5ef57 --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.scss @@ -0,0 +1,33 @@ +@import 'settings'; + +:host { + display: flex; + align-items: baseline; + gap: $size-2; + border-top: 1px dashed var(--color-grey-500); + margin-top: $size-3; + padding-top: $size-3; + font-size: 12px; + + &:empty { + display: none; + } +} + +.title { + font-weight: 500; +} + +.content { + display: flex; + gap: $size-1; + margin-top: $size-2; +} + +.annotation { + cursor: pointer; + + &:hover { + background-color: rgba(0, 0, 0, 0.15); + } +} diff --git a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.spec.ts b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.spec.ts new file mode 100644 index 0000000..8c7eb64 --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.spec.ts @@ -0,0 +1,22 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { MarkdownModule } from 'ngx-markdown'; +import { ChatAnnotationsComponent } from './chat-annotations.component'; + +describe('ChatAnnotationsComponent', () => { + let component: ChatAnnotationsComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + imports: [ChatAnnotationsComponent, MarkdownModule.forRoot()], + }).compileComponents(); + + fixture = TestBed.createComponent(ChatAnnotationsComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.ts b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.ts new file mode 100644 index 0000000..590b745 --- /dev/null +++ b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.ts @@ -0,0 +1,15 @@ +import { Component, Input } from '@angular/core'; +import { MatTooltip } from '@angular/material/tooltip'; +import { MessageContent } from 'openai/resources/beta/threads'; +import { ChatAnnotationComponent } from '../chat-annotation/chat-annotation.component'; + +@Component({ + selector: 'ai-chat-annotations', + standalone: true, + templateUrl: './chat-annotations.component.html', + styleUrl: './chat-annotations.component.scss', + imports: [MatTooltip, ChatAnnotationComponent], +}) +export class ChatAnnotationsComponent { + @Input() message!: MessageContent; +} diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html index f6ca99e..e680f7c 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html @@ -4,8 +4,11 @@ }
- @if (message | messageText) { - + @for (msg of message.content; track msg) { + @if (msg.type === 'text') { + + } + } @if ((message | messageImageFile).length) { diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss index 0be069c..16f476e 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.scss @@ -11,7 +11,6 @@ justify-content: flex-end; .chat-message { - border-bottom-left-radius: 0; background: var(--color-primary-200); border-bottom-right-radius: 0; align-self: flex-end; diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts index d4bf345..739e635 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.spec.ts @@ -1,7 +1,7 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { MarkdownModule } from 'ngx-markdown'; import { ChatMessageComponent } from './chat-message.component'; -import { MarkdownModule } from 'ngx-markdown'; describe('ChatMessageComponent', () => { let component: ChatMessageComponent; diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts index 1f86e9e..c8af12b 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.ts @@ -7,8 +7,9 @@ import { MarkdownComponent } from 'ngx-markdown'; import { ChatAudioComponent } from '../chat-audio/chat-audio.component'; import { NgClass } from '@angular/common'; import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component'; -import { MessageTextPipe } from '../../../pipes/message-text.pipe'; import { MessageImageFilePipe } from '../../../pipes/message-file.pipe'; +import { AnnotationPipe } from '../../../pipes/annotation.pipe'; +import { ChatAnnotationsComponent } from '../chat-annotations/chat-annotations.component'; @Component({ selector: 'ai-chat-message', @@ -20,8 +21,9 @@ import { MessageImageFilePipe } from '../../../pipes/message-file.pipe'; MarkdownComponent, ChatAudioComponent, ChatAvatarComponent, - MessageTextPipe, MessageImageFilePipe, + AnnotationPipe, + ChatAnnotationsComponent, ], }) export class ChatMessageComponent { diff --git a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts index 6756100..9b16a34 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-cloud/chat-cloud.component.spec.ts @@ -1,9 +1,10 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; - -import { ChatCloudComponent } from './chat-cloud.component'; import { HttpClientTestingModule } from '@angular/common/http/testing'; import { MarkdownModule } from 'ngx-markdown'; +import { ChatCloudComponent } from './chat-cloud.component'; +import { AnnotationPipe } from '../../../../pipes/annotation.pipe'; + describe('ChatHomeComponent', () => { let component: ChatCloudComponent; let fixture: ComponentFixture; @@ -15,6 +16,7 @@ describe('ChatHomeComponent', () => { HttpClientTestingModule, MarkdownModule.forRoot(), ], + providers: [AnnotationPipe], }).compileComponents(); fixture = TestBed.createComponent(ChatCloudComponent); diff --git a/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts index 1136e75..a9baf12 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-home/chat-home.component.spec.ts @@ -4,6 +4,7 @@ import { ChatHomeComponent } from './chat-home.component'; import { HttpClientTestingModule } from '@angular/common/http/testing'; import { MarkdownModule } from 'ngx-markdown'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; +import { AnnotationPipe } from '../../../../pipes/annotation.pipe'; describe('ChatHomeComponent', () => { let component: ChatHomeComponent; @@ -17,6 +18,7 @@ describe('ChatHomeComponent', () => { BrowserAnimationsModule, MarkdownModule.forRoot(), ], + providers: [AnnotationPipe], }).compileComponents(); fixture = TestBed.createComponent(ChatHomeComponent); diff --git a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts index 2c2a199..00e19d3 100644 --- a/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts +++ b/apps/spa/src/app/modules/+chat/containers/chat-iframe/chat-iframe.component.spec.ts @@ -4,6 +4,7 @@ import { ChatIframeComponent } from './chat-iframe.component'; import { HttpClientTestingModule } from '@angular/common/http/testing'; import { BrowserAnimationsModule } from '@angular/platform-browser/animations'; import { RouterTestingModule } from '@angular/router/testing'; +import { AnnotationPipe } from '../../../../pipes/annotation.pipe'; describe('ChatIframeComponent', () => { let component: ChatIframeComponent; @@ -17,6 +18,7 @@ describe('ChatIframeComponent', () => { ChatIframeComponent, RouterTestingModule, ], + providers: [AnnotationPipe], }).compileComponents(); fixture = TestBed.createComponent(ChatIframeComponent); diff --git a/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts b/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts index 405ba4a..e7dd335 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat-client.service.ts @@ -10,6 +10,7 @@ import { UploadFilesPayload, UploadFilesResponseDto, } from '@boldare/openai-assistant'; +import { FileObject } from 'openai/resources'; @Injectable({ providedIn: 'root' }) export class ChatClientService { @@ -49,4 +50,10 @@ export class ChatClientService { ), ); } + + retriveFile(fileId: string): Observable { + return this.httpClient.get( + `${this.apiUrl}/files/retrive/${fileId}`, + ); + } } diff --git a/apps/spa/src/app/modules/+chat/shared/chat.service.ts b/apps/spa/src/app/modules/+chat/shared/chat.service.ts index e1c3567..f7bc963 100644 --- a/apps/spa/src/app/modules/+chat/shared/chat.service.ts +++ b/apps/spa/src/app/modules/+chat/shared/chat.service.ts @@ -23,7 +23,6 @@ import { ThreadService } from './thread.service'; import { ChatFilesService } from './chat-files.service'; import { MessageContentService } from '../../../components/controls/message-content/message-content.service'; import { environment } from '../../../../environments/environment'; -import { AnnotationPipe } from '../../../pipes/annotation.pipe'; import { imageFileContentBlock, messageAttachment, @@ -46,7 +45,6 @@ export class ChatService { private readonly threadService: ThreadService, private readonly chatFilesService: ChatFilesService, private readonly messageContentService: MessageContentService, - private readonly annotationPipe: AnnotationPipe, ) { document.body.classList.add('ai-chat'); @@ -70,10 +68,6 @@ export class ChatService { return metadata?.['status'] === ChatMessageStatus.Invisible; } - isTextMessage(message: Message): boolean { - return message.content?.[0]?.type === 'text'; - } - parseMessages(thread: GetThreadResponseDto): Message[] { if (!thread.messages) { return []; @@ -81,10 +75,7 @@ export class ChatService { return thread.messages .reverse() - .filter( - message => - this.isTextMessage(message) && !this.isMessageInvisible(message), - ); + .filter(message => !this.isMessageInvisible(message)); } setInitialValues(): void { @@ -162,7 +153,10 @@ export class ChatService { this.messages$.next([ ...this.messages$.value.slice(0, -1), - messageContentBlock([textContentBlock(content)], ChatRole.User), + messageContentBlock( + [textContentBlock(content), ...imageFileContentList], + ChatRole.User, + ), ]); return [ @@ -180,7 +174,7 @@ export class ChatService { this.isResponding$.next(true); const message = messageContentBlock( - [textContentBlock(data.text.value)], + [textContentBlock('')], ChatRole.Assistant, ); this.addMessage(message); @@ -197,7 +191,10 @@ export class ChatService { if (isTextContentBlock(lastMessageContent)) { ( this.messages$.value[length - 1].content?.[0] as TextContentBlock - ).text = data.text; + ).text.value += data.textDelta.value || ''; + ( + this.messages$.value[length - 1].content?.[0] as TextContentBlock + ).text.annotations = data.text.annotations || []; } }); } @@ -207,13 +204,13 @@ export class ChatService { this.isTyping$.next(false); this.isResponding$.next(false); - const annotationContent = this.annotationPipe.transform(event); - const message = messageContentBlock( - [textContentBlock(annotationContent)], - ChatRole.Assistant, - ); - - this.messages$.next([...this.messages$.value.slice(0, -1), message]); + this.messages$.next([ + ...this.messages$.value.slice(0, -1), + { + ...this.messages$.value.pop(), + annotations: event.annotations, + }, + ]); }); } diff --git a/apps/spa/src/app/pipes/annotation.pipe.ts b/apps/spa/src/app/pipes/annotation.pipe.ts index 975f7c9..363e7ca 100644 --- a/apps/spa/src/app/pipes/annotation.pipe.ts +++ b/apps/spa/src/app/pipes/annotation.pipe.ts @@ -1,29 +1,64 @@ import { Pipe, PipeTransform } from '@angular/core'; -import { MessageWithAnnotations } from '@boldare/openai-assistant'; -import { Message } from 'openai/resources/beta/threads'; +import { + FileCitationAnnotation, + FilePathAnnotation, + MessageContent, +} from 'openai/resources/beta/threads'; import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers'; +export const isFileCitation = (item: { + type: string; +}): item is FileCitationAnnotation => item.type === 'file_citation'; + +export const isFilePath = (item: { + type: string; +}): item is FilePathAnnotation => item.type === 'file_path'; + @Pipe({ standalone: true, name: 'annotation', + pure: false, }) export class AnnotationPipe implements PipeTransform { - transform(event: MessageWithAnnotations<{ message: Message }>): string { - if (!isTextContentBlock(event.data.message.content[0])) { + transform(textContent: MessageContent): string { + if (!isTextContentBlock(textContent)) { return ''; } - const { text } = event.data.message.content[0]; - - if (!event.annotations?.length) { - return text.value; + if (!textContent.text.annotations?.length) { + return textContent.text.value; } - for (const item of event.annotations) { - const { index, annotation } = item; - text.value = text.value.replace(annotation.text, `[${index}]`); + let index = 1; + + for (const annotation of textContent.text.annotations) { + const { text } = annotation; + let fileId = null; + + if (isFileCitation(annotation)) { + fileId = annotation.file_citation.file_id; + } + + if (isFilePath(annotation)) { + fileId = annotation.file_path.file_id; + } + + const annotationBlock = `  + + ${fileId} + ${annotation?.type} + + [${index}] + `; + + textContent.text.value = textContent.text.value.replace( + text, + annotationBlock, + ); + + index++; } - return text.value; + return textContent.text.value; } } diff --git a/apps/spa/src/app/pipes/message-file.pipe.ts b/apps/spa/src/app/pipes/message-file.pipe.ts index e2b75ff..f420c99 100644 --- a/apps/spa/src/app/pipes/message-file.pipe.ts +++ b/apps/spa/src/app/pipes/message-file.pipe.ts @@ -6,6 +6,7 @@ import { ImageFileContentBlock } from 'openai/resources/beta/threads'; @Pipe({ standalone: true, name: 'messageImageFile', + pure: true, }) export class MessageImageFilePipe implements PipeTransform { transform(message: Partial): ImageFileContentBlock[] { diff --git a/apps/spa/src/environments/environment.development.ts b/apps/spa/src/environments/environment.development.ts index 4b76138..80faaea 100644 --- a/apps/spa/src/environments/environment.development.ts +++ b/apps/spa/src/environments/environment.development.ts @@ -12,6 +12,6 @@ export const environment = { isRefreshEnabled: true, isConfigEnabled: false, isAutoOpen: true, - isStreamingEnabled: false, + isStreamingEnabled: true, isImageContentEnabled: true, }; diff --git a/apps/spa/src/styles/_extends/_markdown.scss b/apps/spa/src/styles/_extends/_markdown.scss new file mode 100644 index 0000000..5d63ec1 --- /dev/null +++ b/apps/spa/src/styles/_extends/_markdown.scss @@ -0,0 +1,15 @@ +.annotation { + display: inline-block; + text-align: center; + padding: 2px 4px; + font-size: 11px; + font-weight: 400; + min-width: 14px; + border-radius: 4px; + text-decoration: none; + background-color: rgba(0, 0, 0, 0.1); +} + +.annotation__metadata { + display: none; +} diff --git a/apps/spa/src/styles/_extends/_material.scss b/apps/spa/src/styles/_extends/_material.scss index bd5de81..b1d3105 100644 --- a/apps/spa/src/styles/_extends/_material.scss +++ b/apps/spa/src/styles/_extends/_material.scss @@ -84,4 +84,26 @@ .mat-mdc-text-field-wrapper .mat-mdc-form-field-flex .mat-mdc-floating-label { top: 28px; } + + .popover { + background: var(--color-grey-600); + transition: none; + padding: 4px 12px; + margin-bottom: 4px; + margin-left: -20px; + color: white; + + .mat-mdc-menu-content { + align-items: center; + display: flex; + padding: 0; + font-size: 12px; + } + + .icon { + font-size: inherit; + height: initial; + width: initial; + } + } } diff --git a/apps/spa/src/styles/styles.scss b/apps/spa/src/styles/styles.scss index 00717f8..c0e3c3d 100644 --- a/apps/spa/src/styles/styles.scss +++ b/apps/spa/src/styles/styles.scss @@ -6,6 +6,7 @@ @import '_settings/borders'; @import '_extends/material'; +@import '_extends/markdown'; html, body { diff --git a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts index 815b3db..85045b9 100644 --- a/libs/openai-assistant/src/lib/annotations/annotations.utils.ts +++ b/libs/openai-assistant/src/lib/annotations/annotations.utils.ts @@ -9,6 +9,7 @@ import { AnnotationData, AnnotationType } from './annotations.model'; export const isFileCitation = (item: { type: string; }): item is FileCitationAnnotation => item.type === 'file_citation'; + export const isFilePath = (item: { type: string; }): item is FilePathAnnotation => item.type === 'file_path'; diff --git a/libs/openai-assistant/src/lib/files/files.controller.ts b/libs/openai-assistant/src/lib/files/files.controller.ts index b899215..0104121 100644 --- a/libs/openai-assistant/src/lib/files/files.controller.ts +++ b/libs/openai-assistant/src/lib/files/files.controller.ts @@ -1,5 +1,7 @@ import { Controller, + Get, + Param, Post, UploadedFiles, UseInterceptors, @@ -8,6 +10,7 @@ import { FileFieldsInterceptor } from '@nestjs/platform-express'; import { FilesService } from './files.service'; import { UploadFilesDto, UploadFilesResponseDto } from './files.model'; import { ApiBody, ApiConsumes, ApiResponse, ApiTags } from '@nestjs/swagger'; +import { FileObject } from 'openai/resources'; @ApiTags('Files') @Controller('assistant/files') @@ -26,4 +29,9 @@ export class FilesController { files: await this.filesService.files(uploadedData.files), }; } + + @Get('/retrive/:fileId') + async retriveFile(@Param() params: { fileId: string }): Promise { + return this.filesService.retriveFile(params.fileId); + } } diff --git a/libs/openai-assistant/src/lib/files/files.service.ts b/libs/openai-assistant/src/lib/files/files.service.ts index a247df6..f9e57da 100644 --- a/libs/openai-assistant/src/lib/files/files.service.ts +++ b/libs/openai-assistant/src/lib/files/files.service.ts @@ -22,4 +22,8 @@ export class FilesService { }), ); } + + async retriveFile(fileId: string): Promise { + return await this.provider.files.retrieve(fileId); + } } From 42582b88c9c78bc01581a4da02f4e67830e2766e Mon Sep 17 00:00:00 2001 From: Sebastian Musial Date: Thu, 13 Jun 2024 09:05:50 +0200 Subject: [PATCH 5/5] feat(annotaions): refactor after code review --- .../chat-annotation/chat-annotation.component.spec.ts | 5 +---- .../chat/chat-annotations/chat-annotations.component.html | 5 ++++- .../chat/chat-message/chat-message.component.html | 2 +- .../chat/chat-messages/chat-messages.component.html | 2 +- .../components/chat/chat-tips/chat-tips.component.html | 8 ++++---- .../src/app/components/controls/files/files.service.ts | 5 ++++- .../controls/message-content/message-content.service.ts | 8 ++++---- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts index 53f589b..1517bad 100644 --- a/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts +++ b/apps/spa/src/app/components/chat/chat-annotation/chat-annotation.component.spec.ts @@ -8,10 +8,7 @@ describe('ChatAnnotationComponent', () => { beforeEach(async () => { await TestBed.configureTestingModule({ - imports: [ - ChatAnnotationComponent, - HttpClientTestingModule, - ], + imports: [ChatAnnotationComponent, HttpClientTestingModule], }).compileComponents(); fixture = TestBed.createComponent(ChatAnnotationComponent); diff --git a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html index 62994ff..8c6a040 100644 --- a/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html +++ b/apps/spa/src/app/components/chat/chat-annotations/chat-annotations.component.html @@ -1,7 +1,10 @@ @if (message && message.type === 'text' && message.text.annotations.length) {
Annotations:
- @for (annotation of message.text.annotations; track annotation.text) { + @for ( + annotation of message.text.annotations; + track annotation.text + $index + ) { [{{ $index + 1 }}] diff --git a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html index e680f7c..54306de 100644 --- a/apps/spa/src/app/components/chat/chat-message/chat-message.component.html +++ b/apps/spa/src/app/components/chat/chat-message/chat-message.component.html @@ -4,7 +4,7 @@ }
- @for (msg of message.content; track msg) { + @for (msg of message.content; track $index) { @if (msg.type === 'text') { } diff --git a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html index 950882a..b97dd08 100644 --- a/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html +++ b/apps/spa/src/app/components/chat/chat-messages/chat-messages.component.html @@ -1,5 +1,5 @@
- @for (message of initialMessages.concat(messages); track message) { + @for (message of initialMessages.concat(messages); track message.id) { diff --git a/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html b/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html index da32b73..c156832 100644 --- a/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html +++ b/apps/spa/src/app/components/chat/chat-tips/chat-tips.component.html @@ -1,5 +1,5 @@ -@for (tip of tips; track tip) { - - {{ tip }} - +@for (tip of tips; track $index) { + + {{ tip }} + } diff --git a/apps/spa/src/app/components/controls/files/files.service.ts b/apps/spa/src/app/components/controls/files/files.service.ts index e6a3552..6688115 100644 --- a/apps/spa/src/app/components/controls/files/files.service.ts +++ b/apps/spa/src/app/components/controls/files/files.service.ts @@ -6,9 +6,12 @@ export class FilesService { files$ = new BehaviorSubject([]); add(files: FileList) { + const convertedFiles = Object.keys(files).map( + key => files[key as unknown as number], + ); const updatedFiles = [ ...this.files$.value, - ...Object.keys(files).map(key => files[key as unknown as number]), + ...convertedFiles, ]; this.files$.next(updatedFiles); diff --git a/apps/spa/src/app/components/controls/message-content/message-content.service.ts b/apps/spa/src/app/components/controls/message-content/message-content.service.ts index f943545..9d5a8f6 100644 --- a/apps/spa/src/app/components/controls/message-content/message-content.service.ts +++ b/apps/spa/src/app/components/controls/message-content/message-content.service.ts @@ -10,10 +10,10 @@ export class MessageContentService { constructor(private readonly chatClientService: ChatClientService) {} add(files: FileList) { - const updatedFiles = [ - ...this.data$.value, - ...Object.keys(files).map(key => files[key as unknown as number]), - ]; + const convertedFiles = Object.keys(files).map( + key => files[key as unknown as number], + ); + const updatedFiles = [...this.data$.value, ...convertedFiles]; this.data$.next(updatedFiles); }