From bb24ca16832a59f07cf21b9f0ec33d4bfe9acd2b Mon Sep 17 00:00:00 2001 From: Danny Banks Date: Mon, 7 Oct 2024 23:01:06 -0700 Subject: [PATCH 1/5] feat(ai): add message renderer --- .../amplify_outputs.js | 2 + .../ai-conversation-renderer/index.page.tsx | 89 +++++++++++++++++++ .../AIConversation/AIConversation.tsx | 2 + .../context/MessageRenderContext.tsx | 13 +++ .../AIConversation/context/index.ts | 6 ++ .../AIConversation/createAIConversation.tsx | 2 + .../AIConversation/createProvider.tsx | 61 +++++++------ .../src/components/AIConversation/types.ts | 8 +- .../views/Controls/MessagesControl.tsx | 12 ++- 9 files changed, 163 insertions(+), 32 deletions(-) create mode 100644 examples/next/pages/ui/components/ai/ai-conversation-renderer/amplify_outputs.js create mode 100644 examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx create mode 100644 packages/react-ai/src/components/AIConversation/context/MessageRenderContext.tsx diff --git a/examples/next/pages/ui/components/ai/ai-conversation-renderer/amplify_outputs.js b/examples/next/pages/ui/components/ai/ai-conversation-renderer/amplify_outputs.js new file mode 100644 index 00000000000..2f1016412fd --- /dev/null +++ b/examples/next/pages/ui/components/ai/ai-conversation-renderer/amplify_outputs.js @@ -0,0 +1,2 @@ +import amplifyOutputs from '@environments/ai/gen2/amplify_outputs'; +export default amplifyOutputs; diff --git a/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx b/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx new file mode 100644 index 00000000000..3cead428ccf --- /dev/null +++ b/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx @@ -0,0 +1,89 @@ +import { Amplify } from 'aws-amplify'; +import { createAIHooks, AIConversation } from '@aws-amplify/ui-react-ai'; +import { generateClient } from 'aws-amplify/api'; +import '@aws-amplify/ui-react/styles.css'; +import '@aws-amplify/ui-react-ai/ai-conversation-styles.css'; + +import outputs from './amplify_outputs'; +import type { Schema } from '@environments/ai/gen2/amplify/data/resource'; +import { Authenticator, Card, Text } from '@aws-amplify/ui-react'; +import Image from 'next/image'; + +const client = generateClient({ authMode: 'userPool' }); +const { useAIConversation } = createAIHooks(client); + +Amplify.configure(outputs); + +function arrayBufferToBase64(buffer: ArrayBuffer) { + let binary = ''; + const bytes = new Uint8Array(buffer); + const len = bytes.byteLength; + for (let i = 0; i < len; i++) { + binary += String.fromCharCode(bytes[i]); + } + return window.btoa(binary); +} + +function convertBufferToBase64(buffer: ArrayBuffer, format: string): string { + let base64string = ''; + // Use node-based buffer if available + // fall back on browser if not + if (typeof Buffer !== 'undefined') { + base64string = Buffer.from(new Uint8Array(buffer)).toString('base64'); + } else { + base64string = arrayBufferToBase64(buffer); + } + return `data:image/${format};base64,${base64string}`; +} + +function Chat() { + const [ + { + data: { messages }, + isLoading, + }, + sendMessage, + ] = useAIConversation('pirateChat'); + + return ( + + {message}, + image: (image) => ( + + ), + }} + suggestedPrompts={[ + { + inputText: 'hello', + header: 'hello', + }, + { + inputText: 'how are you?', + header: 'how are you?', + }, + ]} + variant="bubble" + /> + + ); +} + +export default function Example() { + return ( + + + + ); +} diff --git a/packages/react-ai/src/components/AIConversation/AIConversation.tsx b/packages/react-ai/src/components/AIConversation/AIConversation.tsx index 200dd604c62..8688d19d369 100644 --- a/packages/react-ai/src/components/AIConversation/AIConversation.tsx +++ b/packages/react-ai/src/components/AIConversation/AIConversation.tsx @@ -31,6 +31,7 @@ function AIConversationBase({ isLoading, displayText, allowAttachments, + messageRenderer, }: AIConversationBaseProps): JSX.Element { const icons = useIcons('aiConversation'); const defaultAvatars: Avatars = { @@ -64,6 +65,7 @@ function AIConversationBase({ }, displayText, allowAttachments, + messageRenderer, }); const providerProps = { diff --git a/packages/react-ai/src/components/AIConversation/context/MessageRenderContext.tsx b/packages/react-ai/src/components/AIConversation/context/MessageRenderContext.tsx new file mode 100644 index 00000000000..7336e74f95c --- /dev/null +++ b/packages/react-ai/src/components/AIConversation/context/MessageRenderContext.tsx @@ -0,0 +1,13 @@ +import { createContextUtilities } from '@aws-amplify/ui-react-core'; +import { MessageRenderer } from '../types'; + +export const { + MessageRendererContext, + MessageRendererProvider, + useMessageRenderer, +} = createContextUtilities({ + contextName: 'MessageRenderer', + defaultValue: undefined, + errorMessage: + '`useMessageRenderer` must be used with an AIConversation component', +}); diff --git a/packages/react-ai/src/components/AIConversation/context/index.ts b/packages/react-ai/src/components/AIConversation/context/index.ts index 5c56c750018..ca32e10d824 100644 --- a/packages/react-ai/src/components/AIConversation/context/index.ts +++ b/packages/react-ai/src/components/AIConversation/context/index.ts @@ -34,5 +34,11 @@ export { RESPONSE_COMPONENT_PREFIX, } from './ResponseComponentsContext'; export { SendMessageContextProvider } from './SendMessageContext'; +export { + MessageRendererProvider, + MessageRendererContext, + useMessageRenderer, +} from './MessageRenderContext'; +export { AttachmentProvider, AttachmentContext } from './AttachmentContext'; export * from './elements'; diff --git a/packages/react-ai/src/components/AIConversation/createAIConversation.tsx b/packages/react-ai/src/components/AIConversation/createAIConversation.tsx index f894133611a..acd95d3a0be 100644 --- a/packages/react-ai/src/components/AIConversation/createAIConversation.tsx +++ b/packages/react-ai/src/components/AIConversation/createAIConversation.tsx @@ -31,6 +31,7 @@ export function createAIConversation(input: AIConversationInput = {}): { controls, displayText, allowAttachments, + messageRenderer, } = input; const Provider = createProvider({ @@ -42,6 +43,7 @@ export function createAIConversation(input: AIConversationInput = {}): { controls, displayText, allowAttachments, + messageRenderer, }); function AIConversation(props: AIConversationProps): JSX.Element { diff --git a/packages/react-ai/src/components/AIConversation/createProvider.tsx b/packages/react-ai/src/components/AIConversation/createProvider.tsx index 0d0f841eb87..c576c7f03dc 100644 --- a/packages/react-ai/src/components/AIConversation/createProvider.tsx +++ b/packages/react-ai/src/components/AIConversation/createProvider.tsx @@ -5,19 +5,20 @@ import { ElementsProvider } from '@aws-amplify/ui-react-core/elements'; import { AIConversationInput, AIConversationProps } from './types'; import { defaultAIConversationDisplayTextEn } from './displayText'; import { - ConversationDisplayTextProvider, - SuggestedPromptProvider, - ConversationInputContextProvider, - AvatarsProvider, ActionsProvider, - MessageVariantProvider, - MessagesProvider, + AttachmentProvider, + AvatarsProvider, ControlsProvider, + ConversationDisplayTextProvider, + ConversationInputContextProvider, LoadingContextProvider, + MessagesProvider, + MessageRendererProvider, + MessageVariantProvider, ResponseComponentsProvider, SendMessageContextProvider, + SuggestedPromptProvider, } from './context'; -import { AttachmentProvider } from './context/AttachmentContext'; export default function createProvider({ elements, @@ -28,6 +29,7 @@ export default function createProvider({ controls, displayText, allowAttachments, + messageRenderer = {}, }: Pick< AIConversationInput, | 'elements' @@ -38,6 +40,7 @@ export default function createProvider({ | 'controls' | 'displayText' | 'allowAttachments' + | 'messageRenderer' >) { return function Provider({ children, @@ -60,27 +63,29 @@ export default function createProvider({ - - - - - - - - - - {children} - - - - - - - - - + + + + + + + + + + + {children} + + + + + + + + + + diff --git a/packages/react-ai/src/components/AIConversation/types.ts b/packages/react-ai/src/components/AIConversation/types.ts index ba987b8dce1..e611cfbede0 100644 --- a/packages/react-ai/src/components/AIConversation/types.ts +++ b/packages/react-ai/src/components/AIConversation/types.ts @@ -11,7 +11,7 @@ import { } from './views'; import { DisplayTextTemplate } from '@aws-amplify/ui'; import { AIConversationDisplayText } from './displayText'; -import { ConversationMessage, SendMessage } from '../../types'; +import { ConversationMessage, ImageContent, SendMessage } from '../../types'; import { ControlsContextProps } from './context/ControlsContext'; export interface Controls { @@ -32,6 +32,7 @@ export interface AIConversationInput { variant?: MessageVariant; controls?: ControlsContextProps; allowAttachments?: boolean; + messageRenderer?: MessageRenderer; } export interface AIConversationProps { @@ -54,6 +55,11 @@ export interface AIConversation { export type MessageVariant = 'bubble' | 'default'; +export interface MessageRenderer { + text?: (message: string) => React.JSX.Element; + image?: (image: ImageContent) => React.JSX.Element; +} + export interface Avatar { username?: string; avatar?: React.ReactNode; diff --git a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx index deface4b773..14a2effb327 100644 --- a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx +++ b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx @@ -6,6 +6,7 @@ import { MessageVariantContext, RoleContext, useConversationDisplayText, + useMessageRenderer, } from '../../context'; import { AIConversationElements } from '../../context/elements'; import { convertBufferToBase64 } from '../../utils'; @@ -63,17 +64,22 @@ const ContentContainer: typeof View = React.forwardRef( export const MessageControl: MessageControl = ({ message }) => { const responseComponents = React.useContext(ResponseComponentsContext); + const { text, image } = useMessageRenderer(); return ( {message.content.map((content, index) => { if (content.text) { - return ( + return text ? ( + text(content.text) + ) : ( {content.text} ); } else if (content.image) { - return ( + return image ? ( + image(content.image) + ) : ( { content.image?.source.bytes, content.image?.format )} - > + /> ); } else if (content.toolUse) { // For now tool use is limited to custom response components From 3e442b02c36d5b43dc62c5f01c1c05615cb056c0 Mon Sep 17 00:00:00 2001 From: Danny Banks Date: Mon, 7 Oct 2024 23:04:11 -0700 Subject: [PATCH 2/5] Create heavy-dots-applaud.md --- .changeset/heavy-dots-applaud.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .changeset/heavy-dots-applaud.md diff --git a/.changeset/heavy-dots-applaud.md b/.changeset/heavy-dots-applaud.md new file mode 100644 index 00000000000..f013283490e --- /dev/null +++ b/.changeset/heavy-dots-applaud.md @@ -0,0 +1,16 @@ +--- +"@aws-amplify/ui-react-ai": minor +--- + +feat(ai): add message renderer + +```tsx + {message}, + }} +/> +``` From 5ceb1ba34d335b7e2da6a2696334686b7d9a3aeb Mon Sep 17 00:00:00 2001 From: Danny Banks Date: Tue, 8 Oct 2024 09:54:44 -0700 Subject: [PATCH 3/5] chore(ai): fixing tests --- .../views/Controls/MessagesControl.tsx | 12 ++++----- .../__tests__/MessagesControl.spec.tsx | 25 +++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx index 14a2effb327..4b1e45676bb 100644 --- a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx +++ b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx @@ -2,11 +2,11 @@ import React from 'react'; import { withBaseElementProps } from '@aws-amplify/ui-react-core/elements'; import { + MessageRendererContext, MessagesContext, MessageVariantContext, RoleContext, useConversationDisplayText, - useMessageRenderer, } from '../../context'; import { AIConversationElements } from '../../context/elements'; import { convertBufferToBase64 } from '../../utils'; @@ -64,21 +64,21 @@ const ContentContainer: typeof View = React.forwardRef( export const MessageControl: MessageControl = ({ message }) => { const responseComponents = React.useContext(ResponseComponentsContext); - const { text, image } = useMessageRenderer(); + const messageRenderer = React.useContext(MessageRendererContext); return ( {message.content.map((content, index) => { if (content.text) { - return text ? ( - text(content.text) + return messageRenderer?.text ? ( + messageRenderer?.text(content.text) ) : ( {content.text} ); } else if (content.image) { - return image ? ( - image(content.image) + return messageRenderer?.image ? ( + messageRenderer?.image(content.image) ) : ( { const { container } = render(); expect(container.firstChild).toBeEmptyDOMElement(); }); + + it('uses text message renderer if passed', () => { + render( +
{message}
} + > + + + ); + const message = screen.getByTestId('custom-message'); + expect(message).toBeInTheDocument(); + }); + + it('uses image message renderer if passed', () => { + render( + } + > + + + ); + const message = screen.getByTestId('custom-message'); + expect(message).toBeInTheDocument(); + }); }); From b329b77eeaa7379984e2f205e9b14b1441220532 Mon Sep 17 00:00:00 2001 From: Danny Banks Date: Thu, 17 Oct 2024 15:12:07 -0700 Subject: [PATCH 4/5] chore: updating message renderer function signature --- .changeset/heavy-dots-applaud.md | 2 +- .../ai-conversation-renderer/index.page.tsx | 4 +- .../src/components/AIConversation/types.ts | 11 +++-- .../views/Controls/MessagesControl.tsx | 14 ++---- .../__tests__/MessagesControl.spec.tsx | 47 +++---------------- 5 files changed, 23 insertions(+), 55 deletions(-) diff --git a/.changeset/heavy-dots-applaud.md b/.changeset/heavy-dots-applaud.md index f013283490e..26b303ed917 100644 --- a/.changeset/heavy-dots-applaud.md +++ b/.changeset/heavy-dots-applaud.md @@ -10,7 +10,7 @@ feat(ai): add message renderer handleSendMessage={sendMessage} isLoading={isLoading} messageRenderer={{ - text: (message) => {message}, + text: ({text}) => {text}, }} /> ``` diff --git a/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx b/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx index 3cead428ccf..42409609fdd 100644 --- a/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx +++ b/examples/next/pages/ui/components/ai/ai-conversation-renderer/index.page.tsx @@ -53,8 +53,8 @@ function Chat() { isLoading={isLoading} allowAttachments messageRenderer={{ - text: (message) => {message}, - image: (image) => ( + text: ({ text }) => {text}, + image: ({ image }) => ( React.JSX.Element; - image?: (image: ImageContent) => React.JSX.Element; + text?: (input: { text: TextContent }) => React.JSX.Element; + image?: (input: { image: ImageContent }) => React.JSX.Element; } export interface Avatar { diff --git a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx index 4b1e45676bb..5de1fd3f1dc 100644 --- a/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx +++ b/packages/react-ai/src/components/AIConversation/views/Controls/MessagesControl.tsx @@ -70,7 +70,7 @@ export const MessageControl: MessageControl = ({ message }) => { {message.content.map((content, index) => { if (content.text) { return messageRenderer?.text ? ( - messageRenderer?.text(content.text) + messageRenderer.text({ text: content.text }) ) : ( {content.text} @@ -78,7 +78,7 @@ export const MessageControl: MessageControl = ({ message }) => { ); } else if (content.image) { return messageRenderer?.image ? ( - messageRenderer?.image(content.image) + messageRenderer?.image({ image: content.image }) ) : ( { +export const MessagesControl: MessagesControl = () => { const messages = React.useContext(MessagesContext); const controls = React.useContext(ControlsContext); const { getMessageTimestampText } = useConversationDisplayText(); @@ -232,9 +232,7 @@ export const MessagesControl: MessagesControl = ({ renderMessage }) => { return ( {messagesWithRenderableContent?.map((message, index) => { - return renderMessage ? ( - renderMessage(message) - ) : ( + return ( React.ReactNode; - }): JSX.Element; + (): JSX.Element; ActionsBar: ActionsBarControl; Avatar: AvatarControl; Container: AIConversationElements['View']; diff --git a/packages/react-ai/src/components/AIConversation/views/Controls/__tests__/MessagesControl.spec.tsx b/packages/react-ai/src/components/AIConversation/views/Controls/__tests__/MessagesControl.spec.tsx index 888d073a625..a054835a569 100644 --- a/packages/react-ai/src/components/AIConversation/views/Controls/__tests__/MessagesControl.spec.tsx +++ b/packages/react-ai/src/components/AIConversation/views/Controls/__tests__/MessagesControl.spec.tsx @@ -213,44 +213,6 @@ describe('MessagesControl', () => { expect(actionElements).toHaveLength(2); }); - it('renders a MessagesControl element with a custom renderMessage function', () => { - const customMessage = jest.fn((message: ConversationMessage) => ( -
- {message.content.map((content, index) => { - if (content.text) { - return

{content.text}

; - } else if (content.image) { - return ( - - ); - } - })} -
- )); - - render( - - - - ); - - expect(customMessage).toHaveBeenCalledTimes(3); - - const defaultMessageElements = screen.queryAllByTestId('message'); - expect(defaultMessageElements).toHaveLength(0); - - const customMessageElements = screen.queryAllByTestId('custom-message'); - expect(customMessageElements).toHaveLength(3); - }); - it('renders avatars and actions appropriately if the same user sends multiple messages', () => { const { rerender } = render( @@ -392,7 +354,7 @@ describe('MessageControl', () => { it('uses text message renderer if passed', () => { render(
{message}
} + text={({ text }) =>
{text}
} > @@ -404,7 +366,12 @@ describe('MessageControl', () => { it('uses image message renderer if passed', () => { render( } + image={({ image }) => ( + + )} > From 2dd9988549fb5b76002beb9d09b849f59aa6d6f7 Mon Sep 17 00:00:00 2001 From: Danny Banks Date: Thu, 17 Oct 2024 15:22:59 -0700 Subject: [PATCH 5/5] chore: updating type name to not clash --- packages/react-ai/src/components/AIConversation/types.ts | 8 ++++---- packages/react-ai/src/components/AIConversation/utils.ts | 4 ++-- packages/react-ai/src/types.ts | 6 ++++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/react-ai/src/components/AIConversation/types.ts b/packages/react-ai/src/components/AIConversation/types.ts index b32b1dc875f..1a6ccac8487 100644 --- a/packages/react-ai/src/components/AIConversation/types.ts +++ b/packages/react-ai/src/components/AIConversation/types.ts @@ -13,9 +13,9 @@ import { DisplayTextTemplate } from '@aws-amplify/ui'; import { AIConversationDisplayText } from './displayText'; import { ConversationMessage, - ImageContent, + ImageContentBlock, SendMessage, - TextContent, + TextContentBlock, } from '../../types'; import { ControlsContextProps } from './context/ControlsContext'; @@ -61,8 +61,8 @@ export interface AIConversation { export type MessageVariant = 'bubble' | 'default'; export interface MessageRenderer { - text?: (input: { text: TextContent }) => React.JSX.Element; - image?: (input: { image: ImageContent }) => React.JSX.Element; + text?: (input: { text: TextContentBlock }) => React.JSX.Element; + image?: (input: { image: ImageContentBlock }) => React.JSX.Element; } export interface Avatar { diff --git a/packages/react-ai/src/components/AIConversation/utils.ts b/packages/react-ai/src/components/AIConversation/utils.ts index d93401baa66..023cec0be22 100644 --- a/packages/react-ai/src/components/AIConversation/utils.ts +++ b/packages/react-ai/src/components/AIConversation/utils.ts @@ -1,4 +1,4 @@ -import { ImageContent } from '../../types'; +import { ImageContentBlock } from '../../types'; export function formatDate(date: Date): string { const dateString = date.toLocaleDateString('en-US', { @@ -27,7 +27,7 @@ function arrayBufferToBase64(buffer: ArrayBuffer) { export function convertBufferToBase64( buffer: ArrayBuffer, - format: ImageContent['format'] + format: ImageContentBlock['format'] ): string { let base64string = ''; // Use node-based buffer if available diff --git a/packages/react-ai/src/types.ts b/packages/react-ai/src/types.ts index ec7c108ece4..8a887c5020e 100644 --- a/packages/react-ai/src/types.ts +++ b/packages/react-ai/src/types.ts @@ -11,9 +11,11 @@ export type ConversationMessage = NonNullable< export type ConversationMessageContent = ConversationMessage['content'][number]; -export type TextContent = NonNullable; +export type TextContentBlock = NonNullable; -export type ImageContent = NonNullable; +export type ImageContentBlock = NonNullable< + ConversationMessageContent['image'] +>; // Note: the conversation sendMessage function is an overload // that accepts a string OR an object