From 349539719a080caf7d0e9a313371394df466c668 Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Tue, 24 Sep 2024 14:25:11 +0530 Subject: [PATCH 1/7] init: image chat in every chat --- src/app/api/imageInput/route.ts | 8 +++++ src/app/env.mjs | 4 +++ src/components/chat.tsx | 4 +++ src/components/inputBar.tsx | 63 ++++++++++++++++++++++----------- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/src/app/api/imageInput/route.ts b/src/app/api/imageInput/route.ts index 80d38012..14c833d3 100644 --- a/src/app/api/imageInput/route.ts +++ b/src/app/api/imageInput/route.ts @@ -10,6 +10,7 @@ import { NextApiResponse } from "next"; import { StreamingTextResponse, LangChainStream } from "ai"; import { systemPrompt, ellaPrompt } from "@/utils/prompts"; import { chattype } from "@/lib/types"; +// import { ChatAnthropic } from "langchain/chat_models/anthropic"; export const maxDuration = 60; // This function can run for a maximum of 5 seconds export const dynamic = "force-dynamic"; @@ -153,6 +154,13 @@ export async function POST(request: Request, response: NextApiResponse) { }, ], }); + // const anthropic = new ChatAnthropic({ + // anthropicApiKey: env.ANTHROPIC_API_KEY, + // streaming: true, + // modelName: "claude-3-sonnet-20240229", + // callbacks: [handlers] + // }); + // const str = anthropic.call([...msg, message], {}, [handlers]) const str = chat .call([...msg, message], {}, [handlers]) .catch(console.error); diff --git a/src/app/env.mjs b/src/app/env.mjs index 8a64a8fd..2ec4d7ca 100644 --- a/src/app/env.mjs +++ b/src/app/env.mjs @@ -3,6 +3,8 @@ import { z } from "zod"; export const env = createEnv({ server: { + // Anthropic + ANTHROPIC_API_KEY: z.string().min(10), // OpenAI OPEN_AI_API_KEY: z.string().min(10), // LLaMA-2-7B-32K-Instruct (7B) from https://api.together.xyz @@ -62,6 +64,8 @@ export const env = createEnv({ }, runtimeEnv: { + // Anthropic + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, // Clerk (Auth) NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, diff --git a/src/components/chat.tsx b/src/components/chat.tsx index 79d8f81b..4a425a20 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -166,6 +166,7 @@ export default function Chat(props: ChatProps) { }); console.log("messages", messages); + //TODO: handle user incoming from dashboard when invoked a chat useEffect(() => { if (isNewChat === "true" && incomingInput) { //TODO: use types for useQueryState @@ -323,6 +324,9 @@ export default function Chat(props: ChatProps) { )} void; + getRootProps: any; } const InputBar = (props: InputBarProps) => { @@ -478,24 +481,41 @@ const InputBar = (props: InputBarProps) => { return () => clearInterval(interval); }, [isBlinking]); + + useEffect(() => { + if (!isBlinking) { + const resetTimer = setTimeout(() => { + setTranscriptHashTable({}); + }, 5000); // Reset after 5 seconds + + return () => clearTimeout(resetTimer); + } + }, [isBlinking]); + return (
{ + console.log("being dropped", acceptedFiles); + props.onDrop(acceptedFiles); + }} > + - { } chattype={props.chattype} setChatType={props.setChattype} - /> - {/* */} + /> */} + { value={ props.value + (isBlinking ? ".".repeat(displayNumber) : "") } - onChange={handleInputChange} + onChange={(e) => { + handleInputChange(e); + }} onKeyDown={(e) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); @@ -605,13 +627,12 @@ const InputBar = (props: InputBarProps) => { const newAudioId = audioId + 1; setAudioId(newAudioId); setTranscriptHashTable((prev) => ({ - ...prev, [newAudioId]: props.value, })); }} onStopListening={() => { setIsBlinking(false); - setTranscriptHashTable({}); + // setTranscriptHashTable({}); setIsAudioWaveVisible(false); }} // disabled={isRecording || isTranscribing || disableInputs} From 203cb0b632055163865e9be00b16e73136f5f093 Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Tue, 24 Sep 2024 15:09:06 +0530 Subject: [PATCH 2/7] update: build --- src/components/inputBar.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/inputBar.tsx b/src/components/inputBar.tsx index e93bc064..5ac25f4f 100644 --- a/src/components/inputBar.tsx +++ b/src/components/inputBar.tsx @@ -21,7 +21,7 @@ import z from "zod"; import { toast } from "./ui/use-toast"; import usePreferences from "@/store/userPreferences"; import { useImageState } from "@/store/tlDrawImage"; -import ModelSwitcher from "./modelswitcher"; +// import ModelSwitcher from "./modelswitcher"; // import VadAudio from "./vadAudio"; import VadAudio from "./VadAudio"; const isValidImageType = (value: string) => From 37196f640348e8a1ade67016903b1d13ff015279 Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Wed, 25 Sep 2024 10:44:20 +0530 Subject: [PATCH 3/7] update: image-chat --- .github/workflows/main.yml | 1 + package.json | 1 + src/app/api/imageInput/route.ts | 1 + src/app/page.tsx | 59 ++++++++++++- src/components/VadAudio.tsx | 4 + src/components/chat.tsx | 152 +++++++++++++++++++++++++++++--- src/components/chatcard.tsx | 1 - src/components/inputBar.tsx | 2 +- src/components/inputBar2.tsx | 149 ++++++++++++++++++++----------- src/components/room.tsx | 4 +- 10 files changed, 303 insertions(+), 71 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53944f22..ccff1590 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,7 @@ jobs: - name: Lint and fix run: npm run lint:fix env: + ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}} TURSO_DB_URL: ${{secrets.TURSO_DB_URL}} TURSO_DB_AUTH_TOKEN: ${{secrets.TURSO_DB_AUTH_TOKEN}} OPEN_AI_API_KEY: ${{secrets.OPEN_AI_API_KEY}} diff --git a/package.json b/package.json index c105e750..3b33b846 100644 --- a/package.json +++ b/package.json @@ -120,6 +120,7 @@ "terser": "^5.33.0", "tldraw": "2.0.2", "typescript": "5.0.3", + "use-stay-awake": "^0.1.7", "vaul": "0.8.0", "zod": "3.22.4", "zustand": "4.4.6" diff --git a/src/app/api/imageInput/route.ts b/src/app/api/imageInput/route.ts index 14c833d3..e83fd544 100644 --- a/src/app/api/imageInput/route.ts +++ b/src/app/api/imageInput/route.ts @@ -78,6 +78,7 @@ export async function POST(request: Request, response: NextApiResponse) { { status: 400 }, ); } + console.log("imageFile", imageFile); const parts = imageFile.name.split("."); const extension = parts[parts.length - 1]; let awsImageUrl = ""; diff --git a/src/app/page.tsx b/src/app/page.tsx index d92756cb..acc9615b 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -59,6 +59,27 @@ export default function Home() { "model", parseAsString.withDefault("chat"), ); + const [imageUrl, setImageUrl] = useQueryState( + "imageUrl", + parseAsString.withDefault(""), + ); + const [imageName, setImageName] = useQueryState( + "imageName", + parseAsString.withDefault(""), + ); + const [imageType, setImageType] = useQueryState( + "imageType", + parseAsString.withDefault(""), + ); + const [imageSize, setImageSize] = useQueryState( + "imageSize", + parseAsString.withDefault(""), + ); + const [imageExtension, setImageExtension] = useQueryState( + "imageExtension", + parseAsString.withDefault(""), + ); + const [dropzoneActive, setDropzoneActive] = useState(false); const { isSignedIn, orgId, orgSlug, userId } = useAuth(); // if (isSignedIn) { @@ -80,9 +101,27 @@ export default function Home() { }); const data = await res.json(); - router.push( - `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`, - ); + if (dropzoneActive) { + const queryParams = new URLSearchParams(window.location.search); + const params: { [key: string]: string } = {}; + queryParams.forEach((value, key) => { + params[key] = value; + }); + const params2 = { + ...params, + new: "true", + clipboard: "true", + model: chatType, + input: input, + }; + const queryParamsString = new URLSearchParams(params2).toString(); + + router.push(`/dashboard/chat/${data.newChatId}?${queryParamsString}`); + } else { + router.push( + `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`, + ); + } } catch (error) { console.error("Error creating new chat:", error); } @@ -120,6 +159,18 @@ export default function Home() { {isSignedIn && orgId && orgSlug ? (
-
+
void; @@ -24,6 +25,7 @@ export default function VadAudio({ const audioChunks = useRef([]); const timerRef = useRef(null); const startTimeRef = useRef(null); + const device = useStayAwake(); const vad = useMicVAD({ onSpeechEnd: (audio: Float32Array) => { @@ -51,6 +53,7 @@ export default function VadAudio({ const handleStartListening = useCallback(() => { vad.start(); startTimer(); + device.preventSleeping(); onStartListening(); setIsListening(true); audioChunks.current = []; @@ -62,6 +65,7 @@ export default function VadAudio({ vad.pause(); resetDuration(); clearTimer(); + device.allowSleeping(); }, [vad]); const startTimer = () => { diff --git a/src/components/chat.tsx b/src/components/chat.tsx index 4a425a20..949e9e04 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useEffect, useCallback } from "react"; import { ChatType } from "@/lib/types"; -import InputBar from "@/components/inputBar"; +import InputBar, { Schema } from "@/components/inputBar"; import { Message, useChat } from "ai/react"; import Startnewchatbutton from "@/components/startnewchatbutton"; import ChatMessageCombinator from "@/components/chatmessagecombinator"; @@ -164,25 +164,153 @@ export default function Chat(props: ChatProps) { }, sendExtraMessageFields: true, }); - console.log("messages", messages); + + const handleFirstImageMessage = useCallback(async () => { + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + const queryParams: { [key: string]: string } = {}; + params.forEach((value, key) => { + queryParams[key] = value; + }); + const ID = nanoid(); + const imageMessasgeId = nanoid(); + const message: Message = { + id: ID, + role: "user", + content: incomingInput || "", + name: `${props.username},${props.uid}`, + audio: "", + }; + const createFileFromBlobUrl = async ( + blobUrl: string, + fileName: string, + ) => { + const response = await fetch(blobUrl); + const blob = await response.blob(); + return new File([blob], fileName, { type: blob.type }); + }; + + const imageUrl = params.get("imageUrl")!; + const imageName = params.get("imageName")!; + const imageExtension = params.get("imageExtension")!; + + const file = await createFileFromBlobUrl( + imageUrl, + `image.${imageExtension}`, + ); + console.log("Created file from blob URL:", file); + const zodMessage: any = Schema.safeParse({ + imageName: params.get("imageName"), + imageType: params.get("imageType"), + imageSize: Number(params.get("imageSize")), + file: file, + value: input, + userId: props.uid, + orgId: props.orgId, + chatId: props.chatId, + message: [message], + id: ID, + chattype: chattype, + }); + console.log("zodMessageImage Extension:", imageExtension); + // console.log("zodmessage", zodMessage); + // console.log("dropzone", props.dropZoneActive); + console.log("zodMessage", zodMessage, imageExtension); + if (zodMessage.success) { + const zodMSG = JSON.stringify(zodMessage); + const formData = new FormData(); + formData.append("zodMessage", zodMSG); + formData.append("file", file); + const response = await fetch("/api/imageInput", { + method: "POST", + body: formData, + }); + if (response) { + console.log("responce", response); + let assistantMsg = ""; + const reader = response.body?.getReader(); + console.log("reader", reader); + const decoder = new TextDecoder(); + let charsReceived = 0; + let content = ""; + reader + ?.read() + .then(async function processText({ done, value }) { + if (done) { + console.log("Stream complete"); + return; + } + charsReceived += value.length; + const chunk = decoder.decode(value, { stream: true }); + assistantMsg += chunk === "" ? `${chunk} \n` : chunk; + content += chunk === "" ? `${chunk} \n` : chunk; + // console.log("assistMsg", assistantMsg); + setMessages([ + ...messages, + awsImageMessage, + message, + { + ...assistantMessage, + content: assistantMsg, + }, + ]); + reader.read().then(processText); + }) + .then((e) => { + console.error("error", e); + }); + const awsImageMessage = { + role: "user", + subRole: "input-image", + content: `${process.env.NEXT_PUBLIC_IMAGE_PREFIX_URL}imagefolder/${props.chatId}/${ID}.${imageExtension}`, + id: ID, + } as Message; + const assistantMessage: Message = { + id: ID, + role: "assistant", + content: content, + }; + + console.log("image chat", queryParams); + // image chat + } + } + } + }, []); //TODO: handle user incoming from dashboard when invoked a chat useEffect(() => { if (isNewChat === "true" && incomingInput) { //TODO: use types for useQueryState if (incomingInput && chattype !== "tldraw") { - const newMessage = { - id: nanoid(), - role: "user", - content: incomingInput, - name: `${props.username},${props.uid}`, - audio: "", - } as Message; - append(newMessage); + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + console.log("zodMessage", "we made to here", params); + handleFirstImageMessage(); + } else { + const newMessage = { + id: nanoid(), + role: "user", + content: incomingInput, + name: `${props.username},${props.uid}`, + audio: "", + } as Message; + append(newMessage); + } } - setIsFromClipboard("false"); - setIsNewChat("false"); } + setIsFromClipboard("false"); + setIsNewChat("false"); }, [isFromClipboard, isNewChat]); useEffect(() => { diff --git a/src/components/chatcard.tsx b/src/components/chatcard.tsx index 1d737f83..622bf7d0 100644 --- a/src/components/chatcard.tsx +++ b/src/components/chatcard.tsx @@ -53,7 +53,6 @@ const Chatcard = ({ const chatlog = JSON.parse(chat.messages as string) as ChatLog; console.log("chatlog", chatlog.log); const msgs = chatlog.log as ChatEntry[]; - console.log("messages", msgs); const chats = msgs.slice(0, 2); const res = await fetch(`/api/generateTitle/${chat.id}/${org_id}`, { method: "POST", diff --git a/src/components/inputBar.tsx b/src/components/inputBar.tsx index 5ac25f4f..d8a79eb8 100644 --- a/src/components/inputBar.tsx +++ b/src/components/inputBar.tsx @@ -27,7 +27,7 @@ import VadAudio from "./VadAudio"; const isValidImageType = (value: string) => /^image\/(jpeg|png|jpg|webp)$/.test(value); -const Schema = z.object({ +export const Schema = z.object({ imageName: z.any(), imageType: z.string().refine(isValidImageType, { message: "File type must be JPEG, PNG, or WEBP image", diff --git a/src/components/inputBar2.tsx b/src/components/inputBar2.tsx index 4cc56ac1..8b0ddf79 100644 --- a/src/components/inputBar2.tsx +++ b/src/components/inputBar2.tsx @@ -6,21 +6,23 @@ import { Dispatch, FormEvent, SetStateAction, + useCallback, useEffect, useState, } from "react"; import { ChatRequestOptions, CreateMessage, Message, nanoid } from "ai"; -import { PaperPlaneTilt } from "@phosphor-icons/react"; +import { PaperPlaneTilt, UploadSimple } from "@phosphor-icons/react"; import { Button } from "@/components/button"; import { ChatType, chattype } from "@/lib/types"; import { useQueryClient } from "@tanstack/react-query"; import { fetchEventSource } from "@microsoft/fetch-event-source"; import z from "zod"; import { toast } from "./ui/use-toast"; -import { useImageState } from "@/store/tlDrawImage"; import ModelSwitcher from "./modelswitcher"; // import VadAudio from "./vadAudio"; import VadAudio from "./VadAudio"; +import { useDropzone } from "react-dropzone"; +import { X } from "lucide-react"; const isValidImageType = (value: string) => /^image\/(jpeg|png|jpg|webp)$/.test(value); @@ -74,22 +76,26 @@ interface InputBarProps { isLoading?: boolean; chattype?: ChatType; setChattype?: Dispatch>; - setDropzoneActive?: Dispatch>; dropZoneActive?: boolean; onClickOpen?: any; onClickOpenChatSheet?: boolean | any; isHome?: boolean; submitInput?: () => void; + imageUrl: string; + setImageUrl: Dispatch>; + imageName: string; + setImageName: Dispatch>; + imageType: string; + setImageType: Dispatch>; + imageSize: string; + setImageSize: Dispatch>; + setDropzoneActive: Dispatch>; + dropzoneActive: boolean; + imageExtension: string; + setImageExtension: Dispatch>; } const InputBar = (props: InputBarProps) => { - const { - tldrawImageUrl, - tlDrawImage, - setTlDrawImage, - settldrawImageUrl, - onClickOpenChatSheet, - } = useImageState(); const [isAudioWaveVisible, setIsAudioWaveVisible] = useState(false); const [isRecording, setIsRecording] = useState(false); const [isTranscribing, setIsTranscribing] = useState(false); @@ -97,17 +103,6 @@ const InputBar = (props: InputBarProps) => { const [isRagLoading, setIsRagLoading] = useState(false); const queryClient = useQueryClient(); - // const preferences = usePreferences(); - // const { presenceData, updateStatus } = usePresence( - // `channel_${props.chatId}`, - // { - // id: props.userId, - // username: props.username, - // isTyping: false, - // } - // ); - // using local state for development purposes - const handleSubmit = async (e: FormEvent) => { e.preventDefault(); console.log("props.value", props.value); @@ -171,8 +166,6 @@ const InputBar = (props: InputBarProps) => { ?.read() .then(async function processText({ done, value }) { if (done) { - settldrawImageUrl(""); - setTlDrawImage(""); setDisableInputs(false); setIsRagLoading(false); console.log("Stream complete"); @@ -305,30 +298,43 @@ const InputBar = (props: InputBarProps) => { props?.append?.(message as Message); props?.setInput?.(""); }; + const [image, setImage] = useState([]); - const handleAudio = async (audioFile: File) => { - setIsAudioWaveVisible(false); - setIsTranscribing(true); - const f = new FormData(); - f.append("file", audioFile); - // Buffer.from(audioFile) - console.log(audioFile); - try { - const res = await fetch("/api/transcript", { - method: "POST", - body: f, - }); - - // console.log('data', await data.json()); - const data = await res.json(); - console.log("got the data", data); - props?.setInput?.(data.text); - setIsTranscribing(false); - } catch (err) { - console.error("got in error", err); - setIsTranscribing(false); + const onDrop = useCallback(async (acceptedFiles: File[]) => { + if (acceptedFiles && acceptedFiles[0]?.type.startsWith("image/")) { + setImage(acceptedFiles); + props.setImageType(acceptedFiles[0].type); + props.setImageSize(String(acceptedFiles[0].size)); + props.setImageUrl(URL.createObjectURL(acceptedFiles[0])); + props.setImageName(JSON.stringify(acceptedFiles[0].name)); + props.setImageExtension(acceptedFiles[0].name.split(".").pop() || ""); + props.setDropzoneActive(true); + } else { + { + image + ? null + : toast({ + description: ( + 
+                  
+                    Please select a image file.
+                  
+
+ ), + }); + } } - }; + }, []); + const { getRootProps, getInputProps, open } = useDropzone({ + onDrop, + accept: { + "image/jpeg": [], + "image/png": [], + }, + maxFiles: 1, + noClick: true, + noKeyboard: true, + }); const [audioId, setAudioId] = useState(0); const [transcriptHashTable, setTranscriptHashTable] = useState<{ @@ -400,12 +406,40 @@ const InputBar = (props: InputBarProps) => { return ( -
+
+ {/* */} + {props.dropzoneActive ? ( + <> + {" "} +
{ + props.setDropzoneActive(false); //TODO: clear params + props.setImageUrl(""); + props.setImageName(""); + props.setImageType(""); + props.setImageSize(""); + props.setImageExtension(""); + const url = new URL(window.location.href); + window.history.replaceState({}, document.title, url.toString()); + }} + > + Preview + +
+ + ) : null}
{
-
+
{ setChatType={props.setChattype} isHome={props.isHome} /> +
diff --git a/src/components/room.tsx b/src/components/room.tsx index a71bb05f..c810d944 100644 --- a/src/components/room.tsx +++ b/src/components/room.tsx @@ -47,9 +47,7 @@ const RoomWrapper = (props: Props) => { username: props.username, isTyping: false, }, - (presenseUpdate) => { - console.log("presenseUpdate", presenseUpdate); - }, + (presenseUpdate) => {}, ); const dbIds = getUserIdList( props.type === "tldraw" ? props.snapShot : props.chat, From c4e4ff80c4c8574cac77e6391dbcc8b698d1e0bc Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Wed, 25 Sep 2024 10:55:28 +0530 Subject: [PATCH 4/7] update: build issues --- src/components/chat.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/chat.tsx b/src/components/chat.tsx index 949e9e04..1fc3feca 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -184,7 +184,6 @@ export default function Chat(props: ChatProps) { role: "user", content: incomingInput || "", name: `${props.username},${props.uid}`, - audio: "", }; const createFileFromBlobUrl = async ( blobUrl: string, From ecc4c2aca640634a4bd4a15eb146b50e751d46a3 Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Wed, 25 Sep 2024 11:23:08 +0530 Subject: [PATCH 5/7] update: wake-lock --- package.json | 2 +- src/components/VadAudio.tsx | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3b33b846..58eede04 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "react-intersection-observer": "9.5.2", "react-markdown": "8.0.7", "react-mic": "12.4.6", + "react-screen-wake-lock": "^3.0.2", "react-syntax-highlighter": "15.5.0", "react-textarea-autosize": "8.4.1", "remark-gfm": "3.0.1", @@ -120,7 +121,6 @@ "terser": "^5.33.0", "tldraw": "2.0.2", "typescript": "5.0.3", - "use-stay-awake": "^0.1.7", "vaul": "0.8.0", "zod": "3.22.4", "zustand": "4.4.6" diff --git a/src/components/VadAudio.tsx b/src/components/VadAudio.tsx index d5464e6f..29dd51d3 100644 --- a/src/components/VadAudio.tsx +++ b/src/components/VadAudio.tsx @@ -5,8 +5,8 @@ import { useMicVAD, utils } from "@ricky0123/vad-react"; import { Microphone, StopCircle } from "@phosphor-icons/react"; import { Button } from "@/components/button"; import { cn } from "@/lib/utils"; -import useStayAwake from "use-stay-awake"; +import { useWakeLock } from "react-screen-wake-lock"; interface VadAudioProps { onAudioCapture: (audioFile: File) => void; onStartListening: () => void; @@ -25,7 +25,13 @@ export default function VadAudio({ const audioChunks = useRef([]); const timerRef = useRef(null); const startTimeRef = useRef(null); - const device = useStayAwake(); + const { isSupported, released, request, release } = useWakeLock({ + onRequest: () => {}, + onError: () => { + console.error("Error requesting wake lock"); + }, + onRelease: () => {}, + }); const vad = useMicVAD({ onSpeechEnd: (audio: Float32Array) => { @@ -53,7 +59,7 @@ export default function VadAudio({ const handleStartListening = useCallback(() => { vad.start(); startTimer(); - device.preventSleeping(); + request(); onStartListening(); setIsListening(true); audioChunks.current = []; @@ -65,7 +71,7 @@ export default function VadAudio({ vad.pause(); resetDuration(); clearTimer(); - device.allowSleeping(); + release(); }, [vad]); const startTimer = () => { From 1932c27b4cd003099e5f32625e60ba1cc363b809 Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Wed, 25 Sep 2024 11:42:50 +0530 Subject: [PATCH 6/7] update: change wake-lock lib --- package.json | 2 +- src/components/VadAudio.tsx | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 58eede04..c9dd1707 100644 --- a/package.json +++ b/package.json @@ -109,9 +109,9 @@ "react-intersection-observer": "9.5.2", "react-markdown": "8.0.7", "react-mic": "12.4.6", - "react-screen-wake-lock": "^3.0.2", "react-syntax-highlighter": "15.5.0", "react-textarea-autosize": "8.4.1", + "react-use-wake-lock": "^1.0.1", "remark-gfm": "3.0.1", "remark-math": "^6.0.0", "remark-rehype": "10.1.0", diff --git a/src/components/VadAudio.tsx b/src/components/VadAudio.tsx index 29dd51d3..15fe8967 100644 --- a/src/components/VadAudio.tsx +++ b/src/components/VadAudio.tsx @@ -6,7 +6,7 @@ import { Microphone, StopCircle } from "@phosphor-icons/react"; import { Button } from "@/components/button"; import { cn } from "@/lib/utils"; -import { useWakeLock } from "react-screen-wake-lock"; +import useWakeLock from "react-use-wake-lock"; interface VadAudioProps { onAudioCapture: (audioFile: File) => void; onStartListening: () => void; @@ -25,8 +25,7 @@ export default function VadAudio({ const audioChunks = useRef([]); const timerRef = useRef(null); const startTimeRef = useRef(null); - const { isSupported, released, request, release } = useWakeLock({ - onRequest: () => {}, + const { isSupported, isLocked, request, release } = useWakeLock({ onError: () => { console.error("Error requesting wake lock"); }, @@ -59,7 +58,9 @@ export default function VadAudio({ const handleStartListening = useCallback(() => { vad.start(); startTimer(); - request(); + if (isSupported) { + request(); + } onStartListening(); setIsListening(true); audioChunks.current = []; @@ -71,7 +72,9 @@ export default function VadAudio({ vad.pause(); resetDuration(); clearTimer(); - release(); + if (isSupported) { + release(); + } }, [vad]); const startTimer = () => { From 6513171413bea49170dce34f734e22d47cc14e5a Mon Sep 17 00:00:00 2001 From: Prince Baghel Date: Wed, 25 Sep 2024 12:23:17 +0530 Subject: [PATCH 7/7] patch: image-chat --- src/app/page.tsx | 2 +- src/components/chat.tsx | 153 +---------------------------- src/components/inputBar.tsx | 186 +++++++++++++++++++++++++++++++----- 3 files changed, 163 insertions(+), 178 deletions(-) diff --git a/src/app/page.tsx b/src/app/page.tsx index acc9615b..cf9f85ac 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -97,7 +97,7 @@ export default function Home() { try { const res = await fetch(`/api/generateNewChatId/${orgId}`, { method: "POST", - body: JSON.stringify({ type: "chat" }), + body: JSON.stringify({ type: chatType || "chat" }), }); const data = await res.json(); diff --git a/src/components/chat.tsx b/src/components/chat.tsx index 1fc3feca..73065880 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -1,7 +1,7 @@ "use client"; import { useState, useEffect, useCallback } from "react"; import { ChatType } from "@/lib/types"; -import InputBar, { Schema } from "@/components/inputBar"; +import InputBar from "@/components/inputBar"; import { Message, useChat } from "ai/react"; import Startnewchatbutton from "@/components/startnewchatbutton"; import ChatMessageCombinator from "@/components/chatmessagecombinator"; @@ -15,7 +15,6 @@ import { useDropzone } from "react-dropzone"; import { X } from "lucide-react"; import { useImageState } from "@/store/tlDrawImage"; import { useQueryState } from "next-usequerystate"; -import { nanoid } from "ai"; interface ChatProps { orgId: string; @@ -49,10 +48,7 @@ export default function Chat(props: ChatProps) { const [imageUrl, setImageUrl] = useState(""); const [imageName, setImageName] = useState(""); const queryClient = useQueryClient(); - const [isNewChat, setIsNewChat] = useQueryState("new"); - const [isFromClipboard, setIsFromClipboard] = useQueryState("clipboard"); const [incomingModel] = useQueryState("model"); - const [incomingInput] = useQueryState("input"); const [chattype, setChattype] = useState( props?.type || incomingModel || "chat", ); @@ -165,153 +161,6 @@ export default function Chat(props: ChatProps) { sendExtraMessageFields: true, }); - const handleFirstImageMessage = useCallback(async () => { - const params = new URLSearchParams(window.location.search); - if ( - params.get("imageUrl") && - params.get("imageName") && - params.get("imageType") && - params.get("imageSize") - ) { - const queryParams: { [key: string]: string } = {}; - params.forEach((value, key) => { - queryParams[key] = value; - }); - const ID = nanoid(); - const imageMessasgeId = nanoid(); - const message: Message = { - id: ID, - role: "user", - content: incomingInput || "", - name: `${props.username},${props.uid}`, - }; - const createFileFromBlobUrl = async ( - blobUrl: string, - fileName: string, - ) => { - const response = await fetch(blobUrl); - const blob = await response.blob(); - return new File([blob], fileName, { type: blob.type }); - }; - - const imageUrl = params.get("imageUrl")!; - const imageName = params.get("imageName")!; - const imageExtension = params.get("imageExtension")!; - - const file = await createFileFromBlobUrl( - imageUrl, - `image.${imageExtension}`, - ); - console.log("Created file from blob URL:", file); - const zodMessage: any = Schema.safeParse({ - imageName: params.get("imageName"), - imageType: params.get("imageType"), - imageSize: Number(params.get("imageSize")), - file: file, - value: input, - userId: props.uid, - orgId: props.orgId, - chatId: props.chatId, - message: [message], - id: ID, - chattype: chattype, - }); - console.log("zodMessageImage Extension:", imageExtension); - // console.log("zodmessage", zodMessage); - // console.log("dropzone", props.dropZoneActive); - console.log("zodMessage", zodMessage, imageExtension); - if (zodMessage.success) { - const zodMSG = JSON.stringify(zodMessage); - const formData = new FormData(); - formData.append("zodMessage", zodMSG); - formData.append("file", file); - const response = await fetch("/api/imageInput", { - method: "POST", - body: formData, - }); - if (response) { - console.log("responce", response); - let assistantMsg = ""; - const reader = response.body?.getReader(); - console.log("reader", reader); - const decoder = new TextDecoder(); - let charsReceived = 0; - let content = ""; - reader - ?.read() - .then(async function processText({ done, value }) { - if (done) { - console.log("Stream complete"); - return; - } - charsReceived += value.length; - const chunk = decoder.decode(value, { stream: true }); - assistantMsg += chunk === "" ? `${chunk} \n` : chunk; - content += chunk === "" ? `${chunk} \n` : chunk; - // console.log("assistMsg", assistantMsg); - setMessages([ - ...messages, - awsImageMessage, - message, - { - ...assistantMessage, - content: assistantMsg, - }, - ]); - reader.read().then(processText); - }) - .then((e) => { - console.error("error", e); - }); - const awsImageMessage = { - role: "user", - subRole: "input-image", - content: `${process.env.NEXT_PUBLIC_IMAGE_PREFIX_URL}imagefolder/${props.chatId}/${ID}.${imageExtension}`, - id: ID, - } as Message; - const assistantMessage: Message = { - id: ID, - role: "assistant", - content: content, - }; - - console.log("image chat", queryParams); - // image chat - } - } - } - }, []); - - //TODO: handle user incoming from dashboard when invoked a chat - useEffect(() => { - if (isNewChat === "true" && incomingInput) { - //TODO: use types for useQueryState - if (incomingInput && chattype !== "tldraw") { - const params = new URLSearchParams(window.location.search); - if ( - params.get("imageUrl") && - params.get("imageName") && - params.get("imageType") && - params.get("imageSize") - ) { - console.log("zodMessage", "we made to here", params); - handleFirstImageMessage(); - } else { - const newMessage = { - id: nanoid(), - role: "user", - content: incomingInput, - name: `${props.username},${props.uid}`, - audio: "", - } as Message; - append(newMessage); - } - } - } - setIsFromClipboard("false"); - setIsNewChat("false"); - }, [isFromClipboard, isNewChat]); - useEffect(() => { let mainArray: Message[][] = []; let subarray: Message[] = []; diff --git a/src/components/inputBar.tsx b/src/components/inputBar.tsx index d8a79eb8..052b64e0 100644 --- a/src/components/inputBar.tsx +++ b/src/components/inputBar.tsx @@ -6,6 +6,7 @@ import { Dispatch, FormEvent, SetStateAction, + useCallback, useEffect, useState, } from "react"; @@ -24,6 +25,7 @@ import { useImageState } from "@/store/tlDrawImage"; // import ModelSwitcher from "./modelswitcher"; // import VadAudio from "./vadAudio"; import VadAudio from "./VadAudio"; +import { useQueryState } from "next-usequerystate"; const isValidImageType = (value: string) => /^image\/(jpeg|png|jpg|webp)$/.test(value); @@ -100,7 +102,163 @@ const InputBar = (props: InputBarProps) => { const [disableInputs, setDisableInputs] = useState(false); const [isRagLoading, setIsRagLoading] = useState(false); const queryClient = useQueryClient(); + const [isNewChat, setIsNewChat] = useQueryState("new"); + const [isFromClipboard, setIsFromClipboard] = useQueryState("clipboard"); + const [incomingModel] = useQueryState("model"); + const [incomingInput] = useQueryState("input"); + const [chattype, setChattype] = useState( + props?.chattype || incomingModel || "chat", + ); + + const handleFirstImageMessage = useCallback(async () => { + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + const queryParams: { [key: string]: string } = {}; + params.forEach((value, key) => { + queryParams[key] = value; + }); + const ID = nanoid(); + const message: Message = { + id: ID, + role: "user", + content: incomingInput || "", + name: `${props.username},${props.userId}`, + }; + const createFileFromBlobUrl = async ( + blobUrl: string, + fileName: string, + ) => { + const response = await fetch(blobUrl); + const blob = await response.blob(); + return new File([blob], fileName, { type: blob.type }); + }; + + const imageUrl = params.get("imageUrl")!; + const imageExtension = params.get("imageExtension")!; + + const file = await createFileFromBlobUrl( + imageUrl, + `image.${imageExtension}`, + ); + console.log("Created file from blob URL:", file); + const zodMessage: any = Schema.safeParse({ + imageName: params.get("imageName"), + imageType: params.get("imageType"), + imageSize: Number(params.get("imageSize")), + file: file, + value: incomingInput || "", + userId: props.userId, + orgId: props.orgId, + chatId: props.chatId, + message: [message], + id: ID, + chattype: chattype, + }); + console.log("zodMessageImage Extension:", imageExtension); + // console.log("zodmessage", zodMessage); + // console.log("dropzone", props.dropZoneActive); + console.log("zodMessage", zodMessage, imageExtension); + if (zodMessage.success) { + const zodMSG = JSON.stringify(zodMessage); + const formData = new FormData(); + formData.append("zodMessage", zodMSG); + formData.append("file", file); + setIsRagLoading(true); + const response = await fetch("/api/imageInput", { + method: "POST", + body: formData, + }); + if (response && response.status.toString().startsWith("2")) { + console.log("responce", response); + let assistantMsg = ""; + const reader = response.body?.getReader(); + console.log("reader", reader); + const decoder = new TextDecoder(); + let charsReceived = 0; + let content = ""; + reader + ?.read() + .then(async function processText({ done, value }) { + if (done) { + setIsRagLoading(false); + console.log("Stream complete"); + return; + } + charsReceived += value.length; + const chunk = decoder.decode(value, { stream: true }); + assistantMsg += chunk === "" ? `${chunk} \n` : chunk; + content += chunk === "" ? `${chunk} \n` : chunk; + // console.log("assistMsg", assistantMsg); + props.setMessages([ + ...props.messages, + awsImageMessage, + message, + { + ...assistantMessage, + content: assistantMsg, + }, + ]); + reader.read().then(processText); + }) + .then((e) => { + setIsRagLoading(false); + console.error("error", e); + }); + const awsImageMessage = { + role: "user", + subRole: "input-image", + content: `${process.env.NEXT_PUBLIC_IMAGE_PREFIX_URL}imagefolder/${props.chatId}/${ID}.${imageExtension}`, + id: ID, + } as Message; + const assistantMessage: Message = { + id: ID, + role: "assistant", + content: content, + }; + console.log("image chat", queryParams); + // image chat + } else { + //TODO: api thrown some error + setIsRagLoading(false); + } + } + } + }, []); + + useEffect(() => { + if (isNewChat === "true" && incomingInput) { + //TODO: use types for useQueryState + if (incomingInput && chattype !== "tldraw") { + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + console.log("zodMessage", "we made to here", params); + handleFirstImageMessage(); + } else { + const newMessage = { + id: nanoid(), + role: "user", + content: incomingInput, + name: `${props.username},${props.userId}`, + audio: "", + } as Message; + props.append(newMessage); + } + } + } + setIsFromClipboard("false"); + setIsNewChat("false"); + }, [isFromClipboard, isNewChat]); // const ably = useAbly(); // console.log( @@ -319,30 +477,6 @@ const InputBar = (props: InputBarProps) => { props.setInput(""); }; - const handleAudio = async (audioFile: File) => { - setIsAudioWaveVisible(false); - setIsTranscribing(true); - const f = new FormData(); - f.append("file", audioFile); - // Buffer.from(audioFile) - console.log(audioFile); - try { - const res = await fetch("/api/transcript", { - method: "POST", - body: f, - }); - - // console.log('data', await data.json()); - const data = await res.json(); - console.log("got the data", data); - props.setInput(data.text); - setIsTranscribing(false); - } catch (err) { - console.error("got in error", err); - setIsTranscribing(false); - } - }; - const [audioId, setAudioId] = useState(0); const [transcriptHashTable, setTranscriptHashTable] = useState<{ [key: number]: string; @@ -417,7 +551,9 @@ const InputBar = (props: InputBarProps) => { if (countdown > 0) { updateStatus({ isTyping: true, - username: `Echoes is thinking (${countdown--} secs)`, + username: props?.isLoading + ? `Echoes is thinking (${countdown--} secs)` + : `Echoes is typing (${countdown--} secs)`, id: props.userId, }); } else {