diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53944f22..ccff1590 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,6 +19,7 @@ jobs: - name: Lint and fix run: npm run lint:fix env: + ANTHROPIC_API_KEY: ${{secrets.ANTHROPIC_API_KEY}} TURSO_DB_URL: ${{secrets.TURSO_DB_URL}} TURSO_DB_AUTH_TOKEN: ${{secrets.TURSO_DB_AUTH_TOKEN}} OPEN_AI_API_KEY: ${{secrets.OPEN_AI_API_KEY}} diff --git a/package.json b/package.json index c105e750..c9dd1707 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "react-mic": "12.4.6", "react-syntax-highlighter": "15.5.0", "react-textarea-autosize": "8.4.1", + "react-use-wake-lock": "^1.0.1", "remark-gfm": "3.0.1", "remark-math": "^6.0.0", "remark-rehype": "10.1.0", diff --git a/src/app/api/imageInput/route.ts b/src/app/api/imageInput/route.ts index 80d38012..e83fd544 100644 --- a/src/app/api/imageInput/route.ts +++ b/src/app/api/imageInput/route.ts @@ -10,6 +10,7 @@ import { NextApiResponse } from "next"; import { StreamingTextResponse, LangChainStream } from "ai"; import { systemPrompt, ellaPrompt } from "@/utils/prompts"; import { chattype } from "@/lib/types"; +// import { ChatAnthropic } from "langchain/chat_models/anthropic"; export const maxDuration = 60; // This function can run for a maximum of 5 seconds export const dynamic = "force-dynamic"; @@ -77,6 +78,7 @@ export async function POST(request: Request, response: NextApiResponse) { { status: 400 }, ); } + console.log("imageFile", imageFile); const parts = imageFile.name.split("."); const extension = parts[parts.length - 1]; let awsImageUrl = ""; @@ -153,6 +155,13 @@ export async function POST(request: Request, response: NextApiResponse) { }, ], }); + // const anthropic = new ChatAnthropic({ + // anthropicApiKey: env.ANTHROPIC_API_KEY, + // streaming: true, + // modelName: "claude-3-sonnet-20240229", + // callbacks: [handlers] + // }); + // const str = anthropic.call([...msg, message], {}, [handlers]) const str = chat .call([...msg, message], {}, [handlers]) .catch(console.error); diff --git a/src/app/env.mjs b/src/app/env.mjs index 8a64a8fd..2ec4d7ca 100644 --- a/src/app/env.mjs +++ b/src/app/env.mjs @@ -3,6 +3,8 @@ import { z } from "zod"; export const env = createEnv({ server: { + // Anthropic + ANTHROPIC_API_KEY: z.string().min(10), // OpenAI OPEN_AI_API_KEY: z.string().min(10), // LLaMA-2-7B-32K-Instruct (7B) from https://api.together.xyz @@ -62,6 +64,8 @@ export const env = createEnv({ }, runtimeEnv: { + // Anthropic + ANTHROPIC_API_KEY: process.env.ANTHROPIC_API_KEY, // Clerk (Auth) NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: process.env.NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY, diff --git a/src/app/page.tsx b/src/app/page.tsx index d92756cb..cf9f85ac 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -59,6 +59,27 @@ export default function Home() { "model", parseAsString.withDefault("chat"), ); + const [imageUrl, setImageUrl] = useQueryState( + "imageUrl", + parseAsString.withDefault(""), + ); + const [imageName, setImageName] = useQueryState( + "imageName", + parseAsString.withDefault(""), + ); + const [imageType, setImageType] = useQueryState( + "imageType", + parseAsString.withDefault(""), + ); + const [imageSize, setImageSize] = useQueryState( + "imageSize", + parseAsString.withDefault(""), + ); + const [imageExtension, setImageExtension] = useQueryState( + "imageExtension", + parseAsString.withDefault(""), + ); + const [dropzoneActive, setDropzoneActive] = useState(false); const { isSignedIn, orgId, orgSlug, userId } = useAuth(); // if (isSignedIn) { @@ -76,13 +97,31 @@ export default function Home() { try { const res = await fetch(`/api/generateNewChatId/${orgId}`, { method: "POST", - body: JSON.stringify({ type: "chat" }), + body: JSON.stringify({ type: chatType || "chat" }), }); const data = await res.json(); - router.push( - `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`, - ); + if (dropzoneActive) { + const queryParams = new URLSearchParams(window.location.search); + const params: { [key: string]: string } = {}; + queryParams.forEach((value, key) => { + params[key] = value; + }); + const params2 = { + ...params, + new: "true", + clipboard: "true", + model: chatType, + input: input, + }; + const queryParamsString = new URLSearchParams(params2).toString(); + + router.push(`/dashboard/chat/${data.newChatId}?${queryParamsString}`); + } else { + router.push( + `/dashboard/chat/${data.newChatId}?new=true&clipboard=true&model=${chatType}&input=${input}`, + ); + } } catch (error) { console.error("Error creating new chat:", error); } @@ -120,6 +159,18 @@ export default function Home() { {isSignedIn && orgId && orgSlug ? (
-
+
void; onStartListening: () => void; @@ -24,6 +25,12 @@ export default function VadAudio({ const audioChunks = useRef([]); const timerRef = useRef(null); const startTimeRef = useRef(null); + const { isSupported, isLocked, request, release } = useWakeLock({ + onError: () => { + console.error("Error requesting wake lock"); + }, + onRelease: () => {}, + }); const vad = useMicVAD({ onSpeechEnd: (audio: Float32Array) => { @@ -51,6 +58,9 @@ export default function VadAudio({ const handleStartListening = useCallback(() => { vad.start(); startTimer(); + if (isSupported) { + request(); + } onStartListening(); setIsListening(true); audioChunks.current = []; @@ -62,6 +72,9 @@ export default function VadAudio({ vad.pause(); resetDuration(); clearTimer(); + if (isSupported) { + release(); + } }, [vad]); const startTimer = () => { diff --git a/src/components/chat.tsx b/src/components/chat.tsx index 79d8f81b..73065880 100644 --- a/src/components/chat.tsx +++ b/src/components/chat.tsx @@ -15,7 +15,6 @@ import { useDropzone } from "react-dropzone"; import { X } from "lucide-react"; import { useImageState } from "@/store/tlDrawImage"; import { useQueryState } from "next-usequerystate"; -import { nanoid } from "ai"; interface ChatProps { orgId: string; @@ -49,10 +48,7 @@ export default function Chat(props: ChatProps) { const [imageUrl, setImageUrl] = useState(""); const [imageName, setImageName] = useState(""); const queryClient = useQueryClient(); - const [isNewChat, setIsNewChat] = useQueryState("new"); - const [isFromClipboard, setIsFromClipboard] = useQueryState("clipboard"); const [incomingModel] = useQueryState("model"); - const [incomingInput] = useQueryState("input"); const [chattype, setChattype] = useState( props?.type || incomingModel || "chat", ); @@ -164,25 +160,6 @@ export default function Chat(props: ChatProps) { }, sendExtraMessageFields: true, }); - console.log("messages", messages); - - useEffect(() => { - if (isNewChat === "true" && incomingInput) { - //TODO: use types for useQueryState - if (incomingInput && chattype !== "tldraw") { - const newMessage = { - id: nanoid(), - role: "user", - content: incomingInput, - name: `${props.username},${props.uid}`, - audio: "", - } as Message; - append(newMessage); - } - setIsFromClipboard("false"); - setIsNewChat("false"); - } - }, [isFromClipboard, isNewChat]); useEffect(() => { let mainArray: Message[][] = []; @@ -323,6 +300,9 @@ export default function Chat(props: ChatProps) {
)} /^image\/(jpeg|png|jpg|webp)$/.test(value); -const Schema = z.object({ +export const Schema = z.object({ imageName: z.any(), imageType: z.string().refine(isValidImageType, { message: "File type must be JPEG, PNG, or WEBP image", @@ -81,6 +83,9 @@ interface InputBarProps { dropZoneActive: boolean; onClickOpen: any; onClickOpenChatSheet: boolean | any; + getInputProps: any; + onDrop: (acceptedFiles: any) => void; + getRootProps: any; } const InputBar = (props: InputBarProps) => { @@ -97,7 +102,163 @@ const InputBar = (props: InputBarProps) => { const [disableInputs, setDisableInputs] = useState(false); const [isRagLoading, setIsRagLoading] = useState(false); const queryClient = useQueryClient(); + const [isNewChat, setIsNewChat] = useQueryState("new"); + const [isFromClipboard, setIsFromClipboard] = useQueryState("clipboard"); + const [incomingModel] = useQueryState("model"); + const [incomingInput] = useQueryState("input"); + const [chattype, setChattype] = useState( + props?.chattype || incomingModel || "chat", + ); + + const handleFirstImageMessage = useCallback(async () => { + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + const queryParams: { [key: string]: string } = {}; + params.forEach((value, key) => { + queryParams[key] = value; + }); + const ID = nanoid(); + const message: Message = { + id: ID, + role: "user", + content: incomingInput || "", + name: `${props.username},${props.userId}`, + }; + const createFileFromBlobUrl = async ( + blobUrl: string, + fileName: string, + ) => { + const response = await fetch(blobUrl); + const blob = await response.blob(); + return new File([blob], fileName, { type: blob.type }); + }; + + const imageUrl = params.get("imageUrl")!; + const imageExtension = params.get("imageExtension")!; + const file = await createFileFromBlobUrl( + imageUrl, + `image.${imageExtension}`, + ); + console.log("Created file from blob URL:", file); + const zodMessage: any = Schema.safeParse({ + imageName: params.get("imageName"), + imageType: params.get("imageType"), + imageSize: Number(params.get("imageSize")), + file: file, + value: incomingInput || "", + userId: props.userId, + orgId: props.orgId, + chatId: props.chatId, + message: [message], + id: ID, + chattype: chattype, + }); + console.log("zodMessageImage Extension:", imageExtension); + // console.log("zodmessage", zodMessage); + // console.log("dropzone", props.dropZoneActive); + console.log("zodMessage", zodMessage, imageExtension); + if (zodMessage.success) { + const zodMSG = JSON.stringify(zodMessage); + const formData = new FormData(); + formData.append("zodMessage", zodMSG); + formData.append("file", file); + setIsRagLoading(true); + const response = await fetch("/api/imageInput", { + method: "POST", + body: formData, + }); + if (response && response.status.toString().startsWith("2")) { + console.log("responce", response); + let assistantMsg = ""; + const reader = response.body?.getReader(); + console.log("reader", reader); + const decoder = new TextDecoder(); + let charsReceived = 0; + let content = ""; + reader + ?.read() + .then(async function processText({ done, value }) { + if (done) { + setIsRagLoading(false); + console.log("Stream complete"); + return; + } + charsReceived += value.length; + const chunk = decoder.decode(value, { stream: true }); + assistantMsg += chunk === "" ? `${chunk} \n` : chunk; + content += chunk === "" ? `${chunk} \n` : chunk; + // console.log("assistMsg", assistantMsg); + props.setMessages([ + ...props.messages, + awsImageMessage, + message, + { + ...assistantMessage, + content: assistantMsg, + }, + ]); + reader.read().then(processText); + }) + .then((e) => { + setIsRagLoading(false); + console.error("error", e); + }); + const awsImageMessage = { + role: "user", + subRole: "input-image", + content: `${process.env.NEXT_PUBLIC_IMAGE_PREFIX_URL}imagefolder/${props.chatId}/${ID}.${imageExtension}`, + id: ID, + } as Message; + const assistantMessage: Message = { + id: ID, + role: "assistant", + content: content, + }; + + console.log("image chat", queryParams); + // image chat + } else { + //TODO: api thrown some error + setIsRagLoading(false); + } + } + } + }, []); + + useEffect(() => { + if (isNewChat === "true" && incomingInput) { + //TODO: use types for useQueryState + if (incomingInput && chattype !== "tldraw") { + const params = new URLSearchParams(window.location.search); + if ( + params.get("imageUrl") && + params.get("imageName") && + params.get("imageType") && + params.get("imageSize") + ) { + console.log("zodMessage", "we made to here", params); + handleFirstImageMessage(); + } else { + const newMessage = { + id: nanoid(), + role: "user", + content: incomingInput, + name: `${props.username},${props.userId}`, + audio: "", + } as Message; + props.append(newMessage); + } + } + } + setIsFromClipboard("false"); + setIsNewChat("false"); + }, [isFromClipboard, isNewChat]); // const ably = useAbly(); // console.log( @@ -316,30 +477,6 @@ const InputBar = (props: InputBarProps) => { props.setInput(""); }; - const handleAudio = async (audioFile: File) => { - setIsAudioWaveVisible(false); - setIsTranscribing(true); - const f = new FormData(); - f.append("file", audioFile); - // Buffer.from(audioFile) - console.log(audioFile); - try { - const res = await fetch("/api/transcript", { - method: "POST", - body: f, - }); - - // console.log('data', await data.json()); - const data = await res.json(); - console.log("got the data", data); - props.setInput(data.text); - setIsTranscribing(false); - } catch (err) { - console.error("got in error", err); - setIsTranscribing(false); - } - }; - const [audioId, setAudioId] = useState(0); const [transcriptHashTable, setTranscriptHashTable] = useState<{ [key: number]: string; @@ -414,7 +551,9 @@ const InputBar = (props: InputBarProps) => { if (countdown > 0) { updateStatus({ isTyping: true, - username: `Echoes is thinking (${countdown--} secs)`, + username: props?.isLoading + ? `Echoes is thinking (${countdown--} secs)` + : `Echoes is typing (${countdown--} secs)`, id: props.userId, }); } else { @@ -478,24 +617,41 @@ const InputBar = (props: InputBarProps) => { return () => clearInterval(interval); }, [isBlinking]); + + useEffect(() => { + if (!isBlinking) { + const resetTimer = setTimeout(() => { + setTranscriptHashTable({}); + }, 5000); // Reset after 5 seconds + + return () => clearTimeout(resetTimer); + } + }, [isBlinking]); + return (
{ + console.log("being dropped", acceptedFiles); + props.onDrop(acceptedFiles); + }} > + - { } chattype={props.chattype} setChatType={props.setChattype} - /> - {/* */} + /> */} + { value={ props.value + (isBlinking ? ".".repeat(displayNumber) : "") } - onChange={handleInputChange} + onChange={(e) => { + handleInputChange(e); + }} onKeyDown={(e) => { if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); @@ -605,13 +763,12 @@ const InputBar = (props: InputBarProps) => { const newAudioId = audioId + 1; setAudioId(newAudioId); setTranscriptHashTable((prev) => ({ - ...prev, [newAudioId]: props.value, })); }} onStopListening={() => { setIsBlinking(false); - setTranscriptHashTable({}); + // setTranscriptHashTable({}); setIsAudioWaveVisible(false); }} // disabled={isRecording || isTranscribing || disableInputs} diff --git a/src/components/inputBar2.tsx b/src/components/inputBar2.tsx index 4cc56ac1..8b0ddf79 100644 --- a/src/components/inputBar2.tsx +++ b/src/components/inputBar2.tsx @@ -6,21 +6,23 @@ import { Dispatch, FormEvent, SetStateAction, + useCallback, useEffect, useState, } from "react"; import { ChatRequestOptions, CreateMessage, Message, nanoid } from "ai"; -import { PaperPlaneTilt } from "@phosphor-icons/react"; +import { PaperPlaneTilt, UploadSimple } from "@phosphor-icons/react"; import { Button } from "@/components/button"; import { ChatType, chattype } from "@/lib/types"; import { useQueryClient } from "@tanstack/react-query"; import { fetchEventSource } from "@microsoft/fetch-event-source"; import z from "zod"; import { toast } from "./ui/use-toast"; -import { useImageState } from "@/store/tlDrawImage"; import ModelSwitcher from "./modelswitcher"; // import VadAudio from "./vadAudio"; import VadAudio from "./VadAudio"; +import { useDropzone } from "react-dropzone"; +import { X } from "lucide-react"; const isValidImageType = (value: string) => /^image\/(jpeg|png|jpg|webp)$/.test(value); @@ -74,22 +76,26 @@ interface InputBarProps { isLoading?: boolean; chattype?: ChatType; setChattype?: Dispatch>; - setDropzoneActive?: Dispatch>; dropZoneActive?: boolean; onClickOpen?: any; onClickOpenChatSheet?: boolean | any; isHome?: boolean; submitInput?: () => void; + imageUrl: string; + setImageUrl: Dispatch>; + imageName: string; + setImageName: Dispatch>; + imageType: string; + setImageType: Dispatch>; + imageSize: string; + setImageSize: Dispatch>; + setDropzoneActive: Dispatch>; + dropzoneActive: boolean; + imageExtension: string; + setImageExtension: Dispatch>; } const InputBar = (props: InputBarProps) => { - const { - tldrawImageUrl, - tlDrawImage, - setTlDrawImage, - settldrawImageUrl, - onClickOpenChatSheet, - } = useImageState(); const [isAudioWaveVisible, setIsAudioWaveVisible] = useState(false); const [isRecording, setIsRecording] = useState(false); const [isTranscribing, setIsTranscribing] = useState(false); @@ -97,17 +103,6 @@ const InputBar = (props: InputBarProps) => { const [isRagLoading, setIsRagLoading] = useState(false); const queryClient = useQueryClient(); - // const preferences = usePreferences(); - // const { presenceData, updateStatus } = usePresence( - // `channel_${props.chatId}`, - // { - // id: props.userId, - // username: props.username, - // isTyping: false, - // } - // ); - // using local state for development purposes - const handleSubmit = async (e: FormEvent) => { e.preventDefault(); console.log("props.value", props.value); @@ -171,8 +166,6 @@ const InputBar = (props: InputBarProps) => { ?.read() .then(async function processText({ done, value }) { if (done) { - settldrawImageUrl(""); - setTlDrawImage(""); setDisableInputs(false); setIsRagLoading(false); console.log("Stream complete"); @@ -305,30 +298,43 @@ const InputBar = (props: InputBarProps) => { props?.append?.(message as Message); props?.setInput?.(""); }; + const [image, setImage] = useState([]); - const handleAudio = async (audioFile: File) => { - setIsAudioWaveVisible(false); - setIsTranscribing(true); - const f = new FormData(); - f.append("file", audioFile); - // Buffer.from(audioFile) - console.log(audioFile); - try { - const res = await fetch("/api/transcript", { - method: "POST", - body: f, - }); - - // console.log('data', await data.json()); - const data = await res.json(); - console.log("got the data", data); - props?.setInput?.(data.text); - setIsTranscribing(false); - } catch (err) { - console.error("got in error", err); - setIsTranscribing(false); + const onDrop = useCallback(async (acceptedFiles: File[]) => { + if (acceptedFiles && acceptedFiles[0]?.type.startsWith("image/")) { + setImage(acceptedFiles); + props.setImageType(acceptedFiles[0].type); + props.setImageSize(String(acceptedFiles[0].size)); + props.setImageUrl(URL.createObjectURL(acceptedFiles[0])); + props.setImageName(JSON.stringify(acceptedFiles[0].name)); + props.setImageExtension(acceptedFiles[0].name.split(".").pop() || ""); + props.setDropzoneActive(true); + } else { + { + image + ? null + : toast({ + description: ( + 
+                  
+                    Please select a image file.
+                  
+
+ ), + }); + } } - }; + }, []); + const { getRootProps, getInputProps, open } = useDropzone({ + onDrop, + accept: { + "image/jpeg": [], + "image/png": [], + }, + maxFiles: 1, + noClick: true, + noKeyboard: true, + }); const [audioId, setAudioId] = useState(0); const [transcriptHashTable, setTranscriptHashTable] = useState<{ @@ -400,12 +406,40 @@ const InputBar = (props: InputBarProps) => { return ( -
+
+ {/* */} + {props.dropzoneActive ? ( + <> + {" "} +
{ + props.setDropzoneActive(false); //TODO: clear params + props.setImageUrl(""); + props.setImageName(""); + props.setImageType(""); + props.setImageSize(""); + props.setImageExtension(""); + const url = new URL(window.location.href); + window.history.replaceState({}, document.title, url.toString()); + }} + > + Preview + +
+ + ) : null}
{
-
+
{ setChatType={props.setChattype} isHome={props.isHome} /> +
diff --git a/src/components/room.tsx b/src/components/room.tsx index a71bb05f..c810d944 100644 --- a/src/components/room.tsx +++ b/src/components/room.tsx @@ -47,9 +47,7 @@ const RoomWrapper = (props: Props) => { username: props.username, isTyping: false, }, - (presenseUpdate) => { - console.log("presenseUpdate", presenseUpdate); - }, + (presenseUpdate) => {}, ); const dbIds = getUserIdList( props.type === "tldraw" ? props.snapShot : props.chat,