From fec93df01620960cc1319bb12df855219a903426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?th=E1=BB=8Bnh?= Date: Sat, 11 Jan 2025 19:01:32 +0700 Subject: [PATCH] qa prompt v6, extract question prompt v6, use XML to distinguish the complex data segments --- backend/models/chat.py | 36 ++++++ backend/utils/llm.py | 244 ++++++++++++++++++++++++++++++++--------- 2 files changed, 229 insertions(+), 51 deletions(-) diff --git a/backend/models/chat.py b/backend/models/chat.py index 25be8827f..a886f6eec 100644 --- a/backend/models/chat.py +++ b/backend/models/chat.py @@ -62,6 +62,42 @@ def get_sender_name(message: Message) -> str: return '\n'.join(formatted_messages) + @staticmethod + def get_messages_as_xml( + messages: List['Message'], + use_user_name_if_available: bool = False, + use_plugin_name_if_available: bool = False + ) -> str: + sorted_messages = sorted(messages, key=lambda m: m.created_at) + + def get_sender_name(message: Message) -> str: + if message.sender == 'human': + return 'User' + # elif use_plugin_name_if_available and message.plugin_id is not None: + # plugin = next((p for p in plugins if p.id == message.plugin_id), None) + # if plugin: + # return plugin.name RESTORE ME + return message.sender.upper() # TODO: use plugin id + + formatted_messages = [ + f""" + + + {message.created_at.strftime('%d %b %Y at %H:%M UTC')} + + + {get_sender_name(message)} + + + {message.text} + + + """.replace(' ', '').replace('\n\n\n', '\n\n').strip() + for message in sorted_messages + ] + + return '\n'.join(formatted_messages) + class ResponseMessage(Message): ask_for_nps: Optional[bool] = False diff --git a/backend/utils/llm.py b/backend/utils/llm.py index 61b599cb2..ab4cc6bcd 100644 --- a/backend/utils/llm.py +++ b/backend/utils/llm.py @@ -65,9 +65,9 @@ def should_discard_memory(transcript: str) -> bool: ''' You will be given a conversation transcript, and your task is to determine if the conversation is worth storing as a memory or not. It is not worth storing if there are no interesting topics, facts, or information, in that case, output discard = True. - + Transcript: ```{transcript}``` - + {format_instructions}'''.replace(' ', '').strip() ]) chain = prompt | llm_mini | parser @@ -88,13 +88,13 @@ def get_transcript_structure(transcript: str, started_at: datetime, language_cod 'system', '''You are an expert conversation analyzer. Your task is to analyze the conversation and provide structure and clarity to the recording transcription of a conversation. The conversation language is {language_code}. Use the same language {language_code} for your response. - + For the title, use the main topic of the conversation. For the overview, condense the conversation into a summary with the main topics discussed, make sure to capture the key points and important details from the conversation. - For the action items, include a list of commitments, specific tasks or actionable steps from the conversation that the user is planning to do or has to do on that specific day or in future. Remember the speaker is busy so this has to be very efficient and concise, otherwise they might miss some critical tasks. Specify which speaker is responsible for each action item. + For the action items, include a list of commitments, specific tasks or actionable steps from the conversation that the user is planning to do or has to do on that specific day or in future. Remember the speaker is busy so this has to be very efficient and concise, otherwise they might miss some critical tasks. Specify which speaker is responsible for each action item. For the category, classify the conversation into one of the available categories. For Calendar Events, include a list of events extracted from the conversation, that the user must have on his calendar. For date context, this conversation happened on {started_at}. {tz} is the user's timezone, convert it to UTC and respond in UTC. - + Transcript: ```{transcript}``` {format_instructions}'''.replace(' ', '').strip() @@ -119,7 +119,7 @@ def get_transcript_structure(transcript: str, started_at: datetime, language_cod def get_plugin_result(transcript: str, plugin: Plugin) -> str: prompt = f''' Your are an AI with the following characteristics: - Name: ${plugin.name}, + Name: ${plugin.name}, Description: ${plugin.description}, Task: ${plugin.memory_prompt} @@ -151,7 +151,7 @@ def summarize_open_glass(photos: List[MemoryPhoto]) -> Structured: For the title, use the main topic of the scenes. For the overview, condense the descriptions into a brief summary with the main topics discussed, make sure to capture the key points and important details. For the category, classify the scenes into one of the available categories. - + Photos Descriptions: ```{photos_str}``` '''.replace(' ', '').strip() return llm_mini.with_structured_output(Structured).invoke(prompt) @@ -168,7 +168,7 @@ def summarize_experience_text(text: str) -> Structured: For the title, use the main topic of the experience or thought. For the overview, condense the descriptions into a brief summary with the main topics discussed, make sure to capture the key points and important details. For the category, classify the scenes into one of the available categories. - + Text: ```{text}``` '''.replace(' ', '').strip() return llm_mini.with_structured_output(Structured).invoke(prompt) @@ -182,13 +182,13 @@ def get_memory_summary(uid: str, memories: List[Memory]) -> str: prompt = f""" You are an experienced mentor, that helps people achieve their goals and improve their lives. You are advising {user_name} right now, {facts_str} - + The following are a list of {user_name}'s conversations from today, with the transcripts and a slight summary of each, that {user_name} had during his day. {user_name} wants to get a summary of the key action items {user_name} has to take based on today's conversations. Remember {user_name} is busy so this has to be very efficient and concise. Respond in at most 50 words. - + Output your response in plain text, without markdown. No newline character and only use numbers for the action items. ``` ${conversation_history} @@ -252,8 +252,8 @@ def requires_context_v1(messages: List[Message]) -> bool: prompt = f''' Based on the current conversation your task is to determine whether the user is asking a question or a follow up question that requires context outside the conversation to be answered. Take as example: if the user is saying "Hi", "Hello", "How are you?", "Good morning", etc, the answer is False. - - Conversation History: + + Conversation History: {Message.get_messages_as_string(messages)} ''' with_parser = llm_mini.with_structured_output(RequiresContext) @@ -540,7 +540,7 @@ def _get_answer_simple_message_prompt(uid: str, messages: List[Message], plugin: plugin_info = f"Your name is: {plugin.name}, and your personality/description is '{plugin.description}'.\nMake sure to reflect your personality in your response.\n" return f""" - You are an assistant for engaging personal conversations. + You are an assistant for engaging personal conversations. You are made for {user_name}, {facts_str} Use what you know about {user_name}, to continue the conversation, feel free to ask questions, share stories, or just say hi. @@ -598,6 +598,93 @@ def _get_qa_rag_prompt(uid: str, question: str, context: str, plugin: Optional[P user_name, facts_str = get_prompt_facts(uid) facts_str = '\n'.join(facts_str.split('\n')[1:]).strip() + # Use as template (make sure it varies every time): "If I were you $user_name I would do x, y, z." + context = context.replace('\n\n', '\n').strip() + plugin_info = "" + if plugin: + plugin_info = f"Your name is: {plugin.name}, and your personality/description is '{plugin.description}'.\nMake sure to reflect your personality in your response.\n" + + # Ref: https://www.reddit.com/r/perplexity_ai/comments/1hi981d + cited_instruction = """ + - You MUST cite the most relevant that answer the question. \ + - Only cite in not , not . + - Cite in memories using [index] at the end of sentences when needed, for example "You discussed optimizing firmware with your teammate yesterday[1][2]". + - NO SPACE between the last word and the citation. + - Avoid citing irrelevant memories. + """ + + return f""" + + You are an assistant for question-answering tasks. + + + + Write an accurate, detailed, and comprehensive response to the in the most personalized way possible, using the , provided. + + + + - Refine the based on the last before answering it. + - DO NOT use the AI's message in as references to answer the Question. + - Keep the answer concise and high-quality. + - If you don't know the answer or the premise is incorrect, explain why. If the are empty or unhelpful, answer the question as well as you can with existing knowledge. + - It is EXTREMELY IMPORTANT to directly answer the question. + - Use markdown to bold text sparingly, primarily for emphasis within sentences. + {cited_instruction if cited and len(context) > 0 else ""} + {"- Regard the " if len(plugin_info) > 0 else ""}. + + + + {plugin_info} + + + + {question} + + + + {context} + + + + {Message.get_messages_as_xml(messages)} + + + + [Use the following User Facts if relevant to the ] + {facts_str.strip()} + + + + Question's timezone: {tz} + + + + Current date time in UTC: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')} + + + + """.replace(' ', '').replace('\n\n\n', '\n\n').strip() + + +def qa_rag(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, + messages: List[Message] = [], tz: Optional[str] = "UTC") -> str: + prompt = _get_qa_rag_prompt(uid, question, context, plugin, cited, messages, tz) + # print('qa_rag prompt', prompt) + return llm_medium.invoke(prompt).content + +def qa_rag_stream(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, + messages: List[Message] = [], tz: Optional[str] = "UTC", callbacks=[]) -> str: + + prompt = _get_qa_rag_prompt(uid, question, context, plugin, cited, messages, tz) + # print('qa_rag prompt', prompt) + return llm_medium_stream.invoke(prompt, {'callbacks': callbacks}).content + +def _get_qa_rag_prompt_v5(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, + messages: List[Message] = [], tz: Optional[str] = "UTC") -> str: + + user_name, facts_str = get_prompt_facts(uid) + facts_str = '\n'.join(facts_str.split('\n')[1:]).strip() + # Use as template (make sure it varies every time): "If I were you $user_name I would do x, y, z." context = context.replace('\n\n', '\n').strip() plugin_info = "" @@ -659,19 +746,6 @@ def _get_qa_rag_prompt(uid: str, question: str, context: str, plugin: Optional[P """.replace(' ', '').replace('\n\n\n', '\n\n').strip() -def qa_rag(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, - messages: List[Message] = [], tz: Optional[str] = "UTC") -> str: - prompt = _get_qa_rag_prompt(uid, question, context, plugin, cited, messages, tz) - # print('qa_rag prompt', prompt) - return llm_medium.invoke(prompt).content - -def qa_rag_stream(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, - messages: List[Message] = [], tz: Optional[str] = "UTC", callbacks=[]) -> str: - - prompt = _get_qa_rag_prompt(uid, question, context, plugin, cited, messages, tz) - # print('qa_rag prompt', prompt) - return llm_medium_stream.invoke(prompt, {'callbacks': callbacks}).content - def _get_qa_rag_prompt_v4(uid: str, question: str, context: str, plugin: Optional[Plugin] = None, cited: Optional[bool] = False, messages: List[Message] = [], tz: Optional[str] = "UTC") -> str: @@ -872,7 +946,7 @@ def qa_rag_v1(uid: str, question: str, context: str, plugin: Optional[Plugin] = plugin_info = f"Your name is: {plugin.name}, and your personality/description is '{plugin.description}'.\nMake sure to reflect your personality in your response.\n" prompt = f""" - You are an assistant for question-answering tasks. + You are an assistant for question-answering tasks. You answer question in the most personalized way possible, using the context provided. If the user is asking for advice/recommendations, you must always answer, even if there's no context at all. @@ -931,19 +1005,19 @@ def obtain_emotional_message(uid: str, memory: Memory, context: str, emotion: st user_name, facts_str = get_prompt_facts(uid) transcript = memory.get_transcript(False) prompt = f""" - You are a thoughtful and encouraging Friend. + You are a thoughtful and encouraging Friend. Your best friend is {user_name}, {facts_str} - + {user_name} just finished a conversation where {user_name} experienced {emotion}. - + You will be given the conversation transcript, and context from previous related conversations of {user_name}. - + Remember, {user_name} is feeling {emotion}. Use what you know about {user_name}, the transcript, and the related context, to help {user_name} overcome this feeling \ (if bad), or celebrate (if good), by giving advice, encouragement, support, or suggesting the best action to take. - + Make sure the message is nice and short, no more than 20 words. - + Conversation Transcript: {transcript} @@ -1059,20 +1133,20 @@ def trends_extractor(memory: Memory) -> List[Item]: You will be given a finished conversation transcript. You are responsible for extracting the topics of the conversation and classifying each one within one the following categories: {str([e.value for e in TrendEnum]).strip("[]")}. You must identify if the perception is positive or negative, and classify it as "best" or "worst". - + For the specific topics here are the options available, you must classify the topic within one of these options: - ceo_options: {", ".join(ceo_options)} - company_options: {", ".join(company_options)} - software_product_options: {", ".join(software_product_options)} - hardware_product_options: {", ".join(hardware_product_options)} - ai_product_options: {", ".join(ai_product_options)} - + For example, If you identify the topic "Tesla stock has been going up incredibly", you should output: - Category: company - Type: best - Topic: Tesla - + Conversation: {transcript} '''.replace(' ', '').strip() @@ -1110,10 +1184,10 @@ def followup_question_prompt(segments: List[TranscriptSegment]): prompt = f""" You will be given the transcript of an in-progress conversation. Your task as an engaging, fun, and curious conversationalist, is to suggest the next follow-up question to keep the conversation engaging. - + Conversation Transcript: {transcript_str} - + Output your response in plain text, without markdown. Output only the question, without context, be concise and straight to the point. """.replace(' ', '').strip() @@ -1171,6 +1245,74 @@ def extract_question_from_conversation(messages: List[Message]) -> str: if len(user_last_messages) == 0: return "" + prompt = f''' + You will be given a recent conversation between a and an . \ + The conversation may include a few messages exchanged in and partly build up the proper question. \ + Your task is to understand the and identify the question or follow-up question the user is asking. + + You MUST keep the original + + First, determine whether the user is asking a question or a follow-up question. \ + If the user is not asking a question or does not want to follow up, respond with an empty message. \ + For example, if the user says "Hi", "Hello", "How are you?", or "Good morning", the answer should be empty. + + If the contain a complete question, maintain the original version as accurately as possible. \ + Avoid adding unnecessary words. + + You will be provided with between you and the user to help you answer the question. \ + It's super IMPORTANT to refine the question to be in full context with the last messages only. + + Output a WH-question, that is, a question that starts with a WH-word, like "What", "When", "Where", "Who", "Why", "How". + + Example 1: + + + User + + According to WHOOP, my HRV this Sunday was the highest it's been in a month. Here's what I did: + + Attended an outdoor party (cold weather, talked a lot more than usual). + Smoked weed (unusual for me). + Drank lots of relaxing tea. + + Can you prioritize each activity on a 0-10 scale for how much it might have influenced my HRV? + + + + Expected output: "How should each activity (going to a party and talking a lot, smoking weed, and drinking lots of relaxing tea) be prioritized on a scale of 0-10 in terms of their impact on my HRV, considering the recent activities that led to the highest HRV this month?" + + + {Message.get_messages_as_xml(user_last_messages)} + + + + {Message.get_messages_as_xml(messages)} + + + + - today + - my day + - my week + - this week + - this day + - etc. + + '''.replace(' ', '').strip() + # print(prompt) + return llm_mini.with_structured_output(OutputQuestion).invoke(prompt).question + +def extract_question_from_conversation_v5(messages: List[Message]) -> str: + # user last messages + user_message_idx = len(messages) + for i in range(len(messages) - 1, -1, -1): + if messages[i].sender == MessageSender.ai: + break + if messages[i].sender == MessageSender.human: + user_message_idx = i + user_last_messages = messages[user_message_idx:] + if len(user_last_messages) == 0: + return "" + prompt = f''' You will be given a recent conversation within a user and an AI, \ there could be a few messages exchanged, and partly built up the proper question, \ @@ -1390,7 +1532,7 @@ def retrieve_metadata_fields_from_transcript( # TODO: ask it to use max 2 words? to have more standardization possibilities prompt = f''' - You will be given the raw transcript of a conversation, this transcript has about 20% word error rate, + You will be given the raw transcript of a conversation, this transcript has about 20% word error rate, and diarization is also made very poorly. Your task is to extract the most accurate information from the conversation in the output object indicated below. @@ -1403,7 +1545,7 @@ def retrieve_metadata_fields_from_transcript( If one says "yesterday", it means the day before today. If one says "next week", it means the next monday. Do not include dates greater than 2025. - + Conversation Transcript: ``` {transcript} @@ -1466,7 +1608,7 @@ def select_structured_filters(question: str, filters_available: dict) -> dict: prompt = f''' Based on a question asked by the user to an AI, the AI needs to search for the user information related to topics, entities, people, and dates that will help it answering. Your task is to identify the correct fields that can be related to the question and can help answering. - + You must choose for each field, only the ones available in the JSON below. Find as many as possible that can relate to the question asked. ``` @@ -1496,7 +1638,7 @@ def select_structured_filters(question: str, filters_available: dict) -> dict: def extract_question_from_transcript(uid: str, segments: List[TranscriptSegment]) -> str: user_name, facts_str = get_prompt_facts(uid) prompt = f''' - {user_name} is having a conversation. + {user_name} is having a conversation. This is what you know about {user_name}: {facts_str} @@ -1530,23 +1672,23 @@ def provide_advice_message(uid: str, segments: List[TranscriptSegment], context: prompt = f""" You are a brutally honest, very creative, sometimes funny, indefatigable personal life coach who helps people improve their own agency in life, \ - pulling in pop culture references and inspirational business and life figures from recent history, mixed in with references to recent personal memories, - to help drive the point across. - + pulling in pop culture references and inspirational business and life figures from recent history, mixed in with references to recent personal memories, + to help drive the point across. + {facts_str} - + {user_name} just had a conversation and is asking for advice on what to do next. - + In order to answer you must analyize: - The conversation transcript. - The related conversations from previous days. - The facts you know about {user_name}. - + You start all your sentences with: - "If I were you, I would do this..." - "I think you should do x..." - "I believe you need to do y..." - + Your sentences are short, to the point, and very direct, at most 20 words. MUST OUTPUT 20 words or less. @@ -1596,8 +1738,8 @@ def get_proactive_message(uid: str, plugin_prompt: str, params: [str], context: def generate_description(app_name: str, description: str) -> str: prompt = f""" - You are an AI assistant specializing in crafting detailed and engaging descriptions for apps. - You will be provided with the app's name and a brief description which might not be that good. Your task is to expand on the given information, creating a captivating and detailed app description that highlights the app's features, functionality, and benefits. + You are an AI assistant specializing in crafting detailed and engaging descriptions for apps. + You will be provided with the app's name and a brief description which might not be that good. Your task is to expand on the given information, creating a captivating and detailed app description that highlights the app's features, functionality, and benefits. The description should be concise, professional, and not more than 40 words, ensuring clarity and appeal. Respond with only the description, tailored to the app's concept and purpose. App Name: {app_name} Description: {description}