Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Incomplete response after tool usage #1126

Open
Ghat0tkach opened this issue Sep 14, 2024 · 2 comments
Open

Incomplete response after tool usage #1126

Ghat0tkach opened this issue Sep 14, 2024 · 2 comments

Comments

@Ghat0tkach
Copy link

Hi,
I’m using the Phidata assistant with two tools: search_knowledge_base and duckduckgo. I have used the Auto-RAG cookbook example with structured json output and am facing the following issue: I am getting incomplete responses. Even after a successful tool call, the LLM provides incomplete responses. I haven’t set a max token usage either.

Here is a detailed log of the messages

[
    Message(
        role="system",
        content='You are friendly assisntant\n</instructions>\n\nYou also have access to memory from previous interactions with the user but the user has no memories yet.\nIf the user asks about memories, you can let them know that you dont have any memory about the yet, but can add new memories using the `update_memory` tool.\nIf you use the `update_memory` tool, remember to pass on the response to the user.\n\n\nProvide your output as a JSON containing the following fields:\n<json_fields>\n["question", "answer", "table"]\n</json_fields>\nHere are the properties for each field:\n<json_field_properties>\n{\n  "question": {\n    "type": "string"\n  },\n  "answer": {\n    "type": "string"\n  },\n  "table": {\n    "anyOf": [\n      {\n        "type": "string"\n      },\n      {\n        "type": "null"\n      }\n    ],\n    "default": null\n  }\n}\n</json_field_properties>\nStart your response with `{` and end it with `}`.\nYour output will be passed to json.loads() to convert it to a Python object.\nMake sure it only contains valid JSON.',
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=None,
        metrics={},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="user",
        content="/ask can I use honey water to dissolve lye",
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=None,
        metrics={},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="assistant",
        content='\n{ "question": "Can I use honey water to dissolve lye?", "answer": "Using honey water directly to dissolve lye is generally not recommended because it can lead to unwanted reactions and make the lye solution less predictable for soap making. Instead, follow these practical steps:" \n    ,"table": null \n}\n',
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=None,
        metrics={},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="user",
        content="Can you explain the practical steps to dissolve lye properly?",
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=None,
        metrics={},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="assistant",
        content=None,
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=[
            {
                "id": "call_xxxxxxXMI",
                "function": {
                    "arguments": '{"query":"steps to dissolve lye properly"}',
                    "name": "search_knowledge_base",
                },
                "type": "function",
            }
        ],
        metrics={
            "time": 0.6676745410077274,
            "prompt_tokens": 1723,
            "input_tokens": 1723,
            "completion_tokens": 21,
            "output_tokens": 21,
            "total_tokens": 1744,
        },
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="tool",
        content='[search_knowledge_base_answer]',
        name=None,
        tool_call_id="call_xxxxxxXMI",
        tool_call_name="search_knowledge_base",
        tool_call_error=False,
        tool_calls=None,
        metrics={"time": 0.7187862130813301},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="assistant",
        content=None,
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=[
            {
                "id": "call_xxxxxInKc",
                "function": {
                    "arguments": '{"query":"steps to dissolve lye properly","max_results":5}',
                    "name": "duckduckgo_search",
                },
                "type": "function",
            }
        ],
        metrics={
            "time": 0.6753155048936605,
            "prompt_tokens": 2132,
            "input_tokens": 2132,
            "completion_tokens": 27,
            "output_tokens": 27,
            "total_tokens": 2159,
        },
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="tool",
        content='[\n  {\n    "title": "Soap Making Safety Guidelines and How to Safely Handle Lye",\n    "href": "https://lovelygreens.com/soap-making-safety-lye/",\n    "body": "Use lye and heat-resistant equipment. Make soap on a surface that won\'t react with lye. Keep paper towels and kitchen spray on hand. For larger spills, keep a bag of non-clumping kitty litter on hand. When making soap, work near a sink so that water is always available. Mix the lye solution in a well-ventilated place."\n  },\n  {\n    "title": "How to Neutralize Lye Spills When Making Soap (And It\'s Not With ...",\n    "href": "https://angelapalmer.com/how-to-safely-neutralize-lye-and-handle/",\n    "body": "Learn how to neutralize lye spills when making soap, safely. This article is part 3 of the Farm Girl\'s Making Soap Safely Series. Also read Part 1: 8 Common Questions About Making Soap with Lye and Part 2: 7 Must-Know Lye Safety Rules. One of the more serious things we have to deal with while"\n  },\n  {\n    "title": "How to Make a Lye Solution for Cold Process Soap | The Spruce Crafts",\n    "href": "https://www.thesprucecrafts.com/cold-process-soap-lye-water-solution-516815",\n    "body": "Add the Lye to the Water. If your soap recipe calls for adding sugar or salt to the lye solution, add it to the water before you mix in the lye. Stir the water well to make sure the sugar or salt is completely dissolved. Now it\'s time to add the lye to the water in the pitcher. Gradually stir in the lye with your stainless steel spoon, being ..."\n  },\n  {\n    "title": "Back to Basics: Lye Safety Guide | Soap Queen",\n    "href": "https://www.soapqueen.com/bath-and-body-tutorials/tips-and-tricks/back-to-basics-lye-safety-guide/",\n    "body": "Flush immediately with plenty of water for at least 15 minutes. Seek medical attention. If it comes in contact with eyes, flush immediately with water for at least 15 minutes and get medical attention. If inhaled, move to fresh air. Many soapers keep vinegar on hand, believing it neutralizes lye burns."\n  }\n]',
        name=None,
        tool_call_id="call_xxxxxInKc",
        tool_call_name="duckduckgo_search",
        tool_call_error=False,
        tool_calls=None,
        metrics={"time": 1.0803273562341928},
        internal_id=None,
        function_call=None,
    ),
    Message(
        role="assistant",
        content='\n{ "question": "Can you explain the practical steps to dissolve lye properly?", "answer": "Here are the practical steps to dissolve lye properly and safely:"\n    , "table": null }\n\n   ',
        name=None,
        tool_call_id=None,
        tool_call_name=None,
        tool_call_error=False,
        tool_calls=None,
        metrics={
            "time": 0.9308166340924799,
            "prompt_tokens": 2692,
            "input_tokens": 2692,
            "completion_tokens": 46,
            "output_tokens": 46,
            "total_tokens": 2738,
        },
        internal_id=None,
        function_call=None,
    ),
]

the following is the final answer

{ "question": "Can you explain the practical steps to dissolve lye properly?", "answer": "Here are the practical steps to dissolve lye properly and safely:"
    , "table": null }
@jacobweiss2305
Copy link
Contributor

Yeah thats odd. Can you share with me your assistant code? I know you said you were using RAG but I need to be able to replicate this to debug. Or just list out the steps I can try to debug from that?

@Ghat0tkach
Copy link
Author

Thanks @jacobweiss2305 . The odd thing is its very difficult to reproduce. I have created a custom fastapi route for auto-rag cookbook example with structured output.

def get_auto_rag_assistant(
    llm_model: str = "gpt-4-turbo",
    user_id: Optional[str] = None,
    run_id: Optional[str] = None,
    debug_mode: bool = True,
) -> Assistant:
    """Get an Auto RAG Assistant."""

    return Assistant(
        name="auto_rag_assistant",
        run_id=run_id,
        user_id=user_id,
        llm=OpenAIChat(model=llm_model),
        storage=PgAssistantStorage(table_name="auto_rag_assistant_openai", db_url=db_url),
        output_model=response_model,
        knowledge_base=AssistantKnowledge(
            vector_db=PgVector2(
                db_url=db_url,
                collection="auto_rag_documents_openai",
                embedder=OpenAIEmbedder(model="text-embedding-3-small", dimensions=1536),
            ),
            # 3 references are added to the prompt
            num_documents=3,
        ),
        description="You are a helpful Assistant called 'AutoRAG' and your goal is to assist the user in the best way possible.",
        instructions=[
            "Given a user query, first ALWAYS search your knowledge base using the `search_knowledge_base` tool to see if you have relevant information.",
            "If you dont find relevant information in your knowledge base, use the `duckduckgo_search` tool to search the internet.",
            "If you need to reference the chat history, use the `get_chat_history` tool.",
            "If the users question is unclear, ask clarifying questions to get more information.",
            "Carefully read the information you have gathered and provide a clear and concise answer to the user.",
            "Do not use phrases like 'based on my knowledge' or 'depending on the information'.",
        ],
        # Show tool calls in the chat
        show_tool_calls=True,
        # This setting gives the LLM a tool to search the knowledge base for information
        search_knowledge=True,
        # This setting gives the LLM a tool to get chat history
        read_chat_history=True,
        tools=[DuckDuckGo()],
        # This setting tells the LLM to format messages in markdown
        markdown=True,
        # Adds chat history to messages
        add_chat_history_to_messages=True,
        add_datetime_to_instructions=True,
        debug_mode=debug_mode,
    )
class Question(BaseModel):
    text: str
    user_id: str
    llm_model: str
    thread_id: Optional[str] = None


class Assistant_Response(BaseModel):
    question: str
    answer: str
    table: Optional[str] = None


class Answer(Assistant_Response):
    threadId: str
@router.post("/get-result", response_model=Answer)
async def ask_question(question: Question):
    try:
        response = get_answer_from_assistant(question)
        return response
    except ValueError as ve:
        raise HTTPException(status_code=422, detail=str(ve))
    except Exception as e:
        print(e)
        raise HTTPException(status_code=500, detail=str(e))
def get_answer_from_assistant(question: Question) -> Answer:
    assistant_kwargs = {
        "user_id": question.user_id,
        "llm_model": question.llm_model,
        "response_model": Assistant_Response
    }
    if question.thread_id is None:
        assistant = get_auto_rag_assistant(**assistant_kwargs)
        thread_id = assistant.create_run()
    else:
        assistant_kwargs["run_id"] = question.thread_id
        assistant = get_auto_rag_assistant(**assistant_kwargs)
        thread_id = question.thread_id

    response = assistant.run(question.text, stream=False)



    if not isinstance(response, Assistant_Response):
        raise ValueError("Invalid response type from assistant")

    answer = Answer(
        question=response.question,
        answer=response.answer,
        table=response.table,
        threadId=thread_id
    )
    return answer

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

2 participants