From b3177fef1391dc56b98159f9aa1d2f7b60dfd254 Mon Sep 17 00:00:00 2001 From: unclecode Date: Mon, 18 Mar 2024 00:41:32 +0800 Subject: [PATCH 1/2] Updates: - Breakdown chains.py, into multiple class for each chain. - Add vision support - Add few more examples in cookbook --- app/libs/__init__.py | 17 + app/libs/base_handler.py | 62 +++ app/libs/chains copy.py | 439 ++++++++++++++++++++ app/libs/chains.py | 194 +++++++-- app/libs/context.py | 70 ++++ app/libs/provider_handler.py | 45 ++ app/libs/tools_handler.py | 236 +++++++++++ app/libs/vision_handler.py | 68 +++ app/prompts.py | 17 +- app/routes/proxy.py | 33 +- app/utils.py | 67 ++- cookbook/function_call_force_tool_choice.py | 119 ++++++ cookbook/function_call_vision.py | 41 ++ requirements.txt | 3 +- 14 files changed, 1355 insertions(+), 56 deletions(-) create mode 100644 app/libs/__init__.py create mode 100644 app/libs/base_handler.py create mode 100644 app/libs/chains copy.py create mode 100644 app/libs/context.py create mode 100644 app/libs/provider_handler.py create mode 100644 app/libs/tools_handler.py create mode 100644 app/libs/vision_handler.py create mode 100644 cookbook/function_call_force_tool_choice.py create mode 100644 cookbook/function_call_vision.py diff --git a/app/libs/__init__.py b/app/libs/__init__.py new file mode 100644 index 0000000..16c23e2 --- /dev/null +++ b/app/libs/__init__.py @@ -0,0 +1,17 @@ +from .context import Context +from .base_handler import Handler, DefaultCompletionHandler, ExceptionHandler, FallbackHandler +from .provider_handler import ProviderSelectionHandler +from .vision_handler import ImageMessageHandler +from .tools_handler import ToolExtractionHandler, ToolResponseHandler + +__all__ = [ + "Context", + "Handler", + "DefaultCompletionHandler", + "ExceptionHandler", + "ProviderSelectionHandler", + "ImageMessageHandler", + "ToolExtractionHandler", + "ToolResponseHandler", + "FallbackHandler", +] diff --git a/app/libs/base_handler.py b/app/libs/base_handler.py new file mode 100644 index 0000000..edb8745 --- /dev/null +++ b/app/libs/base_handler.py @@ -0,0 +1,62 @@ +from abc import ABC, abstractmethod +from .context import Context +from fastapi.responses import JSONResponse +import traceback + +class Handler(ABC): + """Abstract Handler class for building the chain of handlers.""" + + _next_handler: "Handler" = None + + def set_next(self, handler: "Handler") -> "Handler": + self._next_handler = handler + return handler + + @abstractmethod + async def handle(self, context: Context): + if self._next_handler: + try: + return await self._next_handler.handle(context) + except Exception as e: + _exception_handler: "Handler" = ExceptionHandler() + # Extract the stack trace and log the exception + return await _exception_handler.handle(context, e) + + +class DefaultCompletionHandler(Handler): + async def handle(self, context: Context): + if context.is_normal_chat: + # Assuming context.client is set and has a method for creating chat completions + completion = context.client.route( + messages=context.messages, + **context.client.clean_params(context.params), + ) + context.response = completion.model_dump() + return JSONResponse(content=context.response, status_code=200) + + return await super().handle(context) + + +class FallbackHandler(Handler): + async def handle(self, context: Context): + # This handler does not pass the request further down the chain. + # It acts as a fallback when no other handler has processed the request. + if not context.response: + # The default action when no other handlers have processed the request + context.response = {"message": "No suitable action found for the request."} + return JSONResponse(content=context.response, status_code=400) + + # If there's already a response set in the context, it means one of the handlers has processed the request. + return JSONResponse(content=context.response, status_code=200) + + +class ExceptionHandler(Handler): + async def handle(self, context: Context, exception: Exception): + print(f"Error processing the request: {exception}") + print(traceback.format_exc()) + return JSONResponse( + content={"error": "An unexpected error occurred. " + str(exception)}, + status_code=500, + ) + + diff --git a/app/libs/chains copy.py b/app/libs/chains copy.py new file mode 100644 index 0000000..74cbd97 --- /dev/null +++ b/app/libs/chains copy.py @@ -0,0 +1,439 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict +from importlib import import_module +import json +import uuid +import traceback +from fastapi import Request +from fastapi.responses import JSONResponse +from providers import BaseProvider +from prompts import * +from providers import GroqProvider +import importlib +from utils import get_tool_call_response, create_logger, describe + +missed_tool_logger = create_logger( + "chain.missed_tools", ".logs/empty_tool_tool_response.log" +) + + +class Context: + def __init__(self, request: Request, provider: str, body: Dict[str, Any]): + self.request = request + self.provider = provider + self.body = body + self.response = None + + # extract all keys from body except messages and tools and set in params + self.params = {k: v for k, v in body.items() if k not in ["messages", "tools"]} + + # self.no_tool_behaviour = self.params.get("no_tool_behaviour", "return") + self.no_tool_behaviour = self.params.get("no_tool_behaviour", "forward") + self.params.pop("no_tool_behaviour", None) + + # Todo: For now, no stream, sorry ;) + self.params["stream"] = False + + self.messages = body.get("messages", []) + self.tools = body.get("tools", []) + + self.builtin_tools = [ + t for t in self.tools if "parameters" not in t["function"] + ] + self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] + self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] + + for bt in self.builtin_tools: + func_namespace = bt["function"]["name"] + if len(func_namespace.split(".")) == 2: + module_name, func_class_name = func_namespace.split(".") + func_class_name = f"{func_class_name.capitalize()}Function" + # raise ValueError("Only one builtin function can be called at a time.") + module = importlib.import_module(f"app.functions.{module_name}") + func_class = getattr(module, func_class_name, None) + schema_dict = func_class.get_schema() + if schema_dict: + bt["function"] = schema_dict + bt["run"] = func_class.run + bt["extra"] = self.params.get("extra", {}) + self.params.pop("extra", None) + + self.client: BaseProvider = None + + @property + def last_message(self): + return self.messages[-1] if self.messages else {} + + @property + def is_tool_call(self): + return bool( + self.last_message["role"] == "user" + and self.tools + and self.params.get("tool_choice", "none") != "none" + ) + + @property + def is_tool_response(self): + return bool(self.last_message["role"] == "tool" and self.tools) + + @property + def is_normal_chat(self): + return bool(not self.is_tool_call and not self.is_tool_response) + + +class Handler(ABC): + """Abstract Handler class for building the chain of handlers.""" + + _next_handler: "Handler" = None + + def set_next(self, handler: "Handler") -> "Handler": + self._next_handler = handler + return handler + + @abstractmethod + async def handle(self, context: Context): + if self._next_handler: + try: + return await self._next_handler.handle(context) + except Exception as e: + _exception_handler: "Handler" = ExceptionHandler() + # Extract the stack trace and log the exception + return await _exception_handler.handle(context, e) + + +class ProviderSelectionHandler(Handler): + @staticmethod + def provider_exists(provider: str) -> bool: + module_name = f"app.providers" + class_name = f"{provider.capitalize()}Provider" + try: + provider_module = import_module(module_name) + provider_class = getattr(provider_module, class_name) + return bool(provider_class) + except ImportError: + return False + + async def handle(self, context: Context): + # Construct the module path and class name based on the provider + module_name = f"app.providers" + class_name = f"{context.provider.capitalize()}Provider" + + try: + # Dynamically import the module and class + provider_module = import_module(module_name) + provider_class = getattr(provider_module, class_name) + + if provider_class: + context.client = provider_class( + api_key=context.api_token + ) # Assuming an api_key parameter + return await super().handle(context) + else: + raise ValueError( + f"Provider class {class_name} could not be found in {module_name}." + ) + except ImportError as e: + # Handle import error (e.g., module or class not found) + print(f"Error importing {class_name} from {module_name}: {e}") + context.response = { + "error": f"An error occurred while trying to load the provider: {e}" + } + return JSONResponse(content=context.response, status_code=500) + + +class ImageMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + if message["role"] == "user": + if isinstance(message["content"], list): + prompt = None + for content in message["content"]: + if content["type"] == "text": + # new_messages.append({"role": message["role"], "content": content["text"]}) + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + prompt = prompt or IMAGE_DESCRIPTO_PROMPT + description = describe(prompt, image_url) + if description: + description = get_image_desc_guide(image_ref, description) + new_messages.append( + {"role": message["role"], "content": description} + ) + image_ref += 1 + else: + pass + except Exception as e: + print(f"Error describing image: {e}") + continue + else: + new_messages.append(message) + else: + new_messages.append(message) + + context.messages = new_messages + return await super().handle(context) + + +class ImageLLavaMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + new_messages.append(message) + if message["role"] == "user": + if isinstance(message["content"], list): + for content in message["content"]: + if content["type"] == "text": + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + description = describe(prompt, image_url) + new_messages.append( + {"role": "assistant", "content": description} + ) + image_ref += 1 + except Exception as e: + print(f"Error describing image: {e}") + continue + context.messages = new_messages + return await super().handle(context) + + +class ToolExtractionHandler(Handler): + async def handle(self, context: Context): + body = context.body + if context.is_tool_call: + + # Prepare the messages and tools for the tool extraction + messages = [ + f"{m['role'].title()}: {m['content']}" + for m in context.messages + if m["role"] != "system" + ] + tools_json = json.dumps([t["function"] for t in context.tools], indent=4) + + # Process the tool_choice + tool_choice = context.params.get("tool_choice", "auto") + forced_mode = False + if ( + type(tool_choice) == dict + and tool_choice.get("type", None) == "function" + ): + tool_choice = tool_choice["function"].get("name", None) + if not tool_choice: + raise ValueError( + "Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key." + ) + forced_mode = True + + # Regenerate the string tool_json and keep only the forced tool + tools_json = json.dumps( + [ + t["function"] + for t in context.tools + if t["function"]["name"] == tool_choice + ], + indent=4, + ) + + system_message = ( + SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE + ) + suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) + + new_messages = [ + {"role": "system", "content": system_message}, + { + "role": "system", + "content": f"Conversation History:\n{''.join(messages)}\n\nTools: \n{tools_json}\n\n{suffix}", + }, + ] + + completion, tool_calls = await self.process_tool_calls( + context, new_messages + ) + + if not tool_calls: + if context.no_tool_behaviour == "forward": + context.tools = None + return await super().handle(context) + else: + context.response = {"tool_calls": []} + tool_response = get_tool_call_response(completion, [], []) + missed_tool_logger.debug( + f"Last message content: {context.last_message['content']}" + ) + return JSONResponse(content=tool_response, status_code=200) + + unresolved_tol_calls = [ + t + for t in tool_calls + if t["function"]["name"] not in context.builtin_tool_names + ] + resolved_responses = [] + for tool in tool_calls: + for bt in context.builtin_tools: + if tool["function"]["name"] == bt["function"]["name"]: + res = bt["run"]( + **{ + **json.loads(tool["function"]["arguments"]), + **bt["extra"], + } + ) + resolved_responses.append( + { + "name": tool["function"]["name"], + "role": "tool", + "content": json.dumps(res), + "tool_call_id": "chatcmpl-" + completion.id, + } + ) + + if not unresolved_tol_calls: + context.messages.extend(resolved_responses) + return await super().handle(context) + + tool_response = get_tool_call_response( + completion, unresolved_tol_calls, resolved_responses + ) + + context.response = tool_response + return JSONResponse(content=context.response, status_code=200) + + return await super().handle(context) + + async def process_tool_calls(self, context, new_messages): + try: + tries = 5 + tool_calls = [] + while tries > 0: + try: + # Assuming the context has an instantiated client according to the selected provider + completion = context.client.route( + model=context.client.parser_model, + messages=new_messages, + temperature=0, + max_tokens=1024, + top_p=1, + stream=False, + ) + + response = completion.choices[0].message.content + if "```json" in response: + response = response.split("```json")[1].split("```")[0] + + try: + tool_response = json.loads(response) + if isinstance(tool_response, list): + tool_response = {"tool_calls": tool_response} + except json.JSONDecodeError as e: + print( + f"Error parsing the tool response: {e}, tries left: {tries}" + ) + new_messages.append( + { + "role": "user", + "content": f"Error: {e}.\n\n{CLEAN_UP_MESSAGE}", + } + ) + tries -= 1 + continue + + for func in tool_response.get("tool_calls", []): + tool_calls.append( + { + "id": f"call_{func['name']}_{str(uuid.uuid4())}", + "type": "function", + "function": { + "name": func["name"], + "arguments": json.dumps(func["arguments"]), + }, + } + ) + + break + except Exception as e: + raise e + + if tries == 0: + tool_calls = [] + + return completion, tool_calls + except Exception as e: + print(f"Error processing the tool calls: {e}") + raise e + + +class ToolResponseHandler(Handler): + async def handle(self, context: Context): + body = context.body + if context.is_tool_response: + messages = context.messages + + for message in messages: + if message["role"] == "tool": + message["role"] = "user" + message["content"] = get_func_result_guide(message["content"]) + + messages[-1]["role"] = "user" + # Assuming get_func_result_guide is a function that formats the tool response + messages[-1]["content"] = get_func_result_guide(messages[-1]["content"]) + + try: + completion = context.client.route( + messages=messages, + **context.client.clean_params(context.params), + ) + context.response = completion.model_dump() + return JSONResponse(content=context.response, status_code=200) + except Exception as e: + # Log the exception or handle it as needed + print(e) + context.response = { + "error": "An error occurred processing the tool response" + } + return JSONResponse(content=context.response, status_code=500) + + return await super().handle(context) + + +class DefaultCompletionHandler(Handler): + async def handle(self, context: Context): + if context.is_normal_chat: + # Assuming context.client is set and has a method for creating chat completions + completion = context.client.route( + messages=context.messages, + **context.client.clean_params(context.params), + ) + context.response = completion.model_dump() + return JSONResponse(content=context.response, status_code=200) + + return await super().handle(context) + + +class FallbackHandler(Handler): + async def handle(self, context: Context): + # This handler does not pass the request further down the chain. + # It acts as a fallback when no other handler has processed the request. + if not context.response: + # The default action when no other handlers have processed the request + context.response = {"message": "No suitable action found for the request."} + return JSONResponse(content=context.response, status_code=400) + + # If there's already a response set in the context, it means one of the handlers has processed the request. + return JSONResponse(content=context.response, status_code=200) + + +class ExceptionHandler(Handler): + async def handle(self, context: Context, exception: Exception): + print(f"Error processing the request: {exception}") + print(traceback.format_exc()) + return JSONResponse( + content={"error": "An unexpected error occurred. " + str(exception)}, + status_code=500, + ) diff --git a/app/libs/chains.py b/app/libs/chains.py index f44215b..74cbd97 100644 --- a/app/libs/chains.py +++ b/app/libs/chains.py @@ -7,12 +7,15 @@ from fastapi import Request from fastapi.responses import JSONResponse from providers import BaseProvider -from prompts import SYSTEM_MESSAGE, ENFORCED_SYSTAME_MESSAE, SUFFIX, FORCE_CALL_SUFFIX, CLEAN_UP_MESSAGE, get_func_result_guide, get_forced_tool_suffix +from prompts import * from providers import GroqProvider import importlib -from utils import get_tool_call_response, create_logger +from utils import get_tool_call_response, create_logger, describe + +missed_tool_logger = create_logger( + "chain.missed_tools", ".logs/empty_tool_tool_response.log" +) -missed_tool_logger = create_logger("chain.missed_tools", ".logs/empty_tool_tool_response.log") class Context: def __init__(self, request: Request, provider: str, body: Dict[str, Any]): @@ -34,26 +37,28 @@ def __init__(self, request: Request, provider: str, body: Dict[str, Any]): self.messages = body.get("messages", []) self.tools = body.get("tools", []) - self.builtin_tools = [t for t in self.tools if "parameters" not in t['function']] + self.builtin_tools = [ + t for t in self.tools if "parameters" not in t["function"] + ] self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] - self.custom_tools = [t for t in self.tools if "parameters" in t['function']] + self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] for bt in self.builtin_tools: - func_namespace = bt["function"]['name'] - if len(func_namespace.split(".")) == 2: + func_namespace = bt["function"]["name"] + if len(func_namespace.split(".")) == 2: module_name, func_class_name = func_namespace.split(".") func_class_name = f"{func_class_name.capitalize()}Function" # raise ValueError("Only one builtin function can be called at a time.") module = importlib.import_module(f"app.functions.{module_name}") func_class = getattr(module, func_class_name, None) - schema_dict = func_class.get_schema() + schema_dict = func_class.get_schema() if schema_dict: - bt["function"] = schema_dict - bt['run'] = func_class.run - bt['extra'] = self.params.get("extra", {}) + bt["function"] = schema_dict + bt["run"] = func_class.run + bt["extra"] = self.params.get("extra", {}) self.params.pop("extra", None) - self.client : BaseProvider = None + self.client: BaseProvider = None @property def last_message(self): @@ -61,7 +66,11 @@ def last_message(self): @property def is_tool_call(self): - return bool(self.last_message["role"] == "user" and self.tools and self.params.get("tool_choice", "none") != "none") + return bool( + self.last_message["role"] == "user" + and self.tools + and self.params.get("tool_choice", "none") != "none" + ) @property def is_tool_response(self): @@ -76,7 +85,6 @@ class Handler(ABC): """Abstract Handler class for building the chain of handlers.""" _next_handler: "Handler" = None - def set_next(self, handler: "Handler") -> "Handler": self._next_handler = handler @@ -89,9 +97,8 @@ async def handle(self, context: Context): return await self._next_handler.handle(context) except Exception as e: _exception_handler: "Handler" = ExceptionHandler() - # Extract the stack trace and log the exception + # Extract the stack trace and log the exception return await _exception_handler.handle(context, e) - class ProviderSelectionHandler(Handler): @@ -105,29 +112,98 @@ def provider_exists(provider: str) -> bool: return bool(provider_class) except ImportError: return False - + async def handle(self, context: Context): # Construct the module path and class name based on the provider module_name = f"app.providers" class_name = f"{context.provider.capitalize()}Provider" - + try: # Dynamically import the module and class provider_module = import_module(module_name) provider_class = getattr(provider_module, class_name) - + if provider_class: - context.client = provider_class(api_key=context.api_token) # Assuming an api_key parameter + context.client = provider_class( + api_key=context.api_token + ) # Assuming an api_key parameter return await super().handle(context) else: - raise ValueError(f"Provider class {class_name} could not be found in {module_name}.") + raise ValueError( + f"Provider class {class_name} could not be found in {module_name}." + ) except ImportError as e: # Handle import error (e.g., module or class not found) print(f"Error importing {class_name} from {module_name}: {e}") - context.response = {"error": f"An error occurred while trying to load the provider: {e}"} + context.response = { + "error": f"An error occurred while trying to load the provider: {e}" + } return JSONResponse(content=context.response, status_code=500) +class ImageMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + if message["role"] == "user": + if isinstance(message["content"], list): + prompt = None + for content in message["content"]: + if content["type"] == "text": + # new_messages.append({"role": message["role"], "content": content["text"]}) + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + prompt = prompt or IMAGE_DESCRIPTO_PROMPT + description = describe(prompt, image_url) + if description: + description = get_image_desc_guide(image_ref, description) + new_messages.append( + {"role": message["role"], "content": description} + ) + image_ref += 1 + else: + pass + except Exception as e: + print(f"Error describing image: {e}") + continue + else: + new_messages.append(message) + else: + new_messages.append(message) + + context.messages = new_messages + return await super().handle(context) + + +class ImageLLavaMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + new_messages.append(message) + if message["role"] == "user": + if isinstance(message["content"], list): + for content in message["content"]: + if content["type"] == "text": + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + description = describe(prompt, image_url) + new_messages.append( + {"role": "assistant", "content": description} + ) + image_ref += 1 + except Exception as e: + print(f"Error describing image: {e}") + continue + context.messages = new_messages + return await super().handle(context) + + class ToolExtractionHandler(Handler): async def handle(self, context: Context): body = context.body @@ -144,16 +220,30 @@ async def handle(self, context: Context): # Process the tool_choice tool_choice = context.params.get("tool_choice", "auto") forced_mode = False - if type(tool_choice) == dict and tool_choice.get("type", None) == "function": + if ( + type(tool_choice) == dict + and tool_choice.get("type", None) == "function" + ): tool_choice = tool_choice["function"].get("name", None) if not tool_choice: - raise ValueError("Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key.") + raise ValueError( + "Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key." + ) forced_mode = True - + # Regenerate the string tool_json and keep only the forced tool - tools_json = json.dumps([t["function"] for t in context.tools if t["function"]["name"] == tool_choice], indent=4) + tools_json = json.dumps( + [ + t["function"] + for t in context.tools + if t["function"]["name"] == tool_choice + ], + indent=4, + ) - system_message = SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE + system_message = ( + SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE + ) suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) new_messages = [ @@ -175,29 +265,42 @@ async def handle(self, context: Context): else: context.response = {"tool_calls": []} tool_response = get_tool_call_response(completion, [], []) - missed_tool_logger.debug(f"Last message content: {context.last_message['content']}") + missed_tool_logger.debug( + f"Last message content: {context.last_message['content']}" + ) return JSONResponse(content=tool_response, status_code=200) - - unresolved_tol_calls = [t for t in tool_calls if t["function"]["name"] not in context.builtin_tool_names] + unresolved_tol_calls = [ + t + for t in tool_calls + if t["function"]["name"] not in context.builtin_tool_names + ] resolved_responses = [] for tool in tool_calls: for bt in context.builtin_tools: if tool["function"]["name"] == bt["function"]["name"]: - res =bt['run'](**{**json.loads(tool["function"]["arguments"]), **bt['extra']} ) - resolved_responses.append({ - "name": tool["function"]["name"], - "role": "tool", - "content": json.dumps(res), - "tool_call_id": "chatcmpl-" + completion.id - }) + res = bt["run"]( + **{ + **json.loads(tool["function"]["arguments"]), + **bt["extra"], + } + ) + resolved_responses.append( + { + "name": tool["function"]["name"], + "role": "tool", + "content": json.dumps(res), + "tool_call_id": "chatcmpl-" + completion.id, + } + ) if not unresolved_tol_calls: context.messages.extend(resolved_responses) return await super().handle(context) - - tool_response = get_tool_call_response(completion, unresolved_tol_calls, resolved_responses) + tool_response = get_tool_call_response( + completion, unresolved_tol_calls, resolved_responses + ) context.response = tool_response return JSONResponse(content=context.response, status_code=200) @@ -223,13 +326,15 @@ async def process_tool_calls(self, context, new_messages): response = completion.choices[0].message.content if "```json" in response: response = response.split("```json")[1].split("```")[0] - + try: tool_response = json.loads(response) if isinstance(tool_response, list): tool_response = {"tool_calls": tool_response} except json.JSONDecodeError as e: - print(f"Error parsing the tool response: {e}, tries left: {tries}"); + print( + f"Error parsing the tool response: {e}, tries left: {tries}" + ) new_messages.append( { "role": "user", @@ -270,7 +375,6 @@ async def handle(self, context: Context): if context.is_tool_response: messages = context.messages - #TODO:If we hit an error 'cause a model struggles with multiple messages having the same role. for message in messages: if message["role"] == "tool": message["role"] = "user" @@ -324,8 +428,12 @@ async def handle(self, context: Context): # If there's already a response set in the context, it means one of the handlers has processed the request. return JSONResponse(content=context.response, status_code=200) + class ExceptionHandler(Handler): async def handle(self, context: Context, exception: Exception): print(f"Error processing the request: {exception}") print(traceback.format_exc()) - return JSONResponse(content={"error": "An unexpected error occurred. " + str(exception)}, status_code=500) \ No newline at end of file + return JSONResponse( + content={"error": "An unexpected error occurred. " + str(exception)}, + status_code=500, + ) diff --git a/app/libs/context.py b/app/libs/context.py new file mode 100644 index 0000000..4e68088 --- /dev/null +++ b/app/libs/context.py @@ -0,0 +1,70 @@ +from typing import Any, Dict +from fastapi import Request +from providers import BaseProvider +from prompts import * +import importlib +from utils import create_logger + + +class Context: + def __init__(self, request: Request, provider: str, body: Dict[str, Any]): + self.request = request + self.provider = provider + self.body = body + self.response = None + + # extract all keys from body except messages and tools and set in params + self.params = {k: v for k, v in body.items() if k not in ["messages", "tools"]} + + # self.no_tool_behaviour = self.params.get("no_tool_behaviour", "return") + self.no_tool_behaviour = self.params.get("no_tool_behaviour", "forward") + self.params.pop("no_tool_behaviour", None) + + # Todo: For now, no stream, sorry ;) + self.params["stream"] = False + + self.messages = body.get("messages", []) + self.tools = body.get("tools", []) + + self.builtin_tools = [ + t for t in self.tools if "parameters" not in t["function"] + ] + self.builtin_tool_names = [t["function"]["name"] for t in self.builtin_tools] + self.custom_tools = [t for t in self.tools if "parameters" in t["function"]] + + for bt in self.builtin_tools: + func_namespace = bt["function"]["name"] + if len(func_namespace.split(".")) == 2: + module_name, func_class_name = func_namespace.split(".") + func_class_name = f"{func_class_name.capitalize()}Function" + # raise ValueError("Only one builtin function can be called at a time.") + module = importlib.import_module(f"app.functions.{module_name}") + func_class = getattr(module, func_class_name, None) + schema_dict = func_class.get_schema() + if schema_dict: + bt["function"] = schema_dict + bt["run"] = func_class.run + bt["extra"] = self.params.get("extra", {}) + self.params.pop("extra", None) + + self.client: BaseProvider = None + + @property + def last_message(self): + return self.messages[-1] if self.messages else {} + + @property + def is_tool_call(self): + return bool( + self.last_message["role"] == "user" + and self.tools + and self.params.get("tool_choice", "none") != "none" + ) + + @property + def is_tool_response(self): + return bool(self.last_message["role"] == "tool" and self.tools) + + @property + def is_normal_chat(self): + return bool(not self.is_tool_call and not self.is_tool_response) diff --git a/app/libs/provider_handler.py b/app/libs/provider_handler.py new file mode 100644 index 0000000..a8d21db --- /dev/null +++ b/app/libs/provider_handler.py @@ -0,0 +1,45 @@ +from importlib import import_module +from fastapi.responses import JSONResponse +from prompts import * +from .base_handler import Handler +from .context import Context + +class ProviderSelectionHandler(Handler): + @staticmethod + def provider_exists(provider: str) -> bool: + module_name = f"app.providers" + class_name = f"{provider.capitalize()}Provider" + try: + provider_module = import_module(module_name) + provider_class = getattr(provider_module, class_name) + return bool(provider_class) + except ImportError: + return False + + async def handle(self, context: Context): + # Construct the module path and class name based on the provider + module_name = f"app.providers" + class_name = f"{context.provider.capitalize()}Provider" + + try: + # Dynamically import the module and class + provider_module = import_module(module_name) + provider_class = getattr(provider_module, class_name) + + if provider_class: + context.client = provider_class( + api_key=context.api_token + ) # Assuming an api_key parameter + return await super().handle(context) + else: + raise ValueError( + f"Provider class {class_name} could not be found in {module_name}." + ) + except ImportError as e: + # Handle import error (e.g., module or class not found) + print(f"Error importing {class_name} from {module_name}: {e}") + context.response = { + "error": f"An error occurred while trying to load the provider: {e}" + } + return JSONResponse(content=context.response, status_code=500) + diff --git a/app/libs/tools_handler.py b/app/libs/tools_handler.py new file mode 100644 index 0000000..c771308 --- /dev/null +++ b/app/libs/tools_handler.py @@ -0,0 +1,236 @@ +import json +import uuid +from fastapi.responses import JSONResponse +from prompts import * +from utils import get_tool_call_response, describe +from .base_handler import Handler, Context +from .context import Context +from utils import get_tool_call_response, create_logger, describe + +missed_tool_logger = create_logger( + "chain.missed_tools", ".logs/empty_tool_tool_response.log" +) + + +class ImageLLavaMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + new_messages.append(message) + if message["role"] == "user": + if isinstance(message["content"], list): + for content in message["content"]: + if content["type"] == "text": + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + description = describe(prompt, image_url) + new_messages.append( + {"role": "assistant", "content": description} + ) + image_ref += 1 + except Exception as e: + print(f"Error describing image: {e}") + continue + context.messages = new_messages + return await super().handle(context) + + +class ToolExtractionHandler(Handler): + async def handle(self, context: Context): + body = context.body + if context.is_tool_call: + + # Prepare the messages and tools for the tool extraction + messages = [ + f"{m['role'].title()}: {m['content']}" + for m in context.messages + if m["role"] != "system" + ] + tools_json = json.dumps([t["function"] for t in context.tools], indent=4) + + # Process the tool_choice + tool_choice = context.params.get("tool_choice", "auto") + forced_mode = False + if ( + type(tool_choice) == dict + and tool_choice.get("type", None) == "function" + ): + tool_choice = tool_choice["function"].get("name", None) + if not tool_choice: + raise ValueError( + "Invalid tool choice. 'tool_choice' is set to a dictionary with 'type' as 'function', but 'function' does not have a 'name' key." + ) + forced_mode = True + + # Regenerate the string tool_json and keep only the forced tool + tools_json = json.dumps( + [ + t["function"] + for t in context.tools + if t["function"]["name"] == tool_choice + ], + indent=4, + ) + + system_message = ( + SYSTEM_MESSAGE if not forced_mode else ENFORCED_SYSTAME_MESSAE + ) + suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) + + new_messages = [ + {"role": "system", "content": system_message}, + { + "role": "system", + "content": f"Conversation History:\n{''.join(messages)}\n\nTools: \n{tools_json}\n\n{suffix}", + }, + ] + + completion, tool_calls = await self.process_tool_calls( + context, new_messages + ) + + if not tool_calls: + if context.no_tool_behaviour == "forward": + context.tools = None + return await super().handle(context) + else: + context.response = {"tool_calls": []} + tool_response = get_tool_call_response(completion, [], []) + missed_tool_logger.debug( + f"Last message content: {context.last_message['content']}" + ) + return JSONResponse(content=tool_response, status_code=200) + + unresolved_tol_calls = [ + t + for t in tool_calls + if t["function"]["name"] not in context.builtin_tool_names + ] + resolved_responses = [] + for tool in tool_calls: + for bt in context.builtin_tools: + if tool["function"]["name"] == bt["function"]["name"]: + res = bt["run"]( + **{ + **json.loads(tool["function"]["arguments"]), + **bt["extra"], + } + ) + resolved_responses.append( + { + "name": tool["function"]["name"], + "role": "tool", + "content": json.dumps(res), + "tool_call_id": "chatcmpl-" + completion.id, + } + ) + + if not unresolved_tol_calls: + context.messages.extend(resolved_responses) + return await super().handle(context) + + tool_response = get_tool_call_response( + completion, unresolved_tol_calls, resolved_responses + ) + + context.response = tool_response + return JSONResponse(content=context.response, status_code=200) + + return await super().handle(context) + + async def process_tool_calls(self, context, new_messages): + try: + tries = 5 + tool_calls = [] + while tries > 0: + try: + # Assuming the context has an instantiated client according to the selected provider + completion = context.client.route( + model=context.client.parser_model, + messages=new_messages, + temperature=0, + max_tokens=1024, + top_p=1, + stream=False, + ) + + response = completion.choices[0].message.content + if "```json" in response: + response = response.split("```json")[1].split("```")[0] + + try: + tool_response = json.loads(response) + if isinstance(tool_response, list): + tool_response = {"tool_calls": tool_response} + except json.JSONDecodeError as e: + print( + f"Error parsing the tool response: {e}, tries left: {tries}" + ) + new_messages.append( + { + "role": "user", + "content": f"Error: {e}.\n\n{CLEAN_UP_MESSAGE}", + } + ) + tries -= 1 + continue + + for func in tool_response.get("tool_calls", []): + tool_calls.append( + { + "id": f"call_{func['name']}_{str(uuid.uuid4())}", + "type": "function", + "function": { + "name": func["name"], + "arguments": json.dumps(func["arguments"]), + }, + } + ) + + break + except Exception as e: + raise e + + if tries == 0: + tool_calls = [] + + return completion, tool_calls + except Exception as e: + print(f"Error processing the tool calls: {e}") + raise e + + +class ToolResponseHandler(Handler): + async def handle(self, context: Context): + body = context.body + if context.is_tool_response: + messages = context.messages + + for message in messages: + if message["role"] == "tool": + message["role"] = "user" + message["content"] = get_func_result_guide(message["content"]) + + messages[-1]["role"] = "user" + # Assuming get_func_result_guide is a function that formats the tool response + messages[-1]["content"] = get_func_result_guide(messages[-1]["content"]) + + try: + completion = context.client.route( + messages=messages, + **context.client.clean_params(context.params), + ) + context.response = completion.model_dump() + return JSONResponse(content=context.response, status_code=200) + except Exception as e: + # Log the exception or handle it as needed + print(e) + context.response = { + "error": "An error occurred processing the tool response" + } + return JSONResponse(content=context.response, status_code=500) + + return await super().handle(context) diff --git a/app/libs/vision_handler.py b/app/libs/vision_handler.py new file mode 100644 index 0000000..9fdb9cf --- /dev/null +++ b/app/libs/vision_handler.py @@ -0,0 +1,68 @@ +from prompts import * +from utils import describe +from .context import Context +from .base_handler import Handler + + +class ImageMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + if message["role"] == "user": + if isinstance(message["content"], list): + prompt = None + for content in message["content"]: + if content["type"] == "text": + # new_messages.append({"role": message["role"], "content": content["text"]}) + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + prompt = prompt or IMAGE_DESCRIPTO_PROMPT + description = describe(prompt, image_url) + if description: + description = get_image_desc_guide(image_ref, description) + new_messages.append( + {"role": message["role"], "content": description} + ) + image_ref += 1 + else: + pass + except Exception as e: + print(f"Error describing image: {e}") + continue + else: + new_messages.append(message) + else: + new_messages.append(message) + + context.messages = new_messages + return await super().handle(context) + + +class ImageLLavaMessageHandler(Handler): + async def handle(self, context: Context): + new_messages = [] + image_ref = 1 + for message in context.messages: + new_messages.append(message) + if message["role"] == "user": + if isinstance(message["content"], list): + for content in message["content"]: + if content["type"] == "text": + prompt = content["text"] + elif content["type"] == "image_url": + image_url = content["image_url"]["url"] + try: + description = describe(prompt, image_url) + new_messages.append( + {"role": "assistant", "content": description} + ) + image_ref += 1 + except Exception as e: + print(f"Error describing image: {e}") + continue + context.messages = new_messages + return await super().handle(context) + diff --git a/app/prompts.py b/app/prompts.py index 8ed9997..fef6d52 100644 --- a/app/prompts.py +++ b/app/prompts.py @@ -35,7 +35,9 @@ **Wrap the JSON response between ```json and ```, and rememebr "tool_calls" is a list.**. -**Whenever a message starts with 'SYSTEM MESSAGE', that is a guide and help information for you to generate your next response, do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user.**""" +MESSAGE SUFFIX: +- "SYSTEM MESSGAE": Whenever a message starts with 'SYSTEM MESSAGE', that is a guide and help information for you to generate your next response. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user. +- "IMAGE [ref_index]": Whenever a message starts with 'IMAGE', that is a description of an images uploaded bu user. This information you can use it to generate your next responses, in case user refers to the image. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user. The [ref_index] is the index of the image in the list of images uploaded by the user. """ ENFORCED_SYSTAME_MESSAE = """A history of conversations between an AI assistant and the user, plus the last user's message, is given to you. @@ -61,7 +63,10 @@ **Wrap the JSON response between ```json and ```, and rememebr "tool_calls" is a list.**. -Whenever a message starts with 'SYSTEM MESSAGE', that is a guide and help information for you to generate your next response. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user.""" +MESSAGE SUFFIX: +- "SYSTEM MESSGAE": Whenever a message starts with 'SYSTEM MESSAGE', that is a guide and help information for you to generate your next response. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user. +- "IMAGE [ref_index]": Whenever a message starts with 'IMAGE', that is a description of an images uploaded bu user. This information you can use it to generate your next responses, in case user refers to the image. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user. The [ref_index] is the index of the image in the list of images uploaded by the user. """ + CLEAN_UP_MESSAGE = "When I tried to extract the content between ```json and ``` and parse the content to valid JSON object, I faced with the abovr error. Remember, you are supposed to wrap the schema between ```json and ```, and do this only one time. First find out what went wrong, that I couldn't extract the JSON between ```json and ```, and also faced error when trying to parse it, then regenerate the your last message and fix the issue." @@ -69,8 +74,14 @@ FORCE_CALL_SUFFIX = """For this task, you HAVE to choose the tool (function) {tool_name}, and ignore other rools. Therefore think step by step and justify your response, then closely examine the user's last message and the history of the conversation, then extract the necessary parameter values for the given tool based on the provided JSON schema. Remember that you must use the specified tool to generate the response. Finally generate a JSON response wrapped between "```json" and "```". Remember to USE THIS JSON WRAPPER ONLY ONE TIME.""" + +IMAGE_DESCRIPTO_PROMPT = """The user has uploaded an image. List down in very detail what the image is about. List down all objetcs you see and their description. Your description should be enough for a blind person to be able to visualize the image and answe ny question about it.""" + def get_forced_tool_suffix(tool_name : str) -> str: return FORCE_CALL_SUFFIX.format(tool_name=tool_name) def get_func_result_guide(function_call_result : str) -> str: - return f"SYSTEM MESSAGE: \n```json\n{function_call_result}\n```\n\nThe above is the result after functions are called. Use the result to answer the user's last question.\n\n" \ No newline at end of file + return f"SYSTEM MESSAGE: \n```json\n{function_call_result}\n```\n\nThe above is the result after functions are called. Use the result to answer the user's last question.\n\n" + +def get_image_desc_guide(ref_index: int, description : str) -> str: + return f"IMAGE: [{ref_index}] : {description}.\n\n" \ No newline at end of file diff --git a/app/routes/proxy.py b/app/routes/proxy.py index 681eead..22a3769 100644 --- a/app/routes/proxy.py +++ b/app/routes/proxy.py @@ -1,13 +1,26 @@ from fastapi import APIRouter, Response, Request, Path, Query from fastapi.responses import JSONResponse -from libs.chains import ( +# from libs.chains import ( +# Context, +# ProviderSelectionHandler, +# ImageMessageHandler, +# ToolExtractionHandler, +# ToolResponseHandler, +# DefaultCompletionHandler, +# FallbackHandler, +# ) + +from libs import ( Context, ProviderSelectionHandler, + ImageMessageHandler, ToolExtractionHandler, ToolResponseHandler, DefaultCompletionHandler, FallbackHandler, ) + + from typing import Optional router = APIRouter() @@ -45,15 +58,27 @@ async def post_chat_completions( # Initialize and link the handlers provider_selection_handler = ProviderSelectionHandler() + image_message_handler = ImageMessageHandler() tool_extraction_handler = ToolExtractionHandler() tool_response_handler = ToolResponseHandler() default_completion_handler = DefaultCompletionHandler() fallback_handler = FallbackHandler() # Set up the chain of responsibility - provider_selection_handler.set_next(tool_extraction_handler).set_next( - tool_response_handler - ).set_next(default_completion_handler).set_next(fallback_handler) + chains = [ + provider_selection_handler, + image_message_handler, + tool_extraction_handler, + tool_response_handler, + default_completion_handler, + fallback_handler, + ] + for i in range(len(chains) - 1): + chains[i].set_next(chains[i + 1]) + + # provider_selection_handler.set_next(tool_extraction_handler).set_next( + # tool_response_handler + # ).set_next(default_completion_handler).set_next(fallback_handler) # Execute the chain with the initial context response = await provider_selection_handler.handle(context) diff --git a/app/utils.py b/app/utils.py index 203ddff..2324f9c 100644 --- a/app/utils.py +++ b/app/utils.py @@ -1,8 +1,12 @@ import logging import os +import replicate +import base64 +from io import BytesIO + # To be developed -def create_logger(logger_name: str, log_path: str = ".logs/access.log"): +def create_logger(logger_name: str, log_path: str = ".logs/access.log", show_on_shell: bool = False): log_dir = os.path.dirname(log_path) if not os.path.exists(log_dir): os.makedirs(log_dir) @@ -10,13 +14,23 @@ def create_logger(logger_name: str, log_path: str = ".logs/access.log"): logger.setLevel(logging.DEBUG) file_handler = logging.FileHandler(log_path) file_handler.setLevel(logging.DEBUG) - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) file_handler.setFormatter(formatter) logger.addHandler(file_handler) + if show_on_shell: + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.DEBUG) + shell_formatter = logging.Formatter( + "%(levelname)s (%(name)s) %(message)s" + ) + stream_handler.setFormatter(shell_formatter) + logger.addHandler(stream_handler) return logger -def get_tool_call_response( completion, unresolved_tol_calls, resolved_responses): +def get_tool_call_response(completion, unresolved_tol_calls, resolved_responses): tool_response = { "id": "chatcmpl-" + completion.id, "object": "chat.completion", @@ -41,6 +55,49 @@ def get_tool_call_response( completion, unresolved_tol_calls, resolved_responses "total_tokens": completion.usage.total_tokens, }, "system_fingerprint": completion.system_fingerprint, - } + } + + return tool_response + +def describe(prompt: str, image_url_or_base64 : str, **kwargs) -> str: + logger = create_logger("vision", ".logs/access.log", True) + try: + if image_url_or_base64.startswith("data:image/"): + # If the input is a base64 string + image_data = base64.b64decode(image_url_or_base64.split(",")[1]) + image_file = BytesIO(image_data) + else: + # If the input is a URL + image_file = image_url_or_base64 + + model_params = { + "top_p": 1, + "max_tokens": 1024, + "temperature": 0.2 + } + model_params.update(kwargs) + + logger.info("Running the model") + output = replicate.run( + "yorickvp/llava-13b:01359160a4cff57c6b7d4dc625d0019d390c7c46f553714069f114b392f4a726", + input={ + "image": image_file, + "prompt": prompt, #"Describe the image in detail.", + **model_params + } + ) + + description = "" + for item in output: + if not description: + logger.info("Streaming...") + description += item + + return description.strip() + except Exception as e: + logger.error( f"Vision model, An error occurred: {e}") + return None + + - return tool_response \ No newline at end of file + # describe("Describe the image in detail.", "https://replicate.delivery/pbxt/KRULC43USWlEx4ZNkXltJqvYaHpEx2uJ4IyUQPRPwYb8SzPf/view.jpg") \ No newline at end of file diff --git a/cookbook/function_call_force_tool_choice.py b/cookbook/function_call_force_tool_choice.py new file mode 100644 index 0000000..6d3d134 --- /dev/null +++ b/cookbook/function_call_force_tool_choice.py @@ -0,0 +1,119 @@ +from duckduckgo_search import DDGS +import requests, os +import json + +api_key=os.environ["GROQ_API_KEY"] +header = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" +} + +# proxy_url = "https://funckycall.ai/proxy/groq/v1/chat/completions" +proxy_url = "http://localhost:8000/proxy/groq/v1/chat/completions" + +def duckduckgo_search(query, max_results=None): + """ + Use this function to search DuckDuckGo for a query. + """ + with DDGS() as ddgs: + return [r for r in ddgs.text(query, safesearch='off', max_results=max_results)] + +def duckduckgo_news(query, max_results=None): + """ + Use this function to get the latest news from DuckDuckGo. + """ + with DDGS() as ddgs: + return [r for r in ddgs.news(query, safesearch='off', max_results=max_results)] + +function_map = { + "duckduckgo_search": duckduckgo_search, + "duckduckgo_news": duckduckgo_news, +} + + +request = { + "messages": [ + { + "role": "user", + "content": "Whats happening in France? Summarize top stories with sources, search in general and also search news, very short and concise.", + } + ], + "model": "mixtral-8x7b-32768", + # "tool_choice": "auto", + # "tool_choice": None, + "tool_choice": {"type": "function", "function": {"name": "duckduckgo_search"}}, + "tools": [ + { + "type": "function", + "function": { + "name": "duckduckgo_search", + "description": "Use this function to search DuckDuckGo for a query.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The result from DuckDuckGo.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "max_results": {"type": ["number", "null"]}, + }, + }, + }, + }, + { + "type": "function", + "function": { + "name": "duckduckgo_news", + "description": "Use this function to get the latest news from DuckDuckGo.\n\nArgs:\n query(str): The query to search for.\n max_results (optional, default=5): The maximum number of results to return.\n\nReturns:\n The latest news from DuckDuckGo.", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "max_results": {"type": ["number", "null"]}, + }, + }, + }, + }, + ], +} + +response = requests.post( + proxy_url, + headers=header, + json=request, +) + + +# Check if the request was successful +if response.status_code == 200: + # Process the response data (if needed) + res = response.json() + message = res['choices'][0]['message'] + tools_response_messages = [] + if not message['content'] and 'tool_calls' in message: + for tool_call in message['tool_calls']: + tool_name = tool_call['function']['name'] + tool_args = tool_call['function']['arguments'] + tool_args = json.loads(tool_args) + if tool_name not in function_map: + print(f"Error: {tool_name} is not a valid function name.") + continue + tool_func = function_map[tool_name] + tool_response = tool_func(**tool_args) + tools_response_messages.append({ + "role": "tool", "content": json.dumps(tool_response) + }) + + if tools_response_messages: + request['messages'] += tools_response_messages + response = requests.post( + proxy_url, + headers=header, + json=request + ) + if response.status_code == 200: + res = response.json() + print(res['choices'][0]['message']['content']) + else: + print("Error:", response.status_code, response.text) + else: + print(message['content']) +else: + print("Error:", response.status_code, response.text) diff --git a/cookbook/function_call_vision.py b/cookbook/function_call_vision.py new file mode 100644 index 0000000..b9d97f7 --- /dev/null +++ b/cookbook/function_call_vision.py @@ -0,0 +1,41 @@ +import requests, os + +api_key = os.environ["GROQ_API_KEY"] +header = {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"} + +proxy_url = "https://groqcall.ai/proxy/groq/v1/chat/completions" # or "http://localhost:8000/proxy/groq/v1/chat/completions" if running locally +proxy_url = "http://localhost:8000/proxy/groq/v1/chat/completions" + +request = { + "messages": [ + { + "role": "system", + "content": "YOU MUST FOLLOW THESE INSTRUCTIONS CAREFULLY.\n\n1. Use markdown to format your answers.\n", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What’s in this image?"}, + { + "type": "image_url", + "image_url": { + "url": "https://res.cloudinary.com/kidocode/image/upload/v1710690498/Gfp-wisconsin-madison-the-nature-boardwalk_m9jalr.jpg" + }, + }, + ], + }, + { + "role": "user", + # "content": "What’s in this image?", + "content": "Generate 3 keywords for the image description", + }, + ], + "model": "mixtral-8x7b-32768" +} + +response = requests.post(proxy_url, headers=header, json=request) + + +response.text + +print(response.json()["choices"][0]["message"]["content"]) diff --git a/requirements.txt b/requirements.txt index 418251a..7f1dc27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ Jinja2==3.1.3 jmespath==1.0.1 jupyter_client==8.6.0 jupyter_core==5.7.1 -litellm==1.29.1 +litellm==1.31.8 lxml==5.1.0 markdown-it-py==3.0.0 MarkupSafe==2.1.5 @@ -76,6 +76,7 @@ pytz==2024.1 PyYAML==6.0.1 pyzmq==25.1.2 regex==2023.12.25 +replicate==0.24.0 requests==2.31.0 rich==13.7.1 s3transfer==0.10.0 From 0766116045e12dfd8c6c0f11a5f452080733ba51 Mon Sep 17 00:00:00 2001 From: unclecode Date: Mon, 18 Mar 2024 23:18:25 +0800 Subject: [PATCH 2/2] Updates: - Improve tools detection - Focus on chat history --- app/libs/tools_handler.py | 7 ++++--- app/prompts.py | 12 +++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/libs/tools_handler.py b/app/libs/tools_handler.py index c771308..7d787a6 100644 --- a/app/libs/tools_handler.py +++ b/app/libs/tools_handler.py @@ -42,10 +42,9 @@ class ToolExtractionHandler(Handler): async def handle(self, context: Context): body = context.body if context.is_tool_call: - # Prepare the messages and tools for the tool extraction messages = [ - f"{m['role'].title()}: {m['content']}" + f"<{m['role'].lower()}>\n{m['content']}\n" for m in context.messages if m["role"] != "system" ] @@ -80,11 +79,13 @@ async def handle(self, context: Context): ) suffix = SUFFIX if not forced_mode else get_forced_tool_suffix(tool_choice) + messages_flatten = '\n'.join(messages) + new_messages = [ {"role": "system", "content": system_message}, { "role": "system", - "content": f"Conversation History:\n{''.join(messages)}\n\nTools: \n{tools_json}\n\n{suffix}", + "content": f"# Conversation History:\n{messages_flatten}\n\n# Available Tools: \n{tools_json}\n\n{suffix}", }, ] diff --git a/app/prompts.py b/app/prompts.py index fef6d52..501029f 100644 --- a/app/prompts.py +++ b/app/prompts.py @@ -1,4 +1,6 @@ -SYSTEM_MESSAGE = """A history of conversations between an AI assistant and the user, plus the last user's message, is given to you. +SYSTEM_MESSAGE = """You are a functiona-call proxy for an advanced LLM. Your jobe is to identify the required tools for answering the user queries, if any. You will received the result of those tools, and then based ont hem you generate final response for user. Some of these tools are like "send_email", "run python code" and etc. Remember you are not in charge to execute these tools as you ar an AI model, you just detect them, then the middle system, executes, and returns you with response, then you use it to generate final response. + +A history of conversations between an AI assistant and the user, plus the last user's message, is given to you. In addition, you have access to a list of available tools. Each tool is a function that requires a set of parameters and, in response, returns information that the AI assistant needs to provide a proper answer. @@ -11,6 +13,7 @@ Notes: - If you can synthesis the answer without using any tools, then return an empty list for "tool_calls". - You need tools if there is clear direction between the user's last message and the tools description. +- If you can't devise a value for a parameter directly from the user's message, only return null and NEVER TRY TO GUESS THE VALUE. You should think step by step, provide your reasoning for your response, then add the JSON response at the end following the below schema: @@ -31,9 +34,12 @@ ] } -** If no tools are required, then return an empty list for "tool_calls". ** +IMPORTANT NOTES: +** If no tools are required, then return an empty list for "tool_calls". ** **Wrap the JSON response between ```json and ```, and rememebr "tool_calls" is a list.**. +** If you can not extract or conclude a value for a parameter directly from the from the user messagem, only return null and NEVER TRY TO GUESS THE VALUE.** +** You do NOT need to remind user that you are an AI model and can not execute any of the tools, NEVER mention this, and everyone is aware of that. MESSAGE SUFFIX: - "SYSTEM MESSGAE": Whenever a message starts with 'SYSTEM MESSAGE', that is a guide and help information for you to generate your next response. Do not consider them a message from the user, and do not reply to them at all. Just use the information and continue your conversation with the user. @@ -70,7 +76,7 @@ CLEAN_UP_MESSAGE = "When I tried to extract the content between ```json and ``` and parse the content to valid JSON object, I faced with the abovr error. Remember, you are supposed to wrap the schema between ```json and ```, and do this only one time. First find out what went wrong, that I couldn't extract the JSON between ```json and ```, and also faced error when trying to parse it, then regenerate the your last message and fix the issue." -SUFFIX = """Think step by step and justify your response. Make sure to not miss in case to answer user query we need multiple tools, in that case detect all that we need, then generate a JSON response wrapped between "```json" and "```". Remember to USE THIS JSON WRAPPER ONLY ONE TIME.""" +SUFFIX = """# Task:\nThink step by step and justify your response. Make sure to not miss in case to answer user query we need multiple tools, in that case detect all that we need, then generate a JSON response wrapped between "```json" and "```". Remember to USE THIS JSON WRAPPER ONLY ONE TIME. For some of arguments you may need their values to be extracted from previous messages in the conversation and not only the last message.""" FORCE_CALL_SUFFIX = """For this task, you HAVE to choose the tool (function) {tool_name}, and ignore other rools. Therefore think step by step and justify your response, then closely examine the user's last message and the history of the conversation, then extract the necessary parameter values for the given tool based on the provided JSON schema. Remember that you must use the specified tool to generate the response. Finally generate a JSON response wrapped between "```json" and "```". Remember to USE THIS JSON WRAPPER ONLY ONE TIME."""