Images support in Hal9 app (#455)

apps/hal9/app.py

@@ -13,6 +13,7 @@
 from tools.image import create_image
 from tools.document import document_reply
 from tools.csv import csv_reply
+from tools.image_analyzer import image_analyzer
 
 MODEL = "llama3-groq-70b-8192-tool-use-preview"
 def run(messages, tools):
@@ -42,7 +43,8 @@ def run(messages, tools):
   build_streamlit,
   create_image,
   document_reply,
-  csv_reply
+  csv_reply,
+  image_analyzer
 ]
 
 tools = h9.describe(all_tools, model = "llama")
@@ -51,7 +53,7 @@ def run(messages, tools):
   completion = run(messages, tools)
   h9.complete(completion, messages = messages, tools = all_tools, show = False, model = "llama")
 except Exception as e:
-  h9.event('error', e)
+  h9.event('error', str(e))
   one_tool = h9.describe([generic_reply], model = "llama")
   completion = run(messages, one_tool)
   h9.complete(completion, messages = messages, tools = [generic_reply], show = False, model = "llama")

apps/hal9/tools/image_analyzer.py

@@ -0,0 +1,33 @@
+import openai
+import os
+
+def image_analyzer(image_url, prompt):
+    """Use this tool when the user provides a URL with an image extension such as JPG or PNG.
+Parameters:
+    'image_url' = URL containing the image or photograph.
+    'prompt' = description of what the user wants to analyze in the image. If the user does not specify, it should default to "What's in this image?"
+"""
+    client = openai.AzureOpenAI(
+    azure_endpoint = 'https://openai-hal9.openai.azure.com/',
+    api_key = os.environ['OPENAI_AZURE'],
+    api_version = '2024-02-15-preview',
+    )
+
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url" : {"url": image_url,},
+                    },
+                ],
+            }
+        ],
+    )
+
+    print(response.choices[0].message.content)
+    return response.choices[0].message.content