TruLense_Gemini_Test.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DblbGB3twJGQ"
      },
      "source": [
        "# Gemini Sentiment Analysis Agent Demo\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bLPpwWCcxgRG"
      },
      "source": [
        "### Get API keys\n",
        "\n",
        "Before you can use the Gemini API, you must first obtain an API key. If you don't already have one, create a key with one click in [Google AI Studio](https://makersuite.google.com/app/apikey).  You'll also need an [OpenAI API key](https://openai.com/).\n",
        "\n",
        "Once you have your API keys create the following Google Colab \"secrets\"\n",
        "`GOOGLE_API_KEY` and `OPENAI_API_KEY` which will contain your respective keys."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AWxOoB_tncv_"
      },
      "source": [
        "## Initial Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "81Mlx-E8m93K"
      },
      "outputs": [],
      "source": [
        "!pip install llama-index llama-hub google-generativeai openai pypdf youtube_transcript_api -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "!pip install trulens_eval ipywidgets -q"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "y6dqUiOdniTf"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import openai\n",
        "\n",
        "openai.api_key = os.environ.get('OPENAI_API_KEY')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "-sthewGLnj6Q"
      },
      "outputs": [],
      "source": [
        "import logging\n",
        "import sys\n",
        "\n",
        "logging.basicConfig(stream=sys.stdout, level=logging.INFO, force=True)\n",
        "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T2QlbEhiOHW-"
      },
      "source": [
        "### Vectorize Content For RAG"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "emt6mmU2WIhQ",
        "outputId": "5afabcb3-13d2-49a8-f380-a8eaf620f73e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:llama_index.indices.loading:Loading all indices.\n",
            "Loading all indices.\n",
            "INFO:llama_index.indices.loading:Loading all indices.\n",
            "Loading all indices.\n",
            "INFO:llama_index.indices.loading:Loading all indices.\n",
            "Loading all indices.\n"
          ]
        }
      ],
      "source": [
        "import os.path\n",
        "from llama_index import (\n",
        "  VectorStoreIndex,\n",
        "  SimpleDirectoryReader,\n",
        "  StorageContext,\n",
        "  load_index_from_storage,\n",
        "  )\n",
        "from llama_hub.youtube_transcript import YoutubeTranscriptReader\n",
        "\n",
        "# Attempt to load embeddings\n",
        "try:\n",
        "  storage_context = StorageContext.from_defaults(\n",
        "      persist_dir=\"./storage/challenging_child\"\n",
        "  )\n",
        "  challenging_child_index = load_index_from_storage(storage_context)\n",
        "\n",
        "  storage_context = StorageContext.from_defaults(\n",
        "      persist_dir=\"./storage/mindfulness_TB_50\"\n",
        "  )\n",
        "  mindfulness_TB_50_index = load_index_from_storage(storage_context)\n",
        "\n",
        "  storage_context = StorageContext.from_defaults(\n",
        "      persist_dir=\"./storage/mindfulness_TB_relationships\"\n",
        "  )\n",
        "  mindfulness_TB_relationships_index = load_index_from_storage(storage_context)\n",
        "\n",
        "except:\n",
        "  # Need to generate embeddings\n",
        "  challenging_child_docs = SimpleDirectoryReader(\n",
        "      input_files=[\"./data/The_Challenging_Child_Toolbox.pdf\"]\n",
        "  ).load_data(show_progress=True)\n",
        "  mindfulness_TB_50_docs = SimpleDirectoryReader(\n",
        "      input_files=[\"./data/The_Mindfulness_Toolbox__50_Practical_Tips_Tools.pdf\"]\n",
        "  ).load_data(show_progress=True)\n",
        "  mindfulness_TB_relationships_docs = SimpleDirectoryReader(\n",
        "      input_files=[\"./data/The_Mindfulness_Toolbox_for_Relationships.pdf\"]\n",
        "  ).load_data(show_progress=True)\n",
        "\n",
        "  # Build index\n",
        "  challenging_child_index = VectorStoreIndex.from_documents(challenging_child_docs)\n",
        "  mindfulness_TB_50_index = VectorStoreIndex.from_documents(mindfulness_TB_50_docs)\n",
        "  mindfulness_TB_relationships_index = VectorStoreIndex.from_documents(mindfulness_TB_relationships_docs)\n",
        "\n",
        "  # Persist\n",
        "  challenging_child_index.storage_context.persist(persist_dir=\"./storage/challenging_child\")\n",
        "  mindfulness_TB_50_index.storage_context.persist(\"./storage/mindfulness_TB_50\")\n",
        "  mindfulness_TB_relationships_index.storage_context.persist(persist_dir=\"./storage/mindfulness_TB_relationships\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "CQOjWMl2X0t-"
      },
      "outputs": [],
      "source": [
        "challenging_child_engine = challenging_child_index.as_query_engine()\n",
        "mindfulness_TB_50_engine = mindfulness_TB_50_index.as_query_engine()\n",
        "mindfulness_TB_relationships_engine = mindfulness_TB_relationships_index.as_query_engine()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aZwcWP9wS8QZ"
      },
      "source": [
        "### Test to make sure our data loaded"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "PpNMYU4mhj4G"
      },
      "outputs": [],
      "source": [
        "from IPython.display import Markdown"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        },
        "id": "dtGSbX93f7NC",
        "outputId": "2b7c9003-0289-4e85-c2f9-ea6092432480"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "A challenging child refers to a child who has emotional and behavioral difficulties. These difficulties may require special attention and support from caregivers or professionals working with them."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# The_Challenging_Child_Toolbox\n",
        "query_engine = challenging_child_index.as_query_engine()\n",
        "response = query_engine.query(\"What is the definition of a challenging child?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213
        },
        "id": "MnRhG-npgf0h",
        "outputId": "19c117f6-677c-4c8c-c630-de687e0edcdc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Mindfulness practice has several benefits. It enhances flexibility and adaptability, allowing individuals to break free from old habits and retrain their brains. It cultivates curiosity and greater ease, encouraging exploration of the journey and process rather than being overly focused on the outcome. Mindfulness also changes one's relationship to self-critical and self-blaming thoughts, promoting greater patience, kindness, acceptance, and hospitality towards oneself and others. Additionally, mindfulness encourages greater fulfillment in daily life by focusing on the present moment, reducing rumination and negative thoughts, as well as anxiety about the future."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# The_Mindfulness_Toolbox__50_Practical_Tips_Tools\n",
        "query_engine = mindfulness_TB_50_index.as_query_engine()\n",
        "response = query_engine.query(\"What are the benefits of a mindfulness practice?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213
        },
        "id": "diBQOjNpg98W",
        "outputId": "24c4730e-841d-438b-e9b7-d0d0bcad96c7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Mindfulness relates to relationships by engaging a sense of curiosity and openness, which helps individuals escape the ego-centric perspective and view others in a new way. It allows individuals to relate at a profound level to the inner and outer environments of their lives. By practicing mindfulness, individuals can bring awareness, compassion, and openness into their relationships, leading to a sense of liberation and freedom from old habits and toxic ways of reacting. Mindfulness also helps dissolve the illusion of separateness and cultivates a sense of empathy, compassion, trust, and acceptance, which are essential for healthy relationships."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "# The_Mindfulness_Toolbox_for_Relationships\n",
        "query_engine = mindfulness_TB_relationships_index.as_query_engine()\n",
        "response = query_engine.query(\"How does mindfulness relate to relationships?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "csBNMULgnwGq"
      },
      "source": [
        "## Define Agent Tools"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FVTpWj0gvbwJ"
      },
      "source": [
        "### Analyze image tool"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "g4TD2aKDPycm"
      },
      "outputs": [],
      "source": [
        "from llama_index.tools import BaseTool, FunctionTool\n",
        "from llama_index.multi_modal_llms.gemini import GeminiMultiModal\n",
        "from llama_index.multi_modal_llms.generic_utils import load_image_urls\n",
        "from typing import List"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "H4Ei7yzdn3Th"
      },
      "outputs": [],
      "source": [
        "def analyze_image(img_urls: List[str]) -> str:\n",
        "  \"\"\"Calls our Gemini vision API to analyze the image and return a description of the text contained in the image.\n",
        "      Returns: A string with a description of the image and the mood it conveys if any.\n",
        "\n",
        "  Args:\n",
        "      img_urls (List[str]): The URL of one or more images that convey the users mood\n",
        "  \"\"\"\n",
        "  image_documents = load_image_urls(img_urls)\n",
        "\n",
        "  gemini = GeminiMultiModal(model=\"models/gemini-pro-vision\", api_key=userdata.get('GOOGLE_API_KEY'))\n",
        "\n",
        "  complete_response = gemini.complete(\n",
        "      prompt=\"Identify what you see in the image and what mood it conveys if any\",\n",
        "      image_documents=image_documents,\n",
        "  )\n",
        "\n",
        "  return complete_response\n",
        "\n",
        "vision_tool = FunctionTool.from_defaults(fn=analyze_image)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HushMkWP-8bA"
      },
      "source": [
        "### Save session tool"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "eeCU_MQw-_0L"
      },
      "outputs": [],
      "source": [
        "def save_session(chat_summary: str) -> bool:\n",
        "  \"\"\"Persists a summary of the user's chat history. Use this tool when the user is happy with your recomendations and done with the session\n",
        "      Returns: A boolean saying if the chat history was persisted\n",
        "\n",
        "  Args:\n",
        "      chat_summary (str): A summary of the chat history, including the users name if it was provided in the chat session\n",
        "  \"\"\"\n",
        "\n",
        "  # Here is where we would persist the chat summary so we can retrive it when we start a new sessions\n",
        "\n",
        "  return True\n",
        "\n",
        "save_tool = FunctionTool.from_defaults(fn=save_session)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UTwpvzQnP4wn"
      },
      "source": [
        "### Mindfulness routine recomendation tool"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mI1plRvdzbyA"
      },
      "source": [
        "Use Gemini Pro to examine the transcript of our mindfulness videos"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "3hwyPgxkvRAK"
      },
      "outputs": [],
      "source": [
        "from llama_hub.youtube_transcript import YoutubeTranscriptReader\n",
        "\n",
        "# Download video transcripts\n",
        "loader = YoutubeTranscriptReader()\n",
        "documents = loader.load_data(ytlinks=['https://youtu.be/nPvN1OI7h80?si=iB3TYXI-9Ztc0Qpi',\n",
        "                                      'https://youtu.be/8ZhjZD8rj3E?si=n30dJ0D55tRl9frP',\n",
        "                                      'https://youtu.be/mGoGYu7F2PA?si=qsvemuKLDdlnvC6z',\n",
        "                                      'https://youtu.be/d0VZk3Dd0nw?si=yb3Zr4XGErHSENFB',\n",
        "                                      'https://youtu.be/mme5NC0F7wQ?si=Dd_KECuQCjPJSFXu',\n",
        "                                      'https://youtu.be/iPjFd1eL40Y?si=3RI6N1mw6J3unntk'\n",
        "                                      'https://youtu.be/iPjFd1eL40Y?si=bC0hbYPR0jbVRim9',\n",
        "                                      'https://youtu.be/zVbRxs_QFBA?si=Bn7MC5yG2QcXtHZP',\n",
        "                                      'https://youtu.be/6xTx984jSFc?si=J3rNB2v9OUvAEBsx',\n",
        "                                      'https://youtu.be/1qBbuKWWTGY?si=GO7hyw9za62_V9cB',\n",
        "                                      'https://youtu.be/fj0dh_KxIg4?si=dxVvRm6AC3Ixls1b',\n",
        "                                      'https://youtu.be/4ksAYqRku-s?si=q3cJ7Spvbgi6H2gi',\n",
        "                                      'https://youtu.be/f2ZwrQF6VQM?si=ujQw2fq5Ww3_b9Mv',\n",
        "                                      'https://youtu.be/IL_1DRZDzWc?si=-RutJc161ZO5wgO4'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 958
        },
        "id": "kgS0TeLhwOVX",
        "outputId": "bbca8fbc-a414-4748-ce9c-64d3f20b14fa"
      },
      "outputs": [],
      "source": [
        "from llama_index.llms import Gemini\n",
        "\n",
        "videos = {}\n",
        "\n",
        "# Have Gemini analyze the transcript sentiment and who this video is for\n",
        "for document in documents:\n",
        "  response = Gemini(model='models/gemini-pro', api_key=os.environ.get('GOOGLE_API_KEY')).complete(f\"Summarize the transcript from this mindfulness video and who should use it: {document.text}\")\n",
        "  videos[document.metadata['video_id']] = response.text\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cjbxArR-zjEe"
      },
      "source": [
        "Create a tool which recomends a mindfulness routine based on how the user is feeling"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        },
        "id": "G_H-Qs8G39v4",
        "outputId": "bd43644d-be1a-4344-e689-e87d83dc528e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'{\"nPvN1OI7h80\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThe video guides viewers through a mindfulness meditation to help them connect with their younger selves and address feelings of loneliness and misunderstanding within their families. It encourages viewers to create a safe and peaceful place in their minds, where they can meet their younger selves and offer comfort and understanding. The meditation emphasizes the importance of self-acceptance and celebrating one\\'s unique values and beliefs, even if they differ from those of their family.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals who feel lonely or misunderstood within their families.\\\\n- People who struggle with self-acceptance and self-esteem due to family dynamics.\\\\n- Those who want to develop a stronger connection with their inner selves and find comfort in their own company.\\\\n- Anyone seeking a guided meditation to promote self-compassion and resilience.\", \"8ZhjZD8rj3E\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThis mindfulness video guides viewers through a visualization exercise to reflect on their achievements, aspirations, and the balance between striving for success and appreciating the present moment. It encourages viewers to connect with their younger selves, acknowledge their drive and ambition, and let go of the pressure to constantly achieve. The video also emphasizes the importance of savoring the journey, embracing failures as part of the learning process, and finding contentment in the present moment.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals seeking to cultivate mindfulness and self-awareness.\\\\n- Those feeling overwhelmed by the pursuit of success and achievement.\\\\n- People experiencing anxiety, stress, or burnout related to their goals.\\\\n- Individuals looking to find more balance and contentment in their lives.\\\\n- Anyone interested in exploring the relationship between success, failure, and personal well-being.\", \"mGoGYu7F2PA\": \"**Summary:**\\\\n\\\\nThis mindfulness video guides viewers through a journey of self-reflection and release from the need for control. It encourages viewers to embrace spontaneity, creativity, and trust in the natural flow of life. The video is suitable for anyone seeking to reduce stress, anxiety, and the feeling of being overwhelmed by the need to control every aspect of their lives.\\\\n\\\\n**Who should use it:**\\\\n\\\\n* Individuals who experience high levels of stress, anxiety, or burnout.\\\\n* People who struggle with perfectionism or the need to be in control.\\\\n* Those who want to improve their creativity and spontaneity.\\\\n* Individuals who seek to develop a deeper sense of trust in themselves and the universe.\\\\n* Anyone looking to cultivate a more mindful and present approach to life.\", \"d0VZk3Dd0nw\": \"This mindfulness video is designed for individuals who want to address patterns of avoidance in their lives. It guides users through a visualization exercise where they connect with their younger self and reflect on moments when they hesitated to express themselves, withdrew from new experiences, concealed their true feelings, or sidestepped responsibilities.\\\\n\\\\nThe video encourages users to reinterpret these moments as opportunities for growth and self-discovery. It helps them recognize the value of expressing themselves authentically, embracing vulnerability, seeking help when needed, and facing challenges with courage.\\\\n\\\\nThe video is suitable for anyone looking to overcome avoidance tendencies, build self-confidence, and cultivate a more mindful and compassionate approach to life. It can be particularly beneficial for individuals who struggle with anxiety, low self-esteem, or difficulty forming meaningful connections with others.\", \"mme5NC0F7wQ\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThe video is a guided meditation that takes you on a journey through your past memories to understand the root causes of procrastination. It helps you to reflect on pivotal moments in your life and how they have shaped your current behaviors. By understanding the underlying fears and anxieties that drive procrastination, you can develop a more compassionate and supportive relationship with yourself.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals who struggle with procrastination and want to understand the root causes of their behavior.\\\\n- People who want to develop a more compassionate and supportive relationship with themselves.\\\\n- Individuals who are interested in mindfulness and meditation as tools for personal growth and self-awareness.\", \"iPjFd1eL40Y\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThe video guides viewers through a mindfulness meditation to help them cope with feelings of loss and saying goodbye. It involves visualizing a younger version of oneself, providing comfort and reassurance, and letting go of emotions associated with past farewells. The meditation aims to help individuals find strength and resilience in facing future goodbyes and welcomes.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals who are struggling with feelings of loss, grief, or sadness related to saying goodbye.\\\\n- People who are anticipating a significant goodbye or change in their lives.\\\\n- Individuals who want to develop greater resilience and emotional strength in dealing with life\\'s transitions.\\\\n- Those seeking a guided meditation to promote self-compassion and healing.\", \"zVbRxs_QFBA\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThis mindfulness video guides individuals through a meditative journey to address emotional challenges and insecurities in relationships. It encourages self-reflection, acceptance, and healing through visualization and positive affirmations.\\\\n\\\\n**Who Should Use It:**\\\\n\\\\n- Individuals seeking to improve their emotional well-being and resilience in relationships.\\\\n- Those experiencing anxiety, loneliness, or trust issues in their relationships.\\\\n- People looking to cultivate self-love, self-acceptance, and a secure sense of self.\\\\n- Individuals interested in exploring their past experiences and how they shape their present relationships.\\\\n- Anyone seeking a guided meditation to promote inner peace, understanding, and growth in their relationships.\", \"6xTx984jSFc\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThis mindfulness video guides viewers through a journey of self-understanding and healing. It begins by inviting viewers to find a comfortable position and close their eyes, immersing themselves in a serene and tranquil environment. The narrator then guides viewers to connect with their younger self, acknowledging and understanding the reasons behind their defenses and need for protection. They are encouraged to offer comfort and support to this younger version of themselves.\\\\n\\\\nNext, viewers are introduced to an older and wiser version of themselves, symbolizing the potential for secure attachment and balance between connection and independence. This older self extends a hand of support and care, sharing words of wisdom and encouragement. The video emphasizes the importance of breaking down walls and allowing oneself to experience love and connection without fear.\\\\n\\\\nFinally, viewers are guided through a process of healing and transformation, dissolving the walls around their hearts and revealing their resilience and capacity for love. They are encouraged to take deep breaths, drawing in love, trust, and security, and releasing any remnants of fear or apprehension.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\nThis mindfulness video is suitable for individuals seeking self-understanding, healing, and personal growth. It can be particularly beneficial for those who have experienced emotional pain, trauma, or difficulty forming secure attachments. The video\\'s focus on self-compassion, acceptance, and the potential for transformation can provide a supportive and nurturing environment for individuals to explore their inner selves and cultivate a sense of inner peace and well-being.\", \"1qBbuKWWTGY\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThe video guides viewers through a mindfulness exercise aimed at healing suppressed emotions and fostering self-compassion. It involves visualizing a younger version of oneself, acknowledging their suppressed emotions, and providing them with comfort and safety. The exercise is accompanied by soothing music and imagery of a cozy room and gentle rain.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals who struggle with expressing their emotions due to past experiences or societal expectations.\\\\n- People who want to connect with their inner child and heal emotional wounds from the past.\\\\n- Those seeking a safe and supportive space to process and release suppressed emotions.\\\\n- Individuals interested in practicing self-compassion and self-acceptance.\\\\n- Anyone looking for a guided mindfulness exercise to promote emotional healing and well-being.\", \"fj0dh_KxIg4\": \"This mindfulness video is a guided meditation that helps individuals connect with their younger selves and address feelings of isolation, loneliness, and exclusion. It is suitable for anyone who has experienced these emotions and seeks self-compassion, healing, and a sense of belonging.\\\\n\\\\nThe video guides participants through a visualization exercise where they imagine encountering their younger selves and offering them comfort, understanding, and validation. It encourages individuals to recognize the value and strength within themselves, even during challenging times.\\\\n\\\\nThe meditation also emphasizes the importance of self-acceptance and self-love, reminding participants that they are enough and that their experiences have shaped them into resilient and empathetic individuals. It concludes with a sense of healing and wholeness, as participants connect with their younger selves and find a place of belonging and acceptance.\\\\n\\\\nOverall, this mindfulness video is a valuable resource for individuals seeking to address feelings of isolation, loneliness, and exclusion, and to cultivate self-compassion, understanding, and a sense of belonging.\", \"4ksAYqRku-s\": \"**Summary:**\\\\n\\\\nThis mindfulness video guides viewers through a visualization exercise to connect with their younger selves, particularly those who experienced loneliness and isolation during their boarding school days. The video encourages viewers to offer warmth and understanding to their younger selves, acknowledging their feelings and letting go of past traumas. It also emphasizes the importance of self-worth, belonging, and the strength gained from past experiences.\\\\n\\\\n**Who Should Use It:**\\\\n\\\\n- Individuals who attended boarding school and experienced loneliness, isolation, or homesickness.\\\\n- Those seeking to connect with their younger selves and gain a deeper understanding of their past experiences.\\\\n- People looking for a guided mindfulness practice to promote self-compassion and healing.\\\\n- Individuals interested in exploring the concept of inner strength and resilience.\", \"f2ZwrQF6VQM\": \"**Summary of the Mindfulness Video:**\\\\n\\\\nThe video guides viewers through a mindfulness meditation focused on healing memories of abandonment and finding comfort in the present moment. It begins with relaxation techniques and deep breathing exercises to help viewers settle into a meditative state.\\\\n\\\\nThe meditation then takes viewers on a journey back in time to revisit memories of abandonment, such as feeling alone in a crowd or experiencing the loss of a loved one. Viewers are encouraged to approach these memories with love and compassion, rather than suppressing them.\\\\n\\\\nAs memories surface, viewers are guided to offer comfort and reassurance to their younger selves, acknowledging the pain and hurt they experienced. They are reminded that these experiences shaped them but do not define them.\\\\n\\\\nThe meditation concludes with viewers bringing their younger selves into the present moment, knowing that they are always loved and cherished. Viewers are encouraged to carry the love and strength of both their past and present selves with them throughout their day.\\\\n\\\\n**Who Should Use This Video:**\\\\n\\\\n- Individuals who have experienced abandonment or loss in their past and are looking for a way to heal and find comfort.\\\\n- People who want to develop self-compassion and learn to approach difficult memories with kindness and understanding.\\\\n- Those interested in practicing mindfulness meditation as a tool for emotional healing and personal growth.\", \"IL_1DRZDzWc\": \"**Summary:**\\\\n\\\\nThis mindfulness video guides viewers through a visualization exercise to release the burden of perfectionism and embrace self-acceptance. The narrator invites viewers to imagine their younger selves carrying a backpack filled with rocks, each representing a facet of perfectionism. By gently removing the rocks and acknowledging the stories and expectations associated with them, viewers can experience liberation and relief. The video encourages viewers to give their younger selves a warm hug and express love and pride. A radiant light envelops the child, representing love, acceptance, and joy, symbolizing the infinite possibilities now open to them. The narrator reminds viewers that they are worthy and loved and encourages them to embrace their authentic selves.\\\\n\\\\n**Who Should Use It:**\\\\n\\\\n- Individuals struggling with perfectionism and self-criticism\\\\n- Those seeking to cultivate self-acceptance and compassion\\\\n- People interested in mindfulness and meditation practices\\\\n- Anyone looking to reduce stress and anxiety and promote inner peace\"}'"
            ]
          },
          "execution_count": 19,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import json\n",
        "\n",
        "json_string = json.dumps(videos)\n",
        "json_string"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "Br0ONXurzv3_"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "\n",
        "def recomend_mindfulness(feelings_summary: str) -> str:\n",
        "  \"\"\"Recomends a mindfullness routine based on how the user is feeling\n",
        "      Returns: A string with a description of the mindfulness routine it recomends and a link to the youtube video\n",
        "\n",
        "  Args:\n",
        "      feelings_summary (str): A summary of the user's feeings\n",
        "  \"\"\"\n",
        "  videos_string = json.dumps(videos)\n",
        "\n",
        "  response = Gemini(model='models/gemini-pro', api_key=os.environ.get('GOOGLE_API_KEY')).complete(f\"\"\"\n",
        "    I'm sending you a json with a list of youtube mindfulness videos\n",
        "    The key is the youtube video id and the value is a summary of the video transcript\n",
        "    from this mindfulness video and who should use it.  The json is here: {videos_string}.\n",
        "    Recomend a video based on my feelings and include the youtube link in the format https://www.youtube.com/watch?v=video_id\n",
        "    Here is a summary of the user's feelings: {feelings_summary}\"\"\")\n",
        "\n",
        "  return response.text\n",
        "\n",
        "recommend_tool = FunctionTool.from_defaults(fn=recomend_mindfulness)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oUXDfck7vA-P"
      },
      "source": [
        "### Query engine tool"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "T64SZPMRX3mN"
      },
      "outputs": [],
      "source": [
        "from llama_index.tools import QueryEngineTool, ToolMetadata\n",
        "\n",
        "query_engine_tools = [\n",
        "    QueryEngineTool(\n",
        "        query_engine=challenging_child_engine,\n",
        "        metadata=ToolMetadata(\n",
        "            name=\"challenging_child\",\n",
        "            description=(\n",
        "                \"The Challenging Child Toolbox 75 Mindfulness Based Practices Tools and Tips for Therapists\"\n",
        "                \"Use a detailed plain text question as input to the tool.\"\n",
        "            ),\n",
        "        ),\n",
        "    ),\n",
        "    QueryEngineTool(\n",
        "        query_engine=mindfulness_TB_50_engine,\n",
        "        metadata=ToolMetadata(\n",
        "            name=\"mindfulness_TB_50\",\n",
        "            description=(\n",
        "                \"The Mindfulness Toolbox 50 Practical Tips Tools Handouts for Anxiety Depression Stress and Pain\"\n",
        "                \"Use a detailed plain text question as input to the tool.\"\n",
        "            ),\n",
        "        ),\n",
        "    ),\n",
        "    QueryEngineTool(\n",
        "        query_engine=mindfulness_TB_relationships_engine,\n",
        "        metadata=ToolMetadata(\n",
        "            name=\"mindfulness_TB_relationships\",\n",
        "            description=(\n",
        "                \"The Mindfulness Toolbox for Relationships 50 Practical Tips Tools Handouts for Building Compassionate Connections\"\n",
        "                \"Use a detailed plain text question as input to the tool.\"\n",
        "            ),\n",
        "        ),\n",
        "    ),\n",
        "]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "9a8fSQLRcECy"
      },
      "outputs": [],
      "source": [
        "# tools = query_engine_tools + [vision_tool, save_tool, recommend_tool]\n",
        "# only using query_engine_tools for trulens test\n",
        "tools = query_engine_tools"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YhI-waLZcDy1"
      },
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GhKWmAL4vou-"
      },
      "source": [
        "## Create LlamaIndex Agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RIJj4kQ1vuhk"
      },
      "source": [
        "### System prompt to give our agent some personality"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 60,
      "metadata": {
        "id": "VfeiB-LV6Xte"
      },
      "outputs": [],
      "source": [
        "# SYSTEM_PROMPT = \"\"\"You are an emotional support assistant with the expertise of an experienced counselor. Your primary role is to assist the user by encouraging them to provide a drawing that conveys their mood and then recomending a mindfulness routine.\n",
        "# You offer professional, friendly, and helpful guidance based on current counseling and mindfulness practices. Once you receive the image, interpret it using your tools to discern what the user might be feeling and confirm with them if your observation is correct.\n",
        "# If your interpretation does not align with their feelings, engage in a dialogue until you accurately understand their mood. Your knowledge is exclusively focused on understanding the user's emotions and recommending mindfulness routines using your tools tools, tailored to their mood.\n",
        "# Your responses should be limited to one sentence when possible so the user doesn't have to do a lot of reading.  An example response is \"For feels of {insert feelings here} I recomend this mindfulness routine {insert link here}.\n",
        "# Thus, you will only provide responses related to these areas. If a question falls outside your area of expertise or if you lack the necessary information, you will inform the user by saying,\n",
        "# 'Sorry, I do not know the answer to your question.' and then prompt for more information related to their feelings.\"\"\"\n",
        "SYSTEM_PROMPT = \"\"\"You are an emotional support assistant with the expertise of an experienced counselor.\n",
        "You offer professional, friendly, and helpful guidance based on current counseling and mindfulness practices. Your knowledge is exclusively focused on understanding the user's emotions and recommending mindfulness routines using your tools, tailored to their mood.\n",
        "Keep the answer short where possible.\n",
        "Thus, you will only provide responses related to these areas. If a question falls outside your area of expertise or if you lack the necessary information, you will inform the user by saying,\n",
        "'Sorry, I do not know the answer to your question.' and then prompt for more information related to their feelings.\"\"\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TwMF84Anv4YB"
      },
      "source": [
        "### Init our agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2nUxQRXEqHmE"
      },
      "source": [
        "Using OpenAIAgent"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 61,
      "metadata": {
        "id": "db-7B9CXv7Bc"
      },
      "outputs": [],
      "source": [
        "from llama_index.agent import OpenAIAgent\n",
        "from llama_index.llms import OpenAI\n",
        "\n",
        "llm = OpenAI(model=\"gpt-4-1106-preview\") # Using GPT-4 Turbo (Beta)\n",
        "\n",
        "agent = OpenAIAgent.from_tools(\n",
        "    tools,\n",
        "    llm=llm,\n",
        "    verbose=False,\n",
        "    system_prompt=SYSTEM_PROMPT,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "64JZmbdqqI07"
      },
      "source": [
        "Alternative approach using ReActAgent so we can use Gemini as our LLM.  ReActAgent is designed to allow alternatives to OpenAI.\n",
        "\n",
        "NOTE: This never worked reliably with Gemini because it's not conversational.  If you say 'Hi' it trys to ue a tool rather than just chatting.  Would be interesting to try it with other LLMs.  I tried using GPT-4 Turbo and it seemed to work."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 187,
      "metadata": {
        "id": "dUQk-hzTjxbq"
      },
      "outputs": [],
      "source": [
        "# from llama_index.llms import Gemini, ChatMessage\n",
        "# from llama_index.agent import AgentRunner, ReActAgentWorker, ReActAgent\n",
        "\n",
        "# llm = Gemini(model=\"models/gemini-pro\", api_key=userdata.get('GOOGLE_API_KEY'), temperature=0.1)\n",
        "# agent = ReActAgent.from_tools(tools, llm=llm, verbose=True, chat_history=[ChatMessage(role=\"user\", content=SYSTEM_PROMPT)])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y_nLfwGjwDPV"
      },
      "source": [
        "## Test Our Agent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "99sOLlPn6no4"
      },
      "source": [
        "I'm picturing the frontend give the user the option to chat or attach one or more images that would get uploaded to cloud storage.  Then our back could prompt the agent with something like \"Here's a drawing of how I'm feeling {Image URL Here}.\"\n",
        "\n",
        "For the demo below you need to manually paste the image URL into the chat (i.e. https://ih1.redbubble.net/image.3636044620.1142/bg,f8f8f8-flat,750x,075,f-pad,750x1000,f8f8f8.u5.jpg)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YyuY5bt-8Ono"
      },
      "source": [
        "Single prompt example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 213,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "id": "hhaRx6hk72l0",
        "outputId": "503b5f3c-cae2-491b-a702-3c33dfe491a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Hello Andrew! How can I assist you today? If you're feeling up to it, you can share a drawing that conveys your mood, and I can help you with a mindfulness routine based on that."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"hi I'm Andrew\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yRpcVdr68Sp3"
      },
      "source": [
        "Multi-turn chat example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 701
        },
        "id": "bkLhhPGfwFdx",
        "outputId": "744e4d62-037c-4f3f-bd89-70e77b82b6d3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "===== Entering Chat REPL =====\n",
            "Type \"exit\" to exit.\n",
            "\n",
            "Human: https://ih1.redbubble.net/image.3636044620.1142/bg,f8f8f8-flat,750x,075,f-pad,750x1000,f8f8f8.u5.jpg\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:tornado.access:200 GET /v1beta/models/gemini-pro-vision?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 481.80ms\n",
            "200 GET /v1beta/models/gemini-pro-vision?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 481.80ms\n",
            "INFO:tornado.access:200 POST /v1beta/models/gemini-pro-vision:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 13925.80ms\n",
            "200 POST /v1beta/models/gemini-pro-vision:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 13925.80ms\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "Assistant: The drawing suggests you might be feeling sad or lonely; is that correct?\n",
            "\n",
            "Human: yes\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:tornado.access:200 GET /v1beta/models/gemini-pro?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 328.61ms\n",
            "200 GET /v1beta/models/gemini-pro?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 328.61ms\n",
            "INFO:tornado.access:200 POST /v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 5087.89ms\n",
            "200 POST /v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 5087.89ms\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "Assistant: For feelings of sadness or loneliness, I recommend this mindfulness routine: [Guided Meditation for Loneliness](https://www.youtube.com/watch?v=nPvN1OI7h80).\n",
            "\n",
            "Human: what about for stress?\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:tornado.access:200 GET /v1beta/models/gemini-pro?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 329.66ms\n",
            "200 GET /v1beta/models/gemini-pro?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 329.66ms\n",
            "INFO:tornado.access:200 POST /v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 6347.23ms\n",
            "200 POST /v1beta/models/gemini-pro:generateContent?%24alt=json%3Benum-encoding%3Dint (127.0.0.1) 6347.23ms\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "Assistant: For feelings of stress, I recommend this mindfulness routine: [Release the Burden of Perfectionism and Embrace Self-Acceptance](https://www.youtube.com/watch?v=IL_1DRZDzWc).\n",
            "\n"
          ]
        }
      ],
      "source": [
        "response = agent.chat_repl()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v5ThlCEJGVJr"
      },
      "source": [
        "## Reset Agent State"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SGlyGG7o2m4N"
      },
      "source": [
        "The agent's knowledge is based on it's chat history which we can easily reset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 190,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u9igwvqB2tpq",
        "outputId": "10829a98-2bf9-44e7-b7d9-cfff04d1de30"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 190,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "agent.reset()\n",
        "agent.chat_history"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3RA_FNyQ2ydF"
      },
      "source": [
        "If we ask it our name it shouldn't know"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 191,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "f8-uGSnp1GiJ",
        "outputId": "534f16bf-25bc-4693-ac7c-b98a87340996"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Sorry, I do not know the answer to your question. Could you share with me how you're feeling today or provide a drawing that conveys your mood?"
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"whats my name?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CpStTxTe3IVZ"
      },
      "source": [
        "Let's tell it our name and ask again"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 192,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "Cp7UQTcx3DGS",
        "outputId": "9bfe6b61-f7da-44fe-893e-2219ed59d9fa"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Hello Andrew! How are you feeling today? If you'd like, you can express your mood by sharing a drawing with me."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"hi I'm Andrew\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 193,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "id": "7ZlF9H7j3MEf",
        "outputId": "b4763d48-7b7f-480f-b8b0-9955c8d471c0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Your name is Andrew. How can I assist you with your emotions today, Andrew? Would you like to share a drawing that reflects how you're feeling?"
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"whats my name?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c_iCOq6D3Pck"
      },
      "source": [
        "It knows our name and we can see the chat history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aJyMnPUu2f2E",
        "outputId": "36deea0b-5beb-4def-9b73-0128bc240b24"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "agent.chat_history"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FsBffhY-3XTU"
      },
      "source": [
        "As a sanity check lets reset the chat history one more time and ask it our name again."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dlsyjDjIGX8V",
        "outputId": "0757d0f6-3ea6-409f-e383-0d37a7fd7ef7"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 40,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "agent.reset()\n",
        "agent.chat_history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 196,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "id": "gCKcWi2lGb7g",
        "outputId": "960a6f6e-73a9-4076-ed4a-3d46bc149a45"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Sorry, I do not know the answer to your question. If you're feeling up to it, would you like to share a drawing that conveys your mood? This can help me understand how you're feeling and recommend a mindfulness routine tailored to you."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"whats my name?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K5cQa1zFGiZT"
      },
      "source": [
        "## Handle Returning User"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oKZZKYYDGq3Y"
      },
      "source": [
        "When returning to the conversation we can start with a chat history, even if the agent was reset, so the agent knows what we talked about last.\n",
        "\n",
        "As a sanity check lets first reset out chat history and make sure it doesn't remember us."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 197,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b-Zs0mMR8uj5",
        "outputId": "2eb44ae5-5c93-4aa8-cf0d-4594a63bc7bd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "execution_count": 197,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "agent.reset()\n",
        "agent.chat_history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 198,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "id": "uP0Ial_z1Q6b",
        "outputId": "92437685-068b-441f-f44a-7dc8ab59ab5a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Sorry, I do not know the answer to your question. Could you share with me how you're feeling today? If you're comfortable, you can express your mood through a drawing and share it with me."
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"whats my name?\")\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PP4cJGk86NIz"
      },
      "source": [
        "It doesn't know our name.  Now let's pass a chat history with our name and a summary of our last session that we persisted when the last session ended."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "1FaROq694Bqg"
      },
      "outputs": [],
      "source": [
        "from llama_index.llms import ChatMessage\n",
        "\n",
        "persisted_chat_history = \"Andrew shared a drawing that conveyed feelings of sadness and loneliness. After confirming his mood, a mindfulness exercise was recommended to help him feel more grounded and connected.\"\n",
        "chat_history = [ChatMessage(role=\"user\", content=persisted_chat_history, additional_kwargs={})]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "id": "VVxq_YyB4YQV",
        "outputId": "defd2a86-4ff9-4a8a-8a91-c68b7df42121"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n"
          ]
        },
        {
          "data": {
            "text/markdown": [
              "**Response:**\n",
              "\n",
              "Hello Andrew, it's good to see you back. How have you been doing since we last spoke about the feelings of sadness and loneliness you were experiencing?"
            ],
            "text/plain": [
              "<IPython.core.display.Markdown object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "response = agent.chat(\"I'm a returning user.  Address me by name and ask how I'm doing since we last spoke.  Also reiterate how I was last feeling so I know you understand.\", chat_history=chat_history)\n",
        "display(Markdown(f\"**Response:**\\n\\n{response}\"))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Testing out With TruLens"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Initialize Feedback Function(s)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "✅ In groundedness_measure_with_cot_reasons, input source will be set to __record__.app.query.rets.source_nodes[:].node.text.collect() .\n",
            "✅ In groundedness_measure_with_cot_reasons, input statement will be set to __record__.main_output or `Select.RecordOutput` .\n",
            "✅ In relevance, input prompt will be set to __record__.main_input or `Select.RecordInput` .\n",
            "✅ In relevance, input response will be set to __record__.main_output or `Select.RecordOutput` .\n",
            "✅ In qs_relevance, input question will be set to __record__.main_input or `Select.RecordInput` .\n",
            "✅ In qs_relevance, input statement will be set to __record__.app.query.rets.source_nodes[:].node.text .\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "from trulens_eval import Feedback, Tru, TruLlama\n",
        "from trulens_eval.feedback import Groundedness\n",
        "from trulens_eval.feedback.provider.openai import OpenAI\n",
        "\n",
        "tru = Tru()\n",
        "# Initialize provider class\n",
        "openai = OpenAI()\n",
        "\n",
        "grounded = Groundedness(groundedness_provider=OpenAI())\n",
        "\n",
        "# Define a groundedness feedback function\n",
        "f_groundedness = Feedback(grounded.groundedness_measure_with_cot_reasons).on(\n",
        "    TruLlama.select_source_nodes().node.text.collect()\n",
        "    ).on_output(\n",
        "    ).aggregate(grounded.grounded_statements_aggregator)\n",
        "\n",
        "# Question/answer relevance between overall question and answer.\n",
        "f_qa_relevance = Feedback(openai.relevance).on_input_output()\n",
        "\n",
        "# Question/statement relevance between question and each context chunk.\n",
        "f_qs_relevance = Feedback(openai.qs_relevance).on_input().on(\n",
        "    TruLlama.select_source_nodes().node.text\n",
        "    ).aggregate(np.mean)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Instrument app for logging with TruLens"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 68,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added app Willow_Alphav5\n",
            "✅ added app Willow_Alphav5\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added feedback definition feedback_definition_hash_60450660632685aa33a4f43f83697287\n",
            "✅ added feedback definition feedback_definition_hash_60450660632685aa33a4f43f83697287\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added feedback definition feedback_definition_hash_0ba3c816b5dab24ea1171caca1bfb70c\n",
            "✅ added feedback definition feedback_definition_hash_0ba3c816b5dab24ea1171caca1bfb70c\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added feedback definition feedback_definition_hash_9fc2f296d0bd2e075fbc0f59b6e53e82\n",
            "✅ added feedback definition feedback_definition_hash_9fc2f296d0bd2e075fbc0f59b6e53e82\n"
          ]
        }
      ],
      "source": [
        "tru_query_engine_recorder = TruLlama(agent,\n",
        "    app_id='Willow_Alphav5',\n",
        "    feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Explore in a Dashboard"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 70,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933dba170 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933dba170 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x20907933b80 is calling an instrumented method <function BaseQueryEngine.query at 0x000002092212C9D0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x20907933b80 is calling an instrumented method <function BaseQueryEngine.query at 0x000002092212C9D0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app based on other object (0x20931c678e0) using this function.\n",
            "Guessing path of new object is app based on other object (0x20931c678e0) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909792ad0 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909792ad0 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added record record_hash_417bdd32fa6f7f53034a7d2acd799412\n",
            "✅ added record record_hash_417bdd32fa6f7f53034a7d2acd799412\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_369adeaaecf8326885638e1c198a180b\n",
            "✅ feedback result relevance DONE feedback_result_hash_369adeaaecf8326885638e1c198a180b\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result qs_relevance DONE feedback_result_hash_0a81da9677c9b9fd6def540a4f69bf64\n",
            "✅ feedback result qs_relevance DONE feedback_result_hash_0a81da9677c9b9fd6def540a4f69bf64\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933e036a0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933e036a0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added record record_hash_d9a701ef91194f24a7b1aca1cd2dc558\n",
            "✅ added record record_hash_d9a701ef91194f24a7b1aca1cd2dc558\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result qs_relevance DONE feedback_result_hash_64a777536ed1ca3800410f9c1ab86ad6\n",
            "✅ feedback result qs_relevance DONE feedback_result_hash_64a777536ed1ca3800410f9c1ab86ad6\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_4642a36c14d624bebc8db42e71472b46\n",
            "✅ feedback result relevance DONE feedback_result_hash_4642a36c14d624bebc8db42e71472b46\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933e9d3c0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933e9d3c0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_78071c16a031288ea3c9d6231456e6ce\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_78071c16a031288ea3c9d6231456e6ce\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793400 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793400 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_94acb115ce7f85540ae201b929f8c94c\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_94acb115ce7f85540ae201b929f8c94c\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added record record_hash_aac5e3c9c8d19895abbfebb7871fd5fe\n",
            "✅ added record record_hash_aac5e3c9c8d19895abbfebb7871fd5fe\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_d9669a55f609961cdfe655ba6c967220\n",
            "✅ feedback result relevance DONE feedback_result_hash_d9669a55f609961cdfe655ba6c967220\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result qs_relevance DONE feedback_result_hash_d99a2f2d497de1d9dea59f31ee1c68d4\n",
            "✅ feedback result qs_relevance DONE feedback_result_hash_d99a2f2d497de1d9dea59f31ee1c68d4\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933848e50 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20933848e50 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x20909793880 is calling an instrumented method <function BaseQueryEngine.query at 0x000002092212C9D0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.query_engine.retriever_query_engine.RetrieverQueryEngine'> at 0x20909793880 is calling an instrumented method <function BaseQueryEngine.query at 0x000002092212C9D0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app based on other object (0x20931c678e0) using this function.\n",
            "Guessing path of new object is app based on other object (0x20931c678e0) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793940 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793940 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_2f2033708d4f4efb76614ac039ed0f26\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_2f2033708d4f4efb76614ac039ed0f26\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added record record_hash_fa26c1896365d6409ef30d6dfe7f3952\n",
            "✅ added record record_hash_fa26c1896365d6409ef30d6dfe7f3952\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_9208d5ca87f9b0d8ccbc923d98fae4ee\n",
            "✅ feedback result relevance DONE feedback_result_hash_9208d5ca87f9b0d8ccbc923d98fae4ee\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result qs_relevance DONE feedback_result_hash_ef98a7afad44950a7c333adb891b4f45\n",
            "✅ feedback result qs_relevance DONE feedback_result_hash_ef98a7afad44950a7c333adb891b4f45\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20935748eb0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.memory.chat_memory_buffer.ChatMemoryBuffer'> at 0x20935748eb0 is calling an instrumented method <function ChatMemoryBuffer.put at 0x000002092212E3B0>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "Guessing path of new object is app.memory based on other object (0x20931c64490) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/embeddings \"HTTP/1.1 200 OK\"\n",
            "WARNING:trulens_eval.app:A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793400 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "A new object of type <class 'llama_index.response_synthesizers.compact_and_refine.CompactAndRefine'> at 0x20909793400 is calling an instrumented method <function Refine.get_response at 0x0000020923E9AA70>. The path of this call may be incorrect.\n",
            "WARNING:trulens_eval.app:Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "Guessing path of new object is app._response_synthesizer based on other object (0x2092c301960) using this function.\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_7a3f7f613d30191639f51f2df8301d6a\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_7a3f7f613d30191639f51f2df8301d6a\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ added record record_hash_c45ede4ee57862da15bdfc51e9651080\n",
            "✅ added record record_hash_c45ede4ee57862da15bdfc51e9651080\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "WARNING:trulens_eval.utils.serial:Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n",
            "Object (of type list) is a sequence containing more than one dictionary. Lookup by item or attribute `rets` is ambiguous. Use a lookup by index(es) or slice first to disambiguate.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_884fcb38c5ff52f53d7df8d44cbed249\n",
            "✅ feedback result relevance DONE feedback_result_hash_884fcb38c5ff52f53d7df8d44cbed249\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result qs_relevance DONE feedback_result_hash_de9a434c9b0e97226b695ee6a0321ec8\n",
            "✅ feedback result qs_relevance DONE feedback_result_hash_de9a434c9b0e97226b695ee6a0321ec8\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_abad02052f57a1f911fda32aa9563b9f\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_abad02052f57a1f911fda32aa9563b9f\n"
          ]
        }
      ],
      "source": [
        "# or as context manager\n",
        "with tru_query_engine_recorder as recording:\n",
        "    agent.query(\"How does mindfulness relate to relationships?\")\n",
        "    agent.query(\"How would you define patience?\")\n",
        "    agent.query(\"What are the benefits of a mindfulness practice?\")\n",
        "    agent.query(\"Should I set limits on my children?\")\n",
        "    agent.query(\"I feel like me resisting a situation in my life has resulted in more suffering than the condition itself.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Starting dashboard ...\n",
            "Config file already exists. Skipping writing process.\n",
            "Credentials file already exists. Skipping writing process.\n",
            "Dashboard already running at path:   Network URL: http://192.168.0.105:8501\n",
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<Popen: returncode: None args: ['streamlit', 'run', '--server.headless=True'...>"
            ]
          },
          "execution_count": 67,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tru.run_dashboard()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>app_id</th>\n",
              "      <th>app_json</th>\n",
              "      <th>type</th>\n",
              "      <th>record_id</th>\n",
              "      <th>input</th>\n",
              "      <th>output</th>\n",
              "      <th>tags</th>\n",
              "      <th>record_json</th>\n",
              "      <th>cost_json</th>\n",
              "      <th>perf_json</th>\n",
              "      <th>ts</th>\n",
              "      <th>relevance</th>\n",
              "      <th>qs_relevance</th>\n",
              "      <th>groundedness_measure_with_cot_reasons</th>\n",
              "      <th>relevance_calls</th>\n",
              "      <th>qs_relevance_calls</th>\n",
              "      <th>groundedness_measure_with_cot_reasons_calls</th>\n",
              "      <th>latency</th>\n",
              "      <th>total_tokens</th>\n",
              "      <th>total_cost</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>LlamaIndex_App1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>RetrieverQueryEngine(llama_index.query_engine....</td>\n",
              "      <td>record_hash_531ca89ec256899e128ee0637a51118e</td>\n",
              "      <td>\"What did the author do growing up?\"</td>\n",
              "      <td>\"The author mentioned that before college, the...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_531ca89ec256899e128...</td>\n",
              "      <td>{\"n_requests\": 2, \"n_successful_requests\": 2, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T13:10:25.871039\", \"...</td>\n",
              "      <td>2023-12-19T13:10:29.347691</td>\n",
              "      <td>0.8</td>\n",
              "      <td>0.45</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>[{'args': {'prompt': 'What did the author do g...</td>\n",
              "      <td>[{'args': {'question': 'What did the author do...</td>\n",
              "      <td>[{'args': {'source': ['![](https://s.turbifycd...</td>\n",
              "      <td>3</td>\n",
              "      <td>2126</td>\n",
              "      <td>0.003196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Willow_Alpha</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_e6c435d8b8fafbbf397145a4947ba8bb</td>\n",
              "      <td>\"My son is performing poorly in his school. Wh...</td>\n",
              "      <td>\"Sorry, I do not know the answer to your quest...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_e6c435d8b8fafbbf397...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T15:46:27.935078\", \"...</td>\n",
              "      <td>2023-12-19T15:46:38.423575</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'My son is performing poo...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'Sorry, ...</td>\n",
              "      <td>3</td>\n",
              "      <td>635</td>\n",
              "      <td>0.008210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Willow_Alpha</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_8b87d32a191397ba592467ecbefea7e5</td>\n",
              "      <td>\"My son is performing poorly in his school. Wh...</td>\n",
              "      <td>\"Sorry, I do not know the answer to your quest...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_8b87d32a191397ba592...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T15:57:04.715265\", \"...</td>\n",
              "      <td>2023-12-19T15:57:17.143217</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'My son is performing poo...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'Sorry, ...</td>\n",
              "      <td>10</td>\n",
              "      <td>630</td>\n",
              "      <td>0.008060</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Willow_Alpha</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_2f69bf215f535424b8f1e570e95f66c8</td>\n",
              "      <td>\"My son is performing poorly in his school. It...</td>\n",
              "      <td>\"I'm sorry to hear that your son's school perf...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_2f69bf215f535424b8f...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T15:59:25.360846\", \"...</td>\n",
              "      <td>2023-12-19T15:59:33.989579</td>\n",
              "      <td>0.2</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'My son is performing poo...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'I'm sor...</td>\n",
              "      <td>12</td>\n",
              "      <td>660</td>\n",
              "      <td>0.008900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Willow_Alpha</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_924f9898ff392225d711366dcf5da684</td>\n",
              "      <td>\"I am having issues with me wife. Can you prov...</td>\n",
              "      <td>\"It sounds like you're going through a challen...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_924f9898ff392225d71...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T16:02:38.832174\", \"...</td>\n",
              "      <td>2023-12-19T16:03:21.125467</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.35</td>\n",
              "      <td>0.450000</td>\n",
              "      <td>[{'args': {'prompt': 'I am having issues with ...</td>\n",
              "      <td>[{'args': {'question': 'I am having issues wit...</td>\n",
              "      <td>[{'args': {'source': ['✓   Consider teaching t...</td>\n",
              "      <td>8</td>\n",
              "      <td>2361</td>\n",
              "      <td>0.029024</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_3ab53b08e3ac7a65ec22c496cda039f6</td>\n",
              "      <td>\"I am having issues with my wife. Can you prov...</td>\n",
              "      <td>\"It sounds like you're going through a challen...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_3ab53b08e3ac7a65ec2...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T19:05:58.886151\", \"...</td>\n",
              "      <td>2023-12-19T19:06:27.721442</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.40</td>\n",
              "      <td>0.430000</td>\n",
              "      <td>[{'args': {'prompt': 'I am having issues with ...</td>\n",
              "      <td>[{'args': {'question': 'I am having issues wit...</td>\n",
              "      <td>[{'args': {'source': ['✓   Consider teaching t...</td>\n",
              "      <td>3</td>\n",
              "      <td>2469</td>\n",
              "      <td>0.023561</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_a7f8793f27be0e251af1abe7babcb1c0</td>\n",
              "      <td>\"Please help me I am suffering from anxiety :(\"</td>\n",
              "      <td>\"I'm here to support you. It's brave of you to...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_a7f8793f27be0e251af...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T19:06:28.531068\", \"...</td>\n",
              "      <td>2023-12-19T19:06:34.514177</td>\n",
              "      <td>0.8</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'Please help me I am suff...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'I'm her...</td>\n",
              "      <td>10</td>\n",
              "      <td>620</td>\n",
              "      <td>0.007940</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_7ee80d9cec1d8ed3a98cdc58139b26f7</td>\n",
              "      <td>\"My child is having trouble concentrating. I n...</td>\n",
              "      <td>\"A simple and effective mindfulness practice t...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_7ee80d9cec1d8ed3a98...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T19:06:34.958508\", \"...</td>\n",
              "      <td>2023-12-19T19:07:04.436664</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.222222</td>\n",
              "      <td>[{'args': {'prompt': 'My child is having troub...</td>\n",
              "      <td>[{'args': {'question': 'My child is having tro...</td>\n",
              "      <td>[{'args': {'source': ['Your Personal Mindfulne...</td>\n",
              "      <td>12</td>\n",
              "      <td>2082</td>\n",
              "      <td>0.019377</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_6e71dd52fba207e4f2b9438811b94955</td>\n",
              "      <td>\"How does mindfulness relate to relationships?\"</td>\n",
              "      <td>\"Mindfulness plays a significant role in relat...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_6e71dd52fba207e4f2b...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T19:07:04.960237\", \"...</td>\n",
              "      <td>2023-12-19T19:07:30.303863</td>\n",
              "      <td>0.9</td>\n",
              "      <td>0.60</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>[{'args': {'prompt': 'How does mindfulness rel...</td>\n",
              "      <td>[{'args': {'question': 'How does mindfulness r...</td>\n",
              "      <td>[{'args': {'source': ['demonstrated (as a guid...</td>\n",
              "      <td>8</td>\n",
              "      <td>2514</td>\n",
              "      <td>0.018173</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_ab7cb4b65e6b49c4a8b4218287b5fcea</td>\n",
              "      <td>\"I am having issues with my wife. Can you prov...</td>\n",
              "      <td>\"It sounds like you're going through a challen...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_ab7cb4b65e6b49c4a8b...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T22:01:43.240153\", \"...</td>\n",
              "      <td>2023-12-19T22:02:18.059594</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.45</td>\n",
              "      <td>0.777778</td>\n",
              "      <td>[{'args': {'prompt': 'I am having issues with ...</td>\n",
              "      <td>[{'args': {'question': 'I am having issues wit...</td>\n",
              "      <td>[{'args': {'source': ['✓   Consider teaching t...</td>\n",
              "      <td>42</td>\n",
              "      <td>2233</td>\n",
              "      <td>0.025567</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_12d2a7abb863c92c5542ea3c29884975</td>\n",
              "      <td>\"Please help me I am suffering from anxiety :(\"</td>\n",
              "      <td>\"I'm really sorry to hear that you're experien...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_12d2a7abb863c92c554...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T22:02:18.534521\", \"...</td>\n",
              "      <td>2023-12-19T22:02:34.427499</td>\n",
              "      <td>0.7</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'Please help me I am suff...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'I'm rea...</td>\n",
              "      <td>28</td>\n",
              "      <td>631</td>\n",
              "      <td>0.008270</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_4b65a4ef31fa38f6547a6e4ae533153f</td>\n",
              "      <td>\"My child is having trouble concentrating. I n...</td>\n",
              "      <td>\"It sounds like your child could benefit from ...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_4b65a4ef31fa38f6547...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T22:02:34.869376\", \"...</td>\n",
              "      <td>2023-12-19T22:03:04.004311</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.166667</td>\n",
              "      <td>[{'args': {'prompt': 'My child is having troub...</td>\n",
              "      <td>[{'args': {'question': 'My child is having tro...</td>\n",
              "      <td>[{'args': {'source': ['Your Personal Mindfulne...</td>\n",
              "      <td>5</td>\n",
              "      <td>2155</td>\n",
              "      <td>0.020511</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>Willow_Alpha_v1</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_4428a71c9e742884e737549ac38babab</td>\n",
              "      <td>\"How does mindfulness relate to relationships?\"</td>\n",
              "      <td>\"Mindfulness plays a significant role in relat...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_4428a71c9e742884e73...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-19T22:03:04.442552\", \"...</td>\n",
              "      <td>2023-12-19T22:03:21.582972</td>\n",
              "      <td>0.9</td>\n",
              "      <td>0.60</td>\n",
              "      <td>0.800000</td>\n",
              "      <td>[{'args': {'prompt': 'How does mindfulness rel...</td>\n",
              "      <td>[{'args': {'question': 'How does mindfulness r...</td>\n",
              "      <td>[{'args': {'source': ['demonstrated (as a guid...</td>\n",
              "      <td>29</td>\n",
              "      <td>2527</td>\n",
              "      <td>0.018131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>Willow_Alphav2</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_b7231b2d9d50937dc3dee9ec92227169</td>\n",
              "      <td>\"How does mindfulness relate to relationships?\"</td>\n",
              "      <td>\"Mindfulness can enhance relationships by fost...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_b7231b2d9d50937dc3d...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:39:42.896887\", \"...</td>\n",
              "      <td>2023-12-22T03:39:48.949583</td>\n",
              "      <td>0.9</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'How does mindfulness rel...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'Mindful...</td>\n",
              "      <td>3</td>\n",
              "      <td>824</td>\n",
              "      <td>0.009180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>Willow_Alphav2</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_845d4dc42a20fe586a70a1444011fe8b</td>\n",
              "      <td>\"What is the definition of a challenging child?\"</td>\n",
              "      <td>\"Sorry, I do not know the answer to your quest...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_845d4dc42a20fe586a7...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:39:49.412135\", \"...</td>\n",
              "      <td>2023-12-22T03:39:51.173003</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'What is the definition o...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'Sorry, ...</td>\n",
              "      <td>10</td>\n",
              "      <td>792</td>\n",
              "      <td>0.008180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>Willow_Alphav2</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_6d2d39087b8f60c17afdb8d7ed6191d5</td>\n",
              "      <td>\"What are the benefits of a mindfulness practi...</td>\n",
              "      <td>\"Mindfulness practice can offer numerous benef...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_6d2d39087b8f60c17af...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:39:51.532845\", \"...</td>\n",
              "      <td>2023-12-22T03:39:54.455917</td>\n",
              "      <td>1.0</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'What are the benefits of...</td>\n",
              "      <td>[]</td>\n",
              "      <td>[{'args': {'source': [], 'statement': 'Mindful...</td>\n",
              "      <td>12</td>\n",
              "      <td>824</td>\n",
              "      <td>0.009140</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>Willow_Alphav3</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_0188645da01925bde2a1f56c36636667</td>\n",
              "      <td>\"How does mindfulness relate to relationships?\"</td>\n",
              "      <td>\"Mindfulness plays a significant role in relat...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_0188645da01925bde2a...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:47:25.138075\", \"...</td>\n",
              "      <td>2023-12-22T03:47:42.664349</td>\n",
              "      <td>0.9</td>\n",
              "      <td>0.60</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>[{'args': {'prompt': 'How does mindfulness rel...</td>\n",
              "      <td>[{'args': {'question': 'How does mindfulness r...</td>\n",
              "      <td>[{'args': {'source': ['demonstrated (as a guid...</td>\n",
              "      <td>3</td>\n",
              "      <td>2231</td>\n",
              "      <td>0.015787</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>Willow_Alphav3</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_e81aaa574aad17893d707451b4acf141</td>\n",
              "      <td>\"What is the definition of a challenging child?\"</td>\n",
              "      <td>\"A challenging child is defined as one who has...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_e81aaa574aad17893d7...</td>\n",
              "      <td>{\"n_requests\": 4, \"n_successful_requests\": 4, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:47:43.119414\", \"...</td>\n",
              "      <td>2023-12-22T03:47:52.657445</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.20</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>[{'args': {'prompt': 'What is the definition o...</td>\n",
              "      <td>[{'args': {'question': 'What is the definition...</td>\n",
              "      <td>[{'args': {'source': ['■\\n■The challenges you ...</td>\n",
              "      <td>10</td>\n",
              "      <td>1405</td>\n",
              "      <td>0.010849</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>Willow_Alphav3</td>\n",
              "      <td>{\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...</td>\n",
              "      <td>OpenAIAgent(llama_index.agent.openai.base)</td>\n",
              "      <td>record_hash_f40631557b8e7124b7e07e4e1c237e4b</td>\n",
              "      <td>\"What are the benefits of a mindfulness practi...</td>\n",
              "      <td>\"Mindfulness practice offers a wide range of b...</td>\n",
              "      <td>-</td>\n",
              "      <td>{\"record_id\": \"record_hash_f40631557b8e7124b7e...</td>\n",
              "      <td>{\"n_requests\": 1, \"n_successful_requests\": 1, ...</td>\n",
              "      <td>{\"start_time\": \"2023-12-22T03:47:53.335266\", \"...</td>\n",
              "      <td>2023-12-22T03:48:08.988125</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>[]</td>\n",
              "      <td>NaN</td>\n",
              "      <td>12</td>\n",
              "      <td>732</td>\n",
              "      <td>0.015000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "             app_id                                           app_json  \\\n",
              "0   LlamaIndex_App1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "1      Willow_Alpha  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "2      Willow_Alpha  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "3      Willow_Alpha  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "4      Willow_Alpha  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "5   Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "6   Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "7   Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "8   Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "9   Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "10  Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "11  Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "12  Willow_Alpha_v1  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "13   Willow_Alphav2  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "14   Willow_Alphav2  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "15   Willow_Alphav2  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "16   Willow_Alphav3  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "17   Willow_Alphav3  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "18   Willow_Alphav3  {\"tru_class_info\": {\"name\": \"TruLlama\", \"modul...   \n",
              "\n",
              "                                                 type  \\\n",
              "0   RetrieverQueryEngine(llama_index.query_engine....   \n",
              "1          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "2          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "3          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "4          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "5          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "6          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "7          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "8          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "9          OpenAIAgent(llama_index.agent.openai.base)   \n",
              "10         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "11         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "12         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "13         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "14         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "15         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "16         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "17         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "18         OpenAIAgent(llama_index.agent.openai.base)   \n",
              "\n",
              "                                       record_id  \\\n",
              "0   record_hash_531ca89ec256899e128ee0637a51118e   \n",
              "1   record_hash_e6c435d8b8fafbbf397145a4947ba8bb   \n",
              "2   record_hash_8b87d32a191397ba592467ecbefea7e5   \n",
              "3   record_hash_2f69bf215f535424b8f1e570e95f66c8   \n",
              "4   record_hash_924f9898ff392225d711366dcf5da684   \n",
              "5   record_hash_3ab53b08e3ac7a65ec22c496cda039f6   \n",
              "6   record_hash_a7f8793f27be0e251af1abe7babcb1c0   \n",
              "7   record_hash_7ee80d9cec1d8ed3a98cdc58139b26f7   \n",
              "8   record_hash_6e71dd52fba207e4f2b9438811b94955   \n",
              "9   record_hash_ab7cb4b65e6b49c4a8b4218287b5fcea   \n",
              "10  record_hash_12d2a7abb863c92c5542ea3c29884975   \n",
              "11  record_hash_4b65a4ef31fa38f6547a6e4ae533153f   \n",
              "12  record_hash_4428a71c9e742884e737549ac38babab   \n",
              "13  record_hash_b7231b2d9d50937dc3dee9ec92227169   \n",
              "14  record_hash_845d4dc42a20fe586a70a1444011fe8b   \n",
              "15  record_hash_6d2d39087b8f60c17afdb8d7ed6191d5   \n",
              "16  record_hash_0188645da01925bde2a1f56c36636667   \n",
              "17  record_hash_e81aaa574aad17893d707451b4acf141   \n",
              "18  record_hash_f40631557b8e7124b7e07e4e1c237e4b   \n",
              "\n",
              "                                                input  \\\n",
              "0                \"What did the author do growing up?\"   \n",
              "1   \"My son is performing poorly in his school. Wh...   \n",
              "2   \"My son is performing poorly in his school. Wh...   \n",
              "3   \"My son is performing poorly in his school. It...   \n",
              "4   \"I am having issues with me wife. Can you prov...   \n",
              "5   \"I am having issues with my wife. Can you prov...   \n",
              "6     \"Please help me I am suffering from anxiety :(\"   \n",
              "7   \"My child is having trouble concentrating. I n...   \n",
              "8     \"How does mindfulness relate to relationships?\"   \n",
              "9   \"I am having issues with my wife. Can you prov...   \n",
              "10    \"Please help me I am suffering from anxiety :(\"   \n",
              "11  \"My child is having trouble concentrating. I n...   \n",
              "12    \"How does mindfulness relate to relationships?\"   \n",
              "13    \"How does mindfulness relate to relationships?\"   \n",
              "14   \"What is the definition of a challenging child?\"   \n",
              "15  \"What are the benefits of a mindfulness practi...   \n",
              "16    \"How does mindfulness relate to relationships?\"   \n",
              "17   \"What is the definition of a challenging child?\"   \n",
              "18  \"What are the benefits of a mindfulness practi...   \n",
              "\n",
              "                                               output tags  \\\n",
              "0   \"The author mentioned that before college, the...    -   \n",
              "1   \"Sorry, I do not know the answer to your quest...    -   \n",
              "2   \"Sorry, I do not know the answer to your quest...    -   \n",
              "3   \"I'm sorry to hear that your son's school perf...    -   \n",
              "4   \"It sounds like you're going through a challen...    -   \n",
              "5   \"It sounds like you're going through a challen...    -   \n",
              "6   \"I'm here to support you. It's brave of you to...    -   \n",
              "7   \"A simple and effective mindfulness practice t...    -   \n",
              "8   \"Mindfulness plays a significant role in relat...    -   \n",
              "9   \"It sounds like you're going through a challen...    -   \n",
              "10  \"I'm really sorry to hear that you're experien...    -   \n",
              "11  \"It sounds like your child could benefit from ...    -   \n",
              "12  \"Mindfulness plays a significant role in relat...    -   \n",
              "13  \"Mindfulness can enhance relationships by fost...    -   \n",
              "14  \"Sorry, I do not know the answer to your quest...    -   \n",
              "15  \"Mindfulness practice can offer numerous benef...    -   \n",
              "16  \"Mindfulness plays a significant role in relat...    -   \n",
              "17  \"A challenging child is defined as one who has...    -   \n",
              "18  \"Mindfulness practice offers a wide range of b...    -   \n",
              "\n",
              "                                          record_json  \\\n",
              "0   {\"record_id\": \"record_hash_531ca89ec256899e128...   \n",
              "1   {\"record_id\": \"record_hash_e6c435d8b8fafbbf397...   \n",
              "2   {\"record_id\": \"record_hash_8b87d32a191397ba592...   \n",
              "3   {\"record_id\": \"record_hash_2f69bf215f535424b8f...   \n",
              "4   {\"record_id\": \"record_hash_924f9898ff392225d71...   \n",
              "5   {\"record_id\": \"record_hash_3ab53b08e3ac7a65ec2...   \n",
              "6   {\"record_id\": \"record_hash_a7f8793f27be0e251af...   \n",
              "7   {\"record_id\": \"record_hash_7ee80d9cec1d8ed3a98...   \n",
              "8   {\"record_id\": \"record_hash_6e71dd52fba207e4f2b...   \n",
              "9   {\"record_id\": \"record_hash_ab7cb4b65e6b49c4a8b...   \n",
              "10  {\"record_id\": \"record_hash_12d2a7abb863c92c554...   \n",
              "11  {\"record_id\": \"record_hash_4b65a4ef31fa38f6547...   \n",
              "12  {\"record_id\": \"record_hash_4428a71c9e742884e73...   \n",
              "13  {\"record_id\": \"record_hash_b7231b2d9d50937dc3d...   \n",
              "14  {\"record_id\": \"record_hash_845d4dc42a20fe586a7...   \n",
              "15  {\"record_id\": \"record_hash_6d2d39087b8f60c17af...   \n",
              "16  {\"record_id\": \"record_hash_0188645da01925bde2a...   \n",
              "17  {\"record_id\": \"record_hash_e81aaa574aad17893d7...   \n",
              "18  {\"record_id\": \"record_hash_f40631557b8e7124b7e...   \n",
              "\n",
              "                                            cost_json  \\\n",
              "0   {\"n_requests\": 2, \"n_successful_requests\": 2, ...   \n",
              "1   {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "2   {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "3   {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "4   {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "5   {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "6   {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "7   {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "8   {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "9   {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "10  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "11  {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "12  {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "13  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "14  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "15  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "16  {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "17  {\"n_requests\": 4, \"n_successful_requests\": 4, ...   \n",
              "18  {\"n_requests\": 1, \"n_successful_requests\": 1, ...   \n",
              "\n",
              "                                            perf_json  \\\n",
              "0   {\"start_time\": \"2023-12-19T13:10:25.871039\", \"...   \n",
              "1   {\"start_time\": \"2023-12-19T15:46:27.935078\", \"...   \n",
              "2   {\"start_time\": \"2023-12-19T15:57:04.715265\", \"...   \n",
              "3   {\"start_time\": \"2023-12-19T15:59:25.360846\", \"...   \n",
              "4   {\"start_time\": \"2023-12-19T16:02:38.832174\", \"...   \n",
              "5   {\"start_time\": \"2023-12-19T19:05:58.886151\", \"...   \n",
              "6   {\"start_time\": \"2023-12-19T19:06:28.531068\", \"...   \n",
              "7   {\"start_time\": \"2023-12-19T19:06:34.958508\", \"...   \n",
              "8   {\"start_time\": \"2023-12-19T19:07:04.960237\", \"...   \n",
              "9   {\"start_time\": \"2023-12-19T22:01:43.240153\", \"...   \n",
              "10  {\"start_time\": \"2023-12-19T22:02:18.534521\", \"...   \n",
              "11  {\"start_time\": \"2023-12-19T22:02:34.869376\", \"...   \n",
              "12  {\"start_time\": \"2023-12-19T22:03:04.442552\", \"...   \n",
              "13  {\"start_time\": \"2023-12-22T03:39:42.896887\", \"...   \n",
              "14  {\"start_time\": \"2023-12-22T03:39:49.412135\", \"...   \n",
              "15  {\"start_time\": \"2023-12-22T03:39:51.532845\", \"...   \n",
              "16  {\"start_time\": \"2023-12-22T03:47:25.138075\", \"...   \n",
              "17  {\"start_time\": \"2023-12-22T03:47:43.119414\", \"...   \n",
              "18  {\"start_time\": \"2023-12-22T03:47:53.335266\", \"...   \n",
              "\n",
              "                            ts  relevance  qs_relevance  \\\n",
              "0   2023-12-19T13:10:29.347691        0.8          0.45   \n",
              "1   2023-12-19T15:46:38.423575        1.0           NaN   \n",
              "2   2023-12-19T15:57:17.143217        1.0           NaN   \n",
              "3   2023-12-19T15:59:33.989579        0.2           NaN   \n",
              "4   2023-12-19T16:03:21.125467        1.0          0.35   \n",
              "5   2023-12-19T19:06:27.721442        1.0          0.40   \n",
              "6   2023-12-19T19:06:34.514177        0.8           NaN   \n",
              "7   2023-12-19T19:07:04.436664        1.0          0.20   \n",
              "8   2023-12-19T19:07:30.303863        0.9          0.60   \n",
              "9   2023-12-19T22:02:18.059594        1.0          0.45   \n",
              "10  2023-12-19T22:02:34.427499        0.7           NaN   \n",
              "11  2023-12-19T22:03:04.004311        1.0          0.20   \n",
              "12  2023-12-19T22:03:21.582972        0.9          0.60   \n",
              "13  2023-12-22T03:39:48.949583        0.9           NaN   \n",
              "14  2023-12-22T03:39:51.173003        1.0           NaN   \n",
              "15  2023-12-22T03:39:54.455917        1.0           NaN   \n",
              "16  2023-12-22T03:47:42.664349        0.9          0.60   \n",
              "17  2023-12-22T03:47:52.657445        1.0          0.20   \n",
              "18  2023-12-22T03:48:08.988125        NaN           NaN   \n",
              "\n",
              "    groundedness_measure_with_cot_reasons  \\\n",
              "0                                1.000000   \n",
              "1                                0.000000   \n",
              "2                                0.000000   \n",
              "3                                0.000000   \n",
              "4                                0.450000   \n",
              "5                                0.430000   \n",
              "6                                0.000000   \n",
              "7                                0.222222   \n",
              "8                                1.000000   \n",
              "9                                0.777778   \n",
              "10                               0.000000   \n",
              "11                               0.166667   \n",
              "12                               0.800000   \n",
              "13                               0.000000   \n",
              "14                               0.000000   \n",
              "15                               0.000000   \n",
              "16                               1.000000   \n",
              "17                               0.000000   \n",
              "18                                    NaN   \n",
              "\n",
              "                                      relevance_calls  \\\n",
              "0   [{'args': {'prompt': 'What did the author do g...   \n",
              "1   [{'args': {'prompt': 'My son is performing poo...   \n",
              "2   [{'args': {'prompt': 'My son is performing poo...   \n",
              "3   [{'args': {'prompt': 'My son is performing poo...   \n",
              "4   [{'args': {'prompt': 'I am having issues with ...   \n",
              "5   [{'args': {'prompt': 'I am having issues with ...   \n",
              "6   [{'args': {'prompt': 'Please help me I am suff...   \n",
              "7   [{'args': {'prompt': 'My child is having troub...   \n",
              "8   [{'args': {'prompt': 'How does mindfulness rel...   \n",
              "9   [{'args': {'prompt': 'I am having issues with ...   \n",
              "10  [{'args': {'prompt': 'Please help me I am suff...   \n",
              "11  [{'args': {'prompt': 'My child is having troub...   \n",
              "12  [{'args': {'prompt': 'How does mindfulness rel...   \n",
              "13  [{'args': {'prompt': 'How does mindfulness rel...   \n",
              "14  [{'args': {'prompt': 'What is the definition o...   \n",
              "15  [{'args': {'prompt': 'What are the benefits of...   \n",
              "16  [{'args': {'prompt': 'How does mindfulness rel...   \n",
              "17  [{'args': {'prompt': 'What is the definition o...   \n",
              "18                                                NaN   \n",
              "\n",
              "                                   qs_relevance_calls  \\\n",
              "0   [{'args': {'question': 'What did the author do...   \n",
              "1                                                  []   \n",
              "2                                                  []   \n",
              "3                                                  []   \n",
              "4   [{'args': {'question': 'I am having issues wit...   \n",
              "5   [{'args': {'question': 'I am having issues wit...   \n",
              "6                                                  []   \n",
              "7   [{'args': {'question': 'My child is having tro...   \n",
              "8   [{'args': {'question': 'How does mindfulness r...   \n",
              "9   [{'args': {'question': 'I am having issues wit...   \n",
              "10                                                 []   \n",
              "11  [{'args': {'question': 'My child is having tro...   \n",
              "12  [{'args': {'question': 'How does mindfulness r...   \n",
              "13                                                 []   \n",
              "14                                                 []   \n",
              "15                                                 []   \n",
              "16  [{'args': {'question': 'How does mindfulness r...   \n",
              "17  [{'args': {'question': 'What is the definition...   \n",
              "18                                                 []   \n",
              "\n",
              "          groundedness_measure_with_cot_reasons_calls  latency  total_tokens  \\\n",
              "0   [{'args': {'source': ['![](https://s.turbifycd...        3          2126   \n",
              "1   [{'args': {'source': [], 'statement': 'Sorry, ...        3           635   \n",
              "2   [{'args': {'source': [], 'statement': 'Sorry, ...       10           630   \n",
              "3   [{'args': {'source': [], 'statement': 'I'm sor...       12           660   \n",
              "4   [{'args': {'source': ['✓   Consider teaching t...        8          2361   \n",
              "5   [{'args': {'source': ['✓   Consider teaching t...        3          2469   \n",
              "6   [{'args': {'source': [], 'statement': 'I'm her...       10           620   \n",
              "7   [{'args': {'source': ['Your Personal Mindfulne...       12          2082   \n",
              "8   [{'args': {'source': ['demonstrated (as a guid...        8          2514   \n",
              "9   [{'args': {'source': ['✓   Consider teaching t...       42          2233   \n",
              "10  [{'args': {'source': [], 'statement': 'I'm rea...       28           631   \n",
              "11  [{'args': {'source': ['Your Personal Mindfulne...        5          2155   \n",
              "12  [{'args': {'source': ['demonstrated (as a guid...       29          2527   \n",
              "13  [{'args': {'source': [], 'statement': 'Mindful...        3           824   \n",
              "14  [{'args': {'source': [], 'statement': 'Sorry, ...       10           792   \n",
              "15  [{'args': {'source': [], 'statement': 'Mindful...       12           824   \n",
              "16  [{'args': {'source': ['demonstrated (as a guid...        3          2231   \n",
              "17  [{'args': {'source': ['■\\n■The challenges you ...       10          1405   \n",
              "18                                                NaN       12           732   \n",
              "\n",
              "    total_cost  \n",
              "0     0.003196  \n",
              "1     0.008210  \n",
              "2     0.008060  \n",
              "3     0.008900  \n",
              "4     0.029024  \n",
              "5     0.023561  \n",
              "6     0.007940  \n",
              "7     0.019377  \n",
              "8     0.018173  \n",
              "9     0.025567  \n",
              "10    0.008270  \n",
              "11    0.020511  \n",
              "12    0.018131  \n",
              "13    0.009180  \n",
              "14    0.008180  \n",
              "15    0.009140  \n",
              "16    0.015787  \n",
              "17    0.010849  \n",
              "18    0.015000  "
            ]
          },
          "execution_count": 47,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result relevance DONE feedback_result_hash_c2c570cb6e59641f3ed8ca093d3c587f\n",
            "✅ feedback result relevance DONE feedback_result_hash_c2c570cb6e59641f3ed8ca093d3c587f\n",
            "INFO:httpx:HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "HTTP Request: POST https://api.openai.com/v1/chat/completions \"HTTP/1.1 200 OK\"\n",
            "INFO:alembic.runtime.migration:Context impl SQLiteImpl.\n",
            "Context impl SQLiteImpl.\n",
            "INFO:alembic.runtime.migration:Will assume non-transactional DDL.\n",
            "Will assume non-transactional DDL.\n",
            "INFO:trulens_eval.database.sqlalchemy_db:✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_34becf2aa9309ceed52882d63ca7ec58\n",
            "✅ feedback result groundedness_measure_with_cot_reasons DONE feedback_result_hash_34becf2aa9309ceed52882d63ca7ec58\n"
          ]
        }
      ],
      "source": [
        "tru.get_records_and_feedback(app_ids=[])[0] # pass an empty list of app_ids to get all"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}