Consider system prompt in calculations (#1233)

* Consider system prompt in calculations

* Explicit cast to dict

* Add type ignore

* Rename variable with different type

app/backend/approaches/chatapproach.py

@@ -107,13 +107,16 @@ def get_messages_from_history(
         append_index = len(few_shots) + 1
 
         message_builder.insert_message(self.USER, user_content, index=append_index)
-        total_token_count = message_builder.count_tokens_for_message(dict(message_builder.messages[-1]))  # type: ignore
+
+        total_token_count = 0
+        for existing_message in message_builder.messages:
+            total_token_count += message_builder.count_tokens_for_message(dict(existing_message))  # type: ignore
 
         newest_to_oldest = list(reversed(history[:-1]))
         for message in newest_to_oldest:
             potential_message_count = message_builder.count_tokens_for_message(message)
             if (total_token_count + potential_message_count) > max_tokens:
-                logging.debug("Reached max tokens of %d, history will be truncated", max_tokens)
+                logging.info("Reached max tokens of %d, history will be truncated", max_tokens)
                 break
             message_builder.insert_message(message["role"], message["content"], index=append_index)
             total_token_count += potential_message_count

tests/test_chatapproach.py

@@ -186,8 +186,8 @@ def test_get_messages_from_history_truncated_break_pair(chat_approach):
             {"role": "user", "content": "What happens in a performance review?"},  # 10 tokens
             {
                 "role": "assistant",
-                "content": "During the performance review at Contoso Electronics, the supervisor will discuss the employee's performance over the past year and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals and objectives for the upcoming year [employee_handbook-3.pdf].",
-            },  # 102 tokens
+                "content": "The supervisor will discuss the employee's performance and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals for the upcoming year [employee_handbook-3.pdf].",
+            },  # 87 tokens
             {"role": "user", "content": "Is there a dress code?"},  # 9 tokens
             {
                 "role": "assistant",
@@ -202,7 +202,7 @@ def test_get_messages_from_history_truncated_break_pair(chat_approach):
         {"role": "system", "content": "You are a bot."},
         {
             "role": "assistant",
-            "content": "During the performance review at Contoso Electronics, the supervisor will discuss the employee's performance over the past year and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals and objectives for the upcoming year [employee_handbook-3.pdf].",
+            "content": "The supervisor will discuss the employee's performance and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals for the upcoming year [employee_handbook-3.pdf].",
         },
         {"role": "user", "content": "Is there a dress code?"},
         {
@@ -213,6 +213,36 @@ def test_get_messages_from_history_truncated_break_pair(chat_approach):
     ]
 
 
+def test_get_messages_from_history_system_message(chat_approach):
+    """Tests that the system message token count is considered."""
+    messages = chat_approach.get_messages_from_history(
+        system_prompt="Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.",  # 24 tokens
+        model_id="gpt-35-turbo",
+        history=[
+            {"role": "user", "content": "What happens in a performance review?"},  # 10 tokens
+            {
+                "role": "assistant",
+                "content": "During the performance review at Contoso Electronics, the supervisor will discuss the employee's performance over the past year and provide feedback on areas for improvement. They will also provide an opportunity for the employee to discuss their goals and objectives for the upcoming year. The review is a two-way dialogue between managers and employees, and employees will receive a written summary of their performance review which will include a rating of their performance, feedback, and goals and objectives for the upcoming year [employee_handbook-3.pdf].",
+            },  # 102 tokens
+            {"role": "user", "content": "Is there a dress code?"},  # 9 tokens
+            {
+                "role": "assistant",
+                "content": "Yes, there is a dress code at Contoso Electronics. Look sharp! [employee_handbook-1.pdf]",
+            },  # 26 tokens
+            {"role": "user", "content": "What does a Product Manager do?"},  # 10 tokens
+        ],
+        user_content="What does a Product Manager do?",
+        max_tokens=36,
+    )
+    assert messages == [
+        {
+            "role": "system",
+            "content": "Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers.",
+        },
+        {"role": "user", "content": "What does a Product Manager do?"},
+    ]
+
+
 def test_extract_followup_questions(chat_approach):
     content = "Here is answer to your question.<<What is the dress code?>>"
     pre_content, followup_questions = chat_approach.extract_followup_questions(content)