From 6cf972cd4d9c9eeab5a3239b3e091505e9b4a13c Mon Sep 17 00:00:00 2001 From: ikka Date: Sat, 26 Oct 2024 05:24:27 +0530 Subject: [PATCH] fix: agent goal accuracy (#1583) --- docs/concepts/metrics/available_metrics/agents.md | 4 +++- src/ragas/metrics/_goal_accuracy.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/concepts/metrics/available_metrics/agents.md b/docs/concepts/metrics/available_metrics/agents.md index 3a542bf98..0477c0297 100644 --- a/docs/concepts/metrics/available_metrics/agents.md +++ b/docs/concepts/metrics/available_metrics/agents.md @@ -97,6 +97,7 @@ sample = MultiTurnSample( ) scorer = ToolCallAccuracy() +scorer.llm = your_llm await scorer.multi_turn_ascore(sample) ``` @@ -146,7 +147,8 @@ sample = MultiTurnSample(user_input=[ reference="Table booked at one of the chinese restaurants at 8 pm") scorer = AgentGoalAccuracyWithReference() -await metric.multi_turn_ascore(sample) +scorer.llm = your_llm +await scorer.multi_turn_ascore(sample) ``` diff --git a/src/ragas/metrics/_goal_accuracy.py b/src/ragas/metrics/_goal_accuracy.py index 2c62d85b5..9793df68f 100644 --- a/src/ragas/metrics/_goal_accuracy.py +++ b/src/ragas/metrics/_goal_accuracy.py @@ -114,6 +114,9 @@ class AgentGoalAccuracyWithReference(MetricWithLLM, MultiTurnMetric): ) max_retries: int = 1 + async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float: + raise NotImplementedError + async def _multi_turn_ascore( self, sample: MultiTurnSample, @@ -153,6 +156,9 @@ class AgentGoalAccuracyWithoutReference(MetricWithLLM, MultiTurnMetric): ) max_retries: int = 1 + async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float: + raise NotImplementedError + async def _multi_turn_ascore( self, sample: MultiTurnSample,