From fd5e805b9de89456b4bdb7fe4092957daa00be41 Mon Sep 17 00:00:00 2001 From: Jeffy <36265245+Jeff-67@users.noreply.github.com> Date: Sat, 26 Oct 2024 02:08:25 +0800 Subject: [PATCH] Improve efficiency in factual correctness for precision mode (#1578) ### Description This pull request optimizes the `_single_turn_ascore` function by conditionally skipping unnecessary computations when the mode is set to "precision." Previously, the function calculated `response_reference` regardless of the scoring mode, which consumed resources and potentially increased processing costs. By avoiding the `response_reference` calculation in "precision" mode, this update aims to enhance speed and reduce computational expenses. ### Changes Made - Modified `_single_turn_ascore` function: - **Conditionally calculated `response_reference`**: The `response_reference` is now computed only when the mode is not "precision." --------- Co-authored-by: jeff yang --- src/ragas/metrics/_factual_correctness.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/ragas/metrics/_factual_correctness.py b/src/ragas/metrics/_factual_correctness.py index 5147cb112..7db09cfda 100644 --- a/src/ragas/metrics/_factual_correctness.py +++ b/src/ragas/metrics/_factual_correctness.py @@ -244,13 +244,21 @@ async def _single_turn_ascore( reference_response = await self.verify_claims( premise=reference, hypothesis_list=response_claims, callbacks=callbacks ) - response_reference = await self.verify_claims( - premise=response, hypothesis_list=reference_claims, callbacks=callbacks - ) + + + if self.mode != "precision": + response_reference = await self.verify_claims( + premise=response, hypothesis_list=reference_claims, callbacks=callbacks + ) + else: + response_reference = np.array([]) true_positives = sum(reference_response) false_positives = sum(~reference_response) - false_negatives = sum(~response_reference) + if self.mode != "precision": + false_negatives = sum(~response_reference) + else: + false_negatives = 0 if self.mode == "precision": score = true_positives / (true_positives + false_positives + 1e-8)