fix: validation was incorrect (#75)

explodinggradients · Jul 27, 2023 · 90db3ba · 90db3ba
1 parent 2b9734d
commit 90db3ba
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/src/ragas/validation.py b/src/ragas/validation.py
@@ -47,7 +47,7 @@ def validate_evaluation_modes(ds: Dataset, metrics: list[Metric]):
     for m in metrics:
         required_columns = set(EVALMODE_TO_COLUMNS[m.evaluation_mode])
         available_columns = set(ds.features.keys())
-        if required_columns.symmetric_difference(available_columns):
+        if not required_columns.issubset(available_columns):
             raise ValueError(
                 f"The metric [{m.name}] that that is used requires the following "
                 f"additional columns {list(required_columns - available_columns)} "

diff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.py
@@ -12,15 +12,15 @@
 
 TEST_CASES = [
     CaseToTest("a", "b", ["c"], None, True, [faithfulness], True),
-    CaseToTest("a", "b", ["c"], ["g"], True, [faithfulness], False),
+    CaseToTest("a", "b", ["c"], ["g"], True, [faithfulness], True),
     CaseToTest("a", None, ["c"], None, True, [context_relevancy], True),
     CaseToTest("a", None, "c", None, False, [context_relevancy], True),
     CaseToTest(
         "a", None, [["c"]], None, False, [context_relevancy, answer_relevancy], False
     ),
-    CaseToTest("a", None, ["c"], "g", False, [context_relevancy], False),
-    CaseToTest("a", None, ["c"], [["g"]], False, [context_relevancy], False),
-    CaseToTest(1, None, ["c"], ["g"], False, [context_relevancy], False),
+    CaseToTest("a", None, ["c"], "g", False, [context_relevancy], True),
+    CaseToTest("a", None, ["c"], [["g"]], False, [context_relevancy], True),
+    CaseToTest(1, None, ["c"], ["g"], False, [context_relevancy], True),
     CaseToTest(1, None, None, None, False, [context_relevancy], False),
 ]