diff --git a/bananalyzer/runner/evals.py b/bananalyzer/runner/evals.py index 9a4297d2..7fde70f9 100644 --- a/bananalyzer/runner/evals.py +++ b/bananalyzer/runner/evals.py @@ -12,6 +12,10 @@ def validate_field_match(expected: Result, actual: Result, field: str) -> None: expected_value = expected.get(field, None) actual_value = actual.get(field, None) + # Treat empty string as none + if actual_value is "": + actual_value = None + matcher = get_matcher(expected_value, actual_value) if not matcher(actual_value, expected_value): pytest.fail(f"{expected_value} != {actual_value}") diff --git a/pyproject.toml b/pyproject.toml index 3f2d22f9..ee8ddfed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.poetry] name = "bananalyzer" -version = "0.6.2" +version = "0.6.3" + description = "Open source AI Agent evaluation framework for web tasks 🐒🍌" authors = ["asim-shrestha "] readme = "README.md" diff --git a/tests/test_evals.py b/tests/test_evals.py index 330749a2..6c0e341c 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -83,6 +83,7 @@ def test_is_string_similar(actual, expected, tolerance, expected_result): ({"field": "test"}, {"field": "test!!"}, "field"), ({"field": 123}, {"field": 123}, "field"), ({"field": [1, 2, 3]}, {"field": [1, 2, 3]}, "field"), + ({"field": None}, {"field": ""}, "field"), # None == "" ], ) def test_validate_field_match_pass(expected, actual, field):