refactor(graders): improve parameter validation and streaming support (#62)

* refactor(graders): improve parameter validation and streaming support

- Add threshold validation in common graders (correctness, harmfulness, etc.)
- Fix streaming response handling in text_to_image grader
- Preserve metadata in grader score returns
- Update template default pattern (None with or fallback)
- Update model name in examples from qwen3-max to qwen3-32b

* refactor(text_to_image): simplify streaming response handling

- Extract common score/reason parsing logic
- Remove unused collected_content variable

* refactor(multimodal): improve error handling and streaming support

- Move error handling to aevaluate level for consistent GraderError returns
- Simplify streaming response handling across multimodal graders
- Refactor image_coherence, image_helpfulness, text_to_image graders

* refactor(multimodal): unify structured response parsing and error handling

- Add parse_structured_chat_response utility for streaming/non-streaming responses
- Return GraderError instead of score=0 on exceptions in multimodal graders
- Update tests to verify GraderError behavior
- Move exception handling to aevaluate level for cleaner code

Author: XiaoBoAI

docs/built_in_graders/general.md

@@ -56,7 +56,7 @@ Evaluates how well a response addresses the user's query. Measures whether the a
 | `query` | str | Yes | The user's question or request |
 | `response` | str | Yes | The model's response to evaluate |
 | `context` | str | No | Additional context (e.g., conversation history) |
-| `ground_truth` | str | No | Reference answer for comparison |
+| `reference_response` | str | No | Reference answer for comparison |
 
 **Grading Criteria:**
 - **5**: Comprehensive response with helpful insights
@@ -107,7 +107,7 @@ Detects fabricated information not supported by the provided context or common k
 | `query` | str | Yes | The user's question |
 | `response` | str | Yes | The model's response to evaluate |
 | `context` | str | No | Source documents to verify against |
-| `ground_truth` | str | No | Reference answer |
+| `reference_response` | str | No | Reference answer |
 
 !!! note
     If no context is provided, evaluation is based on common knowledge and factual consistency.
@@ -175,7 +175,7 @@ Identifies harmful, offensive, or inappropriate content in responses. Critical f
 | `query` | str | Yes | The user's input |
 | `response` | str | Yes | The model's response to evaluate |
 | `context` | str | No | Additional context |
-| `ground_truth` | str | No | Reference response |
+| `reference_response` | str | No | Reference response |
 
 **What it detects:**
 - Violence, hatred, discrimination

openjudge/graders/common/correctness.py

@@ -244,7 +244,7 @@ class CorrectnessGrader(LLMGrader):
         >>> from openjudge.graders.common.correctness import CorrectnessGrader
         >>>
         >>> # Initialize grader
-        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-max")
+        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-32b")
         >>> grader = CorrectnessGrader(model=model, threshold=3)
         >>>
         >>> # Good match
@@ -268,7 +268,7 @@ def __init__(
         self,
         model: BaseChatModel | dict,
         threshold: float = 3,
-        template: Optional[PromptTemplate] = DEFAULT_CORRECTNESS_TEMPLATE,
+        template: Optional[PromptTemplate] = None,
         language: LanguageEnum = LanguageEnum.EN,
     ):
         """
@@ -279,7 +279,13 @@ def __init__(
             threshold: Success threshold [1, 5] (default: 3)
             template: PromptTemplate for evaluation prompts (default: DEFAULT_CORRECTNESS_TEMPLATE)
             language: Language for prompts (default: LanguageEnum.EN)
+
+        Raises:
+            ValueError: If threshold is not in range [1, 5]
         """
+        if not 1 <= threshold <= 5:
+            raise ValueError(f"threshold must be in range [1, 5], got {threshold}")
+
         super().__init__(
             name="correctness",
             mode=GraderMode.POINTWISE,
@@ -330,11 +336,11 @@ async def aevaluate(
                 name=self.name,
                 score=result.score,
                 reason=result.reason,
-                metadata={"threshold": self.threshold},
+                metadata={**result.metadata, "threshold": self.threshold},
             )
 
         except Exception as e:
-            logger.error(f"Error evaluating correctness: {e}")
+            logger.exception(f"Error evaluating correctness: {e}")
             return GraderError(
                 name=self.name,
                 error=f"Evaluation error: {str(e)}",

openjudge/graders/common/hallucination.py

@@ -216,7 +216,7 @@ class HallucinationGrader(LLMGrader):
         >>> # Initialize model
         >>> model = OpenAIChatModel(
         ...     api_key="sk-...",
-        ...     model="qwen3-max",
+        ...     model="qwen3-32b",
         ...     temperature=0.1
         ... )
         >>>
@@ -253,7 +253,7 @@ def __init__(
         self,
         model: BaseChatModel | dict,
         threshold: float = 3,
-        template: Optional[PromptTemplate] = DEFAULT_HALLUCINATION_TEMPLATE,
+        template: Optional[PromptTemplate] = None,
         language: LanguageEnum = LanguageEnum.EN,
     ):
         """
@@ -264,7 +264,13 @@ def __init__(
             threshold: Success threshold [1, 5] (default: 3)
             template: PromptTemplate for evaluation prompts (default: DEFAULT_HALLUCINATION_TEMPLATE)
             language: Language for prompts (default: LanguageEnum.EN)
+
+        Raises:
+            ValueError: If threshold is not in range [1, 5]
         """
+        if not 1 <= threshold <= 5:
+            raise ValueError(f"threshold must be in range [1, 5], got {threshold}")
+
         super().__init__(
             name="hallucination",
             mode=GraderMode.POINTWISE,
@@ -322,11 +328,11 @@ async def aevaluate(
                 name=self.name,
                 score=result.score,
                 reason=result.reason,
-                metadata={"threshold": self.threshold},
+                metadata={**result.metadata, "threshold": self.threshold},
             )
 
         except Exception as e:
-            logger.error(f"Error evaluating hallucination: {e}")
+            logger.exception(f"Error evaluating hallucination: {e}")
             return GraderError(
                 name=self.name,
                 error=f"Evaluation error: {str(e)}",

openjudge/graders/common/harmfulness.py

@@ -223,7 +223,7 @@ class HarmfulnessGrader(LLMGrader):
         >>> from openjudge.graders.common.harmfulness import HarmfulnessGrader
         >>>
         >>> # Initialize grader
-        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-max")
+        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-32b")
         >>> grader = HarmfulnessGrader(model=model, threshold=3)
         >>>
         >>> # Safe output
@@ -246,7 +246,7 @@ def __init__(
         self,
         model: BaseChatModel | dict,
         threshold: float = 3,
-        template: Optional[PromptTemplate] = DEFAULT_HARMFULNESS_TEMPLATE,
+        template: Optional[PromptTemplate] = None,
         language: LanguageEnum = LanguageEnum.EN,
     ):
         """
@@ -257,7 +257,13 @@ def __init__(
             threshold: Success threshold [1, 5] (default: 3)
             template: PromptTemplate for evaluation prompts (default: DEFAULT_HARMFULNESS_TEMPLATE)
             language: Language for prompts (default: LanguageEnum.EN)
+
+        Raises:
+            ValueError: If threshold is not in range [1, 5]
         """
+        if not 1 <= threshold <= 5:
+            raise ValueError(f"threshold must be in range [1, 5], got {threshold}")
+
         super().__init__(
             name="harmfulness",
             mode=GraderMode.POINTWISE,
@@ -307,11 +313,11 @@ async def aevaluate(
                 name=self.name,
                 score=result.score,
                 reason=result.reason,
-                metadata={"threshold": self.threshold},
+                metadata={**result.metadata, "threshold": self.threshold},
             )
 
         except Exception as e:
-            logger.error(f"Error evaluating harmfulness: {e}")
+            logger.exception(f"Error evaluating harmfulness: {e}")
             return GraderError(
                 name=self.name,
                 error=f"Evaluation error: {str(e)}",

openjudge/graders/common/instruction_following.py

@@ -238,7 +238,7 @@ class InstructionFollowingGrader(LLMGrader):
         >>> from openjudge.graders.common.instruction_following import InstructionFollowingGrader
         >>>
         >>> # Initialize grader
-        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-max")
+        >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-32b")
         >>> grader = InstructionFollowingGrader(model=model, threshold=3)
         >>>
         >>> # Good adherence
@@ -262,7 +262,7 @@ def __init__(
         self,
         model: BaseChatModel | dict,
         threshold: float = 3,
-        template: Optional[PromptTemplate] = DEFAULT_INSTRUCTION_FOLLOWING_TEMPLATE,
+        template: Optional[PromptTemplate] = None,
         language: LanguageEnum = LanguageEnum.EN,
     ):
         """
@@ -273,7 +273,13 @@ def __init__(
             threshold: Success threshold [1, 5] (default: 3)
             template: PromptTemplate for evaluation prompts (default: DEFAULT_INSTRUCTION_FOLLOWING_TEMPLATE)
             language: Language for prompts (default: LanguageEnum.EN)
+
+        Raises:
+            ValueError: If threshold is not in range [1, 5]
         """
+        if not 1 <= threshold <= 5:
+            raise ValueError(f"threshold must be in range [1, 5], got {threshold}")
+
         super().__init__(
             name="instruction_following",
             mode=GraderMode.POINTWISE,
@@ -318,11 +324,11 @@ async def aevaluate(
                 name=self.name,
                 score=result.score,
                 reason=result.reason,
-                metadata={"threshold": self.threshold},
+                metadata={**result.metadata, "threshold": self.threshold},
             )
 
         except Exception as e:
-            logger.error(f"Error evaluating instruction following: {e}")
+            logger.exception(f"Error evaluating instruction following: {e}")
             return GraderError(
                 name=self.name,
                 error=f"Evaluation error: {str(e)}",

openjudge/graders/common/relevance.py

@@ -217,7 +217,7 @@ class RelevanceGrader(LLMGrader):
 
     Args:
         model: BaseChatModel instance or dict config for OpenAIChatModel
-        threshold: Minimum score [0, 1] to pass (default: 0.7)
+        threshold: Minimum score [1, 5] to pass (default: 3)
         template: Custom evaluation template (default: DEFAULT_RELEVANCE_TEMPLATE)
         language: Prompt language - EN or ZH (default: LanguageEnum.EN)
 
@@ -234,7 +234,7 @@ class RelevanceGrader(LLMGrader):
         >>>
         >>> # Initialize grader
         >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-32b")
-        >>> grader = RelevanceGrader(model=model, threshold=0.7)
+        >>> grader = RelevanceGrader(model=model, threshold=3)
         >>>
         >>> # Relevant response
         >>> result = asyncio.run(grader.aevaluate(
@@ -262,19 +262,25 @@ class RelevanceGrader(LLMGrader):
     def __init__(
         self,
         model: BaseChatModel | dict,
-        threshold: float = 0.7,
-        template: Optional[PromptTemplate] = DEFAULT_RELEVANCE_TEMPLATE,
+        threshold: float = 3,
+        template: Optional[PromptTemplate] = None,
         language: LanguageEnum = LanguageEnum.EN,
     ):
         """
         Initialize RelevanceGrader
 
         Args:
             model: BaseChatModel instance or dict config for OpenAIChatModel
-            threshold: Success threshold [0, 1] (default: 0.7)
+            threshold: Success threshold [1, 5] (default: 3)
             template: PromptTemplate for evaluation prompts (default: DEFAULT_RELEVANCE_TEMPLATE)
             language: Language for prompts (default: LanguageEnum.EN)
+
+        Raises:
+            ValueError: If threshold is not in range [1, 5]
         """
+        if not 1 <= threshold <= 5:
+            raise ValueError(f"threshold must be in range [1, 5], got {threshold}")
+
         super().__init__(
             name="relevance",
             mode=GraderMode.POINTWISE,
@@ -323,11 +329,11 @@ async def aevaluate(
                 name=self.name,
                 score=result.score,
                 reason=result.reason,
-                metadata={"threshold": self.threshold},
+                metadata={**result.metadata, "threshold": self.threshold},
             )
 
         except Exception as e:
-            logger.error(f"Error evaluating relevance: {e}")
+            logger.exception(f"Error evaluating relevance: {e}")
             return GraderError(
                 name=self.name,
                 error=f"Evaluation error: {str(e)}",

openjudge/graders/multimodal/_internal/criteria_utils.py

@@ -87,13 +87,13 @@ def validate_and_sort_rubrics(
     # Sort rubrics by start of range
     sorted_rubrics = sorted(rubrics, key=lambda r: r.score_range[0])
 
-    # Full overlap check
+    # Full overlap check (adjacent ranges like (0,5) and (5,7) are allowed)
     for i in range(len(sorted_rubrics)):
         a_start, a_end = sorted_rubrics[i].score_range
         for j in range(i + 1, len(sorted_rubrics)):
             b_start, b_end = sorted_rubrics[j].score_range
-            # Check if ranges overlap
-            if a_end >= b_start:
+            # Check if ranges overlap (> allows adjacent ranges to touch)
+            if a_end > b_start:
                 raise ValueError(
                     f"Overlapping score ranges: {sorted_rubrics[i].score_range} and {sorted_rubrics[j].score_range}",
                 )
@@ -147,7 +147,7 @@ def construct_params_string(
         >>> construct_params_string(params)
         'Input and Actual Output'
     """
-    params = [PARAM_DISPLAY_NAMES[param] for param in evaluation_params]
+    params = [PARAM_DISPLAY_NAMES.get(param, param.replace("_", " ").title()) for param in evaluation_params]
 
     if len(params) == 1:
         params_str = params[0]
@@ -164,7 +164,7 @@ def get_score_range(rubric: Optional[List[Rubric]]) -> Tuple[int, int]:
     Get the overall score range from rubrics
 
     Args:
-        rubric: List of rubric definitions
+        rubric: List of rubric definitions (does not need to be sorted)
 
     Returns:
         Tuple of (min_score, max_score)
@@ -180,7 +180,9 @@ def get_score_range(rubric: Optional[List[Rubric]]) -> Tuple[int, int]:
     if not rubric:
         return (0, 10)
 
-    return rubric[0].score_range[0], rubric[-1].score_range[1]
+    min_score = min(r.score_range[0] for r in rubric)
+    max_score = max(r.score_range[1] for r in rubric)
+    return (min_score, max_score)
 
 
 __all__ = [

openjudge/graders/multimodal/image_coherence.py

@@ -23,6 +23,7 @@
 from openjudge.models.base_chat_model import BaseChatModel
 from openjudge.models.schema.oai.message import ChatMessage
 from openjudge.models.schema.prompt_template import LanguageEnum, PromptTemplate
+from openjudge.utils.utils import parse_structured_chat_response
 
 # pylint: disable=line-too-long
 
@@ -222,30 +223,27 @@ async def _aevaluate_single_image(
             context_below=context_below or "",
         )
 
-        try:
-            # Format image content for OpenAI API
-            content = [{"type": "text", "text": prompt}]
-
-            if image.url:
-                content.append({"type": "image_url", "image_url": {"url": image.url}})
-            elif image.base64:
-                # Format base64 image with data URL scheme
-                image_format = image.format or "jpeg"
-                data_url = f"data:image/{image_format};base64,{image.base64}"
-                content.append({"type": "image_url", "image_url": {"url": data_url}})
-
-            # Call model without structured output
-            chat_response = await self.model.achat(
-                messages=[{"role": "user", "content": content}],
-                structured_model=GraderScoreCallback,
-            )
-            score = chat_response.parsed["score"]
-            reason = chat_response.parsed["reason"]
-            return score, reason
+        # Format image content for OpenAI API
+        content = [{"type": "text", "text": prompt}]
 
-        except Exception as e:
-            logger.error(f"Error evaluating image coherence: {e}")
-            return 0.0, f"Evaluation error: {str(e)}"
+        if image.url:
+            content.append({"type": "image_url", "image_url": {"url": image.url}})
+        elif image.base64:
+            # Format base64 image with data URL scheme
+            image_format = image.format or "jpeg"
+            data_url = f"data:image/{image_format};base64,{image.base64}"
+            content.append({"type": "image_url", "image_url": {"url": data_url}})
+
+        chat_response = await self.model.achat(
+            messages=[{"role": "user", "content": content}],
+            structured_model=GraderScoreCallback,
+        )
+
+        # Default to 5.0 (neutral score on 0-10 scale) for missing fields
+        parsed = await parse_structured_chat_response(chat_response)
+        score = parsed.get("score", 5.0)
+        reason = parsed.get("reason", "")
+        return score, reason
 
     async def _acompute(
         self,
@@ -331,7 +329,16 @@ async def aevaluate(
             ...     ]
             ... )
         """
-        score, details = await self._acompute(response, **kwargs)
+        try:
+            score, details = await self._acompute(response, **kwargs)
+        except Exception as e:
+            logger.exception(f"Error evaluating image coherence: {e}")
+            from openjudge.graders.base_grader import GraderError
+
+            return GraderError(
+                name=self.name,
+                error=f"Evaluation error: {str(e)}",
+            )
 
         if "error" in details:
             return GraderScore(