feat: update more graders including multimodal graders (#60)

* feat: update more graders including multimodal graders

* feat: update more graders including multimodal graders

Author: jc200808

openjudge/graders/agent/memory/memory_accuracy.py

@@ -180,7 +180,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate memory accuracy",
             model=model,
-            template=template,
+            template=template or DEFAULT_MEMORY_ACCURACY_TEMPLATE,
             language=language,
         )
 

openjudge/graders/agent/tool/tool_call_success.py

@@ -238,10 +238,9 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluates whether tool calls done by an AI agent includes failures or not",
             model=model,
-            template=template,
+            template=template or DEFAULT_TOOL_CALL_SUCCESS_TEMPLATE,
             language=language,
         )
-        self.template = template or DEFAULT_TOOL_CALL_SUCCESS_TEMPLATE
 
     async def aevaluate(
         self,

openjudge/graders/agent/tool/tool_selection.py

@@ -205,7 +205,7 @@ def __init__(
         super().__init__(
             name="tool_selection",
             mode=GraderMode.POINTWISE,
-            description="Evaluate tool selection ",
+            description="Evaluate tool selection",
             model=model,
             template=template or DEFAULT_TOOL_SELECTION_TEMPLATE,
             language=language,

openjudge/graders/agent/trajectory/trajectory_comprehensive.py

@@ -427,7 +427,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Comprehensive evaluation for agent trajectories including step-level and overall problem-solving assessment",
             model=model,
-            template=template,
+            template=template or DEFAULT_TRAJECTORY_COMPREHENSIVE_TEMPLATE,
             language=language,
             structured_model=TrajectoryEvaluationOutput,
             callback=self._create_trajectory_callback(language=language),

openjudge/graders/common/correctness.py

@@ -285,7 +285,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate whether response matches the provided reference response",
             model=model,
-            template=template,
+            template=template or DEFAULT_CORRECTNESS_TEMPLATE,
             language=language,
         )
         self.threshold = threshold

openjudge/graders/common/harmfulness.py

@@ -263,7 +263,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate whether response contains harmful or inappropriate content",
             model=model,
-            template=template,
+            template=template or DEFAULT_HARMFULNESS_TEMPLATE,
             language=language,
         )
         self.threshold = threshold

openjudge/graders/common/instruction_following.py

@@ -279,7 +279,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate whether response follows the given instructions",
             model=model,
-            template=template,
+            template=template or DEFAULT_INSTRUCTION_FOLLOWING_TEMPLATE,
             language=language,
         )
         self.threshold = threshold

openjudge/graders/common/relevance.py

@@ -280,7 +280,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate relevance of response to user query",
             model=model,
-            template=template,
+            template=template or DEFAULT_RELEVANCE_TEMPLATE,
             language=language,
         )
         self.threshold = threshold

openjudge/graders/multimodal/image_coherence.py

@@ -27,7 +27,8 @@
 # pylint: disable=line-too-long
 
 # English Prompt
-IMAGE_COHERENCE_PROMPT_EN = """
+IMAGE_COHERENCE_PROMPT_EN = textwrap.dedent(
+    """
 # Task Description
 You are a multi-modal document evaluation assistant. You will receive an image and its textual context.
 Your task is to evaluate the coherence between the image and the text (context above and below) it accompanies.
@@ -62,9 +63,11 @@
 # Image
 [Insert Image Here]
 """
+).strip()
 
 # Chinese Prompt
-IMAGE_COHERENCE_PROMPT_ZH = """
+IMAGE_COHERENCE_PROMPT_ZH = textwrap.dedent(
+    """
 # 任务描述
 你是一名多模态文档评估助手。你将收到一张图片及其文本背景。
 你的任务是评估图片与其伴随文本（上下文）之间的连贯性。
@@ -99,20 +102,21 @@
 # 图片
 [在此插入图片]
 """
+).strip()
 
 # Build default template from prompts
 DEFAULT_IMAGE_COHERENCE_TEMPLATE = PromptTemplate(
     messages={
         LanguageEnum.EN: [
             ChatMessage(
                 role="user",
-                content=textwrap.dedent(IMAGE_COHERENCE_PROMPT_EN),
+                content=IMAGE_COHERENCE_PROMPT_EN,
             ),
         ],
         LanguageEnum.ZH: [
             ChatMessage(
                 role="user",
-                content=textwrap.dedent(IMAGE_COHERENCE_PROMPT_ZH),
+                content=IMAGE_COHERENCE_PROMPT_ZH,
             ),
         ],
     },
@@ -159,19 +163,20 @@ class ImageCoherenceGrader(LLMGrader):
         GraderScore with normalized coherence score [0, 1]
 
     Example:
+        >>> import asyncio
         >>> from openjudge.model.openai_llm import OpenAIChatModel
         >>> from openjudge.multimodal import ImageCoherenceGrader, MLLMImage
         >>>
         >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-max")
         >>> grader = ImageCoherenceGrader(model=model)
         >>>
-        >>> result = await grader.aevaluate(
+        >>> result = asyncio.run(grader.aevaluate(
         ...     response=[
         ...         "Q3 sales increased 25%.",
         ...         MLLMImage(url="https://example.com/sales_chart.jpg"),
         ...         "Growth driven by new products."
         ...     ]
-        ... )
+        ... ))
         >>> print(result.score)  # 0.95 - image coherent with sales context
     """
 
@@ -198,7 +203,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate image-text coherence",
             model=model,
-            template=template,
+            template=template or DEFAULT_IMAGE_COHERENCE_TEMPLATE,
             language=language,
         )
         self.max_context_size = max_context_size

openjudge/graders/multimodal/image_helpfulness.py

@@ -28,7 +28,8 @@
 # pylint: disable=line-too-long
 
 # English Prompt
-IMAGE_HELPFULNESS_PROMPT_EN = """
+IMAGE_HELPFULNESS_PROMPT_EN = textwrap.dedent(
+    """
 # Task Description
 You are a multi-modal document evaluation assistant. You will receive an image and its textual context.
 Your task is to evaluate the helpfulness of the image in enabling human readers to comprehend the text (context above and below) it accompanies.
@@ -63,9 +64,11 @@
 # Image
 [Insert Image Here]
 """
+).strip()
 
 # Chinese Prompt
-IMAGE_HELPFULNESS_PROMPT_ZH = """
+IMAGE_HELPFULNESS_PROMPT_ZH = textwrap.dedent(
+    """
 # 任务描述
 你是一名多模态文档评估助手。你将收到一张图片及其文本背景。
 你的任务是评估图片对于帮助人类读者理解其伴随文本（上下文）的有用性。
@@ -100,20 +103,21 @@
 # 图片
 [在此插入图片]
 """
+).strip()
 
 # Build default template from prompts
 DEFAULT_IMAGE_HELPFULNESS_TEMPLATE = PromptTemplate(
     messages={
         LanguageEnum.EN: [
             ChatMessage(
                 role="user",
-                content=textwrap.dedent(IMAGE_HELPFULNESS_PROMPT_EN),
+                content=IMAGE_HELPFULNESS_PROMPT_EN,
             ),
         ],
         LanguageEnum.ZH: [
             ChatMessage(
                 role="user",
-                content=textwrap.dedent(IMAGE_HELPFULNESS_PROMPT_ZH),
+                content=IMAGE_HELPFULNESS_PROMPT_ZH,
             ),
         ],
     },
@@ -161,13 +165,14 @@ class ImageHelpfulnessGrader(LLMGrader):
         GraderScore with normalized helpfulness score [0, 1]
 
     Example:
+        >>> import asyncio
         >>> from openjudge.model.openai_llm import OpenAIChatModel
         >>> from openjudge.multimodal import ImageHelpfulnessGrader, MLLMImage
         >>>
         >>> model = OpenAIChatModel(api_key="sk-...", model="qwen3-max")
         >>> grader = ImageHelpfulnessGrader(model=model)
         >>>
-        >>> result = await grader.aevaluate(
+        >>> result = asyncio.run(grader.aevaluate(
         ...     response=[
         ...         "The system architecture has three layers.",
         ...         MLLMImage(url="https://example.com/arch_diagram.jpg"),
@@ -200,7 +205,7 @@ def __init__(
             mode=GraderMode.POINTWISE,
             description="Evaluate image helpfulness for understanding text",
             model=model,
-            template=template,
+            template=template or DEFAULT_IMAGE_HELPFULNESS_TEMPLATE,
             language=language,
         )
         self.max_context_size = max_context_size
@@ -362,4 +367,4 @@ async def aevaluate(
         )
 
 
-__all__ = ["ImageHelpfulnessGrader"]
+__all__ = ["ImageHelpfulnessGrader", "DEFAULT_IMAGE_HELPFULNESS_TEMPLATE"]