Support adk eval --num_runs N as AgentEvaluator

src/google/adk/cli/cli_eval.py

@@ -17,6 +17,7 @@
 import importlib.util
 import logging
 import os
+import statistics
 import sys
 from typing import Any
 from typing import Optional
@@ -34,6 +35,8 @@
 from ..evaluation.eval_case import get_all_tool_calls
 from ..evaluation.eval_case import IntermediateDataType
 from ..evaluation.eval_metrics import EvalMetric
+from ..evaluation.eval_metrics import EvalMetricResult
+from ..evaluation.eval_metrics import EvalStatus
 from ..evaluation.eval_metrics import Interval
 from ..evaluation.eval_metrics import MetricInfo
 from ..evaluation.eval_metrics import MetricValueInfo
@@ -132,6 +135,95 @@ def parse_and_get_evals_to_run(
   return eval_set_to_evals
 
 
+def _generate_final_eval_status(
+    overall_eval_metric_results: list[EvalMetricResult],
+) -> EvalStatus:
+  """Returns final eval status for a case from overall metric results."""
+  final_eval_status = EvalStatus.NOT_EVALUATED
+  for overall_eval_metric_result in overall_eval_metric_results:
+    overall_eval_status = overall_eval_metric_result.eval_status
+    if overall_eval_status == EvalStatus.PASSED:
+      final_eval_status = EvalStatus.PASSED
+    elif overall_eval_status == EvalStatus.NOT_EVALUATED:
+      continue
+    elif overall_eval_status == EvalStatus.FAILED:
+      final_eval_status = EvalStatus.FAILED
+      break
+    else:
+      raise ValueError(f"Unknown eval status: {overall_eval_status}.")
+  return final_eval_status
+
+
+def _aggregate_metric_results(
+    metric_results: list[EvalMetricResult],
+) -> EvalMetricResult:
+  """Aggregates results of the same metric across runs."""
+  if not metric_results:
+    raise ValueError("`metric_results` should not be empty.")
+
+  aggregate_metric_result = metric_results[0].model_copy(deep=True)
+  scores = [m.score for m in metric_results if m.score is not None]
+  if scores:
+    aggregate_metric_result.score = statistics.mean(scores)
+    aggregate_metric_result.eval_status = (
+        EvalStatus.PASSED
+        if aggregate_metric_result.score >= aggregate_metric_result.threshold
+        else EvalStatus.FAILED
+    )
+  else:
+    aggregate_metric_result.score = None
+    aggregate_metric_result.eval_status = EvalStatus.NOT_EVALUATED
+
+  return aggregate_metric_result
+
+
+def aggregate_eval_case_results(
+    eval_results: list[EvalCaseResult],
+) -> list[EvalCaseResult]:
+  """Aggregates EvalCaseResults with the same eval_set_id and eval_id."""
+  eval_results_by_case_id: dict[tuple[str, str], list[EvalCaseResult]] = {}
+  for eval_result in eval_results:
+    key = (eval_result.eval_set_id, eval_result.eval_id)
+    if key not in eval_results_by_case_id:
+      eval_results_by_case_id[key] = []
+    eval_results_by_case_id[key].append(eval_result)
+
+  aggregate_results: list[EvalCaseResult] = []
+  for _, per_case_results in eval_results_by_case_id.items():
+    aggregate_result = per_case_results[0].model_copy(deep=True)
+    metric_results_by_name: dict[str, list[EvalMetricResult]] = {}
+    for per_case_result in per_case_results:
+      for metric_result in per_case_result.overall_eval_metric_results:
+        metric_name = metric_result.metric_name
+        if metric_name not in metric_results_by_name:
+          metric_results_by_name[metric_name] = []
+        metric_results_by_name[metric_name].append(metric_result)
+
+    metric_names_in_order = [
+        metric_result.metric_name
+        for metric_result in aggregate_result.overall_eval_metric_results
+    ]
+    missing_metric_names = sorted(
+        set(metric_results_by_name.keys()) - set(metric_names_in_order)
+    )
+    metric_names_in_order.extend(missing_metric_names)
+
+    aggregate_overall_eval_metric_results: list[EvalMetricResult] = []
+    for metric_name in metric_names_in_order:
+      aggregate_overall_eval_metric_results.append(
+          _aggregate_metric_results(metric_results_by_name[metric_name])
+      )
+    aggregate_result.overall_eval_metric_results = (
+        aggregate_overall_eval_metric_results
+    )
+    aggregate_result.final_eval_status = _generate_final_eval_status(
+        aggregate_overall_eval_metric_results
+    )
+    aggregate_results.append(aggregate_result)
+
+  return sorted(aggregate_results, key=lambda x: (x.eval_set_id, x.eval_id))
+
+
 async def _collect_inferences(
     inference_requests: list[InferenceRequest],
     eval_service: BaseEvalService,

src/google/adk/cli/cli_tools_click.py

@@ -709,6 +709,13 @@ def wrapper(*args, **kwargs):
 )
 @click.argument("eval_set_file_path_or_id", nargs=-1)
 @click.option("--config_file_path", help="Optional. The path to config file.")
+@click.option(
+    "--num_runs",
+    type=click.IntRange(min=1),
+    default=1,
+    show_default=True,
+    help="Optional. Number of times each eval set should be run.",
+)
 @click.option(
     "--print_detailed_results",
     is_flag=True,
@@ -721,6 +728,7 @@ def cli_eval(
     agent_module_file_path: str,
     eval_set_file_path_or_id: list[str],
     config_file_path: str,
+    num_runs: int,
     print_detailed_results: bool,
     eval_storage_uri: Optional[str] = None,
     log_level: str = "INFO",
@@ -777,6 +785,8 @@ def cli_eval(
 
   CONFIG_FILE_PATH: The path to config file.
 
+  NUM_RUNS: Number of times each eval set should be run.
+
   PRINT_DETAILED_RESULTS: Prints detailed results on the console.
   """
   envs.load_dotenv_for_agent(agent_module_file_path, ".")
@@ -801,6 +811,7 @@ def cli_eval(
     from ..evaluation.simulation.user_simulator_provider import UserSimulatorProvider
     from .cli_eval import _collect_eval_results
     from .cli_eval import _collect_inferences
+    from .cli_eval import aggregate_eval_case_results
     from .cli_eval import get_default_metric_info
     from .cli_eval import get_root_agent
     from .cli_eval import parse_and_get_evals_to_run
@@ -920,9 +931,14 @@ def cli_eval(
         metric_evaluator_registry=metric_evaluator_registry,
     )
 
+    repeated_inference_requests = [
+        inference_request.model_copy(deep=True)
+        for inference_request in inference_requests
+    ] * num_runs
     inference_results = asyncio.run(
         _collect_inferences(
-            inference_requests=inference_requests, eval_service=eval_service
+            inference_requests=repeated_inference_requests,
+            eval_service=eval_service,
         )
     )
     eval_results = asyncio.run(
@@ -932,6 +948,7 @@ def cli_eval(
             eval_metrics=eval_metrics,
         )
     )
+    aggregate_eval_results = aggregate_eval_case_results(eval_results)
   except ModuleNotFoundError as mnf:
     raise click.ClickException(MISSING_EVAL_DEPENDENCIES_MESSAGE) from mnf
 
@@ -940,7 +957,7 @@ def cli_eval(
   )
   eval_run_summary = {}
 
-  for eval_result in eval_results:
+  for eval_result in aggregate_eval_results:
     eval_result: EvalCaseResult
 
     if eval_result.eval_set_id not in eval_run_summary:

tests/unittests/cli/utils/test_cli_eval.py

@@ -19,6 +19,11 @@
 from types import SimpleNamespace
 from unittest import mock
 
+from google.adk.cli.cli_eval import aggregate_eval_case_results
+from google.adk.evaluation.eval_metrics import EvalMetricResult
+from google.adk.evaluation.eval_metrics import EvalStatus
+from google.adk.evaluation.eval_result import EvalCaseResult
+
 
 def test_get_eval_sets_manager_local(monkeypatch):
   mock_local_manager = mock.MagicMock()
@@ -49,3 +54,50 @@ def test_get_eval_sets_manager_gcs(monkeypatch):
   )
   assert manager == mock_gcs_manager
   mock_create_gcs.assert_called_once_with("gs://bucket")
+
+
+def test_aggregate_eval_case_results_across_runs():
+  eval_results = [
+      EvalCaseResult(
+          eval_set_id="set1",
+          eval_id="case1",
+          final_eval_status=EvalStatus.PASSED,
+          overall_eval_metric_results=[
+              EvalMetricResult(
+                  metric_name="response_match_score",
+                  threshold=0.8,
+                  score=1.0,
+                  eval_status=EvalStatus.PASSED,
+              )
+          ],
+          eval_metric_result_per_invocation=[],
+          session_id="s1",
+      ),
+      EvalCaseResult(
+          eval_set_id="set1",
+          eval_id="case1",
+          final_eval_status=EvalStatus.FAILED,
+          overall_eval_metric_results=[
+              EvalMetricResult(
+                  metric_name="response_match_score",
+                  threshold=0.8,
+                  score=0.6,
+                  eval_status=EvalStatus.FAILED,
+              )
+          ],
+          eval_metric_result_per_invocation=[],
+          session_id="s2",
+      ),
+  ]
+
+  aggregate_results = aggregate_eval_case_results(eval_results)
+
+  assert len(aggregate_results) == 1
+  assert aggregate_results[0].eval_set_id == "set1"
+  assert aggregate_results[0].eval_id == "case1"
+  assert aggregate_results[0].overall_eval_metric_results[0].score == 0.8
+  assert (
+      aggregate_results[0].overall_eval_metric_results[0].eval_status
+      == EvalStatus.PASSED
+  )
+  assert aggregate_results[0].final_eval_status == EvalStatus.PASSED