Update to strands-agents-evals 0.1.0 API

Author: Aidan Daly

src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/README.md

@@ -28,7 +28,7 @@ Evaluate Strands agents during local development and testing. The integration ca
 
 ```python
 from strands import Agent, tool
-from strands_evals import Dataset, Case
+from strands_evals import Experiment, Case
 from strands_evals.telemetry import StrandsEvalsTelemetry
 from bedrock_agentcore.evaluation import create_strands_evaluator
 
@@ -82,12 +82,13 @@ cases = [
 evaluator = create_strands_evaluator("Builtin.Helpfulness")
 
 # Run evaluations
-dataset = Dataset(cases=cases, evaluator=evaluator)
-report = dataset.run_evaluations(task_fn)
+experiment = Experiment(cases=cases, evaluators=[evaluator])
+reports = experiment.run_evaluations(task_fn)
+report = reports[0]
 
 # View results
-print(f"Average score: {report.average_score:.2f}")
-print(f"Pass rate: {report.pass_rate:.1%}")
+print(f"Overall score: {report.overall_score:.2f}")
+print(f"Pass rate: {sum(report.test_passes) / len(report.test_passes):.1%}")
 ```
 
 ## Production Evaluation with CloudWatch Spans
@@ -128,7 +129,7 @@ spans = fetch_spans_from_cloudwatch(
 ### Evaluation Workflow
 
 ```python
-from strands_evals import Case, Dataset
+from strands_evals import Case, Experiment
 from bedrock_agentcore.evaluation import create_strands_evaluator, fetch_spans_from_cloudwatch
 import time
 
@@ -173,10 +174,11 @@ def task_fn(case):
     }
 
 evaluator = create_strands_evaluator("Builtin.Helpfulness")
-dataset = Dataset(cases=cases, evaluator=evaluator)
-report = dataset.run_evaluations(task_fn)
+experiment = Experiment(cases=cases, evaluators=[evaluator])
+reports = experiment.run_evaluations(task_fn)
+report = reports[0]
 
-print(f"Average score: {report.average_score:.2f}")
+print(f"Overall score: {report.overall_score:.2f}")
 ```
 
 ## Available Evaluators

src/bedrock_agentcore/evaluation/integrations/strands_agents_evals/evaluator.py

@@ -11,9 +11,7 @@
 from typing_extensions import TypeVar
 
 from bedrock_agentcore._utils.endpoints import DEFAULT_REGION
-
 from bedrock_agentcore.evaluation.span_to_adot_serializer import convert_strands_to_adot
-from bedrock_agentcore.evaluation.utils.cloudwatch_span_helper import fetch_spans_from_cloudwatch
 
 logger = logging.getLogger(__name__)
 

src/bedrock_agentcore/evaluation/span_to_adot_serializer/strands_converter.py

@@ -15,8 +15,6 @@
 from .adot_models import (
     ADOTDocumentBuilder,
     ConversationTurn,
-    ResourceInfo,
-    SpanMetadata,
     SpanParser,
     ToolExecution,
 )

src/bedrock_agentcore/evaluation/utils/cloudwatch_span_helper.py

@@ -75,15 +75,15 @@ def query_log_group(
 
             # Poll for completion with exponential backoff
             backoff = initial_backoff
-            for attempt in range(max_attempts):
+            for _attempt in range(max_attempts):
                 result = self.logs_client.get_query_results(queryId=query_id)
 
                 if result["status"] == "Complete":
                     # Check if we hit the 10K result limit
                     statistics = result.get("statistics", {})
                     records_matched = statistics.get("recordsMatched", 0)
                     records_returned = len(result.get("results", []))
-                    
+
                     if records_matched > 10000:
                         logger.warning(
                             "CloudWatch query matched %d records but can only return 10,000. "
@@ -92,7 +92,7 @@ def query_log_group(
                             records_matched,
                             log_group_name,
                         )
-                    
+
                     logger.debug(
                         "CloudWatch query completed: %d results returned, %d records matched",
                         records_returned,

tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_end_to_end.py

@@ -4,7 +4,7 @@
 
 import pytest
 from strands import Agent, tool
-from strands_evals import Case, Dataset
+from strands_evals import Case, Experiment
 from strands_evals.telemetry import StrandsEvalsTelemetry
 
 from bedrock_agentcore.evaluation import create_strands_evaluator
@@ -51,8 +51,9 @@ def task_fn(case):
         # Create evaluator with mocked client
         with patch("boto3.client", return_value=mock_boto_client):
             evaluator = create_strands_evaluator("Builtin.Helpfulness")
-            dataset = Dataset(cases=cases, evaluator=evaluator)
-            report = dataset.run_evaluations(task_fn)
+            experiment = Experiment(cases=cases, evaluators=[evaluator])
+            reports = experiment.run_evaluations(task_fn)
+            report = reports[0]
 
             # Verify results
             assert report.overall_score == 0.85
@@ -80,8 +81,8 @@ def task_fn(case):
 
         with patch("boto3.client", return_value=mock_boto_client):
             evaluator = create_strands_evaluator("Builtin.Helpfulness")
-            dataset = Dataset(cases=cases, evaluator=evaluator)
-            dataset.run_evaluations(task_fn)
+            experiment = Experiment(cases=cases, evaluators=[evaluator])
+            experiment.run_evaluations(task_fn)
 
             # Verify ADOT spans passed through without conversion
             call_args = mock_boto_client.evaluate.call_args[1]
@@ -96,8 +97,9 @@ def task_fn(case):
 
         with patch("boto3.client", return_value=mock_boto_client):
             evaluator = create_strands_evaluator("Builtin.Helpfulness")
-            dataset = Dataset(cases=cases, evaluator=evaluator)
-            report = dataset.run_evaluations(task_fn)
+            experiment = Experiment(cases=cases, evaluators=[evaluator])
+            reports = experiment.run_evaluations(task_fn)
+            report = reports[0]
 
             # Should return 0 score for empty trajectory
             assert report.overall_score == 0.0

tests/bedrock_agentcore/evaluation/integrations/strands_agents_evals/test_evaluator.py

@@ -1,6 +1,5 @@
 """Tests for Strands AgentCore Evaluator."""
 
-from datetime import datetime, timezone
 from unittest.mock import Mock, patch
 
 import pytest
@@ -408,6 +407,3 @@ def test_not_a_dict(self):
         """Test non-dict is invalid."""
         assert _is_valid_adot_document("not a dict") is False
         assert _is_valid_adot_document(None) is False
-
-
-

tests/bedrock_agentcore/evaluation/span_to_adot_serializer/test_adot_models.py

@@ -13,7 +13,6 @@
     ToolExecution,
 )
 
-
 # ==============================================================================
 # Fixtures
 # ==============================================================================

tests/bedrock_agentcore/evaluation/span_to_adot_serializer/test_strands_converter.py

@@ -10,7 +10,6 @@
     StrandsToADOTConverter,
 )
 
-
 # ==============================================================================
 # Fixtures
 # ==============================================================================

tests/bedrock_agentcore/evaluation/utils/test_cloudwatch_span_helper.py

@@ -3,8 +3,6 @@
 from datetime import datetime, timezone
 from unittest.mock import Mock, patch
 
-import pytest
-
 from bedrock_agentcore.evaluation.utils.cloudwatch_span_helper import (
     CloudWatchSpanHelper,
     _is_valid_adot_document,

tests_integ/evaluation/integrations/strands_agents_evals/test_strands_evaluation.py

@@ -9,7 +9,7 @@
 
 import pytest
 from strands import Agent, tool
-from strands_evals import Case, Dataset
+from strands_evals import Case, Experiment
 from strands_evals.telemetry import StrandsEvalsTelemetry
 
 from bedrock_agentcore.evaluation import create_strands_evaluator
@@ -59,8 +59,9 @@ def task_fn(case):
         cases = [Case(input="What is 2+2?", expected_output="4")]
 
         evaluator = create_strands_evaluator("Builtin.Helpfulness", region=REGION)
-        dataset = Dataset(cases=cases, evaluator=evaluator)
-        report = dataset.run_evaluations(task_fn)
+        experiment = Experiment(cases=cases, evaluators=[evaluator])
+        reports = experiment.run_evaluations(task_fn)
+        report = reports[0]
 
         # Verify results
         assert report.overall_score >= 0.0
@@ -84,8 +85,9 @@ def task_fn(case):
         cases = [Case(input="Calculate 5 + 3", expected_output="8")]
 
         evaluator = create_strands_evaluator("Builtin.Accuracy", region=REGION)
-        dataset = Dataset(cases=cases, evaluator=evaluator)
-        report = dataset.run_evaluations(task_fn)
+        experiment = Experiment(cases=cases, evaluators=[evaluator])
+        reports = experiment.run_evaluations(task_fn)
+        report = reports[0]
 
         assert report.overall_score >= 0.0
         assert report.overall_score <= 1.0
@@ -112,8 +114,9 @@ def task_fn(case):
         ]
 
         evaluator = create_strands_evaluator("Builtin.Helpfulness", region=REGION, test_pass_score=0.6)
-        dataset = Dataset(cases=cases, evaluator=evaluator)
-        report = dataset.run_evaluations(task_fn)
+        experiment = Experiment(cases=cases, evaluators=[evaluator])
+        reports = experiment.run_evaluations(task_fn)
+        report = reports[0]
 
         assert report.overall_score >= 0.0
         assert report.overall_score <= 1.0
@@ -131,8 +134,9 @@ def task_fn(case):
         cases = [Case(input="Test", expected_output="Response")]
 
         evaluator = create_strands_evaluator("Builtin.Helpfulness", region=REGION)
-        dataset = Dataset(cases=cases, evaluator=evaluator)
-        report = dataset.run_evaluations(task_fn)
+        experiment = Experiment(cases=cases, evaluators=[evaluator])
+        reports = experiment.run_evaluations(task_fn)
+        report = reports[0]
 
         # Should return 0 score for empty trajectory
         assert report.overall_score == 0.0
@@ -156,8 +160,9 @@ def task_fn(case):
 
         # Test with high threshold
         evaluator = create_strands_evaluator("Builtin.Helpfulness", region=REGION, test_pass_score=0.9)
-        dataset = Dataset(cases=cases, evaluator=evaluator)
-        report = dataset.run_evaluations(task_fn)
+        experiment = Experiment(cases=cases, evaluators=[evaluator])
+        reports = experiment.run_evaluations(task_fn)
+        report = reports[0]
 
         assert report.overall_score >= 0.0
         assert report.overall_score <= 1.0