📄 Move and rename stuff

README.md

@@ -85,8 +85,7 @@ Separately, there are specific `tags` that can be used further filter test inten
 ```python
 import asyncio
 from playwright.async_api import BrowserContext
-from bananalyzer.data.schemas import Example
-from bananalyzer.runner.agent_runner import AgentResult, AgentRunner
+from bananalyzer import AgentResult, AgentRunner, Example
 
 
 class NullAgentRunner(AgentRunner):

bananalyzer/__init__.py

@@ -1,6 +1,6 @@
 from bananalyzer.__version import VERSION
-from bananalyzer.data.examples import get_all_examples
-from bananalyzer.data.schemas import Example, ExampleType
+from bananalyzer.data.example_fetching import get_all_examples
+from bananalyzer.data.example_schemas import ExampleType, Example
 from bananalyzer.runner.agent_runner import AgentRunner, AgentResult
 from bananalyzer.runner.evals import check_match, AllowedJSON
 

bananalyzer/__main__.py

@@ -11,13 +11,13 @@
 from urllib.parse import urlparse
 
 from bananalyzer import AgentRunner
-from bananalyzer.data.banana_seeds import download_har, download_mhtml
-from bananalyzer.data.examples import (
+from bananalyzer.data.example_fetching import (
     download_examples,
     get_examples_path,
     get_test_examples,
     get_training_examples,
 )
+from bananalyzer.data.example_s3 import download_har, download_mhtml
 from bananalyzer.runner.generator import PytestTestGenerator
 from bananalyzer.runner.runner import run_tests
 from bananalyzer.schema import AgentRunnerClass, Args, PytestArgs, XDistArgs

bananalyzer/data/fetch_schemas.py → bananalyzer/data/example_detail_schemas.py

@@ -1,8 +1,7 @@
 import json
 from typing import Dict, Any
 
-from bananalyzer.data.schemas import SchemaName
-from bananalyzer.data.examples import get_examples_path
+from bananalyzer.data.example_fetching import get_examples_path
 
 """
 This file maps a schema_name to a JSON schema or goal string.

bananalyzer/data/examples.py → bananalyzer/data/example_fetching.py

@@ -5,8 +5,8 @@
 from pathlib import Path
 from typing import List, Optional
 
-from bananalyzer.data.banana_seeds import download_examples_from_s3
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_s3 import download_examples_from_s3
+from bananalyzer.data.example_schemas import Example
 
 local_examples_path: Path = Path(__file__).resolve().parent.parent.parent / "static"
 downloaded_examples_path = Path.home() / ".bananalyzer_data"

bananalyzer/data/schemas.py → bananalyzer/data/example_schemas.py

@@ -115,7 +115,7 @@ def handle_json_match(self, result: AllowedJSON, field: Optional[str]) -> None:
 
 class Example(BaseModel):
     id: str
-    url: str | None
+    url: str
     resource_path: Optional[str] = Field(
         description="Local path of a HAR, S3 URL of a HAR directory's tar.gz, or remote URL of MHTML",
         default=None,
@@ -142,7 +142,7 @@ def get_static_url(self) -> str:
 
     @property
     def har_file_path(self) -> Path:
-        from bananalyzer.data.examples import get_examples_path
+        from bananalyzer.data.example_fetching import get_examples_path
 
         if self.source != "har":
             raise ValueError("This example is not a HAR file")
@@ -166,7 +166,7 @@ def har_file_path(self) -> Path:
 
     @model_validator(mode="before")
     def set_schema_and_goal(cls, values: Dict[str, Any]) -> Dict[str, Any]:
-        from bananalyzer.data.fetch_schemas import get_fetch_schema, get_goal
+        from bananalyzer.data.example_detail_schemas import get_fetch_schema, get_goal
 
         if values.get("schema_") is None:
             print(f"Warning: Schema not found for Example {values['id']}")

bananalyzer/runner/agent_runner.py

@@ -2,7 +2,7 @@
 
 from playwright.async_api import Page
 
-from bananalyzer.data.schemas import Example
+from bananalyzer import Example
 from bananalyzer.runner.evals import AllowedJSON
 
 AgentResult = AllowedJSON

bananalyzer/runner/generator.py

@@ -2,7 +2,7 @@
 from urllib.parse import urlparse
 
 from bananalyzer import Example
-from bananalyzer.data.schemas import Eval
+from bananalyzer.data.example_schemas import Eval
 from bananalyzer.runner.runner import BananalyzerTest
 from bananalyzer.schema import MARKER_PREFIX
 
@@ -89,7 +89,7 @@ async def test_{eval_.type}(self, page, result) -> None:
 """
 
     def _generate_class_name(self, example: Example) -> str:
-        domain = urlparse(example.url).hostname
+        domain: str | None = urlparse(example.url).hostname
         if not domain:
             raise ValueError(f"Invalid URL (no domain name): {example.url}")
         domain = domain.replace(".", "_")

bananalyzer/runner/null_agent_wrapper.py

@@ -1,7 +1,8 @@
 import asyncio
+
 from playwright.async_api import Page
 
-from bananalyzer.data.schemas import Example
+from bananalyzer import Example
 from bananalyzer.runner.agent_runner import AgentResult, AgentRunner
 
 

bananalyzer/runner/runner.py

@@ -7,7 +7,7 @@
 import pytest
 from pydantic import BaseModel
 
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_schemas import Example
 from bananalyzer.hooks import BananalyzerPytestPlugin
 from bananalyzer.junit import enrich_report
 from bananalyzer.schema import AgentRunnerClass, PytestArgs, XDistArgs
@@ -43,7 +43,7 @@ def create_test_file(
 import pytest_asyncio
 import asyncio
 
-from bananalyzer.data.examples import get_example_by_url
+from bananalyzer.data.example_fetching import get_example_by_url
 from playwright.async_api import async_playwright
 
 @pytest.fixture(scope="session")

bananalyzer/runner/website_responder.py

@@ -1,8 +1,8 @@
 from abc import ABC, abstractmethod
 from pathlib import Path
 
-from bananalyzer.data.examples import get_examples_path
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_fetching import get_examples_path
+from bananalyzer.data.example_schemas import Example
 
 
 class WebsiteResponder(ABC):

bananalyzer/schema.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel, Field
 from typing import List, Literal, Optional, Union
 
-from bananalyzer.data.schemas import ExampleType
+from pydantic import BaseModel, Field
+
+from bananalyzer.data.example_schemas import ExampleType
 
 XDistDistributionMode = Literal[
     "load", "loadscope", "loadfile", "loadgroup", "worksteal", "no"

static/examples.json

@@ -37206,12 +37206,18 @@
     "evals": [
       {
         "type": "json_match",
-        "expected": [
+        "options": [
           {
             "name": "Sunshine Valley Medical Center",
             "address": "Sunshine Valley Hospital\n5678 Healing Lane\n3rd Floor\nMeadowbrook, CA 90210",
             "phone": "555.123.4567",
             "fax": "555.987.6543"
+          },
+          {
+            "name": "Sunshine Valley Medical Center",
+            "address": "Sunshine Valley Hospital, 5678 Healing Lane, 3rd Floor, Meadowbrook, CA 90210",
+            "phone": "555.123.4567",
+            "fax": "555.987.6543"
           }
         ]
       }

static/re-sync-mhtml.py

@@ -4,8 +4,8 @@
 
 from playwright.async_api import async_playwright
 
-from bananalyzer.data.examples import get_all_examples
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_fetching import get_all_examples
+from bananalyzer.data.example_schemas import Example
 
 """
 Test script to re-load websites and save them as MHTML files.

tests/test_example_eval.py

@@ -5,7 +5,7 @@
 from _pytest.outcomes import Failed
 from pytest_mock import MockFixture
 
-from bananalyzer.data.schemas import Eval, Example
+from bananalyzer.data.example_schemas import Eval, Example
 from bananalyzer.runner.evals import format_new_lines
 
 

tests/test_examples.py

@@ -10,8 +10,8 @@
 import pytest
 from pytest_mock import MockFixture
 
-from bananalyzer.data.banana_seeds import download_har
-from bananalyzer.data.examples import (
+from bananalyzer.data.example_detail_schemas import get_fetch_schema, get_goal
+from bananalyzer.data.example_fetching import (
     download_examples,
     downloaded_examples_path,
     get_all_examples,
@@ -22,8 +22,8 @@
     load_examples_at_path,
     local_examples_path,
 )
-from bananalyzer.data.fetch_schemas import get_fetch_schema, get_goal
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_s3 import download_har
+from bananalyzer.data.example_schemas import Example
 
 
 def test_load_examples_at_path_success(mocker: MockFixture) -> None:

tests/test_generator.py

@@ -1,5 +1,5 @@
 from bananalyzer import Example
-from bananalyzer.data.schemas import Eval
+from bananalyzer.data.example_schemas import Eval
 from bananalyzer.runner.generator import PytestTestGenerator
 
 

tests/test_website_responder.py

@@ -1,7 +1,7 @@
 import pytest
 from pytest_mock import MockFixture
 
-from bananalyzer.data.schemas import Example
+from bananalyzer.data.example_schemas import Example
 from bananalyzer.runner.website_responder import (
     HostedWebsiteResponder,
     StaticFileResponder,