Revert "🕸️ Ensure URL check works"

This reverts commit b5dcc63.

bananalyzer/data/examples.json

@@ -2754,6 +2754,21 @@
       }
     ]
   },
+  {
+    "id": "8fc4b97d-f823-4c31-b50f-ad42671fe2b1",
+    "url": "https://www.shrinerschildrens.org/en/locations?listPage=1",
+    "source": "hosted",
+    "domain": "healthcare",
+    "subdomain": "contact",
+    "type": "navigate",
+    "goal": "You are currently on a listing page. Use pagination controls to travel to the last available listing page if possible ",
+    "evals": [
+      {
+        "type": "end_url_match",
+        "expected": "https://www.shrinerschildrens.org/en/locations?listPage=4"
+      }
+    ]
+  },
   {
     "id": "b572a5e4-717b-41ab-86e3-ec41cac368b3",
     "url": "https://www.sih.net/locations?page=1&perpage=25&q=&serv=&dist=&zip=&sort=Ascending&view=list",

bananalyzer/runner/agent_runner.py

@@ -1,11 +1,11 @@
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Union
 
-from playwright.async_api import Page
+from playwright.async_api import BrowserContext
 
 from bananalyzer.data.schemas import Example
 
-AgentResult = Union[Dict[str, Any], List[Dict[str, Any]], List[str], str]
+AgentResult = Union[Dict[str, Any], List[Dict[str, Any]], str]
 
 
 class AgentRunner(ABC):
@@ -16,7 +16,7 @@ class AgentRunner(ABC):
     @abstractmethod
     async def run(
         self,
-        page: Page,
+        browser_context: BrowserContext,
         eval_context: Example,
     ) -> AgentResult:
         pass

bananalyzer/runner/null_agent_wrapper.py

@@ -1,6 +1,6 @@
 import asyncio
 
-from playwright.async_api import Page
+from playwright.async_api import BrowserContext
 
 from bananalyzer.data.schemas import Example
 from bananalyzer.runner.agent_runner import AgentResult, AgentRunner
@@ -13,16 +13,12 @@ class NullAgentRunner(AgentRunner):
 
     async def run(
         self,
-        page: Page,
+        context: BrowserContext,
         example: Example,
     ) -> AgentResult:
+        page = await context.new_page()
         print(f"Testing {example.get_static_url()}")
         await page.goto(example.get_static_url())
         await asyncio.sleep(0.5)
         print(f"Done testing {example.get_static_url()}")
-
-        # Ensure page is correct
-        if example.evals[0].type == "end_url_match":
-            await page.goto(example.evals[0].expected)
-
-        return example.evals[0].expected
+        return example.evals[0].expected  # type: ignore

bananalyzer/runner/runner.py

@@ -25,13 +25,12 @@ async def test_{example.id.replace("-", "_")}() -> None:
     async with async_playwright() as p:
         browser = await p.chromium.launch(headless={headless})
         context = await browser.new_context()
-        page = await context.new_page()
         example = get_example_by_url("{example.url}")
          
         # The agent is imported into the global context prior to this call 
-        result = await agent.run(page, example)
+        result = await agent.run(context, example)
         for curr_eval in example.evals:
-            curr_eval.eval_results(page, result)
+            curr_eval.eval_results(p, result)
 """,
         example=example,
     )

tests/fixtures/agent_wrapper.py

@@ -0,0 +1,2 @@
+def TestAgentWrapper(AgentWrapper):
+    pass