Merge remote-tracking branch 'origin/main' into forum-test-set

reworkd · Nov 29, 2023 · 177230a · 177230a
2 parents 98b38dc + 35aced4
commit 177230a
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 81 deletions.
diff --git a/bananalyzer/data/fetch_schemas.py b/bananalyzer/data/fetch_schemas.py
@@ -1,4 +1,4 @@
-from typing import Dict, Type
+from typing import Dict, Optional, Type
 
 from pydantic import BaseModel, Field
 
@@ -8,15 +8,22 @@
 
 
 class ContactSchema(BaseModel):
-    name: str
-    website: str = Field(
-        description="An external link to the website if the website provides a link"
+    name: str = Field(
+        description="name of the location (not the hospital system name) *precisely* as it is written on the page (do not edit it, add text, or combine names)"
     )
-    phone: str
-    fax: str = Field(description="Fax number of the location")
-    address: str
-    type: str = Field(
-        description="The type of clinic the location: Hospital, Clinic, etc."
+    address: str = Field(
+        description="complete address of the location including street, city, state, and ZIP",
+    )
+    phone: str = Field(
+        description="phone number of the location (only include the number but retain its formatting)",
+    )
+    fax: Optional[str] = Field(
+        description="fax number of the location (only include the number but retain its formatting)",
+        default=None,
+    )
+    type: Optional[str] = Field(
+        description="the type of location: Neurosurgery, MRI Services, etc. (not all locations will have a type available on the page)",
+        default=None,
     )
 
 

diff --git a/bananalyzer/data/schemas.py b/bananalyzer/data/schemas.py
@@ -12,7 +12,7 @@
 
 GoalType = Literal[
     "fetch",  # Scrape specific JSON information from a single page. Does not require navigation
-    "links",  # Scrape all detail page links from a single page
+    "links",  # Scrape all detail page links from a single listing page
     "click",  # Make a single click on a page
     "navigate",  # Travel to a new page
     "search",  # Search for the answer to a specific query

diff --git a/bananalyzer/hooks.py b/bananalyzer/hooks.py
@@ -30,6 +30,8 @@ def print_field_data(
         perfect = is_perfect and "✅" or "❌"
         table_data.append([field, passed, failed, percentage, perfect])
 
+    table_data.sort(key=lambda row: row[3], reverse=True)  # type: ignore
+
     # Calculate the percentage for the total row
     total_percentage = (
         f"{total_passed / (total_passed + total_failed) * 100:.2f}%"

diff --git a/bananalyzer/runner/null_agent_wrapper.py b/bananalyzer/runner/null_agent_wrapper.py
@@ -1,5 +1,6 @@
 import asyncio
 from random import random
+from typing import Any, cast
 
 from playwright.async_api import Page
 
@@ -12,6 +13,8 @@ class NullAgentRunner(AgentRunner):
     A test agent class that just returns an empty string
     """
 
+    RANDOM_FAILURE_RATE = 0  # You can change this to 0.5 to see random failures
+
     async def run(
         self,
         page: Page,
@@ -29,4 +32,9 @@ async def run(
         ):
             await page.goto(example.evals[0].expected)
 
-        return example.evals[0].expected
+        copy = cast(dict[str, Any], example.evals[0].expected).copy()
+        for key, value in copy.items():
+            if random() < self.RANDOM_FAILURE_RATE:
+                copy[key] = "random"
+
+        return copy
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bananalyzer"
-version = "0.6.4"
+version = "0.6.5"
 
 description = "Open source AI Agent evaluation framework for web tasks 🐒🍌"
 authors = ["asim-shrestha <[email protected]>"]

diff --git a/static/examples.json b/static/examples.json
@@ -525,7 +525,6 @@
         "type": "json_match",
         "expected": {
           "name": "Atlanta Ophthalmology Associates",
-          "website": "https://aoaeye.com/",
           "phone": "404-252-1194",
           "fax": "404-252-1196",
           "address": "5730 Glenridge Drive\nSuite 120\nAtlanta, GA 30328",
@@ -547,7 +546,6 @@
         "type": "json_match",
         "expected": {
           "name": "Baptist Health ENT Partners",
-          "website": null,
           "phone": "334-747-2273",
           "fax": null,
           "address": "2055 East South Blvd; Suite 908\n\nMontgomery, Alabama 36116",
@@ -569,7 +567,6 @@
         "type": "json_match",
         "expected": {
           "name": "MyMichigan Medical Center Clare",
-          "website": null,
           "phone": "(989) 633-5252",
           "fax": null,
           "address": "703 N. McEwan Street\nClare, MI 48617",
@@ -591,7 +588,6 @@
         "type": "json_match",
         "expected": {
           "name": "Montgomery Building",
-          "website": null,
           "phone": "513-936-4510",
           "fax": null,
           "address": "9275 Montgomery Road\nCincinnati, OH 45242",
@@ -613,7 +609,6 @@
         "type": "json_match",
         "expected": {
           "name": "UofL Health \u2013 Jewish Hospital Outpatient Care Center",
-          "website": null,
           "phone": "502-588-7690",
           "fax": "502-540-1414",
           "address": "225 Abraham Flexner Way, Suite 502\nLouisville, KY 40202",
@@ -2152,8 +2147,7 @@
           "address": "160 Falmouth Road Mashpee MA 02649",
           "phone": "774-361-8700",
           "fax": "774-521-3628",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2174,8 +2168,7 @@
           "address": "1690 Harrison Street Batesville, AR 72501",
           "phone": "(870) 793-2371",
           "fax": "(870) 793-7585",
-          "type": null,
-          "website": "https://batesvillebones.org/"
+          "type": null
         }
       }
     ]
@@ -2196,8 +2189,7 @@
           "address": "420 Wilkinson Dr, Dyersburg, TN 38024, USA",
           "phone": "(731) 288-3960",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2218,8 +2210,7 @@
           "address": "500 Medical Park, 2nd Floor Wheeling, WV 26003",
           "phone": "(304) 238-5750",
           "fax": "(304) 797-6186",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2240,8 +2231,7 @@
           "address": "289 North Avenue, Washington, PA 15301",
           "phone": "(724) 942-6100",
           "fax": "(724) 942-6104",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2262,8 +2252,7 @@
           "address": "10010 Falls of Neuse Road, Suite 203 Raleigh, North Carolina 27614",
           "phone": "919-235-6440",
           "fax": "919-350-7093",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2284,8 +2273,7 @@
           "address": "40 Stirling Road, Suite 101 Watchung, NJ 07069",
           "phone": "908-769-8000",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2306,8 +2294,7 @@
           "address": "740 Bragaw Lane Chocowinity, NC 27817",
           "phone": "(252) 946-9562",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2328,8 +2315,7 @@
           "address": "615 Douglas Street, Suite 500 Durham, NC 27705",
           "phone": "919-767-0202",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2350,8 +2336,7 @@
           "address": "3470 Mayland Court Richmond,VA 23233",
           "phone": "(804) 527-4540",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2372,8 +2357,7 @@
           "address": "919 Murfreesboro Road, Franklin, TN 37064",
           "phone": "(615) 936-5000",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2394,8 +2378,7 @@
           "address": "125 Memorial Drive Luray, VA 22835",
           "phone": "540-743-2282",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2416,8 +2399,7 @@
           "address": "Main Hospital, 1550 N 115th St., Seattle, WA 98133",
           "phone": "877.694.4677",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2438,8 +2420,7 @@
           "address": "250 26th Street, Suite 120 / Prairie du Sac, WI 53578",
           "phone": "(608) 643-6060",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2460,8 +2441,7 @@
           "address": "1015 Spring Creek Parkway Zion Crossroads, VA 22942",
           "phone": "434.924.0000",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2482,8 +2462,7 @@
           "address": "2451 University Hospital Dr. Mobile, AL 36617",
           "phone": "(251) 471-7160",
           "fax": "(251) 471-7019",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2504,8 +2483,7 @@
           "address": "UofL Health - Jackson Street Outpatient Center 550 S. Jackson Street, 3rd Floor Louisville, KY 40202",
           "phone": "502-561-5687",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2526,8 +2504,7 @@
           "address": "800 Spruce Street Philadelphia, PA 19107",
           "phone": "215-829-3000",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2548,8 +2525,7 @@
           "address": "10700 Nall Avenue Level 3 Overland Park, KS 66211",
           "phone": "913-588-1227",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2570,8 +2546,7 @@
           "address": "99 E. 86th Ave. Suite C Merrillville, IN 46410",
           "phone": "773-702-6169",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2592,8 +2567,7 @@
           "address": "St. Luke's Building A 1012 East Second Street 4th Floor Duluth, MN 55805",
           "phone": "218.249.7860",
           "fax": "218.249.6370",
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]
@@ -2614,8 +2588,7 @@
           "address": "555 St. Joseph's Boulevard",
           "phone": "607-733-6541",
           "fax": null,
-          "type": null,
-          "website": null
+          "type": null
         }
       }
     ]