Merge pull request #32 from reworkd/rohan

Rename (sub)domain -> (sub)category and add url domain flag

README.md

@@ -108,7 +108,7 @@ class NullAgentRunner(AgentRunner):
 - `--headless`: Run with Playwright headless mode
 - `-id` or `--id`: Run a specific test by id
 - `-i` or `--intent`: Only run tests of a particular intent (fetch, links, etc)
-- `-d` or `--domain`: Only run tests of a particular domain (healthcare, manufacturing, software, etc)
+- `-c` or `--category`: Only run tests of a particular category (healthcare, manufacturing, software, etc)
 - `-n` or `--n`: Number of test workers to use. The default is 1
 - `-skip` or `--skip`: A list of ids to skip tests on, separated by commas
 - `-t` or `--type`: Only run tests of a particular type (links, fetch, etc)

bananalyzer/__main__.py

@@ -8,6 +8,7 @@
 import sys
 from pathlib import Path
 from typing import List
+from urllib.parse import urlparse
 
 from bananalyzer import AgentRunner
 from bananalyzer.data.examples import (
@@ -66,6 +67,13 @@ def parse_args() -> Args:
         default=None,
         help="Filter tests by id",
     )
+    parser.add_argument(
+        "-d",
+        "--domain",
+        type=str,
+        default=None,
+        help="Filter tests by a particular URL domain",
+    )
     parser.add_argument(
         "-i",
         "--intent",
@@ -74,17 +82,17 @@ def parse_args() -> Args:
         help="Filter tests by a particular intent",
     )
     parser.add_argument(
-        "-d",
-        "--domain",
+        "-c",
+        "--category",
         type=str,
         default=None,
-        help="Filter tests by a particular domain",
+        help="Filter tests by a particular category",
     )
     parser.add_argument(
-        "--subdomain",
+        "--subcategory",
         type=str,
         default=None,
-        help="Filter tests by a particular domain",
+        help="Filter tests by a particular category",
     )
     parser.add_argument(
         "-n",
@@ -142,7 +150,8 @@ def parse_args() -> Args:
         intent=args.intent,
         id=args.id,
         domain=args.domain,
-        subdomain=args.subdomain,
+        category=args.category,
+        subcategory=args.subcategory,
         skip=args.skip,
         single_browser_instance=args.single_browser_instance,
         type=args.type,
@@ -243,7 +252,15 @@ def main() -> int:
         ]
     if args.domain:
         filtered_examples = [
-            example for example in filtered_examples if example.domain == args.domain
+            example
+            for example in filtered_examples
+            if ".".join(urlparse(example.url).netloc.split(".")[-2:]) == args.domain
+        ]
+    if args.category:
+        filtered_examples = [
+            example
+            for example in filtered_examples
+            if example.category == args.category
         ]
     if args.skip:
         filtered_examples = [
@@ -253,11 +270,11 @@ def main() -> int:
         filtered_examples = [
             example for example in filtered_examples if example.type in args.type
         ]
-    if args.subdomain:
+    if args.subcategory:
         filtered_examples = [
             example
             for example in filtered_examples
-            if example.subdomain in args.subdomain
+            if example.subcategory in args.subcategory
         ]
 
     # Test we actually have tests to run

bananalyzer/data/schemas.py

@@ -59,8 +59,8 @@ class Example(BaseModel):
     source: Literal["mhtml", "hosted", "url"] = Field(
         description="Source of the website"
     )
-    domain: str = Field(description="Domain of the website")
-    subdomain: str = Field(description="Subdomain of the website")
+    category: str = Field(description="Category of the website")
+    subcategory: str = Field(description="Subcategory of the website")
     type: GoalType = Field(
         description="The high level goal intent the agent is aiming to do"
     )

bananalyzer/schema.py

@@ -16,9 +16,10 @@ class Args(BaseModel):
     headless: bool
     single_browser_instance: bool
     id: Optional[str] = Field(default=None)
-    intent: Optional[GoalType] = Field(default=None)
     domain: Optional[str] = Field(default=None)
-    subdomain: Optional[str] = Field(default=None)
+    intent: Optional[GoalType] = Field(default=None)
+    category: Optional[str] = Field(default=None)
+    subcategory: Optional[str] = Field(default=None)
     skip: List[str]
     type: Optional[str] = Field(default=None)
     download: bool

fetch.ipynb

@@ -99,8 +99,8 @@
     "#     \"id\": str(uuid.uuid4()),\n",
     "#     \"url\": desired_url,\n",
     "#     \"source\": \"mhtml\",\n",
-    "#     \"domain\": \"UPDATE\",\n",
-    "#     \"subdomain\": \"UPDATE\",\n",
+    "#     \"category\": \"UPDATE\",\n",
+    "#     \"subcategory\": \"UPDATE\",\n",
     "#     \"type\": \"fetch\",\n",
     "#     \"goal\": {},\n",
     "#     \"evals\": [\n",
@@ -114,8 +114,8 @@
     "    \"id\": str(uuid.uuid4()),\n",
     "    \"url\": desired_url,\n",
     "    \"source\": \"mhtml\",\n",
-    "    \"domain\": \"healthcare\",\n",
-    "    \"subdomain\": \"contact\",\n",
+    "    \"category\": \"healthcare\",\n",
+    "    \"subcategory\": \"contact\",\n",
     "    \"type\": \"links\",\n",
     "    \"goal\": \"Fetch all of the links to the detail pages of clinics on the current page\",\n",
     "    \"evals\": [{\"type\": \"json_match\", \"expected\": {}}],\n",

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bananalyzer"
-version = "0.6.10"
+version = "0.6.11"
 
 description = "Open source AI Agent evaluation framework for web tasks 🐒🍌"
 authors = ["asim-shrestha <[email protected]>"]