From da37fda59e231e524a00c3c9bb8f26aa734d35f1 Mon Sep 17 00:00:00 2001
From: Daofeng Wu <dolphin.w.e@gmail.com>
Date: Sun, 26 Jan 2025 10:07:36 +0900
Subject: [PATCH] refactor(browser): add action param to web interaction hitl
 call

---
 npiai/core/__test__/captcha_detection.py         | 4 +++-
 npiai/core/hitl.py                               | 3 ++-
 npiai/core/tool/_browser.py                      | 9 +++++----
 npiai/tools/scrapers/web/__test__/interactive.py | 4 +++-
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/npiai/core/__test__/captcha_detection.py b/npiai/core/__test__/captcha_detection.py
index 960bf9f..4f4e60c 100644
--- a/npiai/core/__test__/captcha_detection.py
+++ b/npiai/core/__test__/captcha_detection.py
@@ -1,4 +1,5 @@
 import asyncio
+from typing import Literal
 
 from npiai import BrowserTool, HITL
 from npiai.utils.test_utils import DebugContext
@@ -38,8 +39,9 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str:
-        print(f"[HITL] web_interaction: {message=}, {url=}")
+        print(f"[HITL] web_interaction: {message=}, {url=}, {action=}")
         return "web_interaction"
 
 
diff --git a/npiai/core/hitl.py b/npiai/core/hitl.py
index 1078545..add4e23 100644
--- a/npiai/core/hitl.py
+++ b/npiai/core/hitl.py
@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List
+from typing import List, Literal
 
 from npiai.context import Context
 
@@ -41,4 +41,5 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str: ...
diff --git a/npiai/core/tool/_browser.py b/npiai/core/tool/_browser.py
index 804abd5..9433c45 100644
--- a/npiai/core/tool/_browser.py
+++ b/npiai/core/tool/_browser.py
@@ -55,7 +55,8 @@ async def load_page(
                 locator = self.playwright.page.locator(wait_for_selector)
                 await locator.first.wait_for(state="attached", timeout=timeout)
             except TimeoutError:
-                await self.detect_captcha(ctx)
+                # if the selector is not found, check if a captcha is present
+                force_capcha_detection = True
         # wait for the page to become stable
         elif timeout is not None:
             try:
@@ -68,9 +69,7 @@ async def load_page(
 
         # await self.playwright.page.wait_for_timeout(wait)
 
-        # capcha detection will be done (or unnecessary if elements matched) if selector is provided
-        # so we only do it if no selector is provided
-        if not wait_for_selector and force_capcha_detection:
+        if force_capcha_detection:
             await self.detect_captcha(ctx)
 
     @function
@@ -306,12 +305,14 @@ async def handle_captcha(captcha_type: Literal["none", "captcha", "login"]):
                         tool_name=self.name,
                         message="Would you please help me solve the captcha?",
                         url=url,
+                        action="captcha",
                     )
                 case "login":
                     await ctx.hitl.web_interaction(
                         tool_name=self.name,
                         message="Would you please help me login to the website?",
                         url=url,
+                        action="login",
                     )
 
             return captcha_type
diff --git a/npiai/tools/scrapers/web/__test__/interactive.py b/npiai/tools/scrapers/web/__test__/interactive.py
index 62f3e41..17be77f 100644
--- a/npiai/tools/scrapers/web/__test__/interactive.py
+++ b/npiai/tools/scrapers/web/__test__/interactive.py
@@ -1,4 +1,5 @@
 import asyncio
+from typing import Literal
 
 from npiai.utils.test_utils import DebugContext
 
@@ -42,8 +43,9 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str:
-        print(f"[HITL] web_interaction: {message=}, {url=}")
+        print(f"[HITL] web_interaction: {message=}, {url=}, {action=}")
         return "web_interaction"