From 3fa83ec7f2864c32d2be3d6223b8634f1a2e55f0 Mon Sep 17 00:00:00 2001
From: Daofeng Wu <dolphin.w.e@gmail.com>
Date: Sun, 26 Jan 2025 10:07:36 +0900
Subject: [PATCH] refactor(browser): add action param to web interaction hitl
 call

---
 npiai/cloud/_context.py                          | 4 +---
 npiai/core/__test__/captcha_detection.py         | 4 +++-
 npiai/core/hitl.py                               | 3 ++-
 npiai/core/tool/_browser.py                      | 9 +++++----
 npiai/tools/scrapers/web/__test__/interactive.py | 4 +++-
 5 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/npiai/cloud/_context.py b/npiai/cloud/_context.py
index 426b6c29..3c6de5d0 100644
--- a/npiai/cloud/_context.py
+++ b/npiai/cloud/_context.py
@@ -1,7 +1,6 @@
 import asyncio
 from typing import Dict, TypedDict
 from fastapi import WebSocket
-from mem0 import Memory
 
 from npiai.context import Context
 from npiai.cloud import Client
@@ -20,9 +19,8 @@ class CloudContext(Context):
     def __init__(
         self,
         client: Client,
-        memory: Memory | None = None,
     ):
-        super().__init__(memory=memory)
+        super().__init__()
         self.client = client
         self.ws = None
         self.action_result_queue = asyncio.Queue()
diff --git a/npiai/core/__test__/captcha_detection.py b/npiai/core/__test__/captcha_detection.py
index 960bf9f1..4f4e60cc 100644
--- a/npiai/core/__test__/captcha_detection.py
+++ b/npiai/core/__test__/captcha_detection.py
@@ -1,4 +1,5 @@
 import asyncio
+from typing import Literal
 
 from npiai import BrowserTool, HITL
 from npiai.utils.test_utils import DebugContext
@@ -38,8 +39,9 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str:
-        print(f"[HITL] web_interaction: {message=}, {url=}")
+        print(f"[HITL] web_interaction: {message=}, {url=}, {action=}")
         return "web_interaction"
 
 
diff --git a/npiai/core/hitl.py b/npiai/core/hitl.py
index 10785451..add4e238 100644
--- a/npiai/core/hitl.py
+++ b/npiai/core/hitl.py
@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import List
+from typing import List, Literal
 
 from npiai.context import Context
 
@@ -41,4 +41,5 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str: ...
diff --git a/npiai/core/tool/_browser.py b/npiai/core/tool/_browser.py
index 804abd57..9433c45e 100644
--- a/npiai/core/tool/_browser.py
+++ b/npiai/core/tool/_browser.py
@@ -55,7 +55,8 @@ async def load_page(
                 locator = self.playwright.page.locator(wait_for_selector)
                 await locator.first.wait_for(state="attached", timeout=timeout)
             except TimeoutError:
-                await self.detect_captcha(ctx)
+                # if the selector is not found, check if a captcha is present
+                force_capcha_detection = True
         # wait for the page to become stable
         elif timeout is not None:
             try:
@@ -68,9 +69,7 @@ async def load_page(
 
         # await self.playwright.page.wait_for_timeout(wait)
 
-        # capcha detection will be done (or unnecessary if elements matched) if selector is provided
-        # so we only do it if no selector is provided
-        if not wait_for_selector and force_capcha_detection:
+        if force_capcha_detection:
             await self.detect_captcha(ctx)
 
     @function
@@ -306,12 +305,14 @@ async def handle_captcha(captcha_type: Literal["none", "captcha", "login"]):
                         tool_name=self.name,
                         message="Would you please help me solve the captcha?",
                         url=url,
+                        action="captcha",
                     )
                 case "login":
                     await ctx.hitl.web_interaction(
                         tool_name=self.name,
                         message="Would you please help me login to the website?",
                         url=url,
+                        action="login",
                     )
 
             return captcha_type
diff --git a/npiai/tools/scrapers/web/__test__/interactive.py b/npiai/tools/scrapers/web/__test__/interactive.py
index 62f3e41d..17be77f6 100644
--- a/npiai/tools/scrapers/web/__test__/interactive.py
+++ b/npiai/tools/scrapers/web/__test__/interactive.py
@@ -1,4 +1,5 @@
 import asyncio
+from typing import Literal
 
 from npiai.utils.test_utils import DebugContext
 
@@ -42,8 +43,9 @@ async def web_interaction(
         tool_name: str,
         message: str,
         url: str,
+        action: Literal["captcha", "login"],
     ) -> str:
-        print(f"[HITL] web_interaction: {message=}, {url=}")
+        print(f"[HITL] web_interaction: {message=}, {url=}, {action=}")
         return "web_interaction"