diff --git a/npiai/core/tool/_browser.py b/npiai/core/tool/_browser.py index c24270d..804abd5 100644 --- a/npiai/core/tool/_browser.py +++ b/npiai/core/tool/_browser.py @@ -45,7 +45,7 @@ async def load_page( ctx: Context, url: str, wait_for_selector: str = None, - network_idle_timeout: int | None = None, + timeout: int | None = None, force_capcha_detection: bool = False, ): await self.playwright.page.goto(url) @@ -53,17 +53,15 @@ async def load_page( if wait_for_selector is not None: try: locator = self.playwright.page.locator(wait_for_selector) - await locator.first.wait_for( - state="attached", timeout=network_idle_timeout - ) + await locator.first.wait_for(state="attached", timeout=timeout) except TimeoutError: await self.detect_captcha(ctx) # wait for the page to become stable - elif network_idle_timeout is not None: + elif timeout is not None: try: await self.playwright.page.wait_for_load_state( "networkidle", - timeout=network_idle_timeout, + timeout=timeout, ) except TimeoutError: pass diff --git a/npiai/tools/scrapers/page_analyzer/app.py b/npiai/tools/scrapers/page_analyzer/app.py index f4fcd37..b424d42 100644 --- a/npiai/tools/scrapers/page_analyzer/app.py +++ b/npiai/tools/scrapers/page_analyzer/app.py @@ -272,7 +272,7 @@ async def support_infinite_scroll( await self.load_page( ctx=ctx, url=url, - network_idle_timeout=3000, + timeout=3000, wait_for_selector=items_selector, force_capcha_detection=self.force_captcha_detection, ) @@ -567,7 +567,7 @@ async def infer_similar_items_selector( await self.load_page( ctx, url, - network_idle_timeout=3000, + timeout=3000, force_capcha_detection=self.force_captcha_detection, ) diff --git a/npiai/tools/scrapers/web/app.py b/npiai/tools/scrapers/web/app.py index b8ef56f..3923cd2 100644 --- a/npiai/tools/scrapers/web/app.py +++ b/npiai/tools/scrapers/web/app.py @@ -79,7 +79,7 @@ async def init_data(self, ctx: Context): await self.load_page( ctx=ctx, url=self.url, - network_idle_timeout=3000, + timeout=3000, wait_for_selector=self.items_selector, )