Skip to content

Commit

Permalink
refactor: param name network_idle_timeout -> timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
idiotWu committed Jan 25, 2025
1 parent 4c24888 commit 737d40d
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 9 deletions.
10 changes: 4 additions & 6 deletions npiai/core/tool/_browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,25 +45,23 @@ async def load_page(
ctx: Context,
url: str,
wait_for_selector: str = None,
network_idle_timeout: int | None = None,
timeout: int | None = None,
force_capcha_detection: bool = False,
):
await self.playwright.page.goto(url)

if wait_for_selector is not None:
try:
locator = self.playwright.page.locator(wait_for_selector)
await locator.first.wait_for(
state="attached", timeout=network_idle_timeout
)
await locator.first.wait_for(state="attached", timeout=timeout)
except TimeoutError:
await self.detect_captcha(ctx)
# wait for the page to become stable
elif network_idle_timeout is not None:
elif timeout is not None:
try:
await self.playwright.page.wait_for_load_state(
"networkidle",
timeout=network_idle_timeout,
timeout=timeout,
)
except TimeoutError:
pass
Expand Down
4 changes: 2 additions & 2 deletions npiai/tools/scrapers/page_analyzer/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ async def support_infinite_scroll(
await self.load_page(
ctx=ctx,
url=url,
network_idle_timeout=3000,
timeout=3000,
wait_for_selector=items_selector,
force_capcha_detection=self.force_captcha_detection,
)
Expand Down Expand Up @@ -567,7 +567,7 @@ async def infer_similar_items_selector(
await self.load_page(
ctx,
url,
network_idle_timeout=3000,
timeout=3000,
force_capcha_detection=self.force_captcha_detection,
)

Expand Down
2 changes: 1 addition & 1 deletion npiai/tools/scrapers/web/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ async def init_data(self, ctx: Context):
await self.load_page(
ctx=ctx,
url=self.url,
network_idle_timeout=3000,
timeout=3000,
wait_for_selector=self.items_selector,
)

Expand Down

0 comments on commit 737d40d

Please sign in to comment.