crawler tests: Add --max-urls argument to limit test time

To limit the amount of URLs checked in the crawler tests,
one may now pass --max-urls=N to pytest.

Change-Id: I193336edc5031b3ef9aa174610bf673b28b9f2e1

tests/gui_crawl/conftest.py

@@ -15,18 +15,29 @@
 logger = logging.getLogger()
 
 
+def pytest_addoption(parser):
+ parser.addoption(
+ "--max-urls",
+ action="store",
+ default=int(os.environ.get("GUI_CRAWLER_URL_LIMIT", "0")),
+ type=int,
+ help="Select only N URLs for the crawler tests (0=all).",
+ )
+
+
 @pytest.fixture(name="test_site", scope="session")
 def get_site() -> Generator[Site, None, None]:
  yield from get_site_factory(prefix="crawl_").get_test_site()
 
 
 @pytest.fixture(name="test_crawler", scope="session")
-def crawler(test_site: Site) -> Generator[Crawler, None, None]:
+def crawler(test_site: Site, pytestconfig: pytest.Config) -> Generator[Crawler, None, None]:
  xss_crawl = os.environ.get("XSS_CRAWL", "0") == "1"
  crawler_type = XssCrawler if xss_crawl else Crawler
  test_crawler = crawler_type(
  test_site,
  report_file=os.environ.get("CRAWL_REPORT"),
+ max_urls=pytestconfig.getoption(name="--max-urls"),
  )
  try:
  yield test_crawler

tests/testlib/crawler.py

@@ -179,10 +179,8 @@ def __init__(self, test_site: Site, report_file: str | None, max_urls: int = 0)
  "text/x-sh",
  }
 
- # override value using environment-variable
- maxlen = int(os.environ.get("GUI_CRAWLER_URL_LIMIT", "0")) or max_urls
  # limit minimum value to 0.
- self._max_urls = max(0, maxlen)
+ self._max_urls = max(0, max_urls)
  self._todos = deque([Url(self.site.internal_url)])
 
  async def crawl(self, max_tasks: int) -> None: