Skip to content

Commit

Permalink
Disable subcategory crawling with search queries
Browse files Browse the repository at this point in the history
  • Loading branch information
Gallaecio committed Nov 13, 2024
1 parent 8389d7b commit a5f3be4
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion zyte_spider_templates/spiders/ecommerce.py
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ def validate_direct_item_and_search_queries(self):
if self.search_queries and self.crawl_strategy in {
EcommerceCrawlStrategy.direct_item,
EcommerceCrawlStrategy.full,
EcommerceCrawlStrategy.navigation,
}:
raise ValueError(
f"Cannot combine the {self.crawl_strategy.value!r} value of "
Expand Down Expand Up @@ -311,7 +312,10 @@ def parse_navigation(
else:
yield self.get_nextpage_request(navigation.nextPage)

if self.args.crawl_strategy != EcommerceCrawlStrategy.pagination_only:
if (
self.args.crawl_strategy != EcommerceCrawlStrategy.pagination_only
and not self.args.search_queries
):
for request in navigation.subCategories or []:
yield self.get_subcategory_request(request, page_params=page_params)

Expand Down

0 comments on commit a5f3be4

Please sign in to comment.