Article spider template

docs/conf.py

@@ -70,6 +70,7 @@
 autodoc_pydantic_model_show_validator_members = False
 autodoc_pydantic_model_show_validator_summary = False
 autodoc_pydantic_field_list_validators = False
+autodoc_pydantic_field_show_constraints = False
 
 # sphinx-reredirects
 redirects = {

docs/index.rst

@@ -18,6 +18,7 @@ zyte-spider-templates documentation
 
    templates/index
    E-commerce <templates/e-commerce>
+   Article <templates/article>
    Google search <templates/google-search>
 
 .. toctree::
@@ -34,9 +35,16 @@ zyte-spider-templates documentation
    customization/spiders
    customization/pages
 
+.. toctree::
+   :caption: Reference
+   :hidden:
+
+   reference/settings
+   reference/reqmeta
+   reference/api
+
 .. toctree::
    :caption: All the rest
    :hidden:
 
-   reference/index
    changes

docs/reference/api.rst

@@ -0,0 +1,80 @@
+===
+API
+===
+
+Spiders
+=======
+
+.. autoclass:: zyte_spider_templates.BaseSpider
+
+.. autoclass:: zyte_spider_templates.EcommerceSpider
+
+.. autoclass:: zyte_spider_templates.GoogleSearchSpider
+
+
+Pages
+=====
+
+.. autoclass:: zyte_spider_templates.pages.HeuristicsProductNavigationPage
+
+
+.. _parameter-mixins:
+
+Parameter mixins
+================
+
+.. autopydantic_model:: zyte_spider_templates.params.CustomAttrsInputParam
+    :exclude-members: model_computed_fields
+
+.. autopydantic_model:: zyte_spider_templates.params.CustomAttrsMethodParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.params.CustomAttrsMethod
+
+.. autopydantic_model:: zyte_spider_templates.params.ExtractFromParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.params.ExtractFrom
+
+.. autopydantic_model:: zyte_spider_templates.params.GeolocationParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.params.Geolocation
+
+.. autopydantic_model:: zyte_spider_templates.params.MaxRequestsParam
+    :exclude-members: model_computed_fields
+
+.. autopydantic_model:: zyte_spider_templates.params.UrlParam
+    :exclude-members: model_computed_fields
+
+.. autopydantic_model:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategyParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategy
+
+.. autopydantic_model:: zyte_spider_templates.spiders.serp.SerpItemTypeParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.spiders.serp.SerpItemType
+
+.. autopydantic_model:: zyte_spider_templates.spiders.serp.SerpMaxPagesParam
+    :exclude-members: model_computed_fields
+
+.. autopydantic_model:: zyte_spider_templates.spiders.article.ArticleCrawlStrategyParam
+    :exclude-members: model_computed_fields
+
+.. autoenum:: zyte_spider_templates.spiders.article.ArticleCrawlStrategy
+
+
+.. _middlewares:
+
+Middlewares
+===========
+
+.. autoclass:: zyte_spider_templates.CrawlingLogsMiddleware
+.. autoclass:: zyte_spider_templates.TrackNavigationDepthSpiderMiddleware
+.. autoclass:: zyte_spider_templates.MaxRequestsPerSeedDownloaderMiddleware
+.. autoclass:: zyte_spider_templates.OffsiteRequestsPerSeedMiddleware
+.. autoclass:: zyte_spider_templates.OnlyFeedsMiddleware
+.. autoclass:: zyte_spider_templates.TrackSeedsSpiderMiddleware
+.. autoclass:: zyte_spider_templates.IncrementalCrawlMiddleware

docs/reference/index.rst

@@ -1,61 +0,0 @@
-=========
-Reference
-=========
-
-Spiders
-=======
-
-.. autoclass:: zyte_spider_templates.BaseSpider
-
-.. autoclass:: zyte_spider_templates.EcommerceSpider
-
-.. autoclass:: zyte_spider_templates.GoogleSearchSpider
-
-
-Pages
-=====
-
-.. autoclass:: zyte_spider_templates.pages.HeuristicsProductNavigationPage
-
-
-.. _parameter-mixins:
-
-Parameter mixins
-================
-
-.. autopydantic_model:: zyte_spider_templates.params.CustomAttrsInputParam
-    :exclude-members: model_computed_fields
-
-.. autopydantic_model:: zyte_spider_templates.params.CustomAttrsMethodParam
-    :exclude-members: model_computed_fields
-
-.. autoenum:: zyte_spider_templates.params.CustomAttrsMethod
-
-.. autopydantic_model:: zyte_spider_templates.params.ExtractFromParam
-    :exclude-members: model_computed_fields
-
-.. autoenum:: zyte_spider_templates.params.ExtractFrom
-
-.. autopydantic_model:: zyte_spider_templates.params.GeolocationParam
-    :exclude-members: model_computed_fields
-
-.. autoenum:: zyte_spider_templates.params.Geolocation
-
-.. autopydantic_model:: zyte_spider_templates.params.MaxRequestsParam
-    :exclude-members: model_computed_fields
-
-.. autopydantic_model:: zyte_spider_templates.params.UrlParam
-    :exclude-members: model_computed_fields
-
-.. autopydantic_model:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategyParam
-    :exclude-members: model_computed_fields
-
-.. autoenum:: zyte_spider_templates.spiders.ecommerce.EcommerceCrawlStrategy
-
-.. autopydantic_model:: zyte_spider_templates.spiders.serp.SerpItemTypeParam
-    :exclude-members: model_computed_fields
-
-.. autoenum:: zyte_spider_templates.spiders.serp.SerpItemType
-
-.. autopydantic_model:: zyte_spider_templates.spiders.serp.SerpMaxPagesParam
-    :exclude-members: model_computed_fields

docs/reference/reqmeta.rst

@@ -0,0 +1,112 @@
+.. _meta:
+
+=================
+Request.meta keys
+=================
+
+Keys that can be defined in :attr:`Request.meta <scrapy.http.Request.meta>` for
+zyte-spider-templates.
+
+.. reqmeta:: seed
+
+seed
+====
+
+Default: ``The seed URL (or value) from which the request originated.``
+
+The key is used for :class:`~zyte_spider_templates.OffsiteRequestsPerSeedMiddleware` and
+:class:`~zyte_spider_templates.MaxRequestsPerSeedDownloaderMiddleware`.
+
+The `seed` meta key is used to track and identify the origin of a request. It
+is initially set for each request that originates from the start request and
+can be used to manage domain constraints for subsequent requests. This key can
+also be set to an arbitrary value by the user to identify the seed source.
+
+Here's an example:
+
+.. code-block:: python
+
+    meta = {
+        "seed": "http://example.com",
+    }
+
+.. reqmeta:: is_seed_request
+
+is_seed_request
+===============
+
+Default: ``False``
+
+The key is used for :class:`~zyte_spider_templates.OffsiteRequestsPerSeedMiddleware`.
+
+The `is_seed_request` meta key is a boolean flag that identifies whether the
+request is a start request (i.e., originating from the initial seed URL). When
+set to True, the middleware extracts seed domains from the response.
+
+Example:
+    ::
+
+        meta = {
+            'is_seed_request': True,
+        }
+
+.. reqmeta:: seed_domains
+
+seed_domains
+============
+
+Default: ``Initial URL and redirected URLs``
+
+The key is used for :class:`~zyte_spider_templates.OffsiteRequestsPerSeedMiddleware`.
+
+The `seed_domains` meta key is a list of domains that the middleware uses to
+check whether a request belongs to these domains or not. By default, this list
+includes the initial URL's domain and domains of any redirected URLs `(if there
+was a redirection)`. This list can also be set by the user in the spider to
+specify additional domains for which the middleware should allow requests.
+
+Here's an example:
+
+.. code-block:: python
+
+    meta = {"seed_domains": ["example.com", "another-example.com"]}
+
+.. reqmeta:: is_hop
+
+increase_navigation_depth
+=========================
+
+Default: ``True``
+
+The key is used for :class:`~zyte_spider_templates.TrackNavigationDepthSpiderMiddleware`.
+
+The `increase_navigation_depth` meta key is a boolean flag that determines whether the
+navigation_depth for a request should be increased. By default, the middleware increases
+navigation_depth for all requests. Specific spiders can override this behavior for certain
+types of requests, such as pagination or RSS feeds, by explicitly setting the meta key.
+
+Example:
+    ::
+
+        meta = {
+            'increase_navigation_depth': False,
+        }
+
+.. reqmeta:: only_feeds
+
+only_feeds
+==========
+Default: ``False``
+
+The key is used for :class:`~zyte_spider_templates.OnlyFeedsMiddleware`.
+
+The `only_feeds` meta key is a boolean flag that identifies whether the
+spider should discover all links on the website or extract links from RSS/Atom feeds only.
+
+Example:
+    ::
+
+        meta = {
+            'page_params': {'only_feeds': True}
+        }
+