Add a DEFAULT_ERRBACK_NAME to settings

This will allow the ability to change the non-standard behavior of
sending exceptions to the `parse` method of the spider without
introducing a breaking change to scrapyrt.

It also introduces some documentation of the existing behavior.

docs/source/api.rst

@@ -517,6 +517,13 @@ Encoding that's used to encode log messages.
 
 Default: ``utf-8``.
 
+DEFAULT_ERRBACK_NAME
+~~~~~~~~
+
+The name of the default errback method to call on the spider in case of an exception. The default errback method is ``parse`` to maintain backwards compatibility but it is not standard to scrapy and may interfere with the use of middlewares which implement the ``process_spider_exception`` method. Use a setting of ``None`` if you don't want to use the default scrapy exception handling.
+
+Default: ``parse``. Use the ``parse`` method on scrapy spider to handle exceptions. Be aware that this is non-standard to typical scrapy spiders.
+
 
 Spider settings
 ---------------

scrapyrt/conf/default_settings.py

@@ -31,4 +31,6 @@
 # disable in production
 DEBUG = True
 
-TWISTED_REACTOR = None
+TWISTED_REACTOR = None
+
+DEFAULT_ERRBACK_NAME = 'parse'

scrapyrt/core.py

@@ -120,7 +120,7 @@ def __init__(self, spider_name, request_kwargs,
         # because we need to know if spider has method available
         self.callback_name = request_kwargs.pop('callback', None) or 'parse'
         # do the same for errback
-        self.errback_name = request_kwargs.pop('errback', None)
+        self.errback_name = request_kwargs.pop('errback', None) or app_settings.DEFAULT_ERRBACK_NAME
 
         if request_kwargs.get("url"):
             self.request = self.create_spider_request(deepcopy(request_kwargs))