完善代理池

Author: Boris-code

docs/source_code/proxy.md

@@ -53,9 +53,9 @@ ip:port
 1. 编写代理池：例如在你的项目下创建个my_proxypool.py，实现下面的函数
 
     ```python
-    from feapder.network.proxy_pool import ProxyPool 
+    from feapder.network.proxy_pool import BaseProxyPool 
 
-    class MyProxyPool(ProxyPool):
+    class MyProxyPool(BaseProxyPool):
         def get_proxy(self):
             """
             获取代理

feapder/VERSION

@@ -1 +1 @@
-1.8.7
+1.8.8

feapder/network/proxy_pool/__init__.py

@@ -7,5 +7,5 @@
 @author: Boris
 @email: [email protected]
 """
-from .base import ProxyPool
-from .memory_proxy_pool import MemoryProxyPool
+from .base import BaseProxyPool
+from .proxy_pool import ProxyPool

feapder/network/proxy_pool/base.py

@@ -13,7 +13,7 @@
 from feapder.utils.log import log
 
 
-class ProxyPool:
+class BaseProxyPool:
     @abc.abstractmethod
     def get_proxy(self):
         """

feapder/network/proxy_pool/memory_proxy_pool.py renamed to feapder/network/proxy_pool/proxy_pool.py

@@ -12,12 +12,12 @@
 import requests
 
 import feapder.setting as setting
-from feapder.network.proxy_pool.base import ProxyPool
+from feapder.network.proxy_pool.base import BaseProxyPool
 from feapder.utils import metrics
 from feapder.utils import tools
 
 
-class MemoryProxyPool(ProxyPool):
+class ProxyPool(BaseProxyPool):
     """
     通过API提取代理，存储在内存中，无代理时会自动提取
     API返回的代理以 \r\n 分隔

feapder/network/request.py

@@ -21,7 +21,7 @@
 from feapder.db.redisdb import RedisDB
 from feapder.network import user_agent
 from feapder.network.downloader.base import Downloader, RenderDownloader
-from feapder.network.proxy_pool import ProxyPool
+from feapder.network.proxy_pool import BaseProxyPool
 from feapder.network.response import Response
 from feapder.utils.log import log
 
@@ -31,7 +31,7 @@
 
 class Request:
     user_agent_pool = user_agent
-    proxies_pool: ProxyPool = None
+    proxies_pool: BaseProxyPool = None
 
     cache_db = None  # redis / pika
     cached_redis_key = None  # 缓存response的文件文件夹 response_cached:cached_redis_key:md5

feapder/setting.py

@@ -133,7 +133,7 @@
 PROXY_EXTRACT_API = None  # 代理提取API ，返回的代理分割符为\r\n
 PROXY_ENABLE = True
 PROXY_MAX_FAILED_TIMES = 5  # 代理最大失败次数，超过则不使用，自动删除
-PROXY_POOL = "feapder.network.proxy_pool.MemoryProxyPool"  # 代理池
+PROXY_POOL = "feapder.network.proxy_pool.ProxyPool"  # 代理池
 
 # 随机headers
 RANDOM_HEADERS = True

feapder/templates/project_template/setting.py

@@ -122,7 +122,7 @@
 # PROXY_EXTRACT_API = None  # 代理提取API ，返回的代理分割符为\r\n
 # PROXY_ENABLE = True
 # PROXY_MAX_FAILED_TIMES = 5  # 代理最大失败次数，超过则不使用，自动删除
-# PROXY_POOL = "feapder.network.proxy_pool.MemoryProxyPool"  # 代理池
+# PROXY_POOL = "feapder.network.proxy_pool.ProxyPool"  # 代理池
 #
 # # 随机headers
 # RANDOM_HEADERS = True