Fix the bug.

add update node to Mode2
add timeout to requests.get

Author: wuyingren

.gitignore

@@ -114,3 +114,4 @@ dump.rdb
 settings.py
 .node_number.json
 .topics_tester.json
+rsync_vps.sh

run.py

@@ -36,16 +36,17 @@ def __init__(self):
         self.SQ.open_datebase()
         self.redis_conn=Redis()
         self.load_config()
-
-    def Mode1(self):
-        logging.info('start mode1')
-        #start
+        #base
         self.load_json()
 #         self.update_cookies()
         try:
             self.update_nodes()
         except APIError as e:
             pass
+
+    def Mode1(self):
+        logging.info('start mode1')
+        #start
         self.get_rss()
         self.tasker()
         self.topic_ids_enqueue()
@@ -56,8 +57,7 @@ def Mode1(self):
     def Mode2(self):
         logging.info('start mode2')
         #start
-        self.load_json()
-#         self.update_cookies()
+
         self.get_rss()
         self.topic_ids_enqueue()
         self.tester_tasker()
@@ -128,7 +128,7 @@ def update_nodes(self):
         if not nodes_time_status:
             logging.info('update nodes')
             try:
-                resp=self.s.get('https://www.v2ex.com/api/nodes/all.json')
+                resp=self.s.get('https://www.v2ex.com/api/nodes/all.json', timeout=10)
             except requests.exceptions.RequestException as e:
                 logging.error('update_node failed.')
                 logging.error('proxy_status: %s' % settings.i_proxy_enable)

v2ex_base/log_in.py

@@ -47,7 +47,7 @@ def log_in(self,try_time):
             raise LogError('try time too much.')
         #1
         try:
-            r1=self.s.get('https://www.v2ex.com/signin')
+            r1=self.s.get('https://www.v2ex.com/signin', timeout=10)
         except requests.exceptions.RequestException as e:
             logging.error('log in error')
             logging.error(try_time)
@@ -82,7 +82,7 @@ def log_in(self,try_time):
 
     def save_cookies(self):
         logging.debug('start save_cookies')
-        resp=self.s.get('https://www.v2ex.com/go/flamewar')
+        resp=self.s.get('https://www.v2ex.com/go/flamewar', timeout=10)
         if '登录' in resp.text:
             raise LogError('log failed.')
         with open('.cookies.json','w') as f:

v2ex_spider/base_spider.py

@@ -37,7 +37,7 @@ def __init__(self,url,sleep_time):
     def spider(self):
         logging.debug('start spider.')
         try:
-            resp=self.s.get(self.url)
+            resp=self.s.get(self.url, timeout=10)
         except requests.exceptions.RequestException as e:
             logging.error('spider failed.')
             logging.error('proxy_status: %s' % settings.proxy_enable)

v2ex_spider/rss_spider.py

@@ -87,7 +87,7 @@ def latest_and_hot(self):
         logging.debug('start latest_and_hot')
         for url in self.latest_hot_api:
             try:
-                resp=self.s.get(url)
+                resp=self.s.get(url, timeout=10)
             except requests.exceptions.RequestException as e:
                 logging.error('latest_and_hot error')
                 logging.error('proxy_status: %s' % self.proxy_enable)

v2ex_tester/topic_tester.py

@@ -50,7 +50,7 @@ def web_test(self,t_id,status):
         url='https://www.v2ex.com/t/%s' % str(t_id)
         n_time=int(time.time())
         try:
-            resp=self.s.get(url)
+            resp=self.s.get(url, timeout=10)
         except requests.exceptions.RequestException as e:
             logging.error('web_test failed.')
             logging.error('proxy_status: %s' % settings.proxy_enable)
@@ -85,7 +85,7 @@ def api_test(self,t_id,status):
         url='https://www.v2ex.com/api/topics/show.json?id=%s' % str(t_id)
         n_time=int(time.time())
         try:
-            resp=self.s_a.get(url)
+            resp=self.s_a.get(url, timeout=10)
         except requests.exceptions.RequestException as e:
             logging.error('api_test failed.')
             logging.error('proxy_status: %s' % settings.proxy_enable)
@@ -126,5 +126,5 @@ def start(t_id,sleep_time):
 
 if __name__ == '__main__':
 #     start(1,5)
-    start(367743,5)
+    start(375807,5)
     print('finish!')