From 923aa227e66ded72cdc8c44a1adbc6b3052cb9aa Mon Sep 17 00:00:00 2001 From: chris Date: Fri, 4 Jan 2019 16:09:38 +0800 Subject: [PATCH] fix --- json_requests.py | 11 ++++++----- spider_requests.py | 16 ++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/json_requests.py b/json_requests.py index f68ed63..19f25f6 100644 --- a/json_requests.py +++ b/json_requests.py @@ -60,21 +60,22 @@ def main(): # print('Enter the keyowrd: ', end='') # kw = input() kw = 'correct' - start = time.time() + start_time = time.time() counter = 0 - for i in range(0, 3600, 24): - spider = Spider(kw, start=i) + for start in range(0, 3600, 24): + spider = Spider(kw, start=start) response = spider.get_html() items = spider.test(response) if items: spider.write_into_file(response) print( 'Downloading: {0}.json It costs {1}s'.format( - str(i // 24 + 1), str(time.time() - start)),) + str(start // 24 + 1), str(time.time() - start_time)),) counter += 1 else: break - print('Get {0}. It costs {1}s'.format(counter, str(time.time() - start))) + print('Get {0}. It costs {1}s'.format( + counter, str(time.time() - start_time))) if __name__ == '__main__': diff --git a/spider_requests.py b/spider_requests.py index 7f4c1b9..08bb5dd 100644 --- a/spider_requests.py +++ b/spider_requests.py @@ -52,9 +52,9 @@ def test(self, response): items['path'] = path yield items - def get_html_2(self, item): + def get_html_2(self, items): try: - url = item.get('path') + url = items.get('path') headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36' @@ -117,10 +117,10 @@ def main(): # print('Enter the keyowrd: ', end='') # kw = input() kw = 'correct' - start = time.time() + start_time = time.time() counter = 0 - for i in range(0, 3600, 24): - spider = Spider(kw, start=i) + for start in range(0, 3600, 24): + spider = Spider(kw, start=start) response = spider.get_html() items = spider.test(response) if items: @@ -128,15 +128,15 @@ def main(): format, response = spider.get_html_2(item) if format == 'gif': print('Downloading: {0} It costs {1}s.'.format( - item['path'][:-5], time.time() - start)) + item['path'][:-5], time.time() - start_time)) else: print('Downloading: {0} It costs {1}s.'.format( - item['path'], time.time() - start)) + item['path'], time.time() - start_time)) counter += 1 spider.write_into_file(format, response) else: break - print('Get {0}. It costs {1}s'.format(counter, str(time.time() - start))) + print('Get {0}. It costs {1}s'.format(counter, str(time.time() - start_time))) if __name__ == '__main__':