diff --git a/CHANGES b/CHANGES index 298a59e..2dc8c26 100644 --- a/CHANGES +++ b/CHANGES @@ -48,5 +48,4 @@ 1.13 02/14/2023 -- Added additional unit tests. 1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137. 1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data. -1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume -1.15 04/30/2023 -- Don't retry requests if response code is a client error (4xx). \ No newline at end of file +1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume \ No newline at end of file diff --git a/setup.py b/setup.py index 579bfa0..d2309dd 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='yahoofinancials', - version='1.15', + version='1.14', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 3380366..f72b861 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -181,28 +181,21 @@ def _construct_url(self, symbol, config, params, freq, request_type): # Private method to execute a web scrape request and decrypt the return def _request_handler(self, url, res_field=""): urlopener = UrlOpener() - # Try to open the URL up to 10 times sleeping random time if the server responds with a 5xx code + # Try to open the URL up to 10 times sleeping random time if something goes wrong max_retry = 10 - for i in range(0, max_retry): response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout) - - if response.status_code == 200: - res_content = response.text + if response.status_code != 200: + time.sleep(random.randrange(10, 20)) + response.close() + else: + res_content = response.text response.close() self._cache[url] = loads(res_content).get(res_field) break - - if 400 <= response.status_code < 500: - raise ManagedException("Server replied with client error code, HTTP " + str(response.status_code) + - " code while opening the url: " + str(url) + " . Not retrying.") - if response.status_code >= 500: - time.sleep(random.randrange(10, 20)) - response.close() - if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with server error code, HTTP " + str(response.status_code) + + raise ManagedException("Server replied with HTTP " + str(response.status_code) + " code while opening the url: " + str(url)) @staticmethod @@ -471,16 +464,6 @@ def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): return self._recursive_api_request(hist_obj, up_ticker, clean, i) elif clean: return self._clean_historical_data(re_data, True) - - # Private method, acting as wrapper to call _create_dict_ent() and log exceptions as warnings - def _safe_create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): - try: - return self._create_dict_ent(up_ticker, statement_type, tech_type, report_name, hist_obj) - except ManagedException as e: - logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", - str(up_ticker), str(statement_type), str(e)) - - return {up_ticker: None} # Private Method to take scrapped data and build a data dictionary with, used by get_stock_data() def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): @@ -556,33 +539,30 @@ def get_time_code(self, time_interval): # Public Method to get stock data def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): data = {} - tickers = [self.ticker] if isinstance(self.ticker, str) else self.ticker - - if self.concurrent: - data = self._get_stock_data_concurrently(tickers, statement_type, tech_type, report_name, hist_obj) + if isinstance(self.ticker, str): + dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) else: - for tick in tickers: - dict_ent = self._safe_create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - if dict_ent[tick]: - data.update(dict_ent) - - return data - - def _get_stock_data_concurrently(self, tickers, statement_type='income', tech_type='', report_name='', hist_obj={}): - data = {} - - with Pool(self._get_worker_count()) as pool: - dict_ents = pool.map(partial(self._safe_create_dict_ent, - statement_type=statement_type, - tech_type=tech_type, - report_name=report_name, - hist_obj=hist_obj), tickers) - for dict_ent in dict_ents: - if dict_ent: - data.update(dict_ent) - - pool.close() - pool.join() + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + dict_ents = pool.map(partial(self._create_dict_ent, + statement_type=statement_type, + tech_type=tech_type, + report_name=report_name, + hist_obj=hist_obj), self.ticker) + for dict_ent in dict_ents: + data.update(dict_ent) + pool.close() + pool.join() + else: + for tick in self.ticker: + try: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + except ManagedException: + logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", + str(tick), statement_type, str(ManagedException)) + continue return data # Public Method to get technical stock data