enhancements to address 400 error related performance issues, #148

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
  runs-on: ubuntu-20.04
  strategy:
  matrix:
- python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+ python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
  steps:
  - uses: actions/checkout@v3
  - name: Set up Python ${{ matrix.python-version }}

CHANGES

@@ -48,4 +48,8 @@
 1.13 02/14/2023 -- Added additional unit tests.
 1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137.
 1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data.
-1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume
+1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume
+1.15 06/21/2023 -- Dropped Python 3.6 support
+1.15 06/21/2023 -- Enhanced cache to cover all api requests
+1.15 06/21/2023 -- Enhanced api url to better toggle between query1 and query2 subdomains
+1.15 06/21/2023 -- Minimized sleeps in between 400 errors to less than 10 seconds

README.rst

@@ -16,9 +16,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit
 .. image:: https://static.pepy.tech/badge/yahoofinancials/week
  :target: https://pepy.tech/project/yahoofinancials
 
-Current Version: v1.14
+Current Version: v1.15
 
-Version Released: 02/21/2023
+Version Released: 06/21/2023
 
 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues
 
@@ -47,7 +47,7 @@ A powerful financial data module used for pulling both fundamental and technical
 
 Installation
 -------------
-- yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11.
+- yahoofinancials runs on Python 3.7, 3.8, 3.9, 3.10, and 3.11.
 - This package depends on pytz & requests to work.
 
 1. Installation using pip:

setup.py

@@ -10,11 +10,11 @@
 
 setup(
  name='yahoofinancials',
- version='1.14',
+ version='1.15',
  description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance',
  long_description=long_description,
  url='https://github.com/JECSand/yahoofinancials',
- download_url='https://github.com/JECSand/yahoofinancials/archive/1.14.tar.gz',
+ download_url='https://github.com/JECSand/yahoofinancials/archive/1.15.tar.gz',
  author='Connor Sanders',
  author_email='[email protected]',
  license='MIT',
@@ -32,7 +32,6 @@
  'Topic :: Software Development :: Libraries :: Python Modules',
  'Operating System :: OS Independent',
  'License :: OSI Approved :: MIT License',
- 'Programming Language :: Python :: 3.6',
  'Programming Language :: Python :: 3.7',
  'Programming Language :: Python :: 3.8',
  'Programming Language :: Python :: 3.9',

test/test_yahoofinancials.py

@@ -1,13 +1,12 @@
-# YahooFinancials Unit Tests v1.14
-# Version Released: 02/21/2023
+# YahooFinancials Unit Tests v1.15
+# Version Released: 06/21/2023
 # Author: Connor Sanders
-# Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11
-# Copyright (c) 2023 Connor Sanders
+# Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11
+# Copyright (c) 2023 Connor Sanders <[email protected]>
 # MIT License
 
-from unittest import main as test_main, TestCase
-
-from yahoofinancials import YahooFinancials
+from yahoofinancials import YahooFinancials as yf
+from unittest import main as t_main, TestCase
 
 # Test Configuration Variables
 stocks = ['AAPL', 'MSFT', 'C', 'IL&FSTRANS.NS']
@@ -40,12 +39,12 @@ def check_fundamental(test_data, test_type):
 class TestModule(TestCase):
 
  def setUp(self):
- self.test_yf_stock_single = YahooFinancials('C')
- self.test_yf_stock_multi = YahooFinancials(stocks)
- self.test_yf_treasuries_single = YahooFinancials('^IRX')
- self.test_yf_treasuries_multi = YahooFinancials(us_treasuries)
- self.test_yf_currencies = YahooFinancials(currencies)
- self.test_yf_concurrent = YahooFinancials(stocks, concurrent=True)
+ self.test_yf_stock_single = yf('C')
+ self.test_yf_stock_multi = yf(stocks)
+ self.test_yf_treasuries_single = yf('^IRX')
+ self.test_yf_treasuries_multi = yf(us_treasuries)
+ self.test_yf_currencies = yf(currencies)
+ self.test_yf_concurrent = yf(stocks, concurrent=True)
 
  # Fundamentals Test
  def test_yf_fundamentals(self):
@@ -78,20 +77,34 @@ def test_yf_fundamentals(self):
  # Historical Price Test
  def test_yf_historical_price(self):
  single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly')
- expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12',
-  'low': 46.599998474121094, 'date': 1421038800,
-  'close': 47.61000061035156, 'open': 48.959999084472656}
+ expected = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12',
+ 'low': 46.599998474121094, 'date': 1421038800,
+ 'close': 47.61000061035156, 'open': 48.959999084472656}
  # ignore adjclose as it will change with every dividend paid in the future
  del single_stock_prices['C']['prices'][0]['adjclose']
- self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict)
+ self.assertDictEqual(single_stock_prices['C']['prices'][0], expected)
 
  # Historical Stock Daily Dividend Test
  def test_yf_dividend_price(self):
  single_stock_dividend = self.test_yf_stock_single.get_daily_dividend_data('1986-09-15', '1987-09-15')
- expect_dict = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332},
- {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}]
- }
- self.assertDictEqual(single_stock_dividend, expect_dict)
+ expected = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332},
+ {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}]}
+ self.assertDictEqual(single_stock_dividend, expected)
+
+ # Test concurrent functionality of module
+ def test_yf_concurrency(self):
+ # Multi stock test
+ multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance')
+ multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income')
+ multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly',
+ ['income', 'cash', 'balance'])
+ # Multi stock check
+ result = check_fundamental(multi_balance_sheet_data_qt, 'bal')
+ self.assertEqual(result, True)
+ result = check_fundamental(multi_income_statement_data_qt, 'inc')
+ self.assertEqual(result, True)
+ result = check_fundamental(multi_all_statement_data_qt, 'all')
+ self.assertEqual(result, True)
 
  # Extra Module Methods Test
  def test_yf_module_methods(self):
@@ -112,14 +125,15 @@ def test_yf_module_methods(self):
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
- # Stock Profile Data
- if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services":
+
+ # Stock Financial Data
+ if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD":
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
 
- # Stock Financial Data
- if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD":
+ # Stock Profile Data
+ if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services":
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
@@ -154,21 +168,6 @@ def test_yf_module_methods(self):
  else:
  self.assertEqual(False, True)
 
- # Test concurrent functionality of module
- def test_yf_concurrency(self):
- # Multi stock test
- multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance')
- multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income')
- multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly',
- ['income', 'cash', 'balance'])
- # Multi stock check
- result = check_fundamental(multi_balance_sheet_data_qt, 'bal')
- self.assertEqual(result, True)
- result = check_fundamental(multi_income_statement_data_qt, 'inc')
- self.assertEqual(result, True)
- result = check_fundamental(multi_all_statement_data_qt, 'all')
- self.assertEqual(result, True)
-
 
 if __name__ == "__main__":
- test_main()
+ t_main()

yahoofinancials/etl.py

@@ -182,21 +182,28 @@ def _construct_url(self, symbol, config, params, freq, request_type):
  def _request_handler(self, url, res_field=""):
  urlopener = UrlOpener()
  # Try to open the URL up to 10 times sleeping random time if something goes wrong
+ cur_url = url
  max_retry = 10
  for i in range(0, max_retry):
- response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout)
+ response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout)
  if response.status_code != 200:
- time.sleep(random.randrange(10, 20))
+ time.sleep(random.randrange(1, 5))
  response.close()
+ time.sleep(random.randrange(1, 5))
+ if response.status_code == 404 and i % 2 == 0:
+ if 'query2.' in cur_url:
+ cur_url = cur_url.replace("query2.", "query1.")
+ elif 'query1.' in cur_url:
+ cur_url = cur_url.replace("query1.", "query2.")
  else:
  res_content = response.text
  response.close()
  self._cache[url] = loads(res_content).get(res_field)
  break
  if i == max_retry - 1:
  # Raise a custom exception if we can't get the web page within max_retry attempts
- raise ManagedException("Server replied with HTTP " + str(response.status_code) +
- " code while opening the url: " + str(url))
+ raise ManagedException("Server replied with server error code, HTTP " + str(response.status_code) +
+ " code while opening the url: " + str(cur_url))
 
  @staticmethod
  def _format_raw_fundamental_data(raw_data):
@@ -373,7 +380,7 @@ def _clean_historical_data(self, hist_data, last_attempt=False):
  # Private Static Method to build API url for GET Request
  def _build_api_url(self, hist_obj, up_ticker, v="2", events=None):
  if events is None:
- events = [" div", "split", "earn"]
+ events = ["div", "split", "earn"]
  event_str = ''
  for idx, s in enumerate(events, start=1):
  if idx < len(events):
@@ -390,18 +397,31 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None):
 
  # Private Method to get financial data via API Call
  def _get_api_data(self, api_url, tries=0):
+ if tries == 0 and self._cache.get(api_url):
+ return self._cache[api_url]
+ cur_url = api_url
+ if tries > 0 and tries % 2 == 0:
+ if 'query2.' in cur_url:
+ cur_url = cur_url.replace("query2.", "query1.")
+ elif 'query1.' in cur_url:
+ cur_url = cur_url.replace("query1.", "query2.")
  urlopener = UrlOpener()
- response = urlopener.open(api_url, proxy=self._get_proxy(), timeout=self.timeout)
+ response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout)
  if response.status_code == 200:
  res_content = response.text
  response.close()
- return loads(res_content)
+ data = loads(res_content)
+ self._cache[api_url] = data
+ return data
  else:
  if tries < 5:
- time.sleep(random.randrange(10, 20))
+ time.sleep(random.randrange(1, 5))
+ response.close()
+ time.sleep(random.randrange(1, 5))
  tries += 1
  return self._get_api_data(api_url, tries)
  else:
+ response.close()
  return None
 
  # Private Method to clean API data
@@ -448,8 +468,6 @@ def _clean_api_data(self, api_url):
  # Private Method to Handle Recursive API Request
  def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0):
  v = "2"
- if i == 3: # After 3 tries querying against 'query2.finance.yahoo.com' try 'query1.finance.yahoo.com' instead
- v = "1"
  if clean:
  re_data = self._clean_api_data(self._build_api_url(hist_obj, up_ticker, v))
  cleaned_re_data = self._clean_historical_data(re_data)
@@ -573,7 +591,7 @@ def get_stock_tech_data(self, tech_type):
  return self.get_stock_data(tech_type=tech_type)
 
  # Public Method to get reformatted statement data
- def get_reformatted_stmt_data(self, raw_data, statement_type):
+ def get_reformatted_stmt_data(self, raw_data):
  sub_dict, data_dict = {}, {}
  data_type = raw_data['dataType']
  if isinstance(self.ticker, str):

yahoofinancials/yf.py

@@ -1,13 +1,13 @@
 """
 ==============================
 The Yahoo Financials Module
-Version: 1.14
+Version: 1.15
 ==============================
 
 Author: Connor Sanders
 Email: [email protected]
-Version Released: 02/21/2023
-Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11
+Version Released: 06/21/2023
+Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11
 
 Copyright (c) 2023 Connor Sanders
 MIT License
@@ -44,7 +44,7 @@
 from yahoofinancials.calcs import num_shares_outstanding, eps
 from yahoofinancials.etl import YahooFinanceETL
 
-__version__ = "1.14"
+__version__ = "1.15"
 __author__ = "Connor Sanders"
 
 
@@ -76,7 +76,7 @@ def _run_financial_stmt(self, statement_type, report_num, frequency, reformat):
  report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num]
  if reformat:
  raw_data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj)
- data = self.get_reformatted_stmt_data(raw_data, statement_type)
+ data = self.get_reformatted_stmt_data(raw_data)
  else:
  data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj)
  return data