Refactored session management system to handle cookie and crumbs better.

Added fixes for #167, #166, #160.

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
  runs-on: ubuntu-latest
  strategy:
  matrix:
- python-version: ["3.8", "3.9", "3.10", "3.11"]
+ python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12""]
  steps:
  - uses: actions/checkout@v3
  - name: Set up Python ${{ matrix.python-version }}

README.rst

@@ -47,7 +47,7 @@ A powerful financial data module used for pulling both fundamental and technical
 
 Installation
 -------------
-- yahoofinancials runs on Python 3.8, 3.9, 3.10, and 3.11.
+- yahoofinancials runs on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12
 
 1. Installation using pip:
 

setup.py

@@ -37,10 +37,12 @@
  'Topic :: Software Development :: Libraries :: Python Modules',
  'Operating System :: OS Independent',
  'License :: OSI Approved :: MIT License',
+ 'Programming Language :: Python :: 3.7',
  'Programming Language :: Python :: 3.8',
  'Programming Language :: Python :: 3.9',
  'Programming Language :: Python :: 3.10',
  'Programming Language :: Python :: 3.11',
+ 'Programming Language :: Python :: 3.12',
  ],
  zip_safe=False
 )

yahoofinancials/etl.py

@@ -9,7 +9,7 @@
 import pytz
 
 from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP, USER_AGENTS
-from yahoofinancials.sessions import SessionManager
+from yahoofinancials.sessions import SessionManager, _init_session
 from yahoofinancials.utils import remove_prefix, get_request_config, get_request_category
 
 # track the last get timestamp to add a minimum delay between gets - be nice!
@@ -49,6 +49,16 @@ def open(self, url, request_headers=None, params=None, proxy=None, timeout=30):
  )
  return response
 
+ def get_data(self, url, session, request_headers=None, params=None, proxy=None, timeout=30):
+ response = session.get(
+ url=url,
+ params=params,
+ proxies=proxy,
+ timeout=timeout,
+ user_agent_headers=request_headers or self.request_headers
+ )
+ return response
+
 
 class YahooFinanceETL(object):
 
@@ -179,10 +189,19 @@ def _construct_url(self, symbol, config, params, freq, request_type):
  def _request_handler(self, url, res_field=""):
  urlopener = UrlOpener(self.session)
  # Try to open the URL up to 10 times sleeping random time if something goes wrong
+ open_session = False
  cur_url = url
  max_retry = 10
  for i in range(0, max_retry):
- response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout)
+ if open_session:
+ open_session = False
+ session, crumb = _init_session(None, proxies=self._get_proxy(), timeout=self.timeout)
+ crumb_url = cur_url + "&crumb=" + str(crumb)
+ response = urlopener.get_data(crumb_url, proxy=self._get_proxy(), timeout=self.timeout)
+ else:
+ response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout)
+ if response.status_code == 401:
+ open_session = True
  if response.status_code != 200:
  time.sleep(random.randrange(1, 5))
  response.close()
@@ -192,6 +211,7 @@ def _request_handler(self, url, res_field=""):
  cur_url = cur_url.replace("query2.", "query1.")
  elif 'query1.' in cur_url:
  cur_url = cur_url.replace("query1.", "query2.")
+
  else:
  res_content = response.text
  response.close()
@@ -554,7 +574,7 @@ def get_time_code(self, time_interval):
  # Public Method to get stock data
  def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}):
  data = {}
- if statement_type == 'income' and tech_type == '' and report_name == '':
+ if statement_type == 'income' and tech_type == '' and report_name == '': # temp, so this method doesn't return nulls
  statement_type = 'profile'
  tech_type = 'assetProfile'
  report_name = 'assetProfile'