Added additional unit tests and esg scores as requested in #48.

CHANGES

@@ -43,4 +43,6 @@
 1.10 01/25/2023 -- Fixed new decryption issue.
 1.11 01/26/2023 -- Added a dynamic fix for the decryption issue.
 1.12 01/27/2023 -- Fixed get profile function for #127 and added additional unit test.
-1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping.
+1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping.
+1.13 02/14/2023 -- Added method to retrieve ESG data as requested in #48.
+1.13 02/14/2023 -- Added additional unit tests.

README.rst

@@ -35,9 +35,8 @@ A powerful financial data module used for pulling both fundamental and technical
  balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance')
  print(balance_sheet_data_qt)
 
-- New methods in Version 1.9:
- - get_stock_profile_data()
- - get_financial_data()
+- New methods in Version 1.13:
+ - get_esg_score_data()
 
 
 Installation
@@ -98,8 +97,7 @@ Featured Methods
  - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
 2. get_stock_price_data(reformat=True)
 
- - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
-3. get_stock_earnings_data(reformat=True)
+3. get_stock_earnings_data()
 
  - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
 4. get_summary_data(reformat=True)
@@ -119,12 +117,11 @@ Featured Methods
 
  - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price.
 
-Methods Added in v1.5
-^^^^^^^^^^^^^^^^^^^^^^^
-- get_daily_dividend_data(start_date, end_date)
-
 Additional Module Methods
 ^^^^^^^^^^^^^^^^^^^^^^^^^
+- get_daily_dividend_data(start_date, end_date)
+- get_stock_profile_data()
+- get_financial_data()
 - get_interest_expense()
 - get_operating_income()
 - get_total_operating_expense()

test/test_yahoofinancials.py

@@ -95,19 +95,52 @@ def test_yf_dividend_price(self):
 
  # Extra Module Methods Test
  def test_yf_module_methods(self):
- # Stocks
+ # Stock Current Price
  if isinstance(self.test_yf_stock_single.get_current_price(), float):
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
+ # Stock Net Income
  if isinstance(self.test_yf_stock_single.get_net_income(), float):
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
+ # Stock Profile Data
  if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services":
  self.assertEqual(True, True)
  else:
  self.assertEqual(False, True)
+
+ # Stock Financial Data
+ if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD":
+ self.assertEqual(True, True)
+ else:
+ self.assertEqual(False, True)
+
+ # Stock Summary Data
+ if self.test_yf_stock_single.get_summary_data().get("C").get("currency") == "USD":
+ self.assertEqual(True, True)
+ else:
+ self.assertEqual(False, True)
+
+ # Stock Price Data
+ if self.test_yf_stock_single.get_stock_price_data().get("C").get("exchangeName") == "NYSE":
+ self.assertEqual(True, True)
+ else:
+ self.assertEqual(False, True)
+
+ # Stock Key Statistics
+ if isinstance(self.test_yf_stock_single.get_key_statistics_data().get("C").get("forwardPE"), float):
+ self.assertEqual(True, True)
+ else:
+ self.assertEqual(False, True)
+
+ # Stock ESG SCORES
+ if self.test_yf_stock_single.get_esg_score_data().get("C").get("peerGroup") == "Banks":
+ self.assertEqual(True, True)
+ else:
+ self.assertEqual(False, True)
+
  # Treasuries
  if isinstance(self.test_yf_treasuries_single.get_current_price(), float):
  self.assertEqual(True, True)

yahoofinancials/etl.py

@@ -9,10 +9,9 @@
 import pytz
 import requests as requests
 
-from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP
+from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP, USER_AGENTS
 from yahoofinancials.utils import remove_prefix, get_request_config, get_request_category
 
-
 # track the last get timestamp to add a minimum delay between gets - be nice!
 _lastget = 0
 
@@ -27,20 +26,30 @@ class ManagedException(Exception):
 
 # Class used to get data from urls
 class UrlOpener:
- user_agent_headers = {
- 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'
+
+ request_headers = {
+ "accept": "*/*",
+ "accept-encoding": "gzip, deflate, br",
+ "accept-language": "en-US,en;q=0.9",
+ "origin": "https://finance.yahoo.com",
+ "referer": "https://finance.yahoo.com",
+ "sec-fetch-dest": "empty",
+ "sec-fetch-mode": "cors",
+ "sec-fetch-site": "same-site",
  }
+ user_agent = random.choice(USER_AGENTS)
+ request_headers["User-Agent"] = user_agent
 
  def __init__(self, session=None):
  self._session = session or requests
 
- def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30):
+ def open(self, url, request_headers=None, params=None, proxy=None, timeout=30):
  response = self._session.get(
  url=url,
  params=params,
  proxies=proxy,
  timeout=timeout,
- headers=user_agent_headers or self.user_agent_headers
+ headers=request_headers or self.request_headers
  )
  return response
 
@@ -323,13 +332,6 @@ def _encode_ticker(ticker_str):
  encoded_ticker = ticker_str.replace('=', '%3D')
  return encoded_ticker
 
- # Private method to get time interval code
- def _build_historical_url(self, ticker, hist_oj):
- url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \
- '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \
- hist_oj['interval']
- return url
-
  # Private Method to clean the dates of the newly returns historical stock data into readable format
  def _clean_historical_data(self, hist_data, last_attempt=False):
  data = {}

yahoofinancials/maps.py

@@ -2376,3 +2376,17 @@
  },
  },
 }
+
+USER_AGENTS = [
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 "
+ "Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+ "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 '
+ 'Safari/537.36'
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 "
+ "Safari/537.36",
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 "
+ "Safari/537.36",
+ "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36",
+]

yahoofinancials/utils.py

@@ -1,4 +1,3 @@
-
 def remove_prefix(s, prefix):
  return s[len(prefix):] if s.startswith(prefix) else s
 

yahoofinancials/yf.py

@@ -19,8 +19,7 @@
  - statement_type can be 'income', 'balance', 'cash'.
  - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
 2) get_stock_price_data(reformat=True)
- - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
-3) get_stock_earnings_data(reformat=True)
+3) get_stock_earnings_data()
  - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
 4) get_summary_data(reformat=True)
  - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance.
@@ -112,17 +111,14 @@ def get_key_statistics_data(self, reformat=True):
  def get_stock_profile_data(self, reformat=True):
  if reformat:
  return self.get_clean_data(
- self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile'),
+ self.get_stock_data(statement_type='profile', tech_type='assetProfile', report_name='assetProfile'),
  'earnings')
  else:
- return self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile')
+ return self.get_stock_data(statement_type='profile', tech_type='assetProfile', report_name='assetProfile')
 
  # Public Method for the user to get stock earnings data
- def get_stock_earnings_data(self, reformat=True):
- if reformat:
- return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings')
- else:
- return self.get_stock_tech_data('earnings')
+ def get_stock_earnings_data(self):
+ return self.get_stock_tech_data('earnings')
 
  # Public Method for the user to return financial data
  def get_financial_data(self, reformat=True):
@@ -149,6 +145,10 @@ def get_stock_summary_url(self):
  def get_stock_quote_type_data(self):
  return self.get_stock_tech_data('quoteType')
 
+ # Public Method for the user to get stock quote data
+ def get_esg_score_data(self):
+ return self.get_stock_tech_data('esgScores')
+
  # Public Method for user to get historical price data with
  def get_historical_price_data(self, start_date, end_date, time_interval):
  interval_code = self.get_time_code(time_interval)