diff --git a/CHANGES b/CHANGES index fcf8842..79d8aa1 100644 --- a/CHANGES +++ b/CHANGES @@ -59,3 +59,6 @@ 1.18 12/09/2023 -- Merged in branch from shaunpatterson to fix #164. 1.19 12/12/2023 -- Refactored session management system to handle cookie and crumbs better. 1.19 12/12/2023 -- Added fixes for #167, #166, #160. +1.20 12/16/2023 -- Merged in pull request #171 from bjosun. +1.20 12/17/2023 -- Added optional flat format output param on YahooFinancial class. +1.20 12/17/2023 -- Added get_insight() and get_recommendations() methods. diff --git a/README.rst b/README.rst index 3d675cb..209d372 100644 --- a/README.rst +++ b/README.rst @@ -16,9 +16,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://static.pepy.tech/badge/yahoofinancials/week :target: https://pepy.tech/project/yahoofinancials -Current Version: v1.19 +Current Version: v1.20 -Version Released: 12/12/2023 +Version Released: 12/17/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -26,6 +26,61 @@ Overview -------- A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. +- New analytic methods in v1.20: + - get_insights() + - returns data for: + - 'instrumentInfo' + - 'companySnapshot' + - 'recommendation' + - 'sigDevs' + - 'secReports' + - get_recommendations() + +- Example: + +.. code-block:: python + + print(YahooFinancials('C').get_recommendations()) + +- Example Output: + +.. code-block:: javascript + + { + "C": [ + { + "recommendedSymbols": [ + { + "score": 0.239602, + "symbol": "BAC" + }, + { + "score": 0.225134, + "symbol": "JPM" + }, + { + "score": 0.167669, + "symbol": "WFC" + }, + { + "score": 0.145864, + "symbol": "GS" + }, + { + "score": 0.134071, + "symbol": "F" + } + ], + "symbol": "C" + } + ] + } + +- As of Version 1.20, YahooFinancials supports a new optional parameter called flat_format. + - When `YahooFinancials(flat_format=True)`, financial statement data will return in a dict instead of a list. The keys of the dict will be the reporting dates. + - Default is False, to ensure backwards compatibility. + + - As of Version 1.9, YahooFinancials supports optional parameters for asynchronous execution, proxies, and international requests. .. code-block:: python @@ -41,10 +96,6 @@ A powerful financial data module used for pulling both fundamental and technical balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') print(balance_sheet_data_qt) -- New methods in Version 1.13: - - get_esg_score_data() - - Installation ------------- - yahoofinancials runs on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12 @@ -91,7 +142,7 @@ Module Methods -------------- - The financial data from all methods is returned as JSON. - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -- YahooFinancials works with Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 and runs on all operating systems. (Windows, Mac, Linux). +- YahooFinancials works with Python 3.7, 3.8, 3.9, 3.10, 3.11 and 3.12 and runs on all operating systems. (Windows, Mac, Linux). Featured Methods ^^^^^^^^^^^^^^^^ @@ -134,6 +185,7 @@ Additional Module Methods - get_cost_of_revenue() - get_income_before_tax() - get_income_tax_expense() +- get_esg_score_data() - get_gross_profit() - get_net_income_from_continuing_ops() - get_research_and_development() diff --git a/setup.py b/setup.py index 0a11a2e..3cfb327 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.19', + version='1.20', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.19.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.20.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 6597474..e72b557 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,7 +1,7 @@ -# YahooFinancials Unit Tests v1.19 -# Version Released: 12/12/2023 +# YahooFinancials Unit Tests v1.20 +# Version Released: 12/17/2023 # Author: Connor Sanders -# Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11 +# Tested on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12 # Copyright (c) 2023 Connor Sanders # MIT License @@ -42,6 +42,8 @@ def setUp(self): self.test_yf_treasuries_multi = yf(us_treasuries) self.test_yf_currencies = yf(currencies) self.test_yf_concurrent = yf(stocks, concurrent=True) + self.test_yf_stock_flat = yf('C', flat_format=True) + self.test_yf_stock_analytic = yf('WFC') # Fundamentals Test def test_yf_fundamentals(self): @@ -103,6 +105,35 @@ def test_yf_concurrency(self): result = check_fundamental(multi_all_statement_data_qt, 'all') self.assertEqual(result, True) + # Fundamentals in Flat Format Test + def test_yf_fundamentals_flat(self): + # Single stock test + single_all_statement_data_qt = self.test_yf_stock_flat.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + if ((isinstance(single_all_statement_data_qt.get("incomeStatementHistoryQuarterly").get("C"), dict) and + isinstance(single_all_statement_data_qt.get("balanceSheetHistoryQuarterly").get("C"), dict)) and + isinstance(single_all_statement_data_qt.get("cashflowStatementHistoryQuarterly").get("C"), dict)): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Analytic Methods Test + def test_yf_analytic_methods(self): + + # Get Insights + out = self.test_yf_stock_analytic.get_insights() + if out.get("WFC").get("instrumentInfo").get("technicalEvents").get("sector") == "Financial Services": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Get Recommendations + out = self.test_yf_stock_analytic.get_recommendations() + if isinstance(out.get("WFC"), list): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + # Extra Module Methods Test def test_yf_module_methods(self): diff --git a/yahoofinancials/cache.py b/yahoofinancials/cache.py index 8219083..d2a5fe3 100644 --- a/yahoofinancials/cache.py +++ b/yahoofinancials/cache.py @@ -360,7 +360,6 @@ def initialise(self): continue self.initialised = 0 # failure - def lookup(self, strategy): if self.dummy: return None diff --git a/yahoofinancials/etl.py b/yahoofinancials/data.py similarity index 93% rename from yahoofinancials/etl.py rename to yahoofinancials/data.py index 7b71dad..1975b14 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/data.py @@ -60,7 +60,7 @@ def get_data(self, session, url, request_headers=None, params=None, proxy=None, return response -class YahooFinanceETL(object): +class YahooFinanceData(object): def __init__(self, ticker, **kwargs): self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] @@ -72,6 +72,7 @@ def __init__(self, ticker, **kwargs): self.timeout = kwargs.get("timeout", 30) self.proxies = kwargs.get("proxies") self.session = kwargs.pop("session", None) + self.flat_format = kwargs.get("flat_format", False) self._cache = {} # Minimum interval between Yahoo Finance requests for this instance @@ -167,9 +168,15 @@ def _construct_url(self, symbol, config, params, freq, request_type): params.update({k: v['options'][request_type].get(freq)}) elif k == "modules" and request_type in v['options']: params.update({k: request_type}) + elif k == "symbol": + params.update({k: symbol.lower()}) elif k not in params: + if k == 'reportsCount' and v is None: + continue params.update({k: v['default']}) for k, v in _default_query_params.items(): # general defaults + if k == 'reportsCount' and v is None: + continue if k not in params: params.update({k: v}) if params.get("type"): @@ -183,6 +190,8 @@ def _construct_url(self, symbol, config, params, freq, request_type): for k, v in params.items(): if k != "modules": url += "&" + k + "=" + str(v) + elif params.get("symbol"): + url += "?symbol=" + params.get("symbol") return url # Private method to execute a web scrape request and decrypt the return @@ -269,6 +278,11 @@ def _get_historical_data(self, url, config, tech_type, statement_type): data = self._cache[url] if tech_type == '' and statement_type in ["income", "balance", "cash"]: data = self._format_raw_fundamental_data(data) + elif statement_type == 'analytic': + data = data.get("result") + if tech_type == "recommendations": + if isinstance(data, list) and len(data) > 0: + data[0].get("recommendedSymbols") else: data = self._format_raw_module_data(data, tech_type) return data @@ -519,7 +533,9 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi dict_ent = {} params = {} r_map = get_request_config(tech_type, REQUEST_MAP) - r_cat = get_request_category(tech_type, self.YAHOO_FINANCIAL_TYPES, statement_type) + r_cat = None + if statement_type != 'analytic': + r_cat = get_request_category(tech_type, self.YAHOO_FINANCIAL_TYPES, statement_type) YAHOO_URL = self._construct_url( up_ticker.lower(), r_map, @@ -544,6 +560,17 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi dict_ent = {up_ticker: re_data} return dict_ent + def _retry_create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): + i = 0 + while i < 250: + try: + out = self._create_dict_ent(up_ticker, statement_type, tech_type, report_name, hist_obj) + return out + except: + time.sleep(random.randint(2, 10)) + i += 1 + continue + # Private method to return the stmt_id for the reformat_process def _get_stmt_id(self, statement_type, raw_data): stmt_id = '' @@ -568,8 +595,22 @@ def _reformat_stmt_data_process(raw_data): else: return raw_data + # Private Method for the Flat Reformat Process + @staticmethod + def _reformat_stmt_data_process_flat(raw_data): + final_data = {} + if raw_data is not None: + for date_key, data_item in raw_data.items(): + final_data.update({date_key: data_item}) + return final_data + else: + return raw_data + # Private Method to return subdict entry for the statement reformat process def _get_sub_dict_ent(self, ticker, raw_data): + if self.flat_format: + form_data_dict = self._reformat_stmt_data_process_flat(raw_data[ticker]) + return {ticker: form_data_dict} form_data_list = self._reformat_stmt_data_process(raw_data[ticker]) return {ticker: form_data_list} @@ -581,17 +622,17 @@ def get_time_code(self, time_interval): # Public Method to get stock data def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): data = {} - if statement_type == 'income' and tech_type == '' and report_name == '': # temp, so this method doesn't return nulls + if statement_type == 'income' and tech_type == '' and report_name == '': # temp, so this method doesn't return nulls statement_type = 'profile' tech_type = 'assetProfile' report_name = 'assetProfile' if isinstance(self.ticker, str): - dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + dict_ent = self._retry_create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) data.update(dict_ent) else: if self.concurrent: with Pool(self._get_worker_count()) as pool: - dict_ents = pool.map(partial(self._create_dict_ent, + dict_ents = pool.map(partial(self._retry_create_dict_ent, statement_type=statement_type, tech_type=tech_type, report_name=report_name, diff --git a/yahoofinancials/maps.py b/yahoofinancials/maps.py index e028ac8..5696331 100644 --- a/yahoofinancials/maps.py +++ b/yahoofinancials/maps.py @@ -2375,6 +2375,19 @@ "padTimeSeries": {"required": False, "default": False}, }, }, + "insights": { + "path": "https://query1.finance.yahoo.com/ws/insights/v2/finance/insights", + "response_field": "finance", + "request": { + "symbol": {"required": True, "default": None}, + "reportsCount": {"required": False, "default": None}, + }, + }, + "recommendations": { + "path": "https://query1.finance.yahoo.com/v6/finance/recommendationsbysymbol/{symbol}", + "response_field": "finance", + "request": {}, + }, } USER_AGENTS = [ diff --git a/yahoofinancials/sessions.py b/yahoofinancials/sessions.py index e820102..96fb6d5 100644 --- a/yahoofinancials/sessions.py +++ b/yahoofinancials/sessions.py @@ -207,11 +207,11 @@ def _set_cookie_strategy(self, strategy, have_lock=False): try: if self._cookie_strategy == 'csrf': - logging.info(f'yahoofinancials: toggling cookie strategy {self._cookie_strategy} -> basic') + logging.debug(f'yahoofinancials: toggling cookie strategy {self._cookie_strategy} -> basic') self._session.cookies.clear() self._cookie_strategy = 'basic' else: - logging.info(f'yahoofinancials: toggling cookie strategy {self._cookie_strategy} -> csrf') + logging.debug(f'yahoofinancials: toggling cookie strategy {self._cookie_strategy} -> csrf') self._cookie_strategy = 'csrf' self._cookie = None self._crumb = None @@ -237,7 +237,7 @@ def _load_session_cookies(self): if cookie_dict['age'] > datetime.timedelta(days=1): return False self._session.cookies.update(cookie_dict['cookie']) - logging.info('yahoofinancials: loaded persistent cookie') + logging.debug('yahoofinancials: loaded persistent cookie') def _save_cookie_basic(self, cookie): try: @@ -253,18 +253,16 @@ def _load_cookie_basic(self): # Periodically refresh, 24 hours seems fair. if cookie_dict['age'] > datetime.timedelta(days=1): return None - logging.info('yahoofinancials: loaded persistent cookie') + logging.debug('yahoofinancials: loaded persistent cookie') return cookie_dict['cookie'] def _get_cookie_basic(self, proxy=None, timeout=30): if self._cookie is not None: - logging.info('yahoofinancials: reusing cookie') + logging.debug('yahoofinancials: reusing cookie') return self._cookie - self._cookie = self._load_cookie_basic() if self._cookie is not None: return self._cookie - # To avoid infinite recursion, do NOT use self.get() # - 'allow_redirects' copied from @psychoz971 solution - does it help USA? response = self._session.get( @@ -273,21 +271,20 @@ def _get_cookie_basic(self, proxy=None, timeout=30): proxies=proxy, timeout=timeout, allow_redirects=True) - if not response.cookies: - logging.info("yahoofinancials: response.cookies = None") + logging.debug("yahoofinancials: response.cookies = None") return None self._cookie = list(response.cookies)[0] if self._cookie == '': - logging.info("yahoofinancials: list(response.cookies)[0] = ''") + logging.debug("yahoofinancials: list(response.cookies)[0] = ''") return None self._save_cookie_basic(self._cookie) - logging.info(f"yahoofinancials: fetched basic cookie = {self._cookie}") + logging.debug(f"yahoofinancials: fetched basic cookie = {self._cookie}") return self._cookie def _get_crumb_basic(self, proxy=None, timeout=30): if self._crumb is not None: - logging.info('yahoofinancials: reusing crumb') + logging.debug('yahoofinancials: reusing crumb') return self._crumb cookie = self._get_cookie_basic() @@ -310,10 +307,9 @@ def _get_crumb_basic(self, proxy=None, timeout=30): crumb_response = self._session.get(**get_args) self._crumb = crumb_response.text if self._crumb is None or '' in self._crumb: - logging.info("yahoofinancials: didn't receive crumb") + logging.debug("yahoofinancials: didn't receive crumb") return None - - logging.info(f"yahoofinancials: crumb = '{self._crumb}'") + logging.debug(f"yahoofinancials: crumb = '{self._crumb}'") return self._crumb def _get_cookie_and_crumb_basic(self, proxy, timeout): @@ -323,37 +319,33 @@ def _get_cookie_and_crumb_basic(self, proxy, timeout): def _get_cookie_csrf(self, proxy, timeout): if self._cookie is not None: - logging.info('yahoofinancials: reusing cookie') + logging.debug('yahoofinancials: reusing cookie') return True elif self._load_session_cookies(): - logging.info('yahoofinancials: reusing persistent cookie') + logging.debug('yahoofinancials: reusing persistent cookie') self._cookie = True return True - base_args = { 'headers': self.user_agent_headers, 'proxies': proxy, 'timeout': timeout} - get_args = {**base_args, 'url': 'https://guce.yahoo.com/consent'} if self._session_is_caching: get_args['expire_after'] = self._expire_after response = self._session.get(**get_args) else: response = self._session.get(**get_args) - soup = BeautifulSoup(response.content, 'html.parser') csrfTokenInput = soup.find('input', attrs={'name': 'csrfToken'}) if csrfTokenInput is None: - logging.info('yahoofinancials: Failed to find "csrfToken" in response') + logging.debug('yahoofinancials: Failed to find "csrfToken" in response') return False csrfToken = csrfTokenInput['value'] - logging.info(f'csrfToken = {csrfToken}') + logging.debug(f'csrfToken = {csrfToken}') sessionIdInput = soup.find('input', attrs={'name': 'sessionId'}) sessionId = sessionIdInput['value'] - logging.info(f"sessionId='{sessionId}") - + logging.debug(f"sessionId='{sessionId}") originalDoneUrl = 'https://finance.yahoo.com/' namespace = 'yahoo' data = { @@ -384,7 +376,7 @@ def _get_cookie_csrf(self, proxy, timeout): def _get_crumb_csrf(self, proxy=None, timeout=30): if self._crumb is not None: - logging.info('yahoofinancials: reusing crumb') + logging.debug('yahoofinancials: reusing crumb') return self._crumb if not self._get_cookie_csrf(proxy, timeout): # This cookie stored in session @@ -401,14 +393,14 @@ def _get_crumb_csrf(self, proxy=None, timeout=30): r = self._session.get(**get_args) self._crumb = r.text if self._crumb is None or '' in self._crumb or self._crumb == '': - logging.info("yahoofinancials: didn't receive crumb") + logging.debug("yahoofinancials: didn't receive crumb") return None - logging.info(f"yahoofinancials: crumb = '{self._crumb}'") + logging.debug(f"yahoofinancials: crumb = '{self._crumb}'") return self._crumb def _get_cookie_and_crumb(self, proxy=None, timeout=30): cookie, crumb, strategy = None, None, None - logging.info(f"yahoofinancials: cookie_mode = '{self._cookie_strategy}'") + logging.debug(f"yahoofinancials: cookie_mode = '{self._cookie_strategy}'") with self._cookie_lock: if self._cookie_strategy == 'csrf': crumb = self._get_crumb_csrf() @@ -480,7 +472,7 @@ def _get_proxy(self, proxy): return proxy def get_raw_json(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30): - logging.info(f'yahoofinancials: get_raw_json(): {url}') + logging.debug(f'yahoofinancials: get_raw_json(): {url}') response = self.get(url, user_agent_headers=user_agent_headers, params=params, proxy=proxy, timeout=timeout) response.raise_for_status() return response.json() diff --git a/yahoofinancials/utils.py b/yahoofinancials/utils.py index 622d442..93a52e0 100644 --- a/yahoofinancials/utils.py +++ b/yahoofinancials/utils.py @@ -5,6 +5,10 @@ def remove_prefix(s, prefix): def get_request_config(tech_type, req_map): if tech_type == '': r_map = req_map['fundamentals'] + elif tech_type == 'insights': + r_map = req_map['insights'] + elif tech_type == 'recommendations': + r_map = req_map['recommendations'] else: r_map = req_map['quoteSummary'] return r_map diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 561afb3..26a755a 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.19 +Version: 1.20 ============================== Author: Connor Sanders Email: jecsand@pm.me -Version Released: 12/12/2023 +Version Released: 12/17/2023 Tested on Python 3.7, 3.8, 3.9, 3.10, 3.11, and 3.12 Copyright (c) 2023 Connor Sanders @@ -42,14 +42,14 @@ """ from yahoofinancials.calcs import num_shares_outstanding, eps -from yahoofinancials.etl import YahooFinanceETL +from yahoofinancials.data import YahooFinanceData -__version__ = "1.19" +__version__ = "1.20" __author__ = "Connor Sanders" # Class containing methods to create stock data extracts -class YahooFinancials(YahooFinanceETL): +class YahooFinancials(YahooFinanceData): """ Arguments ---------- @@ -68,6 +68,8 @@ class YahooFinancials(YahooFinanceETL): Defines how long a request will stay open. proxies: str or list, default None, optional Defines any proxies to use during this instantiation. + flat_format: bool, default False, optional + If set to True, returns fundamental data in a flattened format, i.e. without the list of dicts. """ # Private method that handles financial statement extraction @@ -149,6 +151,9 @@ def get_stock_quote_type_data(self): def get_esg_score_data(self): return self.get_stock_tech_data('esgScores') + def _get_analytic_data(self, tech_type): + return self.get_stock_data(statement_type='analytic', tech_type=tech_type) + # Public Method for user to get historical price data with def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) @@ -344,6 +349,12 @@ def get_net_income_from_continuing_ops(self): def get_research_and_development(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') + def get_recommendations(self): + return self._get_analytic_data("recommendations") + + def get_insights(self): + return self._get_analytic_data("insights") + # Calculated Financial Methods def get_earnings_per_share(self): price_data = self.get_current_price()