From 2a7540d1a45cf54d4e5b0f5d82208bc073b12abc Mon Sep 17 00:00:00 2001 From: Connor Sanders <32915591+JECSand@users.noreply.github.com> Date: Thu, 3 May 2018 12:00:05 -0500 Subject: [PATCH 001/108] Updated READ.ME for Contributers --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 403b55e..b9d8ea8 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # yahoofinancials Welcome to Yahoo Financials! +Now Looking for New Contributers! +Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. ## Overview A powerful financial data module used for pulling both fundamental and technical stock data from Yahoo Finance. From a9f612d2470244f0f8eff11d81cf24525d760ce7 Mon Sep 17 00:00:00 2001 From: Connor Sanders <32915591+JECSand@users.noreply.github.com> Date: Thu, 3 May 2018 12:01:07 -0500 Subject: [PATCH 002/108] Updated README.md for contributers - format --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b9d8ea8..05d2735 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # yahoofinancials Welcome to Yahoo Financials! Now Looking for New Contributers! + Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. ## Overview From 544889ce18edea2814bfbc80eb06c035e17c8271 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Mon, 9 Jul 2018 12:20:05 -0600 Subject: [PATCH 003/108] normalize line endings CRLF => LF This makes it sane to merge, rebase, cherry-pick, ... --- setup.py | 48 +- yahoofinancials/__init__.py | 1172 +++++++++++++++++------------------ 2 files changed, 610 insertions(+), 610 deletions(-) diff --git a/setup.py b/setup.py index 015255f..69eeca4 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,24 @@ -try: - from setuptools import setup -except ImportError: - from distutils.core import setup - -setup( - name='yahoofinancials', - version='0.5', - description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', - url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.5.tar.gz', - author='Connor Sanders', - author_email='connor@exceleri.com', - license='MIT', - keywords = ['finance data', 'stocks', 'yahoo finance'], - packages=['yahoofinancials'], - install_requires=[ - "requests", - "beautifulsoup4", - "pytz" - ], - classifiers=[], - zip_safe=False -) +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + name='yahoofinancials', + version='0.5', + description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', + url='https://github.com/JECSand/yahoofinancials', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.5.tar.gz', + author='Connor Sanders', + author_email='connor@exceleri.com', + license='MIT', + keywords = ['finance data', 'stocks', 'yahoo finance'], + packages=['yahoofinancials'], + install_requires=[ + "requests", + "beautifulsoup4", + "pytz" + ], + classifiers=[], + zip_safe=False +) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 28dd7db..5ed3561 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,586 +1,586 @@ -""" -============================== -The Yahoo Financials Module -Version: 0.5 -============================== - -Author: Connor Sanders -Email: connor@exceleri.com -Version Released: 10/22/2017 -Tested on Python 2.7 and 3.5 - -Copyright (c) 2017 Connor Sanders -MIT License - -List of Included Functions: - -1) get_financial_stmts(frequency, statement_type, reformat=True) - - frequency can be either 'annual' or 'quarterly'. - - statement_type can be 'income', 'balance', 'cash'. - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -2) get_stock_price_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -3) get_stock_earnings_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -4) get_stock_summary_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -5) get_stock_quote_type_data() -6) get_historical_stock_data(start_date, end_date, time_interval) - - start_date should be entered in the 'YYYY-MM-DD' format. First day that stock data will be pulled. - - end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. - - time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. - -Usage Examples: -from yahoofinancials import YahooFinancials -#tickers = 'AAPL' -#or -tickers = ['AAPL', 'WFC', 'F'] -yahoo_financials = YahooFinancials(tickers) -balance_sheet_data = yahoo_financials.get_financial_stmts('quarterly', 'balance') -earnings_data = yahoo_financials.get_stock_earnings_data() -historical_stock_prices = yahoo_financials.get_historical_stock_data('2015-01-15', '2017-10-15', 'weekly') -""" - -import sys -import re -from json import loads -import time -from bs4 import BeautifulSoup -import requests -import datetime -from datetime import date -import pytz - - -# Class containing Yahoo Finance ETL Functionality -class YahooFinanceETL(object): - - def __init__(self, ticker): - self.ticker = ticker - - # Meta-data dictionaries for the class to use - YAHOO_FINANCIAL_TYPES = { - 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], - 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], - 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], - 'history': ['history'] - } - - _INTERVAL_DICT = { - 'daily': '1d', - 'weekly': '1wk', - 'monthly': '1mo' - } - - # Base Yahoo Finance URL for the class to build on - _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' - - # private static method to get the appropriate report type identifier - @staticmethod - def get_report_type(frequency): - if frequency == 'annual': - report_num = 1 - else: - report_num = 2 - return report_num - - # Public static method to format date serial string to readable format and vice versa - @staticmethod - def format_date(in_date, convert_type): - if convert_type == 'standard': - form_date = datetime.datetime.fromtimestamp(int(in_date)).strftime('%Y-%m-%d') - else: - split_date = in_date.split('-') - d = date(int(split_date[0]), int(split_date[1]), int(split_date[2])) - form_date = int(time.mktime(d.timetuple())) - return form_date - - # Private Static Method to Convert Eastern Time to UTC - @staticmethod - def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): - utc = pytz.utc - eastern = pytz.timezone('US/Eastern') - date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) - date_eastern = eastern.localize(date_, is_dst=None) - date_utc = date_eastern.astimezone(utc) - return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') - - # private static method to scrap data from yahoo finance - @staticmethod - def _scrape_data(url, tech_type, statement_type): - response = requests.get(url) - time.sleep(7) - soup = BeautifulSoup(response.content, "html.parser") - script = soup.find("script", text=re.compile("root.App.main")).text - data = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) - if tech_type == '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] - elif tech_type != '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"][tech_type] - else: - stores = data["context"]["dispatcher"]["stores"]["HistoricalPriceStore"] - return stores - - # Private static method to determine if a numerical value is in the data object being cleaned - @staticmethod - def _determine_numeric_value(value_dict): - if 'raw' in list(value_dict.keys()): - numerical_val = value_dict['raw'] - else: - numerical_val = None - return numerical_val - - # private static method to format date serial string to readable format and vice versa - def _format_time(self, in_time): - form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') - utc_dt = self._convert_to_utc(form_date_time) - return utc_dt - - # Private method to return the a sub dictionary entry for the earning report cleaning - def _get_cleaned_sub_dict_ent(self, key, val_list): - sub_list = [] - for rec in val_list: - sub_sub_dict = {} - for k, v in rec.items(): - if k == 'date': - sub_sub_dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - sub_sub_dict_ent = {k: numerical_val} - sub_sub_dict.update(sub_sub_dict_ent) - sub_list.append(sub_sub_dict) - sub_ent = {key: sub_list} - return sub_ent - - # Private static method to process raw earnings data and clean - def _clean_earnings_data(self, raw_data): - cleaned_data = {} - earnings_key = 'earningsData' - financials_key = 'financialsData' - for k, v in raw_data.items(): - if k == 'earningsChart': - sub_dict = {} - for k2, v2 in v.items(): - if k2 == 'quarterly': - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - elif k2 == 'currentQuarterEstimate': - numerical_val = self._determine_numeric_value(v2) - sub_ent = {k2: numerical_val} - else: - sub_ent = {k2: v2} - sub_dict.update(sub_ent) - dict_ent = {earnings_key: sub_dict} - cleaned_data.update(dict_ent) - elif k == 'financialsChart': - sub_dict = {} - for k2, v2, in v.items(): - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - sub_dict.update(sub_ent) - dict_ent = {financials_key: sub_dict} - cleaned_data.update(dict_ent) - else: - if k != 'maxAge': - dict_ent = {k: v} - cleaned_data.update(dict_ent) - return cleaned_data - - # Python 2 method for cleaning data due to Unicode - def _clean_process_pytwo(self, raw_data): - cleaned_dict = {} - for k, v in raw_data.items(): - if 'Time' in k: - formatted_utc_time = self._format_time(v) - dict_ent = {k: formatted_utc_time} - elif 'Date' in k : - try: - formatted_date = v['fmt'] - except: - formatted_date = '-' - dict_ent = {k: formatted_date} - elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float) or isinstance(v, unicode): - dict_ent = {k: v} - elif v is None: - dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - dict_ent = {k: numerical_val} - cleaned_dict.update(dict_ent) - return cleaned_dict - - # Python 3 method for cleaning data due to Unicode - def _clean_process_pythree(self, raw_data): - cleaned_dict = {} - for k, v in raw_data.items(): - if 'Time' in k: - formatted_utc_time = self._format_time(v) - dict_ent = {k: formatted_utc_time} - elif 'Date' in k: - try: - formatted_date = v['fmt'] - except: - formatted_date = '-' - dict_ent = {k: formatted_date} - elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float): - dict_ent = {k: v} - elif v is None: - dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - dict_ent = {k: numerical_val} - cleaned_dict.update(dict_ent) - return cleaned_dict - - # Private static method to clean summary and price reports - def _clean_reports(self, raw_data): - if (sys.version_info > (3, 0)): - cleaned_dict = self._clean_process_pythree(raw_data) - else: - cleaned_dict = self._clean_process_pytwo(raw_data) - return cleaned_dict - - # Private method to get time interval code - def _build_historical_url(self, ticker, hist_oj): - url = self._BASE_YAHOO_URL + ticker + '/history?period1=' + str(hist_oj['start']) + '&period2=' + \ - str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ - hist_oj['interval'] - return url - - # Private Method to clean the dates of the newly returns historical stock data into readable format - def _clean_historical_data(self, hist_data): - data = {} - for k, v in hist_data.items(): - if 'date' in k.lower(): - cleaned_date = self.format_date(v, 'standard') - dict_ent = {k: {u'' + 'formatted_date': cleaned_date, 'date': v}} - data.update(dict_ent) - elif isinstance(v, list): - sub_dict_list = [] - for sub_dict in v: - sub_dict[u'' + 'formatted_date'] = self.format_date(sub_dict['date'], 'standard') - sub_dict_list.append(sub_dict) - dict_ent = {k: sub_dict_list} - data.update(dict_ent) - else: - dict_ent = {k: v} - data.update(dict_ent) - return data - - # Private Method to take scrapped data and build a data dictionary with - def _create_dict_ent(self, ticker, statement_type, tech_type, report_name, hist_obj): - up_ticker = ticker.upper() - YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ - up_ticker - if tech_type == '' and statement_type != 'history': - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} - elif tech_type != '' and statement_type != 'history': - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data} - else: - YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - cleaned_re_data = self._clean_historical_data(re_data) - dict_ent = {up_ticker: cleaned_re_data} - return dict_ent - - # Private method to return the stmt_id for the reformat_process - def _get_stmt_id(self, statement_type, raw_data): - stmt_id = '' - i = 0 - for key in list(raw_data.keys()): - if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: - stmt_id = key - i += 1 - if i != 1: - sys.exit(1) - return stmt_id - - # Private Method for the Reformat Process - def _reformat_stmt_data_process(self, raw_data, statement_type): - final_data_list = [] - stmt_id = self._get_stmt_id(statement_type, raw_data) - hashed_data_list = raw_data[stmt_id] - for data_item in hashed_data_list: - data_date = '' - sub_data_dict = {} - for k, v in data_item.items(): - if k == 'endDate': - data_date = v['fmt'] - elif k != 'maxAge': - numerical_val = self._determine_numeric_value(v) - sub_dict_item = {k: numerical_val} - sub_data_dict.update(sub_dict_item) - dict_item = {data_date: sub_data_dict} - final_data_list.append(dict_item) - return final_data_list - - # Private Method to return subdict entry for the statement reformat process - def _get_sub_dict_ent(self, ticker, raw_data, statement_type): - form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) - return {ticker: form_data_list} - - # Public method to get time interval code - def get_time_code(self, time_interval): - interval_code = self._INTERVAL_DICT[time_interval.lower()] - return interval_code - - # Public Method to get stock data - def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): - data = {} - if isinstance(self.ticker, str): - dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - else: - for tick in self.ticker: - dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - return data - - # Public Method to get technical stock data - def get_stock_tech_data(self, tech_type): - return self.get_stock_data(tech_type=tech_type) - - # Public Method to get reformatted statement data - def get_reformatted_stmt_data(self, raw_data, statement_type): - data_dict = {} - sub_dict = {} - dataType = raw_data['dataType'] - if isinstance(self.ticker, str): - sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {dataType: sub_dict} - data_dict.update(dict_ent) - else: - for tick in self.ticker: - sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {dataType: sub_dict} - data_dict.update(dict_ent) - return data_dict - - # Public method to get cleaned summary and price report data - def get_clean_data(self, raw_report_data, report_type): - cleaned_data_dict = {} - if isinstance(self.ticker, str): - if report_type == 'earnings': - cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) - else: - cleaned_data = self._clean_reports(raw_report_data[self.ticker]) - cleaned_data_dict.update({self.ticker: cleaned_data}) - else: - for tick in self.ticker: - if report_type == 'earnings': - cleaned_data = self._clean_earnings_data(raw_report_data[tick]) - else: - cleaned_data = self._clean_reports(raw_report_data[tick]) - cleaned_data_dict.update({tick: cleaned_data}) - return cleaned_data_dict - - -# Class containing methods to create stock data extracts -class YahooFinancials(YahooFinanceETL): - - def __init__(self, ticker): - super(YahooFinancials, self).__init__(ticker) - self.ticker = ticker - - # Private method that handles financial statement extraction - def _run_financial_stmt(self, statement_type, report_num, reformat): - report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] - if reformat: - raw_data = self.get_stock_data(statement_type, report_name=report_name) - data = self.get_reformatted_stmt_data(raw_data, statement_type) - else: - data = self.get_stock_data(statement_type, report_name=report_name) - return data - - # Public Method for the user to get financial statement data - def get_financial_stmts(self, frequency, statement_type, reformat=True): - report_num = self.get_report_type(frequency) - if isinstance(statement_type, str): - data = self._run_financial_stmt(statement_type, report_num, reformat) - else: - data = {} - for stmt_type in statement_type: - re_data = self._run_financial_stmt(stmt_type, report_num, reformat) - data.update(re_data) - return data - - # Public Method for the user to get stock price data - def get_stock_price_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('price'), 'price') - else: - return self.get_stock_tech_data('price') - - # Public Method for the user to get stock earnings data - def get_stock_earnings_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') - else: - return self.get_stock_tech_data('earnings') - - # Public Method for the user to get stock summary data - def get_stock_summary_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') - else: - return self.get_stock_tech_data('summaryDetail') - - # Public Method for the user to get stock quote data - def get_stock_quote_type_data(self): - return self.get_stock_tech_data('quoteType') - - # Public Method for user to get historical stock data with - def get_historical_stock_data(self, start_date, end_date, time_interval): - interval_code = self.get_time_code(time_interval) - start = self.format_date(start_date, 'unixstamp') - end = self.format_date(end_date, 'unixstamp') - hist_obj = {'start': start, 'end': end, 'interval': interval_code} - data = self.get_stock_data('history', hist_obj=hist_obj) - return data - - # Private Method for Functions needing stock_price_data - def _stock_price_data(self, data_field): - if isinstance(self.ticker, str): - return self.get_stock_price_data()[self.ticker][data_field] - else: - data = {} - for tick in self.ticker: - price = self.get_stock_price_data()[tick][data_field] - data.update({tick: price}) - return data - - # Private Method for Functions needing stock_price_data - def _stock_summary_data(self, data_field): - if isinstance(self.ticker, str): - return self.get_stock_summary_data()[self.ticker][data_field] - else: - data = {} - for tick in self.ticker: - price = self.get_stock_summary_data()[tick][data_field] - data.update({tick: price}) - return data - - # Private Method for Functions needing financial statement data - def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): - re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] - if isinstance(self.ticker, str): - try: - date_key = re_data[self.ticker][0].keys()[0] - except: - date_key = list(re_data[self.ticker][0])[0] - data = re_data[self.ticker][0][date_key][field_name] - else: - data = {} - for tick in self.ticker: - date_key = re_data[tick][0].keys()[0] - sub_data = re_data[tick][0][date_key][field_name] - data.update({tick: sub_data}) - return data - - # Public Price Data Methods - def get_current_price(self): - return self._stock_price_data('regularMarketPrice') - - def get_current_change(self): - return self._stock_price_data('regularMarketChange') - - def get_current_percent_change(self): - return self._stock_price_data('regularMarketChangePercent') - - def get_current_volume(self): - return self._stock_price_data('regularMarketVolume') - - def get_prev_close_price(self): - return self._stock_price_data('regularMarketPreviousClose') - - def get_open_price(self): - return self._stock_price_data('regularMarketOpen') - - def get_ten_day_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume10Day') - - def get_three_month_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume3Month') - - def get_stock_exchange(self): - return self._stock_price_data('exchangeName') - - def get_market_cap(self): - return self._stock_price_data('marketCap') - - def get_daily_low(self): - return self._stock_price_data('regularMarketDayLow') - - def get_daily_high(self): - return self._stock_price_data('regularMarketDayHigh') - - def get_currency(self): - return self._stock_price_data('currency') - - # Public Summary Data Methods - def get_yearly_high(self): - return self._stock_summary_data('fiftyTwoWeekHigh') - - def get_yearly_low(self): - return self._stock_summary_data('fiftyTwoWeekLow') - - def get_dividend_yield(self): - return self._stock_summary_data('dividendYield') - - def get_annual_avg_div_yield(self): - return self._stock_summary_data('trailingAnnualDividendYield') - - def get_five_yr_avg_div_yield(self): - return self._stock_summary_data('fiveYearAvgDividendYield') - - def get_dividend_rate(self): - return self._stock_summary_data('dividendRate') - - def get_annual_avg_div_rate(self): - return self._stock_summary_data('trailingAnnualDividendRate') - - def get_50day_moving_avg(self): - return self._stock_summary_data('fiftyDayAverage') - - def get_200day_moving_avg(self): - return self._stock_summary_data('twoHundredDayAverage') - - def get_beta(self): - return self._stock_summary_data('beta') - - def get_payout_ratio(self): - return self._stock_summary_data('payoutRatio') - - def get_pe_ratio(self): - return self._stock_summary_data('trailingPE') - - def get_price_to_sales(self): - return self._stock_summary_data('priceToSalesTrailing12Months') - - def get_exdividend_date(self): - return self._stock_summary_data('exDividendDate') - - # Financial Statement Data Methods - def get_book_value(self): - return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', - 'totalStockholderEquity', 'quarterly') - - def get_ebit(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') - - def get_net_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') - - # Calculated Financial Methods - def get_earnings_per_share(self): - price_data = self.get_current_price() - pe_ratio = self.get_pe_ratio() - if isinstance(self.ticker, str): - return price_data / pe_ratio - else: - data = {} - for tick in self.ticker: - data.update({tick: price_data[tick] / pe_ratio[tick]}) - return data +""" +============================== +The Yahoo Financials Module +Version: 0.5 +============================== + +Author: Connor Sanders +Email: connor@exceleri.com +Version Released: 10/22/2017 +Tested on Python 2.7 and 3.5 + +Copyright (c) 2017 Connor Sanders +MIT License + +List of Included Functions: + +1) get_financial_stmts(frequency, statement_type, reformat=True) + - frequency can be either 'annual' or 'quarterly'. + - statement_type can be 'income', 'balance', 'cash'. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +2) get_stock_price_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +3) get_stock_earnings_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +4) get_stock_summary_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +5) get_stock_quote_type_data() +6) get_historical_stock_data(start_date, end_date, time_interval) + - start_date should be entered in the 'YYYY-MM-DD' format. First day that stock data will be pulled. + - end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. + - time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. + +Usage Examples: +from yahoofinancials import YahooFinancials +#tickers = 'AAPL' +#or +tickers = ['AAPL', 'WFC', 'F'] +yahoo_financials = YahooFinancials(tickers) +balance_sheet_data = yahoo_financials.get_financial_stmts('quarterly', 'balance') +earnings_data = yahoo_financials.get_stock_earnings_data() +historical_stock_prices = yahoo_financials.get_historical_stock_data('2015-01-15', '2017-10-15', 'weekly') +""" + +import sys +import re +from json import loads +import time +from bs4 import BeautifulSoup +import requests +import datetime +from datetime import date +import pytz + + +# Class containing Yahoo Finance ETL Functionality +class YahooFinanceETL(object): + + def __init__(self, ticker): + self.ticker = ticker + + # Meta-data dictionaries for the class to use + YAHOO_FINANCIAL_TYPES = { + 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], + 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], + 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], + 'history': ['history'] + } + + _INTERVAL_DICT = { + 'daily': '1d', + 'weekly': '1wk', + 'monthly': '1mo' + } + + # Base Yahoo Finance URL for the class to build on + _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' + + # private static method to get the appropriate report type identifier + @staticmethod + def get_report_type(frequency): + if frequency == 'annual': + report_num = 1 + else: + report_num = 2 + return report_num + + # Public static method to format date serial string to readable format and vice versa + @staticmethod + def format_date(in_date, convert_type): + if convert_type == 'standard': + form_date = datetime.datetime.fromtimestamp(int(in_date)).strftime('%Y-%m-%d') + else: + split_date = in_date.split('-') + d = date(int(split_date[0]), int(split_date[1]), int(split_date[2])) + form_date = int(time.mktime(d.timetuple())) + return form_date + + # Private Static Method to Convert Eastern Time to UTC + @staticmethod + def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): + utc = pytz.utc + eastern = pytz.timezone('US/Eastern') + date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) + date_eastern = eastern.localize(date_, is_dst=None) + date_utc = date_eastern.astimezone(utc) + return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') + + # private static method to scrap data from yahoo finance + @staticmethod + def _scrape_data(url, tech_type, statement_type): + response = requests.get(url) + time.sleep(7) + soup = BeautifulSoup(response.content, "html.parser") + script = soup.find("script", text=re.compile("root.App.main")).text + data = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) + if tech_type == '' and statement_type != 'history': + stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] + elif tech_type != '' and statement_type != 'history': + stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"][tech_type] + else: + stores = data["context"]["dispatcher"]["stores"]["HistoricalPriceStore"] + return stores + + # Private static method to determine if a numerical value is in the data object being cleaned + @staticmethod + def _determine_numeric_value(value_dict): + if 'raw' in list(value_dict.keys()): + numerical_val = value_dict['raw'] + else: + numerical_val = None + return numerical_val + + # private static method to format date serial string to readable format and vice versa + def _format_time(self, in_time): + form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') + utc_dt = self._convert_to_utc(form_date_time) + return utc_dt + + # Private method to return the a sub dictionary entry for the earning report cleaning + def _get_cleaned_sub_dict_ent(self, key, val_list): + sub_list = [] + for rec in val_list: + sub_sub_dict = {} + for k, v in rec.items(): + if k == 'date': + sub_sub_dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + sub_sub_dict_ent = {k: numerical_val} + sub_sub_dict.update(sub_sub_dict_ent) + sub_list.append(sub_sub_dict) + sub_ent = {key: sub_list} + return sub_ent + + # Private static method to process raw earnings data and clean + def _clean_earnings_data(self, raw_data): + cleaned_data = {} + earnings_key = 'earningsData' + financials_key = 'financialsData' + for k, v in raw_data.items(): + if k == 'earningsChart': + sub_dict = {} + for k2, v2 in v.items(): + if k2 == 'quarterly': + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + elif k2 == 'currentQuarterEstimate': + numerical_val = self._determine_numeric_value(v2) + sub_ent = {k2: numerical_val} + else: + sub_ent = {k2: v2} + sub_dict.update(sub_ent) + dict_ent = {earnings_key: sub_dict} + cleaned_data.update(dict_ent) + elif k == 'financialsChart': + sub_dict = {} + for k2, v2, in v.items(): + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + sub_dict.update(sub_ent) + dict_ent = {financials_key: sub_dict} + cleaned_data.update(dict_ent) + else: + if k != 'maxAge': + dict_ent = {k: v} + cleaned_data.update(dict_ent) + return cleaned_data + + # Python 2 method for cleaning data due to Unicode + def _clean_process_pytwo(self, raw_data): + cleaned_dict = {} + for k, v in raw_data.items(): + if 'Time' in k: + formatted_utc_time = self._format_time(v) + dict_ent = {k: formatted_utc_time} + elif 'Date' in k : + try: + formatted_date = v['fmt'] + except: + formatted_date = '-' + dict_ent = {k: formatted_date} + elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float) or isinstance(v, unicode): + dict_ent = {k: v} + elif v is None: + dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + dict_ent = {k: numerical_val} + cleaned_dict.update(dict_ent) + return cleaned_dict + + # Python 3 method for cleaning data due to Unicode + def _clean_process_pythree(self, raw_data): + cleaned_dict = {} + for k, v in raw_data.items(): + if 'Time' in k: + formatted_utc_time = self._format_time(v) + dict_ent = {k: formatted_utc_time} + elif 'Date' in k: + try: + formatted_date = v['fmt'] + except: + formatted_date = '-' + dict_ent = {k: formatted_date} + elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float): + dict_ent = {k: v} + elif v is None: + dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + dict_ent = {k: numerical_val} + cleaned_dict.update(dict_ent) + return cleaned_dict + + # Private static method to clean summary and price reports + def _clean_reports(self, raw_data): + if (sys.version_info > (3, 0)): + cleaned_dict = self._clean_process_pythree(raw_data) + else: + cleaned_dict = self._clean_process_pytwo(raw_data) + return cleaned_dict + + # Private method to get time interval code + def _build_historical_url(self, ticker, hist_oj): + url = self._BASE_YAHOO_URL + ticker + '/history?period1=' + str(hist_oj['start']) + '&period2=' + \ + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ + hist_oj['interval'] + return url + + # Private Method to clean the dates of the newly returns historical stock data into readable format + def _clean_historical_data(self, hist_data): + data = {} + for k, v in hist_data.items(): + if 'date' in k.lower(): + cleaned_date = self.format_date(v, 'standard') + dict_ent = {k: {u'' + 'formatted_date': cleaned_date, 'date': v}} + data.update(dict_ent) + elif isinstance(v, list): + sub_dict_list = [] + for sub_dict in v: + sub_dict[u'' + 'formatted_date'] = self.format_date(sub_dict['date'], 'standard') + sub_dict_list.append(sub_dict) + dict_ent = {k: sub_dict_list} + data.update(dict_ent) + else: + dict_ent = {k: v} + data.update(dict_ent) + return data + + # Private Method to take scrapped data and build a data dictionary with + def _create_dict_ent(self, ticker, statement_type, tech_type, report_name, hist_obj): + up_ticker = ticker.upper() + YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ + up_ticker + if tech_type == '' and statement_type != 'history': + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} + elif tech_type != '' and statement_type != 'history': + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + dict_ent = {up_ticker: re_data} + else: + YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + cleaned_re_data = self._clean_historical_data(re_data) + dict_ent = {up_ticker: cleaned_re_data} + return dict_ent + + # Private method to return the stmt_id for the reformat_process + def _get_stmt_id(self, statement_type, raw_data): + stmt_id = '' + i = 0 + for key in list(raw_data.keys()): + if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: + stmt_id = key + i += 1 + if i != 1: + sys.exit(1) + return stmt_id + + # Private Method for the Reformat Process + def _reformat_stmt_data_process(self, raw_data, statement_type): + final_data_list = [] + stmt_id = self._get_stmt_id(statement_type, raw_data) + hashed_data_list = raw_data[stmt_id] + for data_item in hashed_data_list: + data_date = '' + sub_data_dict = {} + for k, v in data_item.items(): + if k == 'endDate': + data_date = v['fmt'] + elif k != 'maxAge': + numerical_val = self._determine_numeric_value(v) + sub_dict_item = {k: numerical_val} + sub_data_dict.update(sub_dict_item) + dict_item = {data_date: sub_data_dict} + final_data_list.append(dict_item) + return final_data_list + + # Private Method to return subdict entry for the statement reformat process + def _get_sub_dict_ent(self, ticker, raw_data, statement_type): + form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) + return {ticker: form_data_list} + + # Public method to get time interval code + def get_time_code(self, time_interval): + interval_code = self._INTERVAL_DICT[time_interval.lower()] + return interval_code + + # Public Method to get stock data + def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): + data = {} + if isinstance(self.ticker, str): + dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + else: + for tick in self.ticker: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + return data + + # Public Method to get technical stock data + def get_stock_tech_data(self, tech_type): + return self.get_stock_data(tech_type=tech_type) + + # Public Method to get reformatted statement data + def get_reformatted_stmt_data(self, raw_data, statement_type): + data_dict = {} + sub_dict = {} + dataType = raw_data['dataType'] + if isinstance(self.ticker, str): + sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {dataType: sub_dict} + data_dict.update(dict_ent) + else: + for tick in self.ticker: + sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {dataType: sub_dict} + data_dict.update(dict_ent) + return data_dict + + # Public method to get cleaned summary and price report data + def get_clean_data(self, raw_report_data, report_type): + cleaned_data_dict = {} + if isinstance(self.ticker, str): + if report_type == 'earnings': + cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) + else: + cleaned_data = self._clean_reports(raw_report_data[self.ticker]) + cleaned_data_dict.update({self.ticker: cleaned_data}) + else: + for tick in self.ticker: + if report_type == 'earnings': + cleaned_data = self._clean_earnings_data(raw_report_data[tick]) + else: + cleaned_data = self._clean_reports(raw_report_data[tick]) + cleaned_data_dict.update({tick: cleaned_data}) + return cleaned_data_dict + + +# Class containing methods to create stock data extracts +class YahooFinancials(YahooFinanceETL): + + def __init__(self, ticker): + super(YahooFinancials, self).__init__(ticker) + self.ticker = ticker + + # Private method that handles financial statement extraction + def _run_financial_stmt(self, statement_type, report_num, reformat): + report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] + if reformat: + raw_data = self.get_stock_data(statement_type, report_name=report_name) + data = self.get_reformatted_stmt_data(raw_data, statement_type) + else: + data = self.get_stock_data(statement_type, report_name=report_name) + return data + + # Public Method for the user to get financial statement data + def get_financial_stmts(self, frequency, statement_type, reformat=True): + report_num = self.get_report_type(frequency) + if isinstance(statement_type, str): + data = self._run_financial_stmt(statement_type, report_num, reformat) + else: + data = {} + for stmt_type in statement_type: + re_data = self._run_financial_stmt(stmt_type, report_num, reformat) + data.update(re_data) + return data + + # Public Method for the user to get stock price data + def get_stock_price_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('price'), 'price') + else: + return self.get_stock_tech_data('price') + + # Public Method for the user to get stock earnings data + def get_stock_earnings_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') + else: + return self.get_stock_tech_data('earnings') + + # Public Method for the user to get stock summary data + def get_stock_summary_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') + else: + return self.get_stock_tech_data('summaryDetail') + + # Public Method for the user to get stock quote data + def get_stock_quote_type_data(self): + return self.get_stock_tech_data('quoteType') + + # Public Method for user to get historical stock data with + def get_historical_stock_data(self, start_date, end_date, time_interval): + interval_code = self.get_time_code(time_interval) + start = self.format_date(start_date, 'unixstamp') + end = self.format_date(end_date, 'unixstamp') + hist_obj = {'start': start, 'end': end, 'interval': interval_code} + data = self.get_stock_data('history', hist_obj=hist_obj) + return data + + # Private Method for Functions needing stock_price_data + def _stock_price_data(self, data_field): + if isinstance(self.ticker, str): + return self.get_stock_price_data()[self.ticker][data_field] + else: + data = {} + for tick in self.ticker: + price = self.get_stock_price_data()[tick][data_field] + data.update({tick: price}) + return data + + # Private Method for Functions needing stock_price_data + def _stock_summary_data(self, data_field): + if isinstance(self.ticker, str): + return self.get_stock_summary_data()[self.ticker][data_field] + else: + data = {} + for tick in self.ticker: + price = self.get_stock_summary_data()[tick][data_field] + data.update({tick: price}) + return data + + # Private Method for Functions needing financial statement data + def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): + re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] + if isinstance(self.ticker, str): + try: + date_key = re_data[self.ticker][0].keys()[0] + except: + date_key = list(re_data[self.ticker][0])[0] + data = re_data[self.ticker][0][date_key][field_name] + else: + data = {} + for tick in self.ticker: + date_key = re_data[tick][0].keys()[0] + sub_data = re_data[tick][0][date_key][field_name] + data.update({tick: sub_data}) + return data + + # Public Price Data Methods + def get_current_price(self): + return self._stock_price_data('regularMarketPrice') + + def get_current_change(self): + return self._stock_price_data('regularMarketChange') + + def get_current_percent_change(self): + return self._stock_price_data('regularMarketChangePercent') + + def get_current_volume(self): + return self._stock_price_data('regularMarketVolume') + + def get_prev_close_price(self): + return self._stock_price_data('regularMarketPreviousClose') + + def get_open_price(self): + return self._stock_price_data('regularMarketOpen') + + def get_ten_day_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume10Day') + + def get_three_month_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume3Month') + + def get_stock_exchange(self): + return self._stock_price_data('exchangeName') + + def get_market_cap(self): + return self._stock_price_data('marketCap') + + def get_daily_low(self): + return self._stock_price_data('regularMarketDayLow') + + def get_daily_high(self): + return self._stock_price_data('regularMarketDayHigh') + + def get_currency(self): + return self._stock_price_data('currency') + + # Public Summary Data Methods + def get_yearly_high(self): + return self._stock_summary_data('fiftyTwoWeekHigh') + + def get_yearly_low(self): + return self._stock_summary_data('fiftyTwoWeekLow') + + def get_dividend_yield(self): + return self._stock_summary_data('dividendYield') + + def get_annual_avg_div_yield(self): + return self._stock_summary_data('trailingAnnualDividendYield') + + def get_five_yr_avg_div_yield(self): + return self._stock_summary_data('fiveYearAvgDividendYield') + + def get_dividend_rate(self): + return self._stock_summary_data('dividendRate') + + def get_annual_avg_div_rate(self): + return self._stock_summary_data('trailingAnnualDividendRate') + + def get_50day_moving_avg(self): + return self._stock_summary_data('fiftyDayAverage') + + def get_200day_moving_avg(self): + return self._stock_summary_data('twoHundredDayAverage') + + def get_beta(self): + return self._stock_summary_data('beta') + + def get_payout_ratio(self): + return self._stock_summary_data('payoutRatio') + + def get_pe_ratio(self): + return self._stock_summary_data('trailingPE') + + def get_price_to_sales(self): + return self._stock_summary_data('priceToSalesTrailing12Months') + + def get_exdividend_date(self): + return self._stock_summary_data('exDividendDate') + + # Financial Statement Data Methods + def get_book_value(self): + return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', + 'totalStockholderEquity', 'quarterly') + + def get_ebit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') + + def get_net_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') + + # Calculated Financial Methods + def get_earnings_per_share(self): + price_data = self.get_current_price() + pe_ratio = self.get_pe_ratio() + if isinstance(self.ticker, str): + return price_data / pe_ratio + else: + data = {} + for tick in self.ticker: + data.update({tick: price_data[tick] / pe_ratio[tick]}) + return data From 3321b02d9b88b9afe54c9026c81274b3aa4e543a Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Mon, 9 Jul 2018 12:45:39 -0600 Subject: [PATCH 004/108] show results of various requests ./demo.py AAPL ... 39.0867099762 seconds --- demo.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100755 demo.py diff --git a/demo.py b/demo.py new file mode 100755 index 0000000..5326bd6 --- /dev/null +++ b/demo.py @@ -0,0 +1,34 @@ +#!/usr/bin/python + +from __future__ import print_function +from yahoofinancials import YahooFinancials as YF + +def doit(ticker): + mark = '-' * 64 + tick = YF(ticker) + print(mark) + print(tick.get_stock_summary_data()) + print(mark) + print(tick.get_stock_quote_type_data()) + print(mark) + print(tick.get_stock_price_data()) + print(mark) + print(tick.get_current_price()) + print(mark) + print(tick.get_dividend_rate()) + print(mark) + try: + print(tick._cache.keys()) + print(mark) + except AttributeError: + pass + + + +if __name__ == '__main__': + import sys + import time + ts = sys.argv[1:] or ['MMM', 'AAPL'] + st = time.time() + doit(ts[0] if 1 == len(ts) else ts) + print(time.time() - st, 'seconds') From 8b040b74fa89898f86601fb2b1ab3cfc5c60bfdb Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Mon, 9 Jul 2018 12:35:02 -0600 Subject: [PATCH 005/108] cache to avoid unnecessary yahoo requests, delays ./demo.py AAPL ... 0.904011011124 seconds --- yahoofinancials/__init__.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 5ed3561..04ed24b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -57,6 +57,11 @@ class YahooFinanceETL(object): def __init__(self, ticker): self.ticker = ticker + self._cache = {} + self._lastget = 0 + + # Minimum interval between Yahoo Finance requests for this instance + _MIN_INTERVAL = 7 # Meta-data dictionaries for the class to use YAHOO_FINANCIAL_TYPES = { @@ -105,14 +110,19 @@ def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): date_utc = date_eastern.astimezone(utc) return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') - # private static method to scrap data from yahoo finance - @staticmethod - def _scrape_data(url, tech_type, statement_type): - response = requests.get(url) - time.sleep(7) - soup = BeautifulSoup(response.content, "html.parser") - script = soup.find("script", text=re.compile("root.App.main")).text - data = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) + # private method to scrape data from yahoo finance + def _scrape_data(self, url, tech_type, statement_type): + if not self._cache.get(url): + now = int(time.time()) + if self._lastget and now - self._lastget < self._MIN_INTERVAL: + time.sleep(self._MIN_INTERVAL - (now - self._lastget) + 1) + now = int(time.time()) + self._lastget = now + response = requests.get(url) + soup = BeautifulSoup(response.content, "html.parser") + script = soup.find("script", text=re.compile("root.App.main")).text + self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) + data = self._cache[url] if tech_type == '' and statement_type != 'history': stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] elif tech_type != '' and statement_type != 'history': From dd8eb4e52bdef98318c8b4f1e30524c2ebce71fe Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Mon, 9 Jul 2018 12:26:47 -0600 Subject: [PATCH 006/108] fix using lowercase tickers before: >>> from yahoofinancials import YahooFinancials as YF >>> k = YF('KO') >>> k.get_dividend_rate() 1.56 >>> p = YF('pep') >>> p.get_dividend_rate() Traceback (most recent call last): File "", line 1, in File "yahoofinancials/__init__.py", line 539, in get_dividend_rate return self._stock_summary_data('dividendRate') File "yahoofinancials/__init__.py", line 457, in _stock_summary_data return self.get_stock_summary_data()[self.ticker][data_field] File "yahoofinancials/__init__.py", line 426, in get_stock_summary_data return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') File "yahoofinancials/__init__.py", line 368, in get_clean_data cleaned_data = self._clean_reports(raw_report_data[self.ticker]) KeyError: 'pep' >>> after: >>> from yahoofinancials import YahooFinancials as YF >>> k = YF('KO') >>> k.get_dividend_rate() 1.56 >>> p = YF('pep') >>> p.get_dividend_rate() 3.71 >>> --- yahoofinancials/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 04ed24b..3d11c9b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -56,7 +56,7 @@ class YahooFinanceETL(object): def __init__(self, ticker): - self.ticker = ticker + self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] self._cache = {} self._lastget = 0 @@ -276,8 +276,7 @@ def _clean_historical_data(self, hist_data): return data # Private Method to take scrapped data and build a data dictionary with - def _create_dict_ent(self, ticker, statement_type, tech_type, report_name, hist_obj): - up_ticker = ticker.upper() + def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ up_ticker if tech_type == '' and statement_type != 'history': @@ -390,10 +389,6 @@ def get_clean_data(self, raw_report_data, report_type): # Class containing methods to create stock data extracts class YahooFinancials(YahooFinanceETL): - def __init__(self, ticker): - super(YahooFinancials, self).__init__(ticker) - self.ticker = ticker - # Private method that handles financial statement extraction def _run_financial_stmt(self, statement_type, report_num, reformat): report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] From 5ca99f307dc65949893f732ac10e1b345c557efd Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 18 Jul 2018 23:53:45 -0500 Subject: [PATCH 007/108] version 0.6 bug fixes Fixed reported bug in format_date function. Added exception handler that returns a ticker with no available data with a null value. --- CHANGES | 2 + LICENSE | 2 +- setup.py | 4 +- yahoofinancials/__init__.py | 79 +++++++++++++++++++++++-------------- 4 files changed, 54 insertions(+), 33 deletions(-) diff --git a/CHANGES b/CHANGES index 1d1a47b..3b80215 100644 --- a/CHANGES +++ b/CHANGES @@ -2,3 +2,5 @@ 0.2 10/20/2017 -- Added New Methods and Restructed Module Classes 0.3 10/25/2017 -- Added New Methods and Calculated Measures 0.4 03/06/2018 -- Fixed reported bug on line 470 of init.py by adding a try and except with suggested line of code. +0.6 07/06/2018 -- Fixed reported bug in format_date function. +0.6 07/06/2018 -- Added exception handler that returns a ticker with no available data with a null value. diff --git a/LICENSE b/LICENSE index 599034c..43c1577 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2017 Connor Sanders +Copyright (c) 2018 Connor Sanders Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/setup.py b/setup.py index 69eeca4..120a356 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name='yahoofinancials', - version='0.5', + version='0.6', description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.5.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.6.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 3d11c9b..e8299ca 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,15 +1,15 @@ """ ============================== The Yahoo Financials Module -Version: 0.5 +Version: 0.6 ============================== Author: Connor Sanders -Email: connor@exceleri.com -Version Released: 10/22/2017 +Email: sandersconnor1@gmail.com +Version Released: 7/18/2018 Tested on Python 2.7 and 3.5 -Copyright (c) 2017 Connor Sanders +Copyright (c) 2018 Connor Sanders MIT License List of Included Functions: @@ -93,7 +93,13 @@ def get_report_type(frequency): @staticmethod def format_date(in_date, convert_type): if convert_type == 'standard': - form_date = datetime.datetime.fromtimestamp(int(in_date)).strftime('%Y-%m-%d') + if in_date >= 0: + form_date = datetime.datetime.fromtimestamp(int(in_date)).strftime('%Y-%m-%d') + else: + form_date = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date) + if ' ' not in str(form_date): + return str(form_date) + return str(form_date.date()) else: split_date = in_date.split('-') d = date(int(split_date[0]), int(split_date[1]), int(split_date[2])) @@ -110,7 +116,7 @@ def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): date_utc = date_eastern.astimezone(utc) return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') - # private method to scrape data from yahoo finance + # Private method to scrape data from yahoo finance def _scrape_data(self, url, tech_type, statement_type): if not self._cache.get(url): now = int(time.time()) @@ -140,7 +146,7 @@ def _determine_numeric_value(value_dict): numerical_val = None return numerical_val - # private static method to format date serial string to readable format and vice versa + # Private method to format date serial string to readable format and vice versa def _format_time(self, in_time): form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') utc_dt = self._convert_to_utc(form_date_time) @@ -162,7 +168,7 @@ def _get_cleaned_sub_dict_ent(self, key, val_list): sub_ent = {key: sub_list} return sub_ent - # Private static method to process raw earnings data and clean + # Private method to process raw earnings data and clean def _clean_earnings_data(self, raw_data): cleaned_data = {} earnings_key = 'earningsData' @@ -201,7 +207,7 @@ def _clean_process_pytwo(self, raw_data): if 'Time' in k: formatted_utc_time = self._format_time(v) dict_ent = {k: formatted_utc_time} - elif 'Date' in k : + elif 'Date' in k: try: formatted_date = v['fmt'] except: @@ -240,7 +246,7 @@ def _clean_process_pythree(self, raw_data): cleaned_dict.update(dict_ent) return cleaned_dict - # Private static method to clean summary and price reports + # Private method to clean summary and price reports def _clean_reports(self, raw_data): if (sys.version_info > (3, 0)): cleaned_dict = self._clean_process_pythree(raw_data) @@ -280,15 +286,25 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ up_ticker if tech_type == '' and statement_type != 'history': - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} + except: + re_data = None + dict_ent = {up_ticker: re_data, 'dataType': report_name} elif tech_type != '' and statement_type != 'history': - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + except: + re_data = None dict_ent = {up_ticker: re_data} else: YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - cleaned_re_data = self._clean_historical_data(re_data) + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + cleaned_re_data = self._clean_historical_data(re_data) + except: + cleaned_re_data = None dict_ent = {up_ticker: cleaned_re_data} return dict_ent @@ -307,21 +323,24 @@ def _get_stmt_id(self, statement_type, raw_data): # Private Method for the Reformat Process def _reformat_stmt_data_process(self, raw_data, statement_type): final_data_list = [] - stmt_id = self._get_stmt_id(statement_type, raw_data) - hashed_data_list = raw_data[stmt_id] - for data_item in hashed_data_list: - data_date = '' - sub_data_dict = {} - for k, v in data_item.items(): - if k == 'endDate': - data_date = v['fmt'] - elif k != 'maxAge': - numerical_val = self._determine_numeric_value(v) - sub_dict_item = {k: numerical_val} - sub_data_dict.update(sub_dict_item) - dict_item = {data_date: sub_data_dict} - final_data_list.append(dict_item) - return final_data_list + if raw_data is not None: + stmt_id = self._get_stmt_id(statement_type, raw_data) + hashed_data_list = raw_data[stmt_id] + for data_item in hashed_data_list: + data_date = '' + sub_data_dict = {} + for k, v in data_item.items(): + if k == 'endDate': + data_date = v['fmt'] + elif k != 'maxAge': + numerical_val = self._determine_numeric_value(v) + sub_dict_item = {k: numerical_val} + sub_data_dict.update(sub_dict_item) + dict_item = {data_date: sub_data_dict} + final_data_list.append(dict_item) + return final_data_list + else: + return raw_data # Private Method to return subdict entry for the statement reformat process def _get_sub_dict_ent(self, ticker, raw_data, statement_type): From 37c2f232417581ce5966cda6412865fdf1fbb767 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Thu, 19 Jul 2018 20:20:06 -0600 Subject: [PATCH 008/108] ignore produced files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7e77bd2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.pyc +yahoofinancials/__pycache__ From fd5931045705e97252f0e9288b515b4d2c6c9f65 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Thu, 19 Jul 2018 20:21:37 -0600 Subject: [PATCH 009/108] add method to retrieve summary url --- yahoofinancials/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index e8299ca..57ce434 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -451,6 +451,12 @@ def get_stock_summary_data(self, reformat=True): else: return self.get_stock_tech_data('summaryDetail') + # Public Method for the user to get the yahoo summary url + def get_stock_summary_url(self): + if isinstance(self.ticker, str): + return self._BASE_YAHOO_URL + self.ticker + return {t: self._BASE_YAHOO_URL + t for t in self.ticker} + # Public Method for the user to get stock quote data def get_stock_quote_type_data(self): return self.get_stock_tech_data('quoteType') From 7cc76af9d3541fc542df255485bd06d3097eb579 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 20 Jul 2018 20:45:11 -0600 Subject: [PATCH 010/108] remove excess code, simplify minimize duplication for Python 2 v 3 --- yahoofinancials/__init__.py | 91 +++++++++---------------------------- 1 file changed, 21 insertions(+), 70 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 57ce434..bb00df4 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -91,19 +91,13 @@ def get_report_type(frequency): # Public static method to format date serial string to readable format and vice versa @staticmethod - def format_date(in_date, convert_type): - if convert_type == 'standard': - if in_date >= 0: - form_date = datetime.datetime.fromtimestamp(int(in_date)).strftime('%Y-%m-%d') - else: - form_date = datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date) - if ' ' not in str(form_date): - return str(form_date) - return str(form_date.date()) - else: - split_date = in_date.split('-') - d = date(int(split_date[0]), int(split_date[1]), int(split_date[2])) + def format_date(in_date): + if isinstance(in_date, str): + year, month, day = in_date.split()[0].split('-') + d = date(int(year), int(month), int(day)) form_date = int(time.mktime(d.timetuple())) + else: + form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) return form_date # Private Static Method to Convert Eastern Time to UTC @@ -140,7 +134,7 @@ def _scrape_data(self, url, tech_type, statement_type): # Private static method to determine if a numerical value is in the data object being cleaned @staticmethod def _determine_numeric_value(value_dict): - if 'raw' in list(value_dict.keys()): + if 'raw' in value_dict.keys(): numerical_val = value_dict['raw'] else: numerical_val = None @@ -200,31 +194,8 @@ def _clean_earnings_data(self, raw_data): cleaned_data.update(dict_ent) return cleaned_data - # Python 2 method for cleaning data due to Unicode - def _clean_process_pytwo(self, raw_data): - cleaned_dict = {} - for k, v in raw_data.items(): - if 'Time' in k: - formatted_utc_time = self._format_time(v) - dict_ent = {k: formatted_utc_time} - elif 'Date' in k: - try: - formatted_date = v['fmt'] - except: - formatted_date = '-' - dict_ent = {k: formatted_date} - elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float) or isinstance(v, unicode): - dict_ent = {k: v} - elif v is None: - dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - dict_ent = {k: numerical_val} - cleaned_dict.update(dict_ent) - return cleaned_dict - - # Python 3 method for cleaning data due to Unicode - def _clean_process_pythree(self, raw_data): + # Private method to clean summary and price reports + def _clean_reports(self, raw_data): cleaned_dict = {} for k, v in raw_data.items(): if 'Time' in k: @@ -236,9 +207,10 @@ def _clean_process_pythree(self, raw_data): except: formatted_date = '-' dict_ent = {k: formatted_date} - elif isinstance(v, str) or isinstance(v, int) or isinstance(v, float): + elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): dict_ent = {k: v} - elif v is None: + # Python 2 and Unicode + elif sys.version_info < (3, 0) and isinstance(v, unicode): dict_ent = {k: v} else: numerical_val = self._determine_numeric_value(v) @@ -246,14 +218,6 @@ def _clean_process_pythree(self, raw_data): cleaned_dict.update(dict_ent) return cleaned_dict - # Private method to clean summary and price reports - def _clean_reports(self, raw_data): - if (sys.version_info > (3, 0)): - cleaned_dict = self._clean_process_pythree(raw_data) - else: - cleaned_dict = self._clean_process_pytwo(raw_data) - return cleaned_dict - # Private method to get time interval code def _build_historical_url(self, ticker, hist_oj): url = self._BASE_YAHOO_URL + ticker + '/history?period1=' + str(hist_oj['start']) + '&period2=' + \ @@ -266,19 +230,17 @@ def _clean_historical_data(self, hist_data): data = {} for k, v in hist_data.items(): if 'date' in k.lower(): - cleaned_date = self.format_date(v, 'standard') + cleaned_date = self.format_date(v) dict_ent = {k: {u'' + 'formatted_date': cleaned_date, 'date': v}} - data.update(dict_ent) elif isinstance(v, list): sub_dict_list = [] for sub_dict in v: - sub_dict[u'' + 'formatted_date'] = self.format_date(sub_dict['date'], 'standard') + sub_dict[u'' + 'formatted_date'] = self.format_date(sub_dict['date']) sub_dict_list.append(sub_dict) dict_ent = {k: sub_dict_list} - data.update(dict_ent) else: dict_ent = {k: v} - data.update(dict_ent) + data.update(dict_ent) return data # Private Method to take scrapped data and build a data dictionary with @@ -312,7 +274,7 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi def _get_stmt_id(self, statement_type, raw_data): stmt_id = '' i = 0 - for key in list(raw_data.keys()): + for key in raw_data.keys(): if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: stmt_id = key i += 1 @@ -464,8 +426,8 @@ def get_stock_quote_type_data(self): # Public Method for user to get historical stock data with def get_historical_stock_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) - start = self.format_date(start_date, 'unixstamp') - end = self.format_date(end_date, 'unixstamp') + start = self.format_date(start_date) + end = self.format_date(end_date) hist_obj = {'start': start, 'end': end, 'interval': interval_code} data = self.get_stock_data('history', hist_obj=hist_obj) return data @@ -475,22 +437,14 @@ def _stock_price_data(self, data_field): if isinstance(self.ticker, str): return self.get_stock_price_data()[self.ticker][data_field] else: - data = {} - for tick in self.ticker: - price = self.get_stock_price_data()[tick][data_field] - data.update({tick: price}) - return data + return {tick: self.get_stock_price_data()[tick][data_field] for tick in self.ticker} # Private Method for Functions needing stock_price_data def _stock_summary_data(self, data_field): if isinstance(self.ticker, str): return self.get_stock_summary_data()[self.ticker][data_field] else: - data = {} - for tick in self.ticker: - price = self.get_stock_summary_data()[tick][data_field] - data.update({tick: price}) - return data + return {tick: self.get_stock_summary_data()[tick][data_field] for tick in self.ticker} # Private Method for Functions needing financial statement data def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): @@ -610,7 +564,4 @@ def get_earnings_per_share(self): if isinstance(self.ticker, str): return price_data / pe_ratio else: - data = {} - for tick in self.ticker: - data.update({tick: price_data[tick] / pe_ratio[tick]}) - return data + return {tick: price_data[tick] / pe_ratio[tick] for tick in self.ticker} From f674654ecd0f8fcdbdd3c5942bdb7fcc8d6a0d62 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 20 Jul 2018 21:40:47 -0600 Subject: [PATCH 011/108] track lastget as module not just per instance narrow 'except' matching to hide fewer errors --- yahoofinancials/__init__.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index bb00df4..9295be2 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -52,13 +52,16 @@ import pytz +# track the last get timestamp to add a minimum delay between gets - be nice! +_lastget = 0 + + # Class containing Yahoo Finance ETL Functionality class YahooFinanceETL(object): def __init__(self, ticker): self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] self._cache = {} - self._lastget = 0 # Minimum interval between Yahoo Finance requests for this instance _MIN_INTERVAL = 7 @@ -112,12 +115,13 @@ def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): # Private method to scrape data from yahoo finance def _scrape_data(self, url, tech_type, statement_type): + global _lastget if not self._cache.get(url): now = int(time.time()) - if self._lastget and now - self._lastget < self._MIN_INTERVAL: - time.sleep(self._MIN_INTERVAL - (now - self._lastget) + 1) + if _lastget and now - _lastget < self._MIN_INTERVAL: + time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) now = int(time.time()) - self._lastget = now + _lastget = now response = requests.get(url) soup = BeautifulSoup(response.content, "html.parser") script = soup.find("script", text=re.compile("root.App.main")).text @@ -204,7 +208,7 @@ def _clean_reports(self, raw_data): elif 'Date' in k: try: formatted_date = v['fmt'] - except: + except (KeyError, TypeError): formatted_date = '-' dict_ent = {k: formatted_date} elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): @@ -251,13 +255,13 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi try: re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} - except: + except KeyError: re_data = None dict_ent = {up_ticker: re_data, 'dataType': report_name} elif tech_type != '' and statement_type != 'history': try: re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - except: + except KeyError: re_data = None dict_ent = {up_ticker: re_data} else: @@ -265,7 +269,7 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi try: re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) cleaned_re_data = self._clean_historical_data(re_data) - except: + except KeyError: cleaned_re_data = None dict_ent = {up_ticker: cleaned_re_data} return dict_ent @@ -452,7 +456,7 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): if isinstance(self.ticker, str): try: date_key = re_data[self.ticker][0].keys()[0] - except: + except (IndexError, AttributeError): date_key = list(re_data[self.ticker][0])[0] data = re_data[self.ticker][0][date_key][field_name] else: From 7c02f57d7a0866a524f47d90e03cfbd234a1b419 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Sat, 21 Jul 2018 21:58:43 -0600 Subject: [PATCH 012/108] specify api to demo.py also add --help --- demo.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/demo.py b/demo.py index 5326bd6..38e93d0 100755 --- a/demo.py +++ b/demo.py @@ -1,12 +1,30 @@ #!/usr/bin/python +""" +%(scriptname)s [-h|--help] [api [api [...]]] [ticker [ticker [...]]] + no args: show some information about default tickers (%(defaultargs)s) + ticker(s): show some information about specified ticker(s) + api name(s) and ticker(s): show the specified api(s) result for the ticker(s) + yf and -h: show apis + api name(s) and -h: show help for that api + -h or --help: show usage +""" + from __future__ import print_function +import sys +import time from yahoofinancials import YahooFinancials as YF -def doit(ticker): - mark = '-' * 64 + +DEFAULT_ARGS = ('MMM', 'AAPL') +MODULE_ARGS = ('yf', 'yahoofinancial', 'yahoofinancials') +HELP_ARGS = ('-h', '--help') + +mark = '-' * 64 + + +def defaultapi(ticker): tick = YF(ticker) - print(mark) print(tick.get_stock_summary_data()) print(mark) print(tick.get_stock_quote_type_data()) @@ -16,19 +34,54 @@ def doit(ticker): print(tick.get_current_price()) print(mark) print(tick.get_dividend_rate()) - print(mark) try: - print(tick._cache.keys()) - print(mark) + r = tick._cache.keys() except AttributeError: pass + else: + print(mark) + print(r) +def customapi(queries, ts): + yf = YF(ts[0] if 1 == len(ts) else ts) + for q in queries: + print('%s:' % (q,)) + timeit(lambda: print(getattr(yf, q)())) -if __name__ == '__main__': - import sys - import time - ts = sys.argv[1:] or ['MMM', 'AAPL'] + +def helpapi(queries): + if len(queries) == 1: + print(__doc__ % {'scriptname': sys.argv[0], 'defaultargs': ', '.join(DEFAULT_ARGS)}) + else: + import pydoc + for q in queries: + if q in MODULE_ARGS: + print(pydoc.render_doc(YF, "Help on %s")) + elif q not in HELP_ARGS: + print(pydoc.render_doc(getattr(YF, q), "Help on %s")) + + +def timeit(f, *args): + print(mark) st = time.time() - doit(ts[0] if 1 == len(ts) else ts) - print(time.time() - st, 'seconds') + f(*args) + et = time.time() + print(mark) + print(et - st, 'seconds') + + + +if __name__ == '__main__': + api = set(s for s in dir(YF) if s.startswith('get_')) + api.update(MODULE_ARGS) + api.update(HELP_ARGS) + ts = sys.argv[1:] + queries = [q for q in ts if q in api] + ts = [t for t in ts if not t in queries] or DEFAULT_ARGS + if [h for h in HELP_ARGS if h in queries]: + helpapi(queries) + elif queries: + customapi(queries, ts) + else: + timeit(defaultapi, ts[0] if 1 == len(ts) else ts) From cfa019057380e03bc699af723f6ad03dbe08558d Mon Sep 17 00:00:00 2001 From: alt Date: Fri, 3 Aug 2018 01:24:55 -0500 Subject: [PATCH 013/108] v0.7 push --- CHANGES | 6 +++- README.md | 18 ++++++++-- setup.py | 4 +-- yahoofinancials/__init__.py | 70 ++++++++++++++++++++++++++++++------- 4 files changed, 80 insertions(+), 18 deletions(-) diff --git a/CHANGES b/CHANGES index 3b80215..5d9c370 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,10 @@ 0.1, 10/13/2017 -- Initial release. -0.2 10/20/2017 -- Added New Methods and Restructed Module Classes +0.2 10/20/2017 -- Added New Methods and Restructured Module Classes 0.3 10/25/2017 -- Added New Methods and Calculated Measures 0.4 03/06/2018 -- Fixed reported bug on line 470 of init.py by adding a try and except with suggested line of code. 0.6 07/06/2018 -- Fixed reported bug in format_date function. 0.6 07/06/2018 -- Added exception handler that returns a ticker with no available data with a null value. +0.7 08/03/2018 -- Merged Slyvandb's improvements into the master branch. +0.7 08/03/2018 -- Added a try catch at line 465 to explicitly type cast the dict keys to a list if the initial attempt fails. +0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567 +0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None diff --git a/README.md b/README.md index 05d2735..ae85377 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # yahoofinancials Welcome to Yahoo Financials! -Now Looking for New Contributers! +Current Version: v0.7 +Version Released: 08/03/2018 +New Contributers Welcomed! Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. ## Overview @@ -46,7 +48,19 @@ List of Current Included Methods: * end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. * time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. -### New Methods Added in v0.3 +### 10 New Module Methods Added in v0.7 +* get_interest_expense(): +* get_operating_income(): +* get_total_operating_expense(): +* get_total_revenue(): +* get_cost_of_revenue(): +* get_income_before_tax(): +* get_income_tax_expense(): +* get_gross_profit(): +* get_net_income_from_continuing_ops(): +* get_research_and_development(): + +### Module Methods * get_current_price(): * get_current_change(): * get_current_percent_change(): diff --git a/setup.py b/setup.py index 120a356..50a025f 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name='yahoofinancials', - version='0.6', + version='0.7', description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.6.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.7.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 9295be2..8cc1133 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 0.6 +Version: 0.7 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 7/18/2018 +Version Released: 8/03/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -283,7 +283,7 @@ def _get_stmt_id(self, statement_type, raw_data): stmt_id = key i += 1 if i != 1: - sys.exit(1) + return None return stmt_id # Private Method for the Reformat Process @@ -291,6 +291,8 @@ def _reformat_stmt_data_process(self, raw_data, statement_type): final_data_list = [] if raw_data is not None: stmt_id = self._get_stmt_id(statement_type, raw_data) + if stmt_id is None: + return final_data_list hashed_data_list = raw_data[stmt_id] for data_item in hashed_data_list: data_date = '' @@ -338,17 +340,17 @@ def get_stock_tech_data(self, tech_type): def get_reformatted_stmt_data(self, raw_data, statement_type): data_dict = {} sub_dict = {} - dataType = raw_data['dataType'] + data_type = raw_data['dataType'] if isinstance(self.ticker, str): sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) sub_dict.update(sub_dict_ent) - dict_ent = {dataType: sub_dict} + dict_ent = {data_type: sub_dict} data_dict.update(dict_ent) else: for tick in self.ticker: sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) sub_dict.update(sub_dict_ent) - dict_ent = {dataType: sub_dict} + dict_ent = {data_type: sub_dict} data_dict.update(dict_ent) return data_dict @@ -439,16 +441,16 @@ def get_historical_stock_data(self, start_date, end_date, time_interval): # Private Method for Functions needing stock_price_data def _stock_price_data(self, data_field): if isinstance(self.ticker, str): - return self.get_stock_price_data()[self.ticker][data_field] + return self.get_stock_price_data()[self.ticker].get(data_field, None) else: - return {tick: self.get_stock_price_data()[tick][data_field] for tick in self.ticker} + return {tick: self.get_stock_price_data()[tick].get(data_field, None) for tick in self.ticker} # Private Method for Functions needing stock_price_data def _stock_summary_data(self, data_field): if isinstance(self.ticker, str): - return self.get_stock_summary_data()[self.ticker][data_field] + return self.get_stock_summary_data()[self.ticker].get(data_field, None) else: - return {tick: self.get_stock_summary_data()[tick][data_field] for tick in self.ticker} + return {tick: self.get_stock_summary_data()[tick].get(data_field, None) for tick in self.ticker} # Private Method for Functions needing financial statement data def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): @@ -462,7 +464,10 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): else: data = {} for tick in self.ticker: - date_key = re_data[tick][0].keys()[0] + try: + date_key = re_data[tick][0].keys()[0] + except: + date_key = list(re_data[tick][0].keys())[0] sub_data = re_data[tick][0][date_key][field_name] data.update({tick: sub_data}) return data @@ -561,11 +566,50 @@ def get_ebit(self): def get_net_income(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') + def get_interest_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') + + def get_operating_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') + + def get_total_operating_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') + + def get_total_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalRevenue', 'annual') + + def get_cost_of_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'costOfRevenue', 'annual') + + def get_income_before_tax(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeBeforeTax', 'annual') + + def get_income_tax_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeTaxExpense', 'annual') + + def get_gross_profit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') + + def get_net_income_from_continuing_ops(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncomeFromContinuingOps', 'annual') + + def get_research_and_development(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') + # Calculated Financial Methods def get_earnings_per_share(self): price_data = self.get_current_price() pe_ratio = self.get_pe_ratio() if isinstance(self.ticker, str): - return price_data / pe_ratio + if price_data is not None and pe_ratio is not None: + return price_data / pe_ratio + else: + return None else: - return {tick: price_data[tick] / pe_ratio[tick] for tick in self.ticker} + ret_obj = {} + for tick in self.ticker: + if price_data[tick] is not None and pe_ratio[tick] is not None: + ret_obj.update({tick: price_data[tick] / pe_ratio[tick]}) + else: + ret_obj.update({tick: None}) + return ret_obj From 3a9a41804da3c5e6f1c37669d60f5a84804e4045 Mon Sep 17 00:00:00 2001 From: alt Date: Fri, 3 Aug 2018 01:25:57 -0500 Subject: [PATCH 014/108] v0.7 push --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index ae85377..06d5295 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ # yahoofinancials Welcome to Yahoo Financials! + Current Version: v0.7 + Version Released: 08/03/2018 + New Contributers Welcomed! + Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. ## Overview From ba9af49ab1d0b62ff7f56c4e35d7666f6b0e0161 Mon Sep 17 00:00:00 2001 From: alt Date: Fri, 3 Aug 2018 02:20:09 -0500 Subject: [PATCH 015/108] Readme update - demo instructions --- README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 06d5295..3424a53 100644 --- a/README.md +++ b/README.md @@ -23,12 +23,19 @@ The module uses a web scraping technique for retrieving the data, thus eliminati ```R $ pip install yahoofinancials ``` -2. Installation using github +2. Installation using github: ```R $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials $ python setup.py install ``` +3. Demo using the included demo script: +```R +$ cd yahoofinancials +$ python demo.py -h +$ python demo.py +$ python demo.py WFC C BAC +``` ## Features Financial Data is returned as JSON. From 59b0556144da114d3e7bf8dc80a1710119fa7716 Mon Sep 17 00:00:00 2001 From: alt Date: Mon, 6 Aug 2018 22:30:52 -0500 Subject: [PATCH 016/108] Updated Readme --- README.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 3424a53..d9296bc 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,15 @@ The module uses a web scraping technique for retrieving the data, thus eliminati * the package depends on beautifulsoup4 and requests to work 1. Installation using pip: +* Linux/Mac: ```R $ pip install yahoofinancials ``` -2. Installation using github: +* Windows (If python doesn't work for you in cmd, try running the following command with just py): +```R +> python -m pip install yahoofinancials +``` +2. Installation using github (Mac/Linux): ```R $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials @@ -37,12 +42,12 @@ $ python demo.py $ python demo.py WFC C BAC ``` -## Features -Financial Data is returned as JSON. -Run multiple stocks at once in groupings -Works on most versions of python 2 and 3 -List of Current Included Methods: +## Module Methods +* The financial data from all methods is returned as JSON. +* You can run multiple stocks at once using an inputted array or run an individual stock using an inputted string. +* YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux) +### Featured Methods 1. get_financial_stmts(frequency, statement_type, reformat=True) * frequency can be either 'annual' or 'quarterly'. * statement_type can be 'income', 'balance', 'cash' or a list of several. @@ -71,7 +76,7 @@ List of Current Included Methods: * get_net_income_from_continuing_ops(): * get_research_and_development(): -### Module Methods +### Additional Module Methods * get_current_price(): * get_current_change(): * get_current_percent_change(): From d9d1dc9ecd4be0208894e4172f3ccaffccfe7ec9 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 11 Aug 2018 01:02:41 -0500 Subject: [PATCH 017/108] v0.8 push --- CHANGES | 5 +-- README.md | 12 ++++--- setup.py | 4 +-- yahoofinancials/__init__.py | 66 ++++++++++++++++++++++++++++++++++--- 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index 5d9c370..7e8c402 100644 --- a/CHANGES +++ b/CHANGES @@ -6,5 +6,6 @@ 0.6 07/06/2018 -- Added exception handler that returns a ticker with no available data with a null value. 0.7 08/03/2018 -- Merged Slyvandb's improvements into the master branch. 0.7 08/03/2018 -- Added a try catch at line 465 to explicitly type cast the dict keys to a list if the initial attempt fails. -0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567 -0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None +0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567. +0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None. +0.8 08/11/2018 -- Added a new method to get the current day's shares outstanding called get_num_shares_outstanding() starting on line 617. diff --git a/README.md b/README.md index d9296bc..0a3db82 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # yahoofinancials Welcome to Yahoo Financials! -Current Version: v0.7 +Current Version: v0.8 -Version Released: 08/03/2018 +Version Released: 08/11/2018 New Contributers Welcomed! @@ -64,7 +64,11 @@ $ python demo.py WFC C BAC * end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. * time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. -### 10 New Module Methods Added in v0.7 +### New Module Method Added in v0.8 +* get_num_shares_outstanding(price_type='current'): +price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. + +### Additional Module Methods * get_interest_expense(): * get_operating_income(): * get_total_operating_expense(): @@ -75,8 +79,6 @@ $ python demo.py WFC C BAC * get_gross_profit(): * get_net_income_from_continuing_ops(): * get_research_and_development(): - -### Additional Module Methods * get_current_price(): * get_current_change(): * get_current_percent_change(): diff --git a/setup.py b/setup.py index 50a025f..974ba51 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name='yahoofinancials', - version='0.7', + version='0.8', description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.7.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.8.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 8cc1133..ecfb218 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 0.7 +Version: 0.8 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/03/2018 +Version Released: 8/11/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -201,6 +201,8 @@ def _clean_earnings_data(self, raw_data): # Private method to clean summary and price reports def _clean_reports(self, raw_data): cleaned_dict = {} + if raw_data is None: + return None for k, v in raw_data.items(): if 'Time' in k: formatted_utc_time = self._format_time(v) @@ -441,16 +443,32 @@ def get_historical_stock_data(self, start_date, end_date, time_interval): # Private Method for Functions needing stock_price_data def _stock_price_data(self, data_field): if isinstance(self.ticker, str): + if self.get_stock_price_data()[self.ticker] is None: + return None return self.get_stock_price_data()[self.ticker].get(data_field, None) else: - return {tick: self.get_stock_price_data()[tick].get(data_field, None) for tick in self.ticker} + ret_obj = {} + for tick in self.ticker: + if self.get_stock_price_data()[tick] is None: + ret_obj.update({tick: None}) + else: + ret_obj.update({tick: self.get_stock_price_data()[tick].get(data_field, None)}) + return ret_obj # Private Method for Functions needing stock_price_data def _stock_summary_data(self, data_field): if isinstance(self.ticker, str): + if self.get_stock_summary_data()[self.ticker] is None: + return None return self.get_stock_summary_data()[self.ticker].get(data_field, None) else: - return {tick: self.get_stock_summary_data()[tick].get(data_field, None) for tick in self.ticker} + ret_obj = {} + for tick in self.ticker: + if self.get_stock_summary_data()[tick] is None: + ret_obj.update({tick: None}) + else: + ret_obj.update({tick: self.get_stock_summary_data()[tick].get(data_field, None)}) + return ret_obj # Private Method for Functions needing financial statement data def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): @@ -613,3 +631,43 @@ def get_earnings_per_share(self): else: ret_obj.update({tick: None}) return ret_obj + + def get_num_shares_outstanding(self, price_type='current'): + today_low = self._stock_summary_data('dayHigh') + today_high = self._stock_summary_data('dayLow') + cur_market_cap = self._stock_summary_data('marketCap') + if isinstance(self.ticker, str): + if cur_market_cap is not None: + if price_type == 'current': + current = self.get_current_price() + if current is not None: + today_average = current + else: + return None + else: + if today_high is not None and today_low is not None: + today_average = (today_high + today_low) / 2 + else: + return None + return cur_market_cap / today_average + else: + return None + else: + ret_obj = {} + for tick in self.ticker: + if cur_market_cap[tick] is not None: + if price_type == 'current': + current = self.get_current_price() + if current[tick] is not None: + ret_obj.update({tick: cur_market_cap[tick] / current[tick]}) + else: + ret_obj.update({tick: None}) + else: + if today_low[tick] is not None and today_high[tick] is not None: + today_average = (today_high[tick] + today_low[tick]) / 2 + ret_obj.update({tick: cur_market_cap[tick] / today_average}) + else: + ret_obj.update({tick: None}) + else: + ret_obj.update({tick: None}) + return ret_obj From d3eebd7efdea65a18248cf9e5a621be5f8206742 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 11 Aug 2018 01:04:20 -0500 Subject: [PATCH 018/108] v0.8 push --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0a3db82..c475e36 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ $ python demo.py WFC C BAC ### New Module Method Added in v0.8 * get_num_shares_outstanding(price_type='current'): -price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. + - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. ### Additional Module Methods * get_interest_expense(): From 1d906cff5f10a8a7f5c99cd87de6ec357d58f580 Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 14 Aug 2018 03:10:15 -0500 Subject: [PATCH 019/108] version 0.9 push --- CHANGES | 4 +- README.md | 205 +++++++++++++++++++++++++++++------- setup.py | 8 +- yahoofinancials/__init__.py | 151 ++++++++++++++++++++++---- 4 files changed, 306 insertions(+), 62 deletions(-) diff --git a/CHANGES b/CHANGES index 7e8c402..a2515fe 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,4 @@ -0.1, 10/13/2017 -- Initial release. +0.1 10/13/2017 -- Initial release. 0.2 10/20/2017 -- Added New Methods and Restructured Module Classes 0.3 10/25/2017 -- Added New Methods and Calculated Measures 0.4 03/06/2018 -- Fixed reported bug on line 470 of init.py by adding a try and except with suggested line of code. @@ -9,3 +9,5 @@ 0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567. 0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None. 0.8 08/11/2018 -- Added a new method to get the current day's shares outstanding called get_num_shares_outstanding() starting on line 617. +0.9 08/14/2018 -- Added a new method called get_historical_price_data() to get price data for commodity futures, indexes, currencies, and cryptos in addition to stocks. +0.9 08/14/2018 -- Depreciated the get_historical_stock_data() method and scheduled it's removal for version 1.0. diff --git a/README.md b/README.md index c475e36..a01b15f 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,23 @@ # yahoofinancials Welcome to Yahoo Financials! -Current Version: v0.8 - -Version Released: 08/11/2018 +Current Version: v0.9 +Version Released: 08/14/2018 New Contributers Welcomed! Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. ## Overview -A powerful financial data module used for pulling both fundamental and technical stock data from Yahoo Finance. -The module uses a web scraping technique for retrieving the data, thus eliminating the need for the now discontinued Yahoo Finance API. +A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. +* As of Version 0.9, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, currencies, indexes, and stocks. + ## Installation * yahoofinancials runs fine on most versions of python 2 and 3. * It was built and tested using versions 2.7 and 3.4 -* the package depends on beautifulsoup4 and requests to work +* The package depends on beautifulsoup4 and requests to work. 1. Installation using pip: * Linux/Mac: @@ -44,8 +44,8 @@ $ python demo.py WFC C BAC ## Module Methods * The financial data from all methods is returned as JSON. -* You can run multiple stocks at once using an inputted array or run an individual stock using an inputted string. -* YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux) +* You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. +* YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). ### Featured Methods 1. get_financial_stmts(frequency, statement_type, reformat=True) @@ -59,14 +59,19 @@ $ python demo.py WFC C BAC 4. get_stock_summary_data(reformat=True) * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5. get_stock_quote_type_data() -6. get_historical_stock_data(start_date, end_date, time_interval) - * start_date should be entered in the 'YYYY-MM-DD' format. First day that stock data will be pulled. - * end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. - * time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. +6. get_historical_price_data(start_date, end_date, time_interval) + * New in v0.9. + * This method will pull historical pricing data for stocks, currencies, cryptocurrencies, commodities, and indexes. + * start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. + * end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. + * time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. + * Data response includes relevant pricing event data such as dividends and stock splits. +7. get_num_shares_outstanding(price_type='current'): + * price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -### New Module Method Added in v0.8 -* get_num_shares_outstanding(price_type='current'): - - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. +### Methods Depreciated in v0.9 +* get_historical_stock_data(): + - Scheduled for removal in v1.0 ### Additional Module Methods * get_interest_expense(): @@ -113,8 +118,8 @@ $ python demo.py WFC C BAC ## Usage Examples * The class constructor can take either a single ticker or a list of tickers as it's parameter. -* This makes it easy to construct multiple classes for different groupings of stocks -* Quarterly statement data returns the last 4 periods of data, while annual returns the past 3. +* This makes it easy to initiate multiple classes for different groupings of financial assets. +* Quarterly statement data returns the last 4 periods of data, while annual returns the last 3. ### Single Ticker Example ```R @@ -128,7 +133,7 @@ income_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'in all_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', ['income', 'cash', 'balance']) apple_earnings_data = yahoo_financials.get_stock_earnings_data() apple_net_income = yahoo_financials.get_net_income() -historical_stock_prices = yahoo_financials.get_historical_stock_data('2015-01-15', '2017-10-15', 'weekly') +historical_stock_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') ``` ### Lists of Tickers Example @@ -137,16 +142,25 @@ from yahoofinancials import YahooFinancials tech_stocks = ['AAPL', 'MSFT', 'INTC'] bank_stocks = ['WFC', 'BAC', 'C'] +commodity_futures = ['GC=F', 'SI=F', 'CL=F'] +cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] +currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] yahoo_financials_tech = YahooFinancials(tech_stocks) yahoo_financials_banks = YahooFinancials(bank_stocks) +yahoo_financials_commodities = YahooFinancials(commodity_futures) +yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) +yahoo_financials_currencies = YahooFinancials(currencies) tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') banks_net_ebit = yahoo_financials_banks.get_ebit() -tech_stock_price_data = tech_cash_flow_data.get_stock_price_data() -daily_bank_stock_prices = yahoo_financials_banks.get_historical_stock_data('2008-09-15', '2017-09-15', 'daily') +tech_stock_price_data = yahoo_financials_tech.get_stock_price_data() +daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') +daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') +daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') +daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') ``` ## Examples of Returned JSON Data @@ -265,40 +279,157 @@ print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) } } ``` -4. Monthly Stock Price History Data for Wells Fargo: +4. Monthly Historical Stock Price Data for Wells Fargo: ```R yahoo_financials = YahooFinancials('WFC') -print(yahoo_financials.get_historical_stock_data("2017-09-10", "2017-10-10", "monthly")) +print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) ``` ```javascript { "WFC": { - "prices": [ - { - "volume": 260271600, - "formatted_date": "2017-09-30", - "high": 55.77000045776367, - "adjclose": 54.91999816894531, - "low": 52.84000015258789, - "date": 1506830400, - "close": 54.91999816894531, - "open": 55.15999984741211 + "currency": "USD", + "eventsData": { + "dividends": { + "2018-08-01": { + "amount": 0.43, + "date": 1533821400, + "formatted_date": "2018-08-09" + } } - ], - "eventsData": [], + }, "firstTradeDate": { "date": 76233600, "formatted_date": "1972-06-01" }, - "isPending": false, + "instrumentType": "EQUITY", + "prices": [ + { + "adjclose": 57.19147872924805, + "close": 57.61000061035156, + "date": 1533096000, + "formatted_date": "2018-08-01", + "high": 59.5, + "low": 57.08000183105469, + "open": 57.959999084472656, + "volume": 138922900 + } + ], "timeZone": { "gmtOffset": -14400 + } + } +} +``` +5. Monthly Historical Price Data for EURUSD: +```R +yahoo_financials = YahooFinancials('EURUSD=X') +print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) +``` +```javascript +{ + "EURUSD=X": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1070236800, + "formatted_date": "2003-12-01" + }, + "instrumentType": "CURRENCY", + "prices": [ + { + "adjclose": 1.1394712924957275, + "close": 1.1394712924957275, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 1.169864296913147, + "low": 1.1365960836410522, + "open": 1.168961763381958, + "volume": 0 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } +} +``` +6. Monthly Historical Price Data for BTC-USD: +```R +yahoo_financials = YahooFinancials('BTC-USD') +print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) +``` +```javascript +{ + "BTC-USD": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1279321200, + "formatted_date": "2010-07-16" }, - "id": "1mo15050196001507611600" + "instrumentType": "CRYPTOCURRENCY", + "prices": [ + { + "adjclose": 6285.02001953125, + "close": 6285.02001953125, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 7760.740234375, + "low": 6133.02978515625, + "open": 7736.25, + "volume": 4334347882 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } +} +``` +7. Weekly Historical Price Data for Crude Oil Futures: +```R +yahoo_financials = YahooFinancials('CL=F') +print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) +``` +```javascript +{ + "CL=F": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1522555200, + "formatted_date": "2018-04-01" + }, + "instrumentType": "FUTURE", + "prices": [ + { + "adjclose": 68.58999633789062, + "close": 68.58999633789062, + "date": 1532923200, + "formatted_date": "2018-07-30", + "high": 69.3499984741211, + "low": 66.91999816894531, + "open": 68.37000274658203, + "volume": 683048039 + }, + { + "adjclose": 67.75, + "close": 67.75, + "date": 1533528000, + "formatted_date": "2018-08-06", + "high": 69.91999816894531, + "low": 66.13999938964844, + "open": 68.76000213623047, + "volume": 1102357981 + } + ], + "timeZone": { + "gmtOffset": -14400 + } } } ``` -5. Apple Stock Quote Data: +8. Apple Stock Quote Data: ```R yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) diff --git a/setup.py b/setup.py index 974ba51..31701e3 100644 --- a/setup.py +++ b/setup.py @@ -5,14 +5,14 @@ setup( name='yahoofinancials', - version='0.8', - description='A powerful financial data module used for pulling fundamental and technical stock data from Yahoo Finance', + version='0.9', + description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.8.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.9.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', - keywords = ['finance data', 'stocks', 'yahoo finance'], + keywords = ['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], packages=['yahoofinancials'], install_requires=[ "requests", diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index ecfb218..c82262f 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 0.8 +Version: 0.9 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/11/2018 +Version Released: 8/14/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -25,20 +25,21 @@ 4) get_stock_summary_data(reformat=True) - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5) get_stock_quote_type_data() -6) get_historical_stock_data(start_date, end_date, time_interval) - - start_date should be entered in the 'YYYY-MM-DD' format. First day that stock data will be pulled. - - end_date should be entered in the 'YYYY-MM-DD' format. Last day that stock data will be pulled. +6) get_historical_price_data(start_date, end_date, time_interval) + - Gets historical price data for currencies, stocks, indexes, cryptocurrencies, and commodity futures. + - start_date should be entered in the 'YYYY-MM-DD' format. First day that financial data will be pulled. + - end_date should be entered in the 'YYYY-MM-DD' format. Last day that financial data will be pulled. - time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. Usage Examples: from yahoofinancials import YahooFinancials #tickers = 'AAPL' #or -tickers = ['AAPL', 'WFC', 'F'] +tickers = ['AAPL', 'WFC', 'F', 'JPY=X', 'XRP-USD', 'GC=F'] yahoo_financials = YahooFinancials(tickers) balance_sheet_data = yahoo_financials.get_financial_stmts('quarterly', 'balance') earnings_data = yahoo_financials.get_stock_earnings_data() -historical_stock_prices = yahoo_financials.get_historical_stock_data('2015-01-15', '2017-10-15', 'weekly') +historical_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') """ import sys @@ -224,10 +225,16 @@ def _clean_reports(self, raw_data): cleaned_dict.update(dict_ent) return cleaned_dict + # Private Static Method to ensure ticker is URL encoded + @staticmethod + def _encode_ticker(ticker_str): + encoded_ticker = ticker_str.replace('=', '%3D') + return encoded_ticker + # Private method to get time interval code def _build_historical_url(self, ticker, hist_oj): - url = self._BASE_YAHOO_URL + ticker + '/history?period1=' + str(hist_oj['start']) + '&period2=' + \ - str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ + url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ + '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ hist_oj['interval'] return url @@ -235,7 +242,21 @@ def _build_historical_url(self, ticker, hist_oj): def _clean_historical_data(self, hist_data): data = {} for k, v in hist_data.items(): - if 'date' in k.lower(): + if k == 'eventsData': + event_obj = {} + if isinstance(v, list): + dict_ent = {k: event_obj} + else: + for type_key, type_obj in v.items(): + formatted_type_obj = {} + for date_key, date_obj in type_obj.items(): + formatted_date_key = self.format_date(int(date_key)) + cleaned_date = self.format_date(int(date_obj['date'])) + date_obj.update({u'' + 'formatted_date': cleaned_date}) + formatted_type_obj.update({formatted_date_key: date_obj}) + event_obj.update({type_key: formatted_type_obj}) + dict_ent = {k: event_obj} + elif 'date' in k.lower(): cleaned_date = self.format_date(v) dict_ent = {k: {u'' + 'formatted_date': cleaned_date, 'date': v}} elif isinstance(v, list): @@ -249,6 +270,62 @@ def _clean_historical_data(self, hist_data): data.update(dict_ent) return data + # Private Static Method to build API url for GET Request + @staticmethod + def _build_api_url(hist_obj, up_ticker): + base_url = "https://query1.finance.yahoo.com/v8/finance/chart/" + api_url = base_url + up_ticker + '?symbol= ' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' +\ + str(hist_obj['end']) + '&interval=' + hist_obj['interval'] + api_url += '&events=div|split|earn' + return api_url + + # Private Static Method to get financial data via API Call + @staticmethod + def _get_api_data(api_url): + response = requests.get(api_url) + if sys.version_info < (3, 0): + return loads(response.content) + return loads(response.content.decode('utf-8')) + + # Private Method to clean API data + def _clean_api_data(self, api_url): + raw_data = self._get_api_data(api_url) + ret_obj = {} + ret_obj.update({'eventsData': []}) + results = raw_data['chart']['result'] + if results is None: + return ret_obj + for result in results: + tz_sub_dict = {} + ret_obj.update({'eventsData': result.get('events', {})}) + ret_obj.update({'firstTradeDate': result['meta'].get('firstTradeDate', 'NA')}) + ret_obj.update({'currency': result['meta'].get('currency', 'NA')}) + ret_obj.update({'instrumentType': result['meta'].get('instrumentType', 'NA')}) + tz_sub_dict.update({'gmtOffset': result['meta']['gmtoffset']}) + ret_obj.update({'timeZone': tz_sub_dict}) + timestamp_list = result['timestamp'] + high_price_list = result['indicators']['quote'][0]['high'] + low_price_list = result['indicators']['quote'][0]['low'] + open_price_list = result['indicators']['quote'][0]['open'] + close_price_list = result['indicators']['quote'][0]['close'] + volume_list = result['indicators']['quote'][0]['volume'] + adj_close_list = result['indicators']['adjclose'][0]['adjclose'] + i = 0 + prices_list = [] + for timestamp in timestamp_list: + price_dict = {} + price_dict.update({'date': timestamp}) + price_dict.update({'high': high_price_list[i]}) + price_dict.update({'low': low_price_list[i]}) + price_dict.update({'open': open_price_list[i]}) + price_dict.update({'close': close_price_list[i]}) + price_dict.update({'volume': volume_list[i]}) + price_dict.update({'adjclose': adj_close_list[i]}) + prices_list.append(price_dict) + i += 1 + ret_obj.update({'prices': prices_list}) + return ret_obj + # Private Method to take scrapped data and build a data dictionary with def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ @@ -269,10 +346,15 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi else: YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + api_url = self._build_api_url(hist_obj, up_ticker) + re_data = self._clean_api_data(api_url) cleaned_re_data = self._clean_historical_data(re_data) except KeyError: - cleaned_re_data = None + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + cleaned_re_data = self._clean_historical_data(re_data) + except KeyError: + cleaned_re_data = None dict_ent = {up_ticker: cleaned_re_data} return dict_ent @@ -361,16 +443,28 @@ def get_clean_data(self, raw_report_data, report_type): cleaned_data_dict = {} if isinstance(self.ticker, str): if report_type == 'earnings': - cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) + try: + cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) + except: + cleaned_data = None else: - cleaned_data = self._clean_reports(raw_report_data[self.ticker]) + try: + cleaned_data = self._clean_reports(raw_report_data[self.ticker]) + except: + cleaned_data = None cleaned_data_dict.update({self.ticker: cleaned_data}) else: for tick in self.ticker: if report_type == 'earnings': - cleaned_data = self._clean_earnings_data(raw_report_data[tick]) + try: + cleaned_data = self._clean_earnings_data(raw_report_data[tick]) + except: + cleaned_data = None else: - cleaned_data = self._clean_reports(raw_report_data[tick]) + try: + cleaned_data = self._clean_reports(raw_report_data[tick]) + except: + cleaned_data = None cleaned_data_dict.update({tick: cleaned_data}) return cleaned_data_dict @@ -431,8 +525,19 @@ def get_stock_summary_url(self): def get_stock_quote_type_data(self): return self.get_stock_tech_data('quoteType') - # Public Method for user to get historical stock data with + # Public Method for user to get historical stock data with (SOON TO BE DEPRECIATED IN V1.0) def get_historical_stock_data(self, start_date, end_date, time_interval): + interval_code = self.get_time_code(time_interval) + start = self.format_date(start_date) + end = self.format_date(end_date) + hist_obj = {'start': start, 'end': end, 'interval': interval_code} + data = self.get_stock_data('history', hist_obj=hist_obj) + print("***WARNING: AS OF v0.9 'get_historical_stock_data()' IS DEPRECIATED AND WILL BE REMOVED IN THE " + "v1.0 RELEASE.***\n***PLEASE USE 'get_historical_price_data()' INSTEAD.***") + return data + + # Public Method for user to get historical stock data with (SOON TO BE DEPRECIATED IN V1.0) + def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) start = self.format_date(start_date) end = self.format_date(end_date) @@ -485,9 +590,15 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): try: date_key = re_data[tick][0].keys()[0] except: - date_key = list(re_data[tick][0].keys())[0] - sub_data = re_data[tick][0][date_key][field_name] - data.update({tick: sub_data}) + try: + date_key = list(re_data[tick][0].keys())[0] + except: + date_key = None + if date_key is not None: + sub_data = re_data[tick][0][date_key][field_name] + data.update({tick: sub_data}) + else: + data.update({tick: None}) return data # Public Price Data Methods From 4a61ca9ec067e45988970c1cf56d9e97f0577da3 Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 14 Aug 2018 19:10:12 -0500 Subject: [PATCH 020/108] v0.10 push --- CHANGES | 2 + README.md | 95 ++++++++++++++++++++++++++++++++++--- setup.py | 4 +- yahoofinancials/__init__.py | 24 +++++++--- 4 files changed, 110 insertions(+), 15 deletions(-) diff --git a/CHANGES b/CHANGES index a2515fe..1b1781f 100644 --- a/CHANGES +++ b/CHANGES @@ -11,3 +11,5 @@ 0.8 08/11/2018 -- Added a new method to get the current day's shares outstanding called get_num_shares_outstanding() starting on line 617. 0.9 08/14/2018 -- Added a new method called get_historical_price_data() to get price data for commodity futures, indexes, currencies, and cryptos in addition to stocks. 0.9 08/14/2018 -- Depreciated the get_historical_stock_data() method and scheduled it's removal for version 1.0. +0.10 08/14/2018 -- Added a new Method to get summary data for stocks, indexes, cryptocurrencies, currencies, and commodity futures, get_summary_data(). +0.10 08/14/2018 -- Depreciated the get_stock_summary_data() method and scheduled it's removal for version 1.0. diff --git a/README.md b/README.md index a01b15f..414f90a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # yahoofinancials Welcome to Yahoo Financials! -Current Version: v0.9 +Current Version: v0.10 Version Released: 08/14/2018 @@ -11,8 +11,7 @@ Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title i ## Overview A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. -* As of Version 0.9, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, currencies, indexes, and stocks. - +* As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. ## Installation * yahoofinancials runs fine on most versions of python 2 and 3. @@ -56,12 +55,14 @@ $ python demo.py WFC C BAC * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 3. get_stock_earnings_data(reformat=True) * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -4. get_stock_summary_data(reformat=True) +4. get_summary_data(reformat=True) + * New in v0.10. + * Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5. get_stock_quote_type_data() 6. get_historical_price_data(start_date, end_date, time_interval) * New in v0.9. - * This method will pull historical pricing data for stocks, currencies, cryptocurrencies, commodities, and indexes. + * This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. * start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. * end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. * time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. @@ -69,7 +70,9 @@ $ python demo.py WFC C BAC 7. get_num_shares_outstanding(price_type='current'): * price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -### Methods Depreciated in v0.9 +### Methods Depreciated in v0.9 & v0.10 +* get_stock_summary_data(): + - Scheduled for removal in v1.0 * get_historical_stock_data(): - Scheduled for removal in v1.0 @@ -145,12 +148,16 @@ bank_stocks = ['WFC', 'BAC', 'C'] commodity_futures = ['GC=F', 'SI=F', 'CL=F'] cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] +mutual_funds = ['PRLAX', 'QASGX', 'HISFX'] +us_treasuries = ['^TNX', '^IRX', '^TYX'] yahoo_financials_tech = YahooFinancials(tech_stocks) yahoo_financials_banks = YahooFinancials(bank_stocks) yahoo_financials_commodities = YahooFinancials(commodity_futures) yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) yahoo_financials_currencies = YahooFinancials(currencies) +yahoo_financials_mutualfunds = YahooFinancials(mutual_funds) +yahoo_financials_treasuries = YahooFinancials(us_treasuries) tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') @@ -161,6 +168,8 @@ daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008 daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') +daily_mutualfund_prices = yahoo_financials_mutualfunds.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') +daily_treasury_prices = yahoo_financials_treasuries.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') ``` ## Examples of Returned JSON Data @@ -454,3 +463,77 @@ print(yahoo_financials.get_stock_quote_type_data()) } } ``` +9. U.S. Treasury Current Pricing Data: +```R +yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) +print(yahoo_financials.get_current_price()) +``` +```javascript +{ + "^IRX": 2.033, + "^TNX": 2.895, + "^TYX": 3.062 +} +``` +10. BTC-USD Summary Data: +```R +yahoo_financials = YahooFinancials('BTC-USD') +print(yahoo_financials.get_summary_data()) +``` +```javascript +{ + "BTC-USD": { + "algorithm": "SHA256", + "ask": null, + "askSize": null, + "averageDailyVolume10Day": 545573809, + "averageVolume": 496761640, + "averageVolume10days": 545573809, + "beta": null, + "bid": null, + "bidSize": null, + "circulatingSupply": 17209812, + "currency": "USD", + "dayHigh": 6266.5, + "dayLow": 5891.87, + "dividendRate": null, + "dividendYield": null, + "exDividendDate": "-", + "expireDate": "-", + "fiftyDayAverage": 6989.074, + "fiftyTwoWeekHigh": 19870.62, + "fiftyTwoWeekLow": 2979.88, + "fiveYearAvgDividendYield": null, + "forwardPE": null, + "fromCurrency": "BTC", + "lastMarket": "CCCAGG", + "marketCap": 106325663744, + "maxAge": 1, + "maxSupply": 21000000, + "navPrice": null, + "open": 6263.2, + "openInterest": null, + "payoutRatio": null, + "previousClose": 6263.2, + "priceHint": 2, + "priceToSalesTrailing12Months": null, + "regularMarketDayHigh": 6266.5, + "regularMarketDayLow": 5891.87, + "regularMarketOpen": 6263.2, + "regularMarketPreviousClose": 6263.2, + "regularMarketVolume": 755834368, + "startDate": "2009-01-03", + "strikePrice": null, + "totalAssets": null, + "tradeable": false, + "trailingAnnualDividendRate": null, + "trailingAnnualDividendYield": null, + "twoHundredDayAverage": 8165.154, + "volume": 755834368, + "volume24Hr": 750196480, + "volumeAllCurrencies": 2673437184, + "yield": null, + "ytdReturn": null + } +} +``` diff --git a/setup.py b/setup.py index 31701e3..720d040 100644 --- a/setup.py +++ b/setup.py @@ -5,10 +5,10 @@ setup( name='yahoofinancials', - version='0.9', + version='0.10', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.9.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/0.10.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index c82262f..8762467 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,7 +1,7 @@ """ ============================== The Yahoo Financials Module -Version: 0.9 +Version: 0.10 ============================== Author: Connor Sanders @@ -22,7 +22,7 @@ - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 3) get_stock_earnings_data(reformat=True) - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -4) get_stock_summary_data(reformat=True) +4) get_summary_data(reformat=True) - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5) get_stock_quote_type_data() 6) get_historical_price_data(start_date, end_date, time_interval) @@ -510,6 +510,15 @@ def get_stock_earnings_data(self, reformat=True): # Public Method for the user to get stock summary data def get_stock_summary_data(self, reformat=True): + print("***WARNING: AS OF v0.10 'get_stock_summary_data()' IS DEPRECIATED AND WILL BE REMOVED IN THE " + "v1.0 RELEASE.***\n***PLEASE USE 'get_summary_data()' INSTEAD.***") + if reformat: + return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') + else: + return self.get_stock_tech_data('summaryDetail') + + # Public Method for the user to get stock summary data + def get_summary_data(self, reformat=True): if reformat: return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') else: @@ -563,16 +572,16 @@ def _stock_price_data(self, data_field): # Private Method for Functions needing stock_price_data def _stock_summary_data(self, data_field): if isinstance(self.ticker, str): - if self.get_stock_summary_data()[self.ticker] is None: + if self.get_summary_data()[self.ticker] is None: return None - return self.get_stock_summary_data()[self.ticker].get(data_field, None) + return self.get_summary_data()[self.ticker].get(data_field, None) else: ret_obj = {} for tick in self.ticker: - if self.get_stock_summary_data()[tick] is None: + if self.get_summary_data()[tick] is None: ret_obj.update({tick: None}) else: - ret_obj.update({tick: self.get_stock_summary_data()[tick].get(data_field, None)}) + ret_obj.update({tick: self.get_summary_data()[tick].get(data_field, None)}) return ret_obj # Private Method for Functions needing financial statement data @@ -720,7 +729,8 @@ def get_gross_profit(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') def get_net_income_from_continuing_ops(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncomeFromContinuingOps', 'annual') + return self._financial_statement_data('income', 'incomeStatementHistory', + 'netIncomeFromContinuingOps', 'annual') def get_research_and_development(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') From 2e799b46b4da65ad0c1004799518d831f2ba5b45 Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:29:36 -0500 Subject: [PATCH 021/108] v1.0 pre push --- CHANGES | 30 +- README.md | 539 ---------------------------------- README.rst | 551 +++++++++++++++++++++++++++++++++++ demo.py | 2 +- setup.cfg | 2 +- setup.py | 30 +- test/test_yahoofinancials.py | 105 +++++++ yahoofinancials/__init__.py | 55 ++-- 8 files changed, 726 insertions(+), 588 deletions(-) delete mode 100644 README.md create mode 100644 README.rst create mode 100644 test/test_yahoofinancials.py diff --git a/CHANGES b/CHANGES index 1b1781f..edec957 100644 --- a/CHANGES +++ b/CHANGES @@ -1,15 +1,19 @@ -0.1 10/13/2017 -- Initial release. -0.2 10/20/2017 -- Added New Methods and Restructured Module Classes -0.3 10/25/2017 -- Added New Methods and Calculated Measures -0.4 03/06/2018 -- Fixed reported bug on line 470 of init.py by adding a try and except with suggested line of code. -0.6 07/06/2018 -- Fixed reported bug in format_date function. -0.6 07/06/2018 -- Added exception handler that returns a ticker with no available data with a null value. -0.7 08/03/2018 -- Merged Slyvandb's improvements into the master branch. -0.7 08/03/2018 -- Added a try catch at line 465 to explicitly type cast the dict keys to a list if the initial attempt fails. -0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567. -0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None. -0.8 08/11/2018 -- Added a new method to get the current day's shares outstanding called get_num_shares_outstanding() starting on line 617. -0.9 08/14/2018 -- Added a new method called get_historical_price_data() to get price data for commodity futures, indexes, currencies, and cryptos in addition to stocks. -0.9 08/14/2018 -- Depreciated the get_historical_stock_data() method and scheduled it's removal for version 1.0. +0.1 10/13/2017 -- Initial release. +0.2 10/20/2017 -- Added New Methods and Restructured Module Classes +0.3 10/25/2017 -- Added New Methods and Calculated Measures +0.4 03/06/2018 -- Fixed reported bug on line 470 of init.py by adding a try and except with suggested line of code. +0.6 07/06/2018 -- Fixed reported bug in format_date function. +0.6 07/06/2018 -- Added exception handler that returns a ticker with no available data with a null value. +0.7 08/03/2018 -- Merged Slyvandb's improvements into the master branch. +0.7 08/03/2018 -- Added a try catch at line 465 to explicitly type cast the dict keys to a list if the initial attempt fails. +0.7 08/03/2018 -- Added 10 new income statement history methods beginning at line 567. +0.7 08/03/2018 -- Added a fix for trevorwelch's open issue involving the unnecessary sys.exit(1) on line 286 by replacing it with return None. +0.8 08/11/2018 -- Added a new method to get the current day's shares outstanding called get_num_shares_outstanding() starting on line 617. +0.9 08/14/2018 -- Added a new method called get_historical_price_data() to get price data for commodity futures, indexes, currencies, and cryptos in addition to stocks. +0.9 08/14/2018 -- Depreciated the get_historical_stock_data() method and scheduled it's removal for version 1.0. 0.10 08/14/2018 -- Added a new Method to get summary data for stocks, indexes, cryptocurrencies, currencies, and commodity futures, get_summary_data(). 0.10 08/14/2018 -- Depreciated the get_stock_summary_data() method and scheduled it's removal for version 1.0. +1.0 08/22/2018 -- Removed the get_historical_stock_data() method. +1.0 08/22/2018 -- Removed the get_stock_summary_data() method. +1.0 08/22/2018 -- Removed the requests dependency and replaced it with urllib. +1.0 08/22/2018 -- Updated README.md to README.rst diff --git a/README.md b/README.md deleted file mode 100644 index 414f90a..0000000 --- a/README.md +++ /dev/null @@ -1,539 +0,0 @@ -# yahoofinancials -Welcome to Yahoo Financials! - -Current Version: v0.10 - -Version Released: 08/14/2018 - -New Contributers Welcomed! - -Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. - -## Overview -A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. -* As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. - -## Installation -* yahoofinancials runs fine on most versions of python 2 and 3. -* It was built and tested using versions 2.7 and 3.4 -* The package depends on beautifulsoup4 and requests to work. - -1. Installation using pip: -* Linux/Mac: -```R -$ pip install yahoofinancials -``` -* Windows (If python doesn't work for you in cmd, try running the following command with just py): -```R -> python -m pip install yahoofinancials -``` -2. Installation using github (Mac/Linux): -```R -$ git clone https://github.com/JECSand/yahoofinancials.git -$ cd yahoofinancials -$ python setup.py install -``` -3. Demo using the included demo script: -```R -$ cd yahoofinancials -$ python demo.py -h -$ python demo.py -$ python demo.py WFC C BAC -``` - -## Module Methods -* The financial data from all methods is returned as JSON. -* You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -* YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). - -### Featured Methods -1. get_financial_stmts(frequency, statement_type, reformat=True) - * frequency can be either 'annual' or 'quarterly'. - * statement_type can be 'income', 'balance', 'cash' or a list of several. - * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -2. get_stock_price_data(reformat=True) - * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -3. get_stock_earnings_data(reformat=True) - * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -4. get_summary_data(reformat=True) - * New in v0.10. - * Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. - * reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -5. get_stock_quote_type_data() -6. get_historical_price_data(start_date, end_date, time_interval) - * New in v0.9. - * This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. - * start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. - * end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. - * time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. - * Data response includes relevant pricing event data such as dividends and stock splits. -7. get_num_shares_outstanding(price_type='current'): - * price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. - -### Methods Depreciated in v0.9 & v0.10 -* get_stock_summary_data(): - - Scheduled for removal in v1.0 -* get_historical_stock_data(): - - Scheduled for removal in v1.0 - -### Additional Module Methods -* get_interest_expense(): -* get_operating_income(): -* get_total_operating_expense(): -* get_total_revenue(): -* get_cost_of_revenue(): -* get_income_before_tax(): -* get_income_tax_expense(): -* get_gross_profit(): -* get_net_income_from_continuing_ops(): -* get_research_and_development(): -* get_current_price(): -* get_current_change(): -* get_current_percent_change(): -* get_current_volume(): -* get_prev_close_price(): -* get_open_price(): -* get_ten_day_avg_daily_volume(): -* get_three_month_avg_daily_volume(): -* get_stock_exchange(): -* get_market_cap(): -* get_daily_low(): -* get_daily_high(): -* get_currency(): -* get_yearly_high(): -* get_yearly_low(): -* get_dividend_yield(): -* get_annual_avg_div_yield(): -* get_five_yr_avg_div_yield(): -* get_dividend_rate(): -* get_annual_avg_div_rate(): -* get_50day_moving_avg(): -* get_200day_moving_avg(): -* get_beta(): -* get_payout_ratio(): -* get_pe_ratio(): -* get_price_to_sales(): -* get_exdividend_date(): -* get_book_value(): -* get_ebit(): -* get_net_income(): -* get_earnings_per_share(): - -## Usage Examples -* The class constructor can take either a single ticker or a list of tickers as it's parameter. -* This makes it easy to initiate multiple classes for different groupings of financial assets. -* Quarterly statement data returns the last 4 periods of data, while annual returns the last 3. - -### Single Ticker Example -```R -from yahoofinancials import YahooFinancials - -ticker = 'AAPL' -yahoo_financials = YahooFinancials(ticker) - -balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') -income_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'income') -all_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', ['income', 'cash', 'balance']) -apple_earnings_data = yahoo_financials.get_stock_earnings_data() -apple_net_income = yahoo_financials.get_net_income() -historical_stock_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') -``` - -### Lists of Tickers Example -```R -from yahoofinancials import YahooFinancials - -tech_stocks = ['AAPL', 'MSFT', 'INTC'] -bank_stocks = ['WFC', 'BAC', 'C'] -commodity_futures = ['GC=F', 'SI=F', 'CL=F'] -cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] -currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] -mutual_funds = ['PRLAX', 'QASGX', 'HISFX'] -us_treasuries = ['^TNX', '^IRX', '^TYX'] - -yahoo_financials_tech = YahooFinancials(tech_stocks) -yahoo_financials_banks = YahooFinancials(bank_stocks) -yahoo_financials_commodities = YahooFinancials(commodity_futures) -yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) -yahoo_financials_currencies = YahooFinancials(currencies) -yahoo_financials_mutualfunds = YahooFinancials(mutual_funds) -yahoo_financials_treasuries = YahooFinancials(us_treasuries) - -tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') -bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') - -banks_net_ebit = yahoo_financials_banks.get_ebit() -tech_stock_price_data = yahoo_financials_tech.get_stock_price_data() -daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -daily_mutualfund_prices = yahoo_financials_mutualfunds.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -daily_treasury_prices = yahoo_financials_treasuries.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') -``` - -## Examples of Returned JSON Data -1. Annual Income Statement Data for Apple: -```R -yahoo_financials = YahooFinancials('AAPL') -print(yahoo_financials.get_financial_stmts('annual', 'income')) -``` -```javascript -{ - "incomeStatementHistory": { - "AAPL": [ - { - "2016-09-24": { - "minorityInterest": null, - "otherOperatingExpenses": null, - "netIncomeFromContinuingOps": 45687000000, - "totalRevenue": 215639000000, - "totalOtherIncomeExpenseNet": 1348000000, - "discontinuedOperations": null, - "incomeTaxExpense": 15685000000, - "extraordinaryItems": null, - "grossProfit": 84263000000, - "netIncome": 45687000000, - "sellingGeneralAdministrative": 14194000000, - "interestExpense": null, - "costOfRevenue": 131376000000, - "researchDevelopment": 10045000000, - "netIncomeApplicableToCommonShares": 45687000000, - "effectOfAccountingCharges": null, - "incomeBeforeTax": 61372000000, - "otherItems": null, - "operatingIncome": 60024000000, - "ebit": 61372000000, - "nonRecurring": null, - "totalOperatingExpenses": 0 - } - } - ] - } -} -``` -2. Annual Balance Sheet Data for Apple: -```R -yahoo_financials = YahooFinancials('AAPL') -print(yahoo_financials.get_financial_stmts('annual', 'balance')) -``` -```javascript -{ - "balanceSheetHistory": { - "AAPL": [ - { - "2016-09-24": { - "otherCurrentLiab": 8080000000, - "otherCurrentAssets": 8283000000, - "goodWill": 5414000000, - "shortTermInvestments": 46671000000, - "longTermInvestments": 170430000000, - "cash": 20484000000, - "netTangibleAssets": 119629000000, - "totalAssets": 321686000000, - "otherLiab": 36074000000, - "totalStockholderEquity": 128249000000, - "inventory": 2132000000, - "retainedEarnings": 96364000000, - "intangibleAssets": 3206000000, - "totalCurrentAssets": 106869000000, - "otherStockholderEquity": 634000000, - "shortLongTermDebt": 11605000000, - "propertyPlantEquipment": 27010000000, - "deferredLongTermLiab": 2930000000, - "netReceivables": 29299000000, - "otherAssets": 8757000000, - "longTermDebt": 75427000000, - "totalLiab": 193437000000, - "commonStock": 31251000000, - "accountsPayable": 59321000000, - "totalCurrentLiabilities": 79006000000 - } - } - ] - } -} -``` -3. Quarterly Cash Flow Statement Data for Citigroup: -```R -yahoo_financials = YahooFinancials('C') -print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) -``` -```javascript -{ - "cashflowStatementHistoryQuarterly": { - "C": [ - { - "2017-06-30": { - "totalCashFromOperatingActivities": -18505000000, - "effectOfExchangeRate": -117000000, - "totalCashFromFinancingActivities": 39798000000, - "netIncome": 3872000000, - "dividendsPaid": -760000000, - "salePurchaseOfStock": -1781000000, - "capitalExpenditures": -861000000, - "changeToLiabilities": -7626000000, - "otherCashflowsFromInvestingActivities": 82000000, - "totalCashflowsFromInvestingActivities": -22508000000, - "netBorrowings": 33586000000, - "depreciation": 901000000, - "changeInCash": -1332000000, - "changeToNetincome": 1444000000, - "otherCashflowsFromFinancingActivities": 8753000000, - "changeToOperatingActivities": -17096000000, - "investments": -23224000000 - } - } - ] - } -} -``` -4. Monthly Historical Stock Price Data for Wells Fargo: -```R -yahoo_financials = YahooFinancials('WFC') -print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -``` -```javascript -{ - "WFC": { - "currency": "USD", - "eventsData": { - "dividends": { - "2018-08-01": { - "amount": 0.43, - "date": 1533821400, - "formatted_date": "2018-08-09" - } - } - }, - "firstTradeDate": { - "date": 76233600, - "formatted_date": "1972-06-01" - }, - "instrumentType": "EQUITY", - "prices": [ - { - "adjclose": 57.19147872924805, - "close": 57.61000061035156, - "date": 1533096000, - "formatted_date": "2018-08-01", - "high": 59.5, - "low": 57.08000183105469, - "open": 57.959999084472656, - "volume": 138922900 - } - ], - "timeZone": { - "gmtOffset": -14400 - } - } -} -``` -5. Monthly Historical Price Data for EURUSD: -```R -yahoo_financials = YahooFinancials('EURUSD=X') -print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -``` -```javascript -{ - "EURUSD=X": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1070236800, - "formatted_date": "2003-12-01" - }, - "instrumentType": "CURRENCY", - "prices": [ - { - "adjclose": 1.1394712924957275, - "close": 1.1394712924957275, - "date": 1533078000, - "formatted_date": "2018-07-31", - "high": 1.169864296913147, - "low": 1.1365960836410522, - "open": 1.168961763381958, - "volume": 0 - } - ], - "timeZone": { - "gmtOffset": 3600 - } - } -} -``` -6. Monthly Historical Price Data for BTC-USD: -```R -yahoo_financials = YahooFinancials('BTC-USD') -print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -``` -```javascript -{ - "BTC-USD": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1279321200, - "formatted_date": "2010-07-16" - }, - "instrumentType": "CRYPTOCURRENCY", - "prices": [ - { - "adjclose": 6285.02001953125, - "close": 6285.02001953125, - "date": 1533078000, - "formatted_date": "2018-07-31", - "high": 7760.740234375, - "low": 6133.02978515625, - "open": 7736.25, - "volume": 4334347882 - } - ], - "timeZone": { - "gmtOffset": 3600 - } - } -} -``` -7. Weekly Historical Price Data for Crude Oil Futures: -```R -yahoo_financials = YahooFinancials('CL=F') -print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) -``` -```javascript -{ - "CL=F": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1522555200, - "formatted_date": "2018-04-01" - }, - "instrumentType": "FUTURE", - "prices": [ - { - "adjclose": 68.58999633789062, - "close": 68.58999633789062, - "date": 1532923200, - "formatted_date": "2018-07-30", - "high": 69.3499984741211, - "low": 66.91999816894531, - "open": 68.37000274658203, - "volume": 683048039 - }, - { - "adjclose": 67.75, - "close": 67.75, - "date": 1533528000, - "formatted_date": "2018-08-06", - "high": 69.91999816894531, - "low": 66.13999938964844, - "open": 68.76000213623047, - "volume": 1102357981 - } - ], - "timeZone": { - "gmtOffset": -14400 - } - } -} -``` -8. Apple Stock Quote Data: -```R -yahoo_financials = YahooFinancials('AAPL') -print(yahoo_financials.get_stock_quote_type_data()) -``` -```javascript -{ - "AAPL": { - "underlyingExchangeSymbol": null, - "exchangeTimezoneName": "America/New_York", - "underlyingSymbol": null, - "headSymbol": null, - "shortName": "Apple Inc.", - "symbol": "AAPL", - "uuid": "8b10e4ae-9eeb-3684-921a-9ab27e4d87aa", - "gmtOffSetMilliseconds": "-14400000", - "exchange": "NMS", - "exchangeTimezoneShortName": "EDT", - "messageBoardId": "finmb_24937", - "longName": "Apple Inc.", - "market": "us_market", - "quoteType": "EQUITY" - } -} -``` -9. U.S. Treasury Current Pricing Data: -```R -yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) -print(yahoo_financials.get_current_price()) -``` -```javascript -{ - "^IRX": 2.033, - "^TNX": 2.895, - "^TYX": 3.062 -} -``` -10. BTC-USD Summary Data: -```R -yahoo_financials = YahooFinancials('BTC-USD') -print(yahoo_financials.get_summary_data()) -``` -```javascript -{ - "BTC-USD": { - "algorithm": "SHA256", - "ask": null, - "askSize": null, - "averageDailyVolume10Day": 545573809, - "averageVolume": 496761640, - "averageVolume10days": 545573809, - "beta": null, - "bid": null, - "bidSize": null, - "circulatingSupply": 17209812, - "currency": "USD", - "dayHigh": 6266.5, - "dayLow": 5891.87, - "dividendRate": null, - "dividendYield": null, - "exDividendDate": "-", - "expireDate": "-", - "fiftyDayAverage": 6989.074, - "fiftyTwoWeekHigh": 19870.62, - "fiftyTwoWeekLow": 2979.88, - "fiveYearAvgDividendYield": null, - "forwardPE": null, - "fromCurrency": "BTC", - "lastMarket": "CCCAGG", - "marketCap": 106325663744, - "maxAge": 1, - "maxSupply": 21000000, - "navPrice": null, - "open": 6263.2, - "openInterest": null, - "payoutRatio": null, - "previousClose": 6263.2, - "priceHint": 2, - "priceToSalesTrailing12Months": null, - "regularMarketDayHigh": 6266.5, - "regularMarketDayLow": 5891.87, - "regularMarketOpen": 6263.2, - "regularMarketPreviousClose": 6263.2, - "regularMarketVolume": 755834368, - "startDate": "2009-01-03", - "strikePrice": null, - "totalAssets": null, - "tradeable": false, - "trailingAnnualDividendRate": null, - "trailingAnnualDividendYield": null, - "twoHundredDayAverage": 8165.154, - "volume": 755834368, - "volume24Hr": 750196480, - "volumeAllCurrencies": 2673437184, - "yield": null, - "ytdReturn": null - } -} -``` diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..03721f6 --- /dev/null +++ b/README.rst @@ -0,0 +1,551 @@ +=============== +yahoofinancials +=============== + +Welcome to Yahoo Financials! + +Current Version: v1.0 + +Version Released: 08/22/2018 + +New Contributers Welcomed! + +Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. + +Overview +-------- +A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. +- As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. + +Installation +------------ +- yahoofinancials runs fine on most versions of python 2 and 3. +- It was built and tested using versions 2.7 and 3.5 +- The package depends on beautifulsoup4 and pytz to work. + +1. Installation using pip: +- Linux/Mac: +.. code:: bash + $ pip install yahoofinancials + +- Windows (If python doesn't work for you in cmd, try running the following command with just py): +.. code:: bash + > python -m pip install yahoofinancials + +2. Installation using github (Mac/Linux): +.. code:: bash + $ git clone https://github.com/JECSand/yahoofinancials.git + $ cd yahoofinancials + $ python setup.py install + +3. Demo using the included demo script: +.. code:: bash + $ cd yahoofinancials + $ python demo.py -h + $ python demo.py + $ python demo.py WFC C BAC + +4. Test using the included unit testing script: +.. code:: bash + $ cd yahoofinancials + $ python test/test_yahoofinancials.py + +Module Methods +-------------- +- The financial data from all methods is returned as JSON. +- You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. +- YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). + +Featured Methods +^^^^^^^^^^^^^^^^ +1. get_financial_stmts(frequency, statement_type, reformat=True) + - frequency can be either 'annual' or 'quarterly'. + - statement_type can be 'income', 'balance', 'cash' or a list of several. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +2. get_stock_price_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +3. get_stock_earnings_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +4. get_summary_data(reformat=True) + - New in v0.10. + - Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +5. get_stock_quote_type_data() +6. get_historical_price_data(start_date, end_date, time_interval) + - New in v0.9. + - This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. + - start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. + - end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. + - time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. + - Data response includes relevant pricing event data such as dividends and stock splits. +7. get_num_shares_outstanding(price_type='current'): + - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. + +Methods Removed in V1.0 +^^^^^^^^^^^^^^^^^^^^^^^ +- get_stock_summary_data(): +- get_historical_stock_data(): + +Additional Module Methods +^^^^^^^^^^^^^^^^^^^^^^^^^ +- get_interest_expense(): +- get_operating_income(): +- get_total_operating_expense(): +- get_total_revenue(): +- get_cost_of_revenue(): +- get_income_before_tax(): +- get_income_tax_expense(): +- get_gross_profit(): +- get_net_income_from_continuing_ops(): +- get_research_and_development(): +- get_current_price(): +- get_current_change(): +- get_current_percent_change(): +- get_current_volume(): +- get_prev_close_price(): +- get_open_price(): +- get_ten_day_avg_daily_volume(): +- get_three_month_avg_daily_volume(): +- get_stock_exchange(): +- get_market_cap(): +- get_daily_low(): +- get_daily_high(): +- get_currency(): +- get_yearly_high(): +- get_yearly_low(): +- get_dividend_yield(): +- get_annual_avg_div_yield(): +- get_five_yr_avg_div_yield(): +- get_dividend_rate(): +- get_annual_avg_div_rate(): +- get_50day_moving_avg(): +- get_200day_moving_avg(): +- get_beta(): +- get_payout_ratio(): +- get_pe_ratio(): +- get_price_to_sales(): +- get_exdividend_date(): +- get_book_value(): +- get_ebit(): +- get_net_income(): +- get_earnings_per_share(): + +Usage Examples +-------------- +- The class constructor can take either a single ticker or a list of tickers as it's parameter. +- This makes it easy to initiate multiple classes for different groupings of financial assets. +- Quarterly statement data returns the last 4 periods of data, while annual returns the last 3. + +Single Ticker Example +^^^^^^^^^^^^^^^^^^^^^ +.. code:: python + from yahoofinancials import YahooFinancials + + ticker = 'AAPL' + yahoo_financials = YahooFinancials(ticker) + + balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') + income_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'income') + all_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', ['income', 'cash', 'balance']) + apple_earnings_data = yahoo_financials.get_stock_earnings_data() + apple_net_income = yahoo_financials.get_net_income() + historical_stock_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') + +Lists of Tickers Example +^^^^^^^^^^^^^^^^^^^^^^^^ +.. code:: python + from yahoofinancials import YahooFinancials + + tech_stocks = ['AAPL', 'MSFT', 'INTC'] + bank_stocks = ['WFC', 'BAC', 'C'] + commodity_futures = ['GC=F', 'SI=F', 'CL=F'] + cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] + currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] + mutual_funds = ['PRLAX', 'QASGX', 'HISFX'] + us_treasuries = ['^TNX', '^IRX', '^TYX'] + + yahoo_financials_tech = YahooFinancials(tech_stocks) + yahoo_financials_banks = YahooFinancials(bank_stocks) + yahoo_financials_commodities = YahooFinancials(commodity_futures) + yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) + yahoo_financials_currencies = YahooFinancials(currencies) + yahoo_financials_mutualfunds = YahooFinancials(mutual_funds) + yahoo_financials_treasuries = YahooFinancials(us_treasuries) + + tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') + bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') + + banks_net_ebit = yahoo_financials_banks.get_ebit() + tech_stock_price_data = yahoo_financials_tech.get_stock_price_data() + daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_mutualfund_prices = yahoo_financials_mutualfunds.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_treasury_prices = yahoo_financials_treasuries.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + +Examples of Returned JSON Data +------------------------------ +1. Annual Income Statement Data for Apple: +.. code:: python + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_financial_stmts('annual', 'income')) + +.. code:: javascript + { + "incomeStatementHistory": { + "AAPL": [ + { + "2016-09-24": { + "minorityInterest": null, + "otherOperatingExpenses": null, + "netIncomeFromContinuingOps": 45687000000, + "totalRevenue": 215639000000, + "totalOtherIncomeExpenseNet": 1348000000, + "discontinuedOperations": null, + "incomeTaxExpense": 15685000000, + "extraordinaryItems": null, + "grossProfit": 84263000000, + "netIncome": 45687000000, + "sellingGeneralAdministrative": 14194000000, + "interestExpense": null, + "costOfRevenue": 131376000000, + "researchDevelopment": 10045000000, + "netIncomeApplicableToCommonShares": 45687000000, + "effectOfAccountingCharges": null, + "incomeBeforeTax": 61372000000, + "otherItems": null, + "operatingIncome": 60024000000, + "ebit": 61372000000, + "nonRecurring": null, + "totalOperatingExpenses": 0 + } + } + ] + } + } + +2. Annual Balance Sheet Data for Apple: +.. code:: python + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_financial_stmts('annual', 'balance')) + +.. code:: javascript + { + "balanceSheetHistory": { + "AAPL": [ + { + "2016-09-24": { + "otherCurrentLiab": 8080000000, + "otherCurrentAssets": 8283000000, + "goodWill": 5414000000, + "shortTermInvestments": 46671000000, + "longTermInvestments": 170430000000, + "cash": 20484000000, + "netTangibleAssets": 119629000000, + "totalAssets": 321686000000, + "otherLiab": 36074000000, + "totalStockholderEquity": 128249000000, + "inventory": 2132000000, + "retainedEarnings": 96364000000, + "intangibleAssets": 3206000000, + "totalCurrentAssets": 106869000000, + "otherStockholderEquity": 634000000, + "shortLongTermDebt": 11605000000, + "propertyPlantEquipment": 27010000000, + "deferredLongTermLiab": 2930000000, + "netReceivables": 29299000000, + "otherAssets": 8757000000, + "longTermDebt": 75427000000, + "totalLiab": 193437000000, + "commonStock": 31251000000, + "accountsPayable": 59321000000, + "totalCurrentLiabilities": 79006000000 + } + } + ] + } + } + +3. Quarterly Cash Flow Statement Data for Citigroup: +.. code:: python + yahoo_financials = YahooFinancials('C') + print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) + +.. code:: javascript + { + "cashflowStatementHistoryQuarterly": { + "C": [ + { + "2017-06-30": { + "totalCashFromOperatingActivities": -18505000000, + "effectOfExchangeRate": -117000000, + "totalCashFromFinancingActivities": 39798000000, + "netIncome": 3872000000, + "dividendsPaid": -760000000, + "salePurchaseOfStock": -1781000000, + "capitalExpenditures": -861000000, + "changeToLiabilities": -7626000000, + "otherCashflowsFromInvestingActivities": 82000000, + "totalCashflowsFromInvestingActivities": -22508000000, + "netBorrowings": 33586000000, + "depreciation": 901000000, + "changeInCash": -1332000000, + "changeToNetincome": 1444000000, + "otherCashflowsFromFinancingActivities": 8753000000, + "changeToOperatingActivities": -17096000000, + "investments": -23224000000 + } + } + ] + } + } + +4. Monthly Historical Stock Price Data for Wells Fargo: +.. code:: python + yahoo_financials = YahooFinancials('WFC') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + +.. code:: javascript + { + "WFC": { + "currency": "USD", + "eventsData": { + "dividends": { + "2018-08-01": { + "amount": 0.43, + "date": 1533821400, + "formatted_date": "2018-08-09" + } + } + }, + "firstTradeDate": { + "date": 76233600, + "formatted_date": "1972-06-01" + }, + "instrumentType": "EQUITY", + "prices": [ + { + "adjclose": 57.19147872924805, + "close": 57.61000061035156, + "date": 1533096000, + "formatted_date": "2018-08-01", + "high": 59.5, + "low": 57.08000183105469, + "open": 57.959999084472656, + "volume": 138922900 + } + ], + "timeZone": { + "gmtOffset": -14400 + } + } + } + +5. Monthly Historical Price Data for EURUSD: +.. code:: python + yahoo_financials = YahooFinancials('EURUSD=X') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + +.. code:: javascript + { + "EURUSD=X": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1070236800, + "formatted_date": "2003-12-01" + }, + "instrumentType": "CURRENCY", + "prices": [ + { + "adjclose": 1.1394712924957275, + "close": 1.1394712924957275, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 1.169864296913147, + "low": 1.1365960836410522, + "open": 1.168961763381958, + "volume": 0 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } + } + +6. Monthly Historical Price Data for BTC-USD: +.. code:: python + yahoo_financials = YahooFinancials('BTC-USD') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + +.. code:: javascript + { + "BTC-USD": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1279321200, + "formatted_date": "2010-07-16" + }, + "instrumentType": "CRYPTOCURRENCY", + "prices": [ + { + "adjclose": 6285.02001953125, + "close": 6285.02001953125, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 7760.740234375, + "low": 6133.02978515625, + "open": 7736.25, + "volume": 4334347882 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } + } + +7. Weekly Historical Price Data for Crude Oil Futures: +.. code:: python + yahoo_financials = YahooFinancials('CL=F') + print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) + +.. code:: javascript + { + "CL=F": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1522555200, + "formatted_date": "2018-04-01" + }, + "instrumentType": "FUTURE", + "prices": [ + { + "adjclose": 68.58999633789062, + "close": 68.58999633789062, + "date": 1532923200, + "formatted_date": "2018-07-30", + "high": 69.3499984741211, + "low": 66.91999816894531, + "open": 68.37000274658203, + "volume": 683048039 + }, + { + "adjclose": 67.75, + "close": 67.75, + "date": 1533528000, + "formatted_date": "2018-08-06", + "high": 69.91999816894531, + "low": 66.13999938964844, + "open": 68.76000213623047, + "volume": 1102357981 + } + ], + "timeZone": { + "gmtOffset": -14400 + } + } + } + +8. Apple Stock Quote Data: +.. code:: python + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_stock_quote_type_data()) + +.. code:: javascript + { + "AAPL": { + "underlyingExchangeSymbol": null, + "exchangeTimezoneName": "America/New_York", + "underlyingSymbol": null, + "headSymbol": null, + "shortName": "Apple Inc.", + "symbol": "AAPL", + "uuid": "8b10e4ae-9eeb-3684-921a-9ab27e4d87aa", + "gmtOffSetMilliseconds": "-14400000", + "exchange": "NMS", + "exchangeTimezoneShortName": "EDT", + "messageBoardId": "finmb_24937", + "longName": "Apple Inc.", + "market": "us_market", + "quoteType": "EQUITY" + } + } + +9. U.S. Treasury Current Pricing Data: +.. code:: python + yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) + print(yahoo_financials.get_current_price()) + +.. code:: javascript + { + "^IRX": 2.033, + "^TNX": 2.895, + "^TYX": 3.062 + } + +10. BTC-USD Summary Data: +.. code:: python + yahoo_financials = YahooFinancials('BTC-USD') + print(yahoo_financials.get_summary_data()) + +.. code:: javascript + { + "BTC-USD": { + "algorithm": "SHA256", + "ask": null, + "askSize": null, + "averageDailyVolume10Day": 545573809, + "averageVolume": 496761640, + "averageVolume10days": 545573809, + "beta": null, + "bid": null, + "bidSize": null, + "circulatingSupply": 17209812, + "currency": "USD", + "dayHigh": 6266.5, + "dayLow": 5891.87, + "dividendRate": null, + "dividendYield": null, + "exDividendDate": "-", + "expireDate": "-", + "fiftyDayAverage": 6989.074, + "fiftyTwoWeekHigh": 19870.62, + "fiftyTwoWeekLow": 2979.88, + "fiveYearAvgDividendYield": null, + "forwardPE": null, + "fromCurrency": "BTC", + "lastMarket": "CCCAGG", + "marketCap": 106325663744, + "maxAge": 1, + "maxSupply": 21000000, + "navPrice": null, + "open": 6263.2, + "openInterest": null, + "payoutRatio": null, + "previousClose": 6263.2, + "priceHint": 2, + "priceToSalesTrailing12Months": null, + "regularMarketDayHigh": 6266.5, + "regularMarketDayLow": 5891.87, + "regularMarketOpen": 6263.2, + "regularMarketPreviousClose": 6263.2, + "regularMarketVolume": 755834368, + "startDate": "2009-01-03", + "strikePrice": null, + "totalAssets": null, + "tradeable": false, + "trailingAnnualDividendRate": null, + "trailingAnnualDividendYield": null, + "twoHundredDayAverage": 8165.154, + "volume": 755834368, + "volume24Hr": 750196480, + "volumeAllCurrencies": 2673437184, + "yield": null, + "ytdReturn": null + } + } diff --git a/demo.py b/demo.py index 38e93d0..b1aac4b 100755 --- a/demo.py +++ b/demo.py @@ -25,7 +25,7 @@ def defaultapi(ticker): tick = YF(ticker) - print(tick.get_stock_summary_data()) + print(tick.get_summary_data()) print(mark) print(tick.get_stock_quote_type_data()) print(mark) diff --git a/setup.cfg b/setup.cfg index b88034e..5aef279 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,2 @@ [metadata] -description-file = README.md +description-file = README.rst diff --git a/setup.py b/setup.py index 720d040..6f1a32d 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,46 @@ +import codecs try: from setuptools import setup except ImportError: from distutils.core import setup + + +with codecs.open('README.rst', encoding='utf-8') as f: + long_description = f.read() setup( name='yahoofinancials', - version='0.10', + version='1.0', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', + long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/0.10.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.0.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', keywords = ['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], packages=['yahoofinancials'], install_requires=[ - "requests", "beautifulsoup4", "pytz" ], - classifiers=[], + classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'Intended Audience :: Financial and Insurance Industry', + 'Topic :: Office/Business :: Financial :: Investment', + 'Topic :: Software Development :: Libraries :: Python Modules', + 'Operating System :: OS Independent', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + ], zip_safe=False ) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py new file mode 100644 index 0000000..a9b6234 --- /dev/null +++ b/test/test_yahoofinancials.py @@ -0,0 +1,105 @@ +# YahooFinancials Unit Tests v1.0 +# Version Released: 08/22/2018 +# Author: Connor Sanders +# Tested on Python 2.7 and 3.5 +# Copyright (c) 2018 Connor Sanders +# MIT License + +import sys +import datetime +from yahoofinancials import YahooFinancials + +if sys.version_info < (2, 7): + from unittest2 import main as test_main, SkipTest, TestCase +else: + from unittest import main as test_main, SkipTest, TestCase + + +# Test Configuration Variables +stocks = ['AAPL', 'MSFT', 'C'] +currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] +us_treasuries = ['^TNX', '^IRX', '^TYX'] + + +# Global function to check Fundamental Test results +def check_fundamental(test_data, test_type): + if test_type == 'bal': + if 'balanceSheetHistoryQuarterly' in test_data and test_data['balanceSheetHistoryQuarterly']['C'] is not None: + return True + else: + return False + elif test_type == 'inc': + if 'incomeStatementHistoryQuarterly' in test_data and \ + test_data['incomeStatementHistoryQuarterly']['C'] is not None: + return True + else: + return False + elif test_type == 'all': + if 'balanceSheetHistoryQuarterly' in test_data and 'incomeStatementHistoryQuarterly' in test_data and \ + 'cashflowStatementHistoryQuarterly' in test_data: + return True + else: + return False + + +# Main Test Module Class +class TestModule(TestCase): + + def setUp(self): + self.test_yf_stock_single = YahooFinancials('C') + self.test_yf_stock_multi = YahooFinancials(stocks) + self.test_yf_treasuries_single = YahooFinancials('^IRX') + self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) + self.test_yf_currencies = YahooFinancials(currencies) + + # Fundamentals Test + def test_yf_fundamentals(self): + # Single stock test + single_balance_sheet_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'balance') + single_income_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'income') + single_all_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Multi stock test + multi_balance_sheet_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'balance') + multi_income_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'income') + multi_all_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Single stock check + result = check_fundamental(single_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(single_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(single_all_statement_data_qt, 'all') + self.assertEqual(result, True) + # Multi stock check + result = check_fundamental(multi_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(multi_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(multi_all_statement_data_qt, 'all') + self.assertEqual(result, True) + + # Historical Price Test + def test_yf_historical_price(self): + single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') + expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.060001373291016} + self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) + + # Extra Module Methods Test + def test_yf_module_methods(self): + # Stocks + if isinstance(self.test_yf_stock_single.get_current_price(), float): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + # Treasuries + if isinstance(self.test_yf_treasuries_single.get_current_price(), float): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + +if __name__ == "__main__": + test_main() diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 8762467..d9ab5b3 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 0.10 +Version: 1.0 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/14/2018 +Version Released: 8/22/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -47,16 +47,24 @@ from json import loads import time from bs4 import BeautifulSoup -import requests import datetime from datetime import date import pytz +try: + from urllib import FancyURLopener +except: + from urllib.request import FancyURLopener # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 +# Class used to open urls for financial data +class UrlOpener(FancyURLopener): + version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11' + + # Class containing Yahoo Finance ETL Functionality class YahooFinanceETL(object): @@ -67,7 +75,7 @@ def __init__(self, ticker): # Minimum interval between Yahoo Finance requests for this instance _MIN_INTERVAL = 7 - # Meta-data dictionaries for the class to use + # Meta-data dictionaries for the classes to use YAHOO_FINANCIAL_TYPES = { 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], @@ -75,6 +83,7 @@ def __init__(self, ticker): 'history': ['history'] } + # Interval value translation dictionary _INTERVAL_DICT = { 'daily': '1d', 'weekly': '1wk', @@ -123,10 +132,13 @@ def _scrape_data(self, url, tech_type, statement_type): time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) now = int(time.time()) _lastget = now - response = requests.get(url) - soup = BeautifulSoup(response.content, "html.parser") + urlopener = UrlOpener() + response = urlopener.open(url) + response_content = response.read() + soup = BeautifulSoup(response_content, "html.parser") script = soup.find("script", text=re.compile("root.App.main")).text self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) + response.close() data = self._cache[url] if tech_type == '' and statement_type != 'history': stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] @@ -282,10 +294,13 @@ def _build_api_url(hist_obj, up_ticker): # Private Static Method to get financial data via API Call @staticmethod def _get_api_data(api_url): - response = requests.get(api_url) + urlopener = UrlOpener() + response = urlopener.open(api_url) + res_content = response.read() + response.close() if sys.version_info < (3, 0): - return loads(response.content) - return loads(response.content.decode('utf-8')) + return loads(res_content) + return loads(res_content.decode('utf-8')) # Private Method to clean API data def _clean_api_data(self, api_url): @@ -508,15 +523,6 @@ def get_stock_earnings_data(self, reformat=True): else: return self.get_stock_tech_data('earnings') - # Public Method for the user to get stock summary data - def get_stock_summary_data(self, reformat=True): - print("***WARNING: AS OF v0.10 'get_stock_summary_data()' IS DEPRECIATED AND WILL BE REMOVED IN THE " - "v1.0 RELEASE.***\n***PLEASE USE 'get_summary_data()' INSTEAD.***") - if reformat: - return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') - else: - return self.get_stock_tech_data('summaryDetail') - # Public Method for the user to get stock summary data def get_summary_data(self, reformat=True): if reformat: @@ -534,18 +540,7 @@ def get_stock_summary_url(self): def get_stock_quote_type_data(self): return self.get_stock_tech_data('quoteType') - # Public Method for user to get historical stock data with (SOON TO BE DEPRECIATED IN V1.0) - def get_historical_stock_data(self, start_date, end_date, time_interval): - interval_code = self.get_time_code(time_interval) - start = self.format_date(start_date) - end = self.format_date(end_date) - hist_obj = {'start': start, 'end': end, 'interval': interval_code} - data = self.get_stock_data('history', hist_obj=hist_obj) - print("***WARNING: AS OF v0.9 'get_historical_stock_data()' IS DEPRECIATED AND WILL BE REMOVED IN THE " - "v1.0 RELEASE.***\n***PLEASE USE 'get_historical_price_data()' INSTEAD.***") - return data - - # Public Method for user to get historical stock data with (SOON TO BE DEPRECIATED IN V1.0) + # Public Method for user to get historical price data with def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) start = self.format_date(start_date) From 1b1276a1b49c1f966b351e18f9a2349ce199e90d Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:32:35 -0500 Subject: [PATCH 022/108] v1.0 pre push2 --- README.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 03721f6..f52af15 100644 --- a/README.rst +++ b/README.rst @@ -15,6 +15,7 @@ Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title i Overview -------- A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. + - As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. Installation @@ -24,21 +25,26 @@ Installation - The package depends on beautifulsoup4 and pytz to work. 1. Installation using pip: + - Linux/Mac: + .. code:: bash $ pip install yahoofinancials - Windows (If python doesn't work for you in cmd, try running the following command with just py): + .. code:: bash > python -m pip install yahoofinancials 2. Installation using github (Mac/Linux): + .. code:: bash $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials $ python setup.py install 3. Demo using the included demo script: + .. code:: bash $ cd yahoofinancials $ python demo.py -h @@ -46,6 +52,7 @@ Installation $ python demo.py WFC C BAC 4. Test using the included unit testing script: + .. code:: bash $ cd yahoofinancials $ python test/test_yahoofinancials.py @@ -59,26 +66,33 @@ Module Methods Featured Methods ^^^^^^^^^^^^^^^^ 1. get_financial_stmts(frequency, statement_type, reformat=True) + - frequency can be either 'annual' or 'quarterly'. - statement_type can be 'income', 'balance', 'cash' or a list of several. - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 2. get_stock_price_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 3. get_stock_earnings_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 4. get_summary_data(reformat=True) + - New in v0.10. - Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5. get_stock_quote_type_data() + 6. get_historical_price_data(start_date, end_date, time_interval) + - New in v0.9. - This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. - start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. - end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. - time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. - Data response includes relevant pricing event data such as dividends and stock splits. -7. get_num_shares_outstanding(price_type='current'): +7. get_num_shares_outstanding(price_type='current') + - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. Methods Removed in V1.0 @@ -138,6 +152,7 @@ Usage Examples Single Ticker Example ^^^^^^^^^^^^^^^^^^^^^ + .. code:: python from yahoofinancials import YahooFinancials @@ -153,6 +168,7 @@ Single Ticker Example Lists of Tickers Example ^^^^^^^^^^^^^^^^^^^^^^^^ + .. code:: python from yahoofinancials import YahooFinancials @@ -186,7 +202,9 @@ Lists of Tickers Example Examples of Returned JSON Data ------------------------------ + 1. Annual Income Statement Data for Apple: + .. code:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) @@ -226,6 +244,7 @@ Examples of Returned JSON Data } 2. Annual Balance Sheet Data for Apple: + .. code:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) @@ -268,6 +287,7 @@ Examples of Returned JSON Data } 3. Quarterly Cash Flow Statement Data for Citigroup: + .. code:: python yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) @@ -302,6 +322,7 @@ Examples of Returned JSON Data } 4. Monthly Historical Stock Price Data for Wells Fargo: + .. code:: python yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) @@ -343,6 +364,7 @@ Examples of Returned JSON Data } 5. Monthly Historical Price Data for EURUSD: + .. code:: python yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) @@ -376,6 +398,7 @@ Examples of Returned JSON Data } 6. Monthly Historical Price Data for BTC-USD: + .. code:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) @@ -409,6 +432,7 @@ Examples of Returned JSON Data } 7. Weekly Historical Price Data for Crude Oil Futures: + .. code:: python yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) @@ -452,6 +476,7 @@ Examples of Returned JSON Data } 8. Apple Stock Quote Data: + .. code:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) @@ -477,6 +502,7 @@ Examples of Returned JSON Data } 9. U.S. Treasury Current Pricing Data: + .. code:: python yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) @@ -489,6 +515,7 @@ Examples of Returned JSON Data } 10. BTC-USD Summary Data: + .. code:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) From d3a479ed472a95c2e4006e832744d56e910e9603 Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:39:09 -0500 Subject: [PATCH 023/108] v1.0 pre push3 --- README.rst | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/README.rst b/README.rst index f52af15..bb49f9f 100644 --- a/README.rst +++ b/README.rst @@ -28,24 +28,24 @@ Installation - Linux/Mac: -.. code:: bash +.. code-block:: bash $ pip install yahoofinancials - Windows (If python doesn't work for you in cmd, try running the following command with just py): -.. code:: bash +.. code-block:: bash > python -m pip install yahoofinancials 2. Installation using github (Mac/Linux): -.. code:: bash +.. code-block:: bash $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials $ python setup.py install 3. Demo using the included demo script: -.. code:: bash +.. code-block:: bash $ cd yahoofinancials $ python demo.py -h $ python demo.py @@ -53,7 +53,7 @@ Installation 4. Test using the included unit testing script: -.. code:: bash +.. code-block:: bash $ cd yahoofinancials $ python test/test_yahoofinancials.py @@ -153,7 +153,7 @@ Usage Examples Single Ticker Example ^^^^^^^^^^^^^^^^^^^^^ -.. code:: python +.. code-block:: python from yahoofinancials import YahooFinancials ticker = 'AAPL' @@ -169,7 +169,7 @@ Single Ticker Example Lists of Tickers Example ^^^^^^^^^^^^^^^^^^^^^^^^ -.. code:: python +.. code-block:: python from yahoofinancials import YahooFinancials tech_stocks = ['AAPL', 'MSFT', 'INTC'] @@ -205,11 +205,11 @@ Examples of Returned JSON Data 1. Annual Income Statement Data for Apple: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) -.. code:: javascript +.. code-block:: javascript { "incomeStatementHistory": { "AAPL": [ @@ -245,11 +245,11 @@ Examples of Returned JSON Data 2. Annual Balance Sheet Data for Apple: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) -.. code:: javascript +.. code-block:: javascript { "balanceSheetHistory": { "AAPL": [ @@ -288,11 +288,11 @@ Examples of Returned JSON Data 3. Quarterly Cash Flow Statement Data for Citigroup: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) -.. code:: javascript +.. code-block:: javascript { "cashflowStatementHistoryQuarterly": { "C": [ @@ -323,11 +323,11 @@ Examples of Returned JSON Data 4. Monthly Historical Stock Price Data for Wells Fargo: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code:: javascript +.. code-block:: javascript { "WFC": { "currency": "USD", @@ -365,11 +365,11 @@ Examples of Returned JSON Data 5. Monthly Historical Price Data for EURUSD: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code:: javascript +.. code-block:: javascript { "EURUSD=X": { "currency": "USD", @@ -399,11 +399,11 @@ Examples of Returned JSON Data 6. Monthly Historical Price Data for BTC-USD: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code:: javascript +.. code-block:: javascript { "BTC-USD": { "currency": "USD", @@ -433,11 +433,11 @@ Examples of Returned JSON Data 7. Weekly Historical Price Data for Crude Oil Futures: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) -.. code:: javascript +.. code-block:: javascript { "CL=F": { "currency": "USD", @@ -477,11 +477,11 @@ Examples of Returned JSON Data 8. Apple Stock Quote Data: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) -.. code:: javascript +.. code-block:: javascript { "AAPL": { "underlyingExchangeSymbol": null, @@ -503,11 +503,11 @@ Examples of Returned JSON Data 9. U.S. Treasury Current Pricing Data: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) -.. code:: javascript +.. code-block:: javascript { "^IRX": 2.033, "^TNX": 2.895, @@ -516,11 +516,11 @@ Examples of Returned JSON Data 10. BTC-USD Summary Data: -.. code:: python +.. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) -.. code:: javascript +.. code-block:: javascript { "BTC-USD": { "algorithm": "SHA256", From 247cd0b2780da2a80cc676874c6f2c9b9e43fb2f Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:49:39 -0500 Subject: [PATCH 024/108] v1.0 pre push4 --- README.rst | 50 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index bb49f9f..715ed26 100644 --- a/README.rst +++ b/README.rst @@ -28,24 +28,24 @@ Installation - Linux/Mac: -.. code-block:: bash +.. code-block:: $ pip install yahoofinancials - Windows (If python doesn't work for you in cmd, try running the following command with just py): -.. code-block:: bash +.. code-block:: > python -m pip install yahoofinancials 2. Installation using github (Mac/Linux): -.. code-block:: bash +.. code-block:: $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials $ python setup.py install 3. Demo using the included demo script: -.. code-block:: bash +.. code-block:: $ cd yahoofinancials $ python demo.py -h $ python demo.py @@ -53,7 +53,7 @@ Installation 4. Test using the included unit testing script: -.. code-block:: bash +.. code-block:: $ cd yahoofinancials $ python test/test_yahoofinancials.py @@ -205,11 +205,13 @@ Examples of Returned JSON Data 1. Annual Income Statement Data for Apple: +Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) -.. code-block:: javascript +Result: +.. code-block:: { "incomeStatementHistory": { "AAPL": [ @@ -245,11 +247,13 @@ Examples of Returned JSON Data 2. Annual Balance Sheet Data for Apple: +Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) -.. code-block:: javascript +Result: +.. code-block:: { "balanceSheetHistory": { "AAPL": [ @@ -288,11 +292,13 @@ Examples of Returned JSON Data 3. Quarterly Cash Flow Statement Data for Citigroup: +Code: .. code-block:: python yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) -.. code-block:: javascript +Result: +.. code-block:: { "cashflowStatementHistoryQuarterly": { "C": [ @@ -323,11 +329,13 @@ Examples of Returned JSON Data 4. Monthly Historical Stock Price Data for Wells Fargo: +Code: .. code-block:: python yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code-block:: javascript +Result: +.. code-block:: { "WFC": { "currency": "USD", @@ -365,11 +373,13 @@ Examples of Returned JSON Data 5. Monthly Historical Price Data for EURUSD: +Code: .. code-block:: python yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code-block:: javascript +Result: +.. code-block:: { "EURUSD=X": { "currency": "USD", @@ -399,11 +409,13 @@ Examples of Returned JSON Data 6. Monthly Historical Price Data for BTC-USD: +Code: .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -.. code-block:: javascript +Result: +.. code-block:: { "BTC-USD": { "currency": "USD", @@ -433,11 +445,13 @@ Examples of Returned JSON Data 7. Weekly Historical Price Data for Crude Oil Futures: +Code: .. code-block:: python yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) -.. code-block:: javascript +Result: +.. code-block:: { "CL=F": { "currency": "USD", @@ -477,11 +491,13 @@ Examples of Returned JSON Data 8. Apple Stock Quote Data: +Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) -.. code-block:: javascript +Result: +.. code-block:: { "AAPL": { "underlyingExchangeSymbol": null, @@ -503,11 +519,13 @@ Examples of Returned JSON Data 9. U.S. Treasury Current Pricing Data: +Code: .. code-block:: python yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) -.. code-block:: javascript +Result: +.. code-block:: { "^IRX": 2.033, "^TNX": 2.895, @@ -516,11 +534,13 @@ Examples of Returned JSON Data 10. BTC-USD Summary Data: +Code: .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) -.. code-block:: javascript +Result: +.. code-block:: { "BTC-USD": { "algorithm": "SHA256", From 084dae7a011ffbd2056403006f007caf9405edaa Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:52:14 -0500 Subject: [PATCH 025/108] v1.0 pre push5 --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 715ed26..8ed7c93 100644 --- a/README.rst +++ b/README.rst @@ -26,12 +26,12 @@ Installation 1. Installation using pip: -- Linux/Mac: +Linux/Mac: .. code-block:: $ pip install yahoofinancials -- Windows (If python doesn't work for you in cmd, try running the following command with just py): +Windows (If python doesn't work for you in cmd, try running the following command with just py): .. code-block:: > python -m pip install yahoofinancials From 664e560cff8e2ef63064b45a92230cf0c999a21d Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:56:26 -0500 Subject: [PATCH 026/108] v1.0 pre push6 --- README.rst | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 8ed7c93..3f82250 100644 --- a/README.rst +++ b/README.rst @@ -26,26 +26,30 @@ Installation 1. Installation using pip: -Linux/Mac: +- Linux/Mac: + +.. code-block:: bash -.. code-block:: $ pip install yahoofinancials -Windows (If python doesn't work for you in cmd, try running the following command with just py): +- Windows (If python doesn't work for you in cmd, try running the following command with just py): .. code-block:: + > python -m pip install yahoofinancials 2. Installation using github (Mac/Linux): -.. code-block:: +.. code-block:: bash + $ git clone https://github.com/JECSand/yahoofinancials.git $ cd yahoofinancials $ python setup.py install 3. Demo using the included demo script: -.. code-block:: +.. code-block:: bash + $ cd yahoofinancials $ python demo.py -h $ python demo.py @@ -53,7 +57,8 @@ Windows (If python doesn't work for you in cmd, try running the following comman 4. Test using the included unit testing script: -.. code-block:: +.. code-block:: bash + $ cd yahoofinancials $ python test/test_yahoofinancials.py @@ -154,6 +159,7 @@ Single Ticker Example ^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python + from yahoofinancials import YahooFinancials ticker = 'AAPL' @@ -170,6 +176,7 @@ Lists of Tickers Example ^^^^^^^^^^^^^^^^^^^^^^^^ .. code-block:: python + from yahoofinancials import YahooFinancials tech_stocks = ['AAPL', 'MSFT', 'INTC'] @@ -207,11 +214,13 @@ Examples of Returned JSON Data Code: .. code-block:: python + yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) Result: .. code-block:: + { "incomeStatementHistory": { "AAPL": [ @@ -249,11 +258,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) Result: .. code-block:: + { "balanceSheetHistory": { "AAPL": [ @@ -294,11 +305,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) Result: .. code-block:: + { "cashflowStatementHistoryQuarterly": { "C": [ @@ -331,11 +344,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: .. code-block:: + { "WFC": { "currency": "USD", @@ -375,11 +390,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: .. code-block:: + { "EURUSD=X": { "currency": "USD", @@ -411,11 +428,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: .. code-block:: + { "BTC-USD": { "currency": "USD", @@ -447,11 +466,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) Result: .. code-block:: + { "CL=F": { "currency": "USD", @@ -493,11 +514,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) Result: .. code-block:: + { "AAPL": { "underlyingExchangeSymbol": null, @@ -521,11 +544,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) Result: .. code-block:: + { "^IRX": 2.033, "^TNX": 2.895, @@ -536,11 +561,13 @@ Result: Code: .. code-block:: python + yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) Result: .. code-block:: + { "BTC-USD": { "algorithm": "SHA256", From 15daf87902fb7cb92ecd278dca331f186b22b0d0 Mon Sep 17 00:00:00 2001 From: alt Date: Tue, 21 Aug 2018 23:58:37 -0500 Subject: [PATCH 027/108] v1.0 pre push7 --- README.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.rst b/README.rst index 3f82250..77a9426 100644 --- a/README.rst +++ b/README.rst @@ -213,12 +213,14 @@ Examples of Returned JSON Data 1. Annual Income Statement Data for Apple: Code: + .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) Result: + .. code-block:: { @@ -257,12 +259,14 @@ Result: 2. Annual Balance Sheet Data for Apple: Code: + .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) Result: + .. code-block:: { @@ -304,12 +308,14 @@ Result: 3. Quarterly Cash Flow Statement Data for Citigroup: Code: + .. code-block:: python yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) Result: + .. code-block:: { @@ -343,12 +349,14 @@ Result: 4. Monthly Historical Stock Price Data for Wells Fargo: Code: + .. code-block:: python yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: + .. code-block:: { @@ -389,12 +397,14 @@ Result: 5. Monthly Historical Price Data for EURUSD: Code: + .. code-block:: python yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: + .. code-block:: { @@ -427,12 +437,14 @@ Result: 6. Monthly Historical Price Data for BTC-USD: Code: + .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) Result: + .. code-block:: { @@ -465,12 +477,14 @@ Result: 7. Weekly Historical Price Data for Crude Oil Futures: Code: + .. code-block:: python yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) Result: + .. code-block:: { @@ -513,12 +527,14 @@ Result: 8. Apple Stock Quote Data: Code: + .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) Result: + .. code-block:: { @@ -543,12 +559,14 @@ Result: 9. U.S. Treasury Current Pricing Data: Code: + .. code-block:: python yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) Result: + .. code-block:: { @@ -560,12 +578,14 @@ Result: 10. BTC-USD Summary Data: Code: + .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) Result: + .. code-block:: { From efeb94c1f02ee165535810dabc5050af3739d63d Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 00:01:43 -0500 Subject: [PATCH 028/108] v1.0 pre push8 --- README.rst | 40 ++++++++++------------------------------ 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/README.rst b/README.rst index 77a9426..e34ea62 100644 --- a/README.rst +++ b/README.rst @@ -212,16 +212,14 @@ Examples of Returned JSON Data 1. Annual Income Statement Data for Apple: -Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'income')) -Result: -.. code-block:: +.. code-block:: javascript { "incomeStatementHistory": { @@ -258,16 +256,14 @@ Result: 2. Annual Balance Sheet Data for Apple: -Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_financial_stmts('annual', 'balance')) -Result: -.. code-block:: +.. code-block:: javascript { "balanceSheetHistory": { @@ -307,16 +303,14 @@ Result: 3. Quarterly Cash Flow Statement Data for Citigroup: -Code: .. code-block:: python yahoo_financials = YahooFinancials('C') print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) -Result: -.. code-block:: +.. code-block:: javascript { "cashflowStatementHistoryQuarterly": { @@ -348,16 +342,14 @@ Result: 4. Monthly Historical Stock Price Data for Wells Fargo: -Code: .. code-block:: python yahoo_financials = YahooFinancials('WFC') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -Result: -.. code-block:: +.. code-block:: javascript { "WFC": { @@ -396,16 +388,14 @@ Result: 5. Monthly Historical Price Data for EURUSD: -Code: .. code-block:: python yahoo_financials = YahooFinancials('EURUSD=X') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -Result: -.. code-block:: +.. code-block:: javascript { "EURUSD=X": { @@ -436,16 +426,14 @@ Result: 6. Monthly Historical Price Data for BTC-USD: -Code: .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) -Result: -.. code-block:: +.. code-block:: javascript { "BTC-USD": { @@ -476,16 +464,14 @@ Result: 7. Weekly Historical Price Data for Crude Oil Futures: -Code: .. code-block:: python yahoo_financials = YahooFinancials('CL=F') print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) -Result: -.. code-block:: +.. code-block:: javascript { "CL=F": { @@ -526,16 +512,14 @@ Result: 8. Apple Stock Quote Data: -Code: .. code-block:: python yahoo_financials = YahooFinancials('AAPL') print(yahoo_financials.get_stock_quote_type_data()) -Result: -.. code-block:: +.. code-block:: javascripts { "AAPL": { @@ -558,16 +542,14 @@ Result: 9. U.S. Treasury Current Pricing Data: -Code: .. code-block:: python yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) print(yahoo_financials.get_current_price()) -Result: -.. code-block:: +.. code-block:: javascript { "^IRX": 2.033, @@ -577,16 +559,14 @@ Result: 10. BTC-USD Summary Data: -Code: .. code-block:: python yahoo_financials = YahooFinancials('BTC-USD') print(yahoo_financials.get_summary_data()) -Result: -.. code-block:: +.. code-block:: javascript { "BTC-USD": { From 72fa5efe3f0ff8bcae90464389fed16d15e566af Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 00:20:35 -0500 Subject: [PATCH 029/108] v1.0 pre push10 --- .travis.yml | 23 +++++++++++++++++++++++ MANIFEST.in | 1 + 2 files changed, 24 insertions(+) create mode 100644 .travis.yml create mode 100644 MANIFEST.in diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9e1ed04 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +language: python + +python: + - "2.6" + - "2.7" + - "pypy" + - "pypy3" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "3.7" + +install: + - if [ "$TRAVIS_PYTHON_VERSION" == "2.6" ]; then pip --quiet install argparse unittest2; fi + - python setup.py install + +script: + - nosetests --with-coverage --cover-package=yahoofinancials + +after_success: + - pip install --quiet coveralls + - coveralls diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9561fb1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include README.rst From de16996f1a49afb1c1a4873ee8511ee03e43121e Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 00:40:11 -0500 Subject: [PATCH 030/108] v1.0 pre push11 --- .travis.yml | 1 - test/test_yahoofinancials.py | 11 ++++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9e1ed04..4d51204 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,6 @@ python: - "3.4" - "3.5" - "3.6" - - "3.7" install: - if [ "$TRAVIS_PYTHON_VERSION" == "2.6" ]; then pip --quiet install argparse unittest2; fi diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index a9b6234..1683e9d 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -82,9 +82,14 @@ def test_yf_fundamentals(self): # Historical Price Test def test_yf_historical_price(self): single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.060001373291016} + if sys.version_info < (3, 0): + expect_dict = {'high': 49.099998474121094, 'volume': 125737200, u'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.959999084472656} + else: + expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.959999084472656} self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) # Extra Module Methods Test From 168c8794ee42cfe48e5bae4b7d8a1ffda5635c66 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 00:46:28 -0500 Subject: [PATCH 031/108] v1.0 pre push12 --- .travis.yml | 1 - setup.py | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d51204..973c920 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: python python: - - "2.6" - "2.7" - "pypy" - "pypy3" diff --git a/setup.py b/setup.py index 6f1a32d..5b55651 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup except ImportError: from distutils.core import setup - + with codecs.open('README.rst', encoding='utf-8') as f: long_description = f.read() @@ -33,14 +33,12 @@ 'Operating System :: OS Independent', 'License :: OSI Approved :: MIT License', 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.6' ], zip_safe=False ) From 09519c81c43c1442b2190645a4029104bdccb6cc Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 01:08:54 -0500 Subject: [PATCH 032/108] v1.0 pre push13 --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index e34ea62..277e00b 100644 --- a/README.rst +++ b/README.rst @@ -2,6 +2,9 @@ yahoofinancials =============== +.. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master + :target: https://travis-ci.org/JECSand/yahoofinancials + Welcome to Yahoo Financials! Current Version: v1.0 From e421296ec0c5d75e7d9c9459cc864430321f405d Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 01:22:16 -0500 Subject: [PATCH 033/108] v1.0 pre push14 --- README.rst | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 277e00b..a8aed88 100644 --- a/README.rst +++ b/README.rst @@ -2,18 +2,16 @@ yahoofinancials =============== +A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, etf, and US Treasury financial data from Yahoo Finance. + .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Welcome to Yahoo Financials! - Current Version: v1.0 Version Released: 08/22/2018 -New Contributers Welcomed! - -Email sandersconnor1@gmail.com with YAHOOFINANCIALS-{Your-Name} as email title if interested. +Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues Overview -------- @@ -86,14 +84,12 @@ Featured Methods - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 4. get_summary_data(reformat=True) - - New in v0.10. - Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 5. get_stock_quote_type_data() 6. get_historical_price_data(start_date, end_date, time_interval) - - New in v0.9. - This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. - start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. - end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. From 5d1429031c83ee0a210e17bfe535d580fbfe11e0 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 22 Aug 2018 01:25:08 -0500 Subject: [PATCH 034/108] pre prod update --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a8aed88..3c4e38d 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ yahoofinancials =============== -A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, etf, and US Treasury financial data from Yahoo Finance. +A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, ETF, and US Treasury financial data from Yahoo Finance. .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials From 65f66c91430ff28c4be0b99002698f057ace3e71 Mon Sep 17 00:00:00 2001 From: alt Date: Thu, 23 Aug 2018 18:28:41 -0500 Subject: [PATCH 035/108] 1.1 push --- CHANGES | 1 + README.rst | 4 ++-- setup.cfg | 2 -- setup.py | 4 ++-- yahoofinancials/__init__.py | 6 +++--- 5 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 setup.cfg diff --git a/CHANGES b/CHANGES index edec957..2321154 100644 --- a/CHANGES +++ b/CHANGES @@ -17,3 +17,4 @@ 1.0 08/22/2018 -- Removed the get_stock_summary_data() method. 1.0 08/22/2018 -- Removed the requests dependency and replaced it with urllib. 1.0 08/22/2018 -- Updated README.md to README.rst +1.1 08/23/2018 -- Fixed net income python 3 error diff --git a/README.rst b/README.rst index 3c4e38d..95fe4c2 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Current Version: v1.0 +Current Version: v1.1 -Version Released: 08/22/2018 +Version Released: 08/23/2018 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 5aef279..0000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.rst diff --git a/setup.py b/setup.py index 5b55651..48301a0 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.0', + version='1.1', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.0.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.1.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index d9ab5b3..37a532d 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.0 +Version: 1.1 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/22/2018 +Version Released: 8/23/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -585,7 +585,7 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): if isinstance(self.ticker, str): try: date_key = re_data[self.ticker][0].keys()[0] - except (IndexError, AttributeError): + except (IndexError, AttributeError, TypeError): date_key = list(re_data[self.ticker][0])[0] data = re_data[self.ticker][0][date_key][field_name] else: From a0e57eaf77acc3425c686c7225688fdd3342c4c1 Mon Sep 17 00:00:00 2001 From: alt Date: Thu, 23 Aug 2018 19:07:20 -0500 Subject: [PATCH 036/108] update unit test with #10 bug fix test --- dist/yahoofinancials-1.1.tar.gz | Bin 0 -> 23437 bytes test/test_yahoofinancials.py | 4 + yahoofinancials.egg-info/PKG-INFO | 651 ++++++++++++++++++ yahoofinancials.egg-info/SOURCES.txt | 11 + yahoofinancials.egg-info/dependency_links.txt | 1 + yahoofinancials.egg-info/not-zip-safe | 1 + yahoofinancials.egg-info/requires.txt | 2 + yahoofinancials.egg-info/top_level.txt | 1 + 8 files changed, 671 insertions(+) create mode 100644 dist/yahoofinancials-1.1.tar.gz create mode 100644 yahoofinancials.egg-info/PKG-INFO create mode 100644 yahoofinancials.egg-info/SOURCES.txt create mode 100644 yahoofinancials.egg-info/dependency_links.txt create mode 100644 yahoofinancials.egg-info/not-zip-safe create mode 100644 yahoofinancials.egg-info/requires.txt create mode 100644 yahoofinancials.egg-info/top_level.txt diff --git a/dist/yahoofinancials-1.1.tar.gz b/dist/yahoofinancials-1.1.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..f2b4b8a0855ee23934d4ad3a5aee280184b643d7 GIT binary patch literal 23437 zcmb5#W0Nod*Cgh#=N{X(ZQHhO+qP}nwr$(C?S1lAHk;i|rIMd;zI9#Q2QLZ&;!(^` z3+T_-!pWJ=!@%6m&eX!zz}CpZz}kt1ftG>R*}xItrpHTfOT3Yu+wV*1oZf8ho))!J zBatJlNeYJeIy4y2%+#q7<1d(oKgh;|7~?R5Z11P0(st*LU)=TIZjQdx&dSb;%Z=2^ zW~b|pl$ikyX)Mc=S0nkI_uDugB(G>+oS#V%9m!v4-kq_E3an1h-|-!>MiR;gSHa&7 zlDDd_zDMAF4huBp)JhK=IO6XQ+ebC@>7Vi6%)`RLhqyCp9P3|aY06w*VdF;{Y)Rb} z{hv=yoi~~I+|%C(->e~{mW<88wf*1zvHbIQ{?pFW+23qg_#e}p@YiMK88z)r4*0zv z)kmFpXf5~DpPlWm+}}f^+xO}DP0^%Z58c_RUc&xd)3;yXTm+7~5>>q?-)8Pzdz20! zdH&OA=2n+r=s(U1!n3N6?Z?%w!QOi3JL@|~7No*Ppg`o)TW6N0kQ6()5UDk!)T)hX z4Q)s?TYDPqT2v)-69j$>D+UxGFUKBxQiC0-xg6-b2H-&iffla1?T#HSO||)^D}43O zE-o%LS^J(&c(PmXv%9b%C;udp-PvEtcGn&qgL7Kcd2njrq5U|o*KJ4BuEAj4-P{Qg-hGebIW^Xp^xWe)p0RHn7e*WH{R+okW=6Pel^EiZ6+QlNqewLX`B)_#W^ zQvmWCLAs#+LA&NX!Ku4FA7fcnN{_d9nR`8v&tnQ}TUC9!?)O)o zcD5e@HzN2CP0&jACg3yyr(JBs!ytmg2yNqt9fsd6_F&2;fN0lHi5*4dT#>|CAB$$M zi!VU@dz$U)Iv;~0n)Ye0Kpdl*5V3c4)(4xfnhAgZRH)DgwzRgoT&}y$1>*b?jUS1< zb?Co%4EE5rry%?i$)&a`G}&1V4t7PHy>0+bJeOw?fA!tez2^IX1Z5W5h9uVH5ZJ#x z$#SCzaFhy+LBT;L$i4%Mh+7|^Lmj&Tei*Yf#-}*6%kFok9GmxR^!4wzP{TET_}fh& z2#)PZRm*);r29Gk1#>~4o1IJp@q+h5Dy7p4OMwU>!0NW;m2-EthLo^$0G=Y1vd-@< zkDbz4TcNP?!}NA(S}MbF-e7(v&>8*3e!gIRV_$|Pis|)j5xq;$` zOkf-90zAT$hNLqBNb7VqWq5`4#gD^Nx%DRv(&?)Jbk}?iAjci~4bDEF-iQ_>6Xr}n z8X(Pg35D-^;->=U(`R(>XUvEsK!`BvlP)bHJZu=N-v>;j9XjG16712aApg7Wm>P>> z8p7G4fM?(~`+N>4vfb3NM)2C|s`b{>`6w$YphwodhI{H+;3o$ZLxXi~@y`%&9b=Q@ zG`DsJ|HCjl<6&4lpMq2((D%{pQtFeZFBK+|EFt6q$}pVu_YXbZ1h@w{yt_NI+aoSH z^wK|qBv;hegMVu3*AN8%e)%2*Re(l~hwcn7LSDHs?0}Lt*UU8wAbc&y?Mdfsn?-H` zsnV7ogX4Bjw#0Twl-D8G{suG5`6b~2n6t0fM{q~Y+F^ls>7awk(hj$Q?%&&zPP9dV zI40T>ykVO$_&J*|xv zRDD$?ka-C%=bae~p;^c0ww62Bi~`+niCVw??QG+);8>mT_HfR}(OIiK-IU{7NIk;wD zc|wGfyRX|>v6f_n0XH>5SsdSayT8cEe*X8b6DUtK@SLe@;ej7pMxZJFO(v+|0Eify z;i;POCbYcUt%|A2r>H_>3hKJz#NQuGz{_ll?HN|&ZCeLqjl6xsdRH^oNM}0U7H|^# z8)FqKnA)GyN&A}5&L3-gx?My9q=EV}rZ=+3d9jbk|)R6aN6s+x^$lF=iCt z!urvK&`#z)jfqTrx@JJ!vmrC?#|nBAqpJ$w6chRP&@w53?QvK)Z9#gThUl=JngFJK zQKUKmBMN(~MIKmZL+2Zim`7vAUJ-$$T8ChP_KEBs=RkAcMFR;`GWW_A-pc7M6=5{4 zkv!${vwHH5WSCzvom4wiH}ys1!Meen#2}Je1gLuW_Uq%!^oNj90Bh;Dd3dD5yN>Y7 zd>SUuJE9*HjZmQ$8JX&wfTadXSFYa?t1ImUR2_U2jeH~kvs$i1WM9XFt(JT3O`tPS z0_S*-){vZhw?GXVeFX~ql=jElqdzysD$xi!1k%~(ly)JV5*?4ok#ucJPp-CRL}kRQ z|EXA_L(8p|GP9Os-wmd>)GqA>1Z`q+7xW$qWOu!@2NMZ6r4>ortCfdzVI`lrErK^C z2I~D34BU+7=??uk!%OY;aGkdF1 zF>SVnN7u_`q;S;#?s~H|68eQCTRPrad{3TPPcXqUN_$=8(Apwk6S7`sjK$}*R(93k_f-%UCOCGyyfmoi~UZWX(GKlJwv zWEvc=O|rj7M+S&FTA%}TRmK?3zc-}SL24jlyZC_w{Jg52t z9KRtRpmh^7rML;;CxDajajSw(9%3e>!pXy8$&ovJas^Q=?q&px2UUjhGxuKTTzhnv zxCw3WlPi zKfjCazqi=>=MRQ2E^k;rCKRS!`V}t*%>U#NJpGiRg{u22zB$if5)x9h1Xw6bf1PD-ccvEy>3lclD$FXe-x`s+5z50*z6)Mj7c<{JAU|Qfxc=)J@F-zW0L{D|| z%IuJ=q;@8Ws0Qmxys>FluDI>=AbTBeyqhkdwL9HVIpD0k4aN%!fNz?> z8BJz~zuPTuQ3kQlFj%E&0^Rqxb!ru)($wo8n90 ziF0wrNYzcz%I|Fr01U7zquz7K7SyT@>#p9?sNMH1`;e#j8tTuPs~|)JXBUvyTY60e z?=E3!FAM}q5k0uEwR4tnZNPp%Mld)b?$qle(bdR(iZlGYJam7~!U zS9eR3EQ0dWr-Xl#E@gZiO&H5pR*p`u&viO<=o`D=?hep}m?k6$a zti&=lvB1wx3!3rFN>)UrVylOffDhOQGO6I5i;KJAlvR<{S)N@HM@+Ssi3KfIuh==q zR&J~*xjkGLnsHwP>6Z$?-jFkPn!J=V?;*L(J>6>Y+ z`9_$pAxNo=<^!Vr2>F~gXQ(o$d6rZ`sSq(D@e6F4Qh>1nMRgQ3K484M9{HA*PnAff zDFg`kkdu;q9mkF745xXTR>NqLbAaLgUJLZEyBQ0Oi2v~HXFU$AwD2m^*T1m?(@|ng zZ*>+v2jD&Buqx#&B4P0K!-`#R)+H~{T7~KnwAW^8353yU+ayEbm?fffi{F$hG9(7c zkvOA%y^WFYBIIqhK~ zWN<2IGUL#MObcF(TS78WW*4xy>;(l=SY*8VK~SDx&n;VA>Lx*g2wyi)OrT-=AbjTOC_1P`@-;2P>^Ea{zh zGjIZglEWF#02mqOo0l%vo@|IVGj5TPq2>1cuO)N7DEm%49$7(J?^>N7CFO=5Ek{3X9n?3g3 z7KXOgo$Al(el|BPCn=D*WwPj0xqpU4k`+0wJ5-b{vn~ME@C8mnf93$m)24M``}mQ<6=%IU}x-tU)WZ5f+Z>|WI`%h&tRW776j=Kz^t{zB9w1lf&viBM+ zK!%F;?e&G7gGgJ(>AT_fVD)fud1EG6MEW3U%ho5uIN@JNDrIam>QXZgNUNlTLC91T z%rAo%RtliW%=GV{J2Ev?L;+(^6@XR*aGH~{0pc{DRHA3@vY|-WPztyxr7$Zgf}a36 ze=(G9z+-(O$xM+3P#HGdYe<;|)xA>5X?) z%P;x>dwzgxL^6oDcE-|u{rNlCieve9jonBRK5omEeZ-_-^0BBDE~}yoyS0`krC5KR zXB!r%pat=8IDZ5Ctayhn5j zE<1FmhnwWw3&q2TnCR#%h(3mT5J`c=toEJ(i9!OXqKV zjHoYwRVt`L%7L(qF-GMDn5gl>l`5zvs}nV`&FeD43@iJWyC5IFC4-}Q8d4~@rnNL| zj|ba*46@R@BF?2nBx>r@SnsQ!w{V=J_4>xu4U8rR~?O`7S0T z_r6-jvUCkbiL5ytr$4EEN%l=!Z`6NhSzbVb&gSf0X9`lAuK?~QekDXhYjD=DI`eky zu_-HIDzAMrJT8ED7EE=LvQW?&k)!tsP@yA~dV-)4&sIS%9~m6FBVR+atV0>&<2*@g;j-$3Z)Ao6R@b~7cg*sE6C5+ zT4OpNL*^zeg+7W+b=a=+U6s@0JY)O{L~y|bR7oxppHXEOrG=6YfUbh z^uCp|TVUpLD(p5v^~y8iy;wiJ6_)yBX&QUzo=^C@SUCFU=n^z6tH5&|4OsE&pNC>9 z0@~h3V?%Td+m;?2nR>1(SNRS6G_`B|p&)LzRQB=QUYUvErPXgD9eShFZ88XRP?Vcg zaZpl>wEW^JA#Y~i%-_K|#LCXVYTqVJ)l65P3bjELU7~YQENstz26cDZc9(aL{rap? zrz1sUq(ptEhl8ZOjojqMMp zAzV5#_b`|Lx&aa;SA+~&`{QrFsw#|({M?B2)B8(XuNAQWtQkhC0jT0aeY(?Y{pnUI zAc)*9r0Y@Lv-_#J=;+Hi+C41zyVjN6dz;#SJG?ObX`7UeLeJ`Qpyvw3Y3`4bN7&5< z_?Ltt{uMx-A@5~>I>+XeY9nvBdL-hiUZSJ9aYK1^N?CXfu2oR_zf#R*&NalW?f=%& zBkFEbyq7l9&|%V8mD@~3-B_8|SUqd4&F3Y7{_0Ksp*`*!^MK*F97PS3?K3?3Njj4L zY3V7JoPFELJ$RVzoBi5N|3R~z|C~~L^}1N_nf~r!{~fhVnf>~m{+-RSjr{$Gk#b<+ z=CCS%M!)yw&iqzd&c5s9lKxm`Uv?#q0pOTwT_~44O_F3{$IvUhxM|(fwpzXK?dJX@zPU?()tak+)xJKN{`f}7a@LxHIC%18 z0?mH@$`k8y|C&DaZlpGu{`^8d{7!$&e)kMU_l_Sg6E@3F{AA>w`nK7aPcr`U4xa#n zztBUz#$@$%Vi?TcmDiEZ!A~%^k&0)Vl;8PA{tCigcarY=VhJ`Izw5~7uKeVV{Z9_dLG&yhG1zc zsj5P|g?)grl%}Z9P@QW$RX^qYsMz0lb`8$!Q0K#`gSE|V9Od$~ zWpB`uvIM31HSx(@XWVc=)DOMIS`c+O?eEf<*0S96aCz;Ne&9s&+-z@h`7wo8Ch4PL;XS@gjGo&aF zs?@aY|3!~kGIs9S;cuz)@#;_hqsP3a&|^$IY78(U%Pb7FOb=+OemdTmzkCS2`}L+F znyj~wd;`>-ZoTy$*aXg(o4n*~p=IECKnO!lrE+-em!&%+s10pBSCelmDSn)oP(lYI zzxDKq2N3L?=Ji#e)6)HiR=hX)RBVxs^-_=iYNvTLjYRPjr`#tUM@}{X?f?&A8mWnj z4?Oh@q}8CWF;C--Ti(3@sPR!@L>Q6^JG7}(-G@}_xn?yCNi?fPXwGMV|7F17aX56f ziZNw%@Co)0Sm7?8Y0v&$`2(U9?D;O-co$KFq-cg)K=*H~NyS>AKp*3+@?SDmO?6}< z?h3^xQrqB$t_In_D0ii{Lm>523tmKaCTL`pSC9%mnQ>9%KKAJnKn*$Jr_`lZWmzx+ z#EIph8svlI^B2s+y)kA*W*GSfeSm%3N<1dUV7C7g=pNDlA3)6cCV8sUq~>xp_%w%d zU)AnTD+m|R`{)vF^O%UO7>g}c825GbT6Wh>M7hTN&yM@3bR_5n%xD)$80Mt)NQG#^ z$FbJ2bsA#OFd)V z@eKYqy&YE!QNl+~nxkB@4WIOClZ2eqV$IBKsqQ!RfIE_&*&^=r zSZ}NyH+@=t#O~e&aE6vYb+7aH42t(~bs@0)?`G?7V*Zv!i~DiL3P<^q;bRrRhMNdB zvS={$q?TmG3|2@#(9lNuK9!Lq4Q)rCs9Sw%l(*;^v!epw1PkTo&@wTO^j9{fdN;u~FL9#;o3y@e!Rlr&h+(JdXY4QuwOQa3mO>0%iKjF0V&(KB| zjliGYH(8{DmduC+R6f3xX^A)*$nq(kPp!dm%S_Qu`t|DstaBajpV z%f{J0jiKmN)g!v7tRqDSX|}t^y9}P$`Gf7$ltHe!qZTzaYD<{+*;<>bc6mIz=ON-9 zLLZ?8Z%C0!n!vUn==11W&^QVP^8u}{XV|2dTl6P?WcutM3e5Lrdug;=~B+S3~Xv`vNw>8|6we>P;F1-(Jl31P#stS11t}?WsF~Y6dzXU$2L%*2xRD|%$IYyyC&J%{91qHg-)HqC2BJ?LMST2jdK)@=vU8 zu zahiWn%mhp$R|Kg;R#70c?i^T~4G>2ZfXQ3FHrXpKH>eE6k~@#3%STN()LXnpMikNh zFidfzo9&9i!ap=y6Rai6En?P;UXrHkXb@%~9(Euib%0(o^Pf%)H|y2nBH~b~ASNSF zR1g>HS;JPA-=ynJX35%bqn!gdsHt3GynX@c?CTn-koD?6F;uBJ;}gK)_JM1IDHGtM zBE_wEKa)Jy&nt64vynNNC89D(eb-)>-CKjs5zHa#qMq3OyvY#ZX>RMW%Mx0VW_7)6 zNf7h7p`w>M?2uP$aH3;+9w`v#2(>nzv(@2|D2~7+4VjBP+$1gzyty=)B(5>F3QIoh ze|By%;(cjh!(PGRO=3Y=kyT}G5RO#MrHS59j74{u2Ox@46I~(Q0gXVf&7fj9@qy-^ zcq-w0+4)3sz0F{ik?bpJ0*=P(W)@bxNk(qjUV3N6zVtMswHx&EfqH_V3QX@b4wpRU zs|30e9-{lej>S%0+tC?8C8}t!JfmjZYVANijgu;fbH$i&l`I{}U-l1Nx2Jf#~8OL0cC$`D2_K0Mk!8R)?Swq2bM z>SS-27x8z^bWIYA1wedXdYGq(yY=Z;aSw9P2LB)+yES9Kg|M*Mh26i-RM?~zW!zXA zFxn=s8<<|5a^!0KkQ0_vq|b=_rd-MTIhry>dQ-ER$wdb9HTHZsP)0PBUWq<|Y@na< z{1T{R=;XuxsR=Pf2|hRn|~3-m%W$$(d`flnPy~)sU2M zxS1_&b$h>py5^nqGNs(iS9!bvJp&W5O-SEv0-gd~JC3hIGg5T^AJS=(`3|!%g_!R* z|L;#keR(C7`#y8&KEOv{HLzN0rY58$%#u+(FEARa9!<6Ha&W3-C`5Kyx z+GIX3#-E7aWpkD$lNveDB_BqUkQr4#bkCdylpQRlx3u{gv*61z=gNEwh9`Gz1t+(~1ATnZRjv!p)!2EOt-wj8|Av6I-i$j) zvch@}h2t+$IO)Mod$e^{tPc!@%Yfx*CZFb^ai&EvhcY+HsO%YN_I$lCB6D-C(;cC` z%q{>J$7M1&o7;lurF%djQYXh`q;SzA78e6C91C9^G}BSia!{2l_$FBJy&@OWG-Y0( z*PhNVQ4k_G?F=epZc5C8&ih&LnO%xo&3ScUGD0?!R!~h<*~RRx`ynA!R+(=Ak(8&| z@~c)>dnl1$LYe2aGV&WX_LP)KwDI0IGv%nHiDwW8Oco1&jpz7za5Q#PzLF>5++P16 zE{YAX$ z0|jRcII$h;|ALB+HoPXJVh+Zhc*R{6mzd&;A{M0L4J=OiQ=$J{L%7egWKPc4lJd;4 zxl^B)v&7|mWnZ#p0Y71-<~E}X^mP6|pkfwjx1HJzRk+;HJjud5KbB?X!Zg^5iAsn7 zEDTc2#ioG6gavvf?J8>QaY?CQSpOWP$3dbjBlSG+yZ3!GVZ`Fw;S%x}X?wOl z1;#1=VoE7vlSz-dd0<8*B^*MgqEJBvoUlp|ZDwY{L?ksEQdTE>d;_9pJZ5>2sUy<7 zfL3i}!G(`f!g491GMkD76N!sFEq@!HQ6_9d1K&ciwIKK_%9B79o#)eqH4D>%cc2>x z-pcTEl?g>A^l1`nGqOEDaymiLgJg7c!8`2v37#>UNcoXF1A>&2WEeh1AF5QU|tNSu+ae<*%=n>*%5a{ z#88le#i0Av*gYVf_EZ+`5`BQ_8Jdv@R=KE!sHdC53ii|!x!cd=*#ProB%+n^E7+z~ zXh1p?d`qn0ckW}ppd2;H;^La_OoOSKd&9YBEVk3HrQSta1`SrKpArpEL4U`S zmW-koZu2F8-NcXhXb3fqx)o>M_8nFw1#G3&FZ#zh@Xox6PEuxa8Uu3lUOq}Rgc466 zw3((>J}w;6SdqS7`G6`u@I`oCfw;at#TJ|+oG-def!^|NlPt`rj+ZkCa6Pxm1u?&;0WIhQsc=Z5Q2*uUfzzndRsH03eW9) zjDoX@GUX(5Cwa#J5Kqp*L0x8GuFq9Fl^=RdwQ94Rd_S)@_96^|`|n+m8h8~D^pj0v}*JvJ5C>tn_%EFnAG zVXq=!7#yOCl2Oln4mMX{L@B-Ra?0>1yKnb*l29!gIHN)~PP0I((BZ z^d@WP9u23VzkJs(Iae}2I=JZ5rVHKf&+Ep=YueiD!)mGEA=cgSvS-N5oV1W5A6lX4 zIny1^)^=**+T0~duT**fO|^8-{Y~~|Rgk#?I{|3?iWQ#Z)2Qf+K7K4N*4KBoATUVl z%c(q(M|m`*=_vXq3-xhmODOjvoS+8Bv^kKgKygUvCTE~vEZ}T(u;yr4P;+jmYkY2W z!1E@qmfQE=Elvj-$Q7Wz*^TID)ad4(AK2ib>NN_n#lOWb>QojSS2VJV!WBRFw` zxwnd&x%V!WQ29TTO{Y%QM=b4qRMEj`Zcx1x|KUW@DNM_)N5d~n%`42GRhH+oX?wr# z?>erygg@A4DUaVV?O!$_hT^;6U#H(TJHuZjiyb_^zdY&*sA-NoKMl2m4#KL?&+NZD z3SKNMtel=nIdG%CQ%5_7og7(6b>K2;ls$r}fwQJOYLvY1HY#t)i1$a0hwat^@RMzJ*dt4} zi8caP&I|edt{hed+djA;PqBc)KDQp(Z~A}sQ$?$|C4&Q@0mclB*YSjltg$pVb}o+& z#dw&)pMVJ|P1sK&gZM>_xAztIYrgL*O&i5Ohe1HcbMzWE$^d4$C>{4L^}|v0MMoW7 znxP~K-A9WDbOaBq)DataTmfySU$)BLCo`tABI@rDqCLKDub674Hg!QAEb&|Co3H1+jxFre2> z0NQ_IX3_ZYKqV~T`^OKB%B28MS|hOy{f`!^H}&stcib>kpgvpDZYg3LXBFG=1hU(E z{v9;ry~yEnDenS|*I`sV)`9Ukf@1)2A{AyrclfwyR)vPm=KTHM&G|e#1-@WVe{Bren{x3MVsHXpgRP=Q|E|BaR~3<~cK9Fm&K1%7PGq9acO*O$LtxomVCpLz+@j z6fLgrZO@HeLKai8u2VoI*A?;FfjxTW#x{Xg@HoQe)RuL-2z{@?hzuo2B%>;>oL-B@ zgR$C$_H`C!B&w&5IV_QGkwV86RjfmPK$+pdwNo!3Pt|5z0)Uz$kuD9Zi9Z1;quDMA zRZ<^6mMEK~7oY_TSo4ce+{XY=?MsZKjz~}~9BOn8Ktc=x5wO0nu@%*wwX=N49o7gC zyNh&)6iQA?2{j5XtbQ3Yyaf+fRuzQ-!_DSvtHXr!?;3ylMJ$8!p~Or}X;Lv&Jj*po zl}ggN3W%VZTk7_Z`rHl^1=3@Dk0~G>m-zHz12sIcbwYC+$=xCWF~!E#C!Y7QZbz4O znIZWU2Z9x05fyWJfHdG}5AamA6ZJJ4quEOtsHKvZfsveT94e{l%DJd9e zXmcB;28`jmkpd3JLM#5@Pis(*uqsd!z$GRZlxw>c?a)rjxWlovF$sqSb%urP)rob3 zY{59F2#yI2A}krymtdcO&HLAuxjJ2XIxl2}zF>|r-MVojQ|os|_f783%P;foml@^v z+44<}tjmw{jXf+%D|7VqR*om((iMR?*}YsR$70$^mn)^)pamL#;#37X=Ku2Em<0+e!Eoz z->VfUNtaDxdUa(Ql!h#LRX*xl@to8dNB&` zhs}<}BIf$Em1*O6&f2RZgYMrx>ymsz&v6ZQGFXoc`lvi-j_FVq0}|Tq$m*S{sPI#D zw3mYe!XNxCRFq;R=QhrE6mA50FjG9k9yhXt2UV5^dOz^mfHx^E4IIdqC>RhNroL__EdyMS-lTrHFyYrEe zpi1%Q6P{iLA6_A?clEsCO;)7c%ECwD^?8S|@k!Glty!k96bpM;D!^#;jOuJUjubD< z91s+7zc;JB-@I$aw6&FiTvf6rh|e_Oleg3J9Xjk$^T^^$br)?bIvTv)wF_kz&O}Om z76hC<8rmN}ekFWvP1Gk@RJU@5W5|PiSFHW;40~dXbe2CuU>~AUY&D#xLV3A@fk+jS zacqcwYh|+GAOye*M=lscRvB!fGnyk*x(clr3Z1B~|9%}UqAo(M$zWfB-XL1LAb3DI zjPDpkknK1(sZVr^2)Q213Zr&Y+O!MCqusz>jYe)%NKf)=5QYbmxY4 z>f?cGR&25U8COa*No}gJxEX=XiG370+5khd02is0T?r}eb#TvajHStx-4-mwt%CTuuDEp5wNXJsI59+vd37VwJ;||l9ZTk0 z2=ub?fn$Ya2}tjRbYk*NCnM;s840D|Hh=1=I0!sKs@N1VP1|)K&I^~xVhdurj&==z(#qf^m66~}L?)(`yo6AG9hlX0=Pu(N$V@kso+|HE8{LC1Q z$(P~a>3^bV!CA*{tG^q@#g(ei`i^efhkS$V0KMH%#_-PZaO4TiUiSnK=u+$c1&RBh zfIweW$Kluc^S_0gu8<(N{l}c=wcc^I_0ieGMU-Eyb>%iti>S&HuSP?D&9{>DH$3Y( zmbQaL?=@{^{(wyq1*|!Lv55|YUI()QUHnYS9Whno>B|S69tTg*EFlcDR?r6JX3dSA zA)Z{miV3eIZ}f}H6GqzYub4IfBEf>(MO;b8D4&xV#jW%$XG)Xn-$d=smqUyh>#pw* zKH5Qm+7=hL_rVdpjSeV}x+oVQyCk-!0x59iPP~TaK1St*AEv63W2*p+*-Pg??0+dx zBA|n1J8{bAix~rVJiA$ZVqJ8~6kE1{@!5D6C*L zveNXxc$(84#~L%q>Ak@%ayhf*<=q*1%Zv&!oL$^dJxckD7bojI?l5x> zD1`Wd+oN^Fci3}Q30^XL^>!~{nwk#6DA3?3*3qe$5c1+MU7$570G6U*)9)FgqjBs`clI0qiMp<=mn{ZF#&%&*toaSlotxpXL~6u+ zKqEiWyvS}pMhJw3oCFf8g0nOe#c4*8@)?yo!;KZKp_m_JBEAx{@A_j{ zuX1q`dxWWaxUs=ZuBv5|pZc6Tnw-Vc8MmG9pfKGh>~72#gO4{1OJ`UI9+2k!T2qF< z>%%n|xh3lv`BB|SVXy}^faws2dWT>Ry?>L3{r;chy&p~u6!r zm{rgW+^|YwWYMJ%Mbny-M=X!)Nm9%m9^$U-cKG*U36ByPviQ<`W%ZqzpzQ^ytltGv z+N(eB1yj1NN?2(IA(fz@yS}Dt09Oqmeu43|7~iT5Hu5~UU~&Hd?$P%9zJEQgo>W=l zo#RI#=EG?116O{=0ZgF+9KD+MYCv&Bfhhnndu$)mFu~;!XN5W#4rPvR)$6_mg zGu>1#1l&kBmq3m$7z)L17oSJ7<1ol3itR!jiakiF4IRBP=r~$21L^47qNxxOZq6+3 zfu9)-eTwsTVRTyQW{sP&Njc11HBD^HEND7GWUIR-DX(_B3gdNQ0964Tlu2KpTlTgG zwI*uQ#G;Fv^8tQ6sg?<=q*OljcQ$M&w3YD0xbbCVSVLYoom(BuLe?U2;#sHn&)Rl~ znVSF78Xn4PyX&rIZfzy}q|+lOhQ&&ujwrs~%Cc#bQgQ$LBiKPXc5KW`GYPGO5Atgc zk_owb&dp&BPdtMxT2Rd)t9~vi6u1tBbx>mn_=}q^YazVB>X~r2wT{EzCM?I4w45u^ z=)-hAJP_;1)d#~3EU|>~sbwEaHcJ~6IR~*5#Z-a}4!!RVi!t zb&W|)v*aKs*)ofp%Zy_#79FCrt)iGQu{@1k3tqv*c^v$TH8R7@xrC}ikNd#LZq}NW za&k%Rn#!WauSLeQ7d+P;UGu2?tF)`@W1T?9pr#ULj&3=w9R$9+EDD zdlRvrg05(wgkb8rR1alJE@M96h_azjjBnDJt9uvQ)}W+g`T&4vRcFzBGh(1g=wwp2CG%nM z%+-AQ>iehNF_N=;C8<41+o_UX&(L0@w9gUw_KK->je??!O&l5P%v)AQ`Bvf%ul`Wb z)3;}vW@vdJkq3ui8;UVvB+{yvCWUIa` zetFB^7J`)10h93T!sO7fDD)kFzB|VF)~1lek=FJOG3X--kE`d~*?VTnu8Ao7s8_ju zCU&szdMqC?51s(}T6a7xiK3ge-}*VfN8<_gQ`6l!2!xP;sA(7)xdJ?ukc*D^l5 zMX4nldFVP^fT&a|XWSUFv}x@}5SqoI*Pl7w?wp~7pb^wtJZIiS)_ZodPuy)7RjIpn zigC!`rD-)2`g=Lq*7Kc^2L;WDWfLRgcf_CJftcS;2kDFonn$tJoKXtJU0l3PPPj{I zti@*ril!QYs=l8i0!H?tMH0f3!**qy(3PZh(~IUqkAW=aMQ!hu_&}Typu+fd!ky(1 z$w-=|er(vFvvs@L3h!H`cUA$qib_|}RSQ~9(S7_$ydH54DJ=heM8Gb|ZxK;CuRdo) zfx-Vmu`q9*F zVe@5PVBc1)3NHpI4e-95YJ5(wh@oXtv~6lH&n2J`I8Zqu@mj%tZ*vJECKA6huU-k; zqSb_%sVh4pc5$M`KQDWr@lp;9d8jKL6&d;y1szIdUHIqBBe&2$Ygr|{U17*03id@(THiqzbYQFL znH?e8s#t4th*dvZy?au%M829=}%Ec z>};VJX@mP^c#l%T!ydM-(aRIWt0T#ChW%WK4Ffr@g-1C4wk<%eZ7;d40|WO=5`kRg z)aXQWUvrMWOnO@XIehni2Oe;yT@BPVkn<-z)ZW*rvFKT=6Wf}2h$mQ&$2YwG=T0Zq ziXfIkdx)~1BHn>r-~#QEaus5-KHaRbZk~iP2=YC+B7i?!E`BkQeZ3I@w7op7myw4D zh48OEg`3hpDZ%0-3JjSj4w1zP@Tja!M{%Q)iu$?x^~Tguo>cpZ+Bx<1n_xc-qyXl3 z@s%`gW0oRFZ55k~|1aKDeRqc1TiTmb7W>|B-J94k&=f1uj)1!uvYRkH1bh8_xo&Lr zW+lDQV-HS)B%;cxM(002ex>?TaeIZ5kcLe%k0?cQZ-g!Nn^;tQAJka)|5=r20mUpr zm=Q2qPj7dHNi$06Yr$aFEpo%Nk}lJjHcfProS!4=y0+kF)bz?j81CPm4oPJA?O0o0 zUF)-7n>NY+>oou89=pjtTh4%djGrF<*i5R)r;RRjNF!QCLp54k0t(Sni$*Q|tZ30V?R{x(t%McuwT8rL)E8?!gqTt#E4Ab2(bV$blLw7gQJ#-`8 z4bt5$4T3Zx(j5cH3>{K}NDnE9pfsEB|M&j(;W}O?>w4Go-p3;>#AgOII(0ZW+0VZ0 zh_|9?_uoo1+|Dmna*5{@e~LuCA+03S^qjRGEkK@JoO>1cFzFLuM`<|}-uRR36iL9F z8UXx>&PBB@wXj0eDiSul8KXhdk~eX`7bn^{C=pjteSD#Cm3z4MPMNHoyz2LX%I}ll zL+cN*OJwIO6fV_$!o+04MBHWN(yI&Wli%=o5z4E0P#;B_rQYhi9v24H)@5GTb?D6% zv2II;B0tvI4*_Sd-}bF~-YPOP3HrSLC_tEFn}VwqX<0mm6*OmlXW1CwcCE$aoL-(i zQk(Aa4nU0!$yhLt=*E0agAsVZ4c}fn21Q>s!=k&|L!Sr6$rXq%&M#_1TxDIpEuwr!isY>Efs`2h;xbR}NqL5aB)6Fo4$_xB6o)!h@K z>5gn-Tr@ikdmF+b>oL8*`zs3DnaFJmLFclzeV3AQ3-8Wq4Od1H@P667Dd4Cs3&C~Z zN4x{^Hgj!p>}M*I(`IYxjcWF&{m4^M``cc(;jA`}U%oFODST$tegUMu((FmCSES63 zb(1n7`@AD=R(H_zHi`6;E~ec`&W@UkH*(M1kNXTpQi~|+9Ev8@ryg$!P73zw)MOqy zb{4>r(9U%puKLv^gQ^dYFJ}6JGb_C>u^M@s-{;HN@8<78S3G8OzKZDQ{##>XQZ1sVnjsy6L>k7~A)Oyr??)CSu2F8&&+nuRTB@>u>%R zjYCc&Q*P`c7rTL8F87G0sqD33KP}J#wpOuVyd%F?W@6t2@tEF3*F=cYE2*V2nkbOY ztVp574%3#{rn{w_XE|L9GV&N6f>+1w;GGdDH@Hl!7b`Hzi1Yt6^RX+-JI=96CL4?| z%{(VJ8$_yVEQ9j!(L7E^IaDG zk?6fvUfJ1T&CO|*bP0~6Z_By(eLjq~pwv{sS5-p&PJSZTRHg3Vx8^Ff;J@q5@D zB{V;;pZZGp9{{BMsduwTFD@j@yjOy@lHxx#kcrdkWz#pKhC@zP14?M0 zeM=`GvBpQ3_c>G?)PqbD3;y*ae{!=H$(T|h*Bq{;!|_if1nSEuOhlJ@F=&IEiEtYg zEqylQ;bHmPn^?tGMN{IQuqjUIWI3uUB06$?w3Vn%6K!7<(^u6O(Q}jK!=UWMGkwBi zI&=t4pdk5#;8xd_S0$EPLqiw$DnXwn9Y<=@*_;(1e`(j&#|fC@mL74NNJcnZF%X#v zt;nFp@vuMQ4DgDw5KmsJDIs=gQB#jeGd{OLQ$f(|X?}3&ko{E#mTHTOrCxiXJrI(A zmVjJom61z(ItCBXzTy@RwD&^V7hu$ZbR*k{-x8p|`d#yT+!eRp<+Cluq{REk}v zL}`_aI*QHtaPD)E&4jVK15n_o=BD^IQLw+sg_Pd9q5lK|AB+9B+FqQG=#Gd=7V9<< zLCh33w+u;{VQ8j#cvC#af7Lb;8t+)yxiZ_riz}1|CxL zYgvg~p{9sgk!Ci88np&49(G?Lf&e?NeA^<_<9MlL2tDl+5wyViZ$FuYI|~U!;-Jb| zn=m=rl4NI(UF<_r6+P>KNtpmI-PZeN{x=)iDjV|Z3dvUfax-jJ&XOPh7ExdYWk(ZD z^A&AlCf0%sNBI#p4s8q+BGEphhy*Jm(5F24!Xvh17@HS3dZpcc7IVNqfDj)EoZ&%# zqgHkpj###66$$9oM(_FdqPXZ04YhBOoI)vO{u)KTFXr8Vg9SGJBmyIfiiobm?d2ai z{qMc)1xtOOzq;C?GJN-zjr9MouEA(7EH?~wID4tIU)NJA1~~MWNAz;8m{AA3qq3WV z!p+O&7j^svn?`gfW=>6A2o?0S>Y7(QAC~qgbL|3K*3&;aMrKV7Z&27QZRbI3xr)h& zdw(PlY*CVZ-myI-Hp1|D?Z=K{61bVH2(I>zQ4pUhwWDxBd?(jMtfmpMu-k(-I$6Ao zWgQisqsU%!F9%o!Vtj%Gdc7M|YqnvqLp902*Zbf_L_!g=4=-z2pDgi9Gnp(AN!!dR zxt}J%NbgokZ-rY{lHK~$t{K4=%jbK)W=aYM&5l%GUg<}+cx&Sig=PbRcV8TS zN)C>YNzR(5s%s82+&5m4+L%=wA6V&g_^g3~OS>A#B$GYX?P=%RTc!|LPi|B%{}}(V zrZl^Q7+v*Uv>0A>+&CN1vVAw;h%N{#=nWw0hy5T=vX~bv3zW(Z@Y|{J|EYRcp?)P1 z(!lJ_GT=l+Ho#W>sF3*JrB6a}9!g>uU)CrZNW&a9&6GlrxOT^aNGX^9fvQ#AOA_gl z(rUH><$`@AU-Gw3p}6b{Szg|oX`SqZk~vNU?HgSLngONvClLW~?JnpXU4@iYql)_M zmYK!3HfprSK@;5#en(cJjb=8Egi-?YA{a_;LS8I`TfLWtQaSi)wAbZL+_+DM<|RO% z;3wSmE2EHe+L(=XQGjNF?NZ|3&elok&}i}9`p;O@^C5Bp;m=os>vZq$c2Oew%|d5* za`AKFPdM7qZod;A2X_!ZaTY_MU>^N9{r=M(`zZfgZ1+R^V=Z5&@d8G%iG;ommakbvz5a7$M9YL zKLJ;?*knnFSRzJD*fu#H03!u;GS$pcoSd)fB4()9K0bPA%+k!F+64^u864JMmmC%W zGlm_(-h65Z9~?lY(t^FFnEo>F1qMcai<#h!iiq&VJ<>_Bfz_Aq^N;yKl!(~v02uPMcI6*?T%i}rW;<}-EQv;X5%kQB~?4)|HuQ;xd( zU?|EGHX{m;1|J<(cL|FQ48sg|Ez8&P0YFO4i<8~+oPeJ ze-PYBy8gqO=1x)y&nWbrhpn(q+=TV>)=~V1^{*e;>FFLrIfo+5NqNz^9Kr#lJpIFW z`Jhr)tDTm$#)qiAxa3XUpF7@f`!9hx+D0cQ)x&9V`VlId8gZ|aHddBoTTMJZuQa~h z+!q$YGc*)zC53$bb-R!(xzC8M+!6*|19q>{;8Aje#dV`9I~PB_ro#pBu-D@g4j`BrzX>*b14%L-VAy%AHFP8@mq z3@V%6Bq-^WCDNI@&yx#{k7bVp>Mx#w21Wb|p zakSJ6WE|)3kC)P3`f=0>&l&V*euGsI6)qN~@~Gvspd3;QjJzTA_id+79XM$7x3bLm zljT=u5yB-&ax#2JVkZ;f7B`{A%hX+}Q1hn>ccdiF%_mA54HrtqMyk+%wox&0RrjJz zB-G~brw+{LV~aYZ3{(rW(ccE{Mb_6{8?dOK>@qRHpsxolnO^)Ro zV@?!}ErAdj3voQtdL_@O>rxJb1|^}9If|=j2F5qxNTj|ZOE0~3OqlJ+WyX1^&4p|| zm#|%QGg+uNU%&Q?EaAE@SL=YR$0LU8h1_NdM)STP}Ve*(;0eyW)Dr0T;thC zyYoL9vVx@)i?tah`)Tyc8bD5<8a@nHV5eHu_~OG*1^!mj7Du7ngKPuj`Wm*kb@Ms> zo|5TlB#R0Sk7#+rUoC=#nwjnfYQ3wf55-?{>#@b`CA4G7QPCT?rV6;F+Uc)MAg_6} zyAIXgiWFH~r*dIFH=EzhHMOq~jU@9>R{mine2HFIiMa&nEZ#Alm}ivIUJ!L8{75Q} zV4SAGgx8`kIH6+URJAL<^Z!r@H@7yx`CM5a0zO&b_VkzD-_fa8W^QxFTy(tpY%R7r z1|0L1Giu)LUprq+iE?b9voqiwF9r8sgC+gB)H*gQxW-=l&L)n~Peto8evAs@NXFU1 zZ|IG&3{0PeUlAL4l4ISO&S?c%vDE?*U2@OFea(<5F!*Jxrbp;DY0@{}ux4?+>6~9) zz9JK=iHR$qpFlR|mkGipna;TzWBZ|!^QIp8$>84G62xAX|C)vK^@M?XWuJ~NdOCfk z27BH$sokp+AUn$cM%`HpffC68x-0aHOmz)@l73&;Qw-9NFC!b&l6kSU+yY*SfZG>L zBBr|Oz-rm-u1#(|Tjx(elB{x6vn3k_KSPF_i*(#ovJiBTnULK7IMVw9QBEcgM^(qK z^*uR)t;`v0LUMHI;``d#cG31O7Lsw!QltZv4GrzBJnu1+>@Jq7j#d7dV&uHui2~x4 zB3*t@HO^IJW0Xik4)&@~KfW$Z-w&55xF0wXO9lwHW07b{>(CaIxTfMTMxQ{I@z$^a#_t z5Uo@hNALsYXe6RVyQAZ)RKca%4nkkg$?`?y5e3`ownZ}{^0@lss#u1n6@5o?R}Q_~ z`lum-*s{Y=t5|CC;D$P`K0gNtp*ZZC&BZK`?c91NVlCYGh2jwJ$U|E#`^1JNGkkUb zNLqoQ)cAdbTjamRcNfFZOYxmQT7LTpCOWAFp@d(fVYxSpK0^%3PY5vM5IOAB!($~|N}23i^%l3W8y zKzffdusR*6#CQJ?n8v(w!7)6Bxd#uDJm+oF$0dq}2A5;eW2v2a=;h>IHI(4|L& zy}DSOeus3Y4)1B-=kqV;=I1QH#T~=?a?6+6`&jafM@eI$ks8Ee)4?l^G8|WYHjzYn zDLjR3+#)#n6W8^=0^PlQnToAti3&`XWbLB2*xcx9t|SHb_uQFsNb}G#>)7ZRcEpsT zV#XMe^U$0TcD@KIrQY5iW-|?QzYlzS80!+WXiJGUH}PPDLtEl(hrOF-E7`Hy&(GO< zdCT19n#txAY@-33Wk=@q;|=7Bq4(9iwO>@;DQs!IX%EfmHe6j`8UErK4$k=+^JjHv zV9wTOPD?T@J$+bzDVIR@CG~bUd&xDj?|hG_n$q{Hj9Y;c`9YG(Sfb(WpRQVu9Y!ua zlDa7^ip2-lm_f_ zd>=E^;fj|$pu@bhH}3{!m{J*iM*nHK0RZ(6N*{a(tLYPsC;b?+k22Ey$&u#;eD;%w z9{~6OoJR4C{+f8^PruqcZ3ZH&y>-_vqq9{G@Zv`v-+_|tFXv2JD#baPvzsW58L^Yz zeq*(R-I5U+0O)pCUlQ&;W72d+fprpdKyFk(E_$;tQPBBoEGA-DUc0=_-AlNQ1n38< z8X9fb>156Em9ncCnoP)apSIhNu+M&2uT^52O$qy)fH!*b_N1bz=VB2XyTDGdt;eC4WQj- zL<`8W^L6|Hk-O&)`{o4kUj}?10sz;hgnGEoAI0)pPM9D3W-@{|ZpujPP6%z)E7%a# z^Txt!JPRbASRYtzAG8Q$eqcVrZ^|s0pIi(w+Q-wMIVMf0pDmYs%ZU}yzF$*6U$aa7 zfJaUBkCeY9pPwDSsnUgqycdb6gCOOHfll@I{+iR%kkVSDfUVSOVY>>fwkBct`|n)v z^_~CIC!^${XC|zEKZ@x_Sq{4a7&UrcS1%rO^!g<7D9}oKoN{uTZx6<5*qicn5EbC? z1k-y*5+N0u&LAP%5x8unyKO&wG2Y?nbZV9#(qz>P%8FXRQ;Dy{=k-1^`?#9QxvfmQ z2Rql_fQ(OGbA9R+9_+NhPfvrGEI$FQpHJ!myJ##RKUUvC?JOr^H@Ek-c8Fm$@ z*EApq*gLY8UL{GpblJaR(%G&F1@cuwP(C#*p6kXm5rmT1G~NC}-uWcaia)Cq z&3jb8sASDfdS48H;lhm7`Q$kUIE)4B!*kypCXJh-|2c|tou(B@PEV?EKZ`sP=t8$e zEr6q}mE}jHKU3OF72_puM!z5k)7nI?XJ1s2h6lnrrw(fDVB;#rmT`nI@K8}w^bs?F zJwr>j-;N~FYzgeCFQ9JofmX&Coo}8DYgizSmXHN6U6F7kGzLy!BM06mY7YZlG|O>* zGbdQCai1^pGTtj)pm3pqu+D=O1elBwXN@PTy-&Zu@ zGf7M>2rnVhS`C4DzUDQ`qAm$EH1TJ^C?FFwpUHZFzf zM^NDP2X0|{$4L3E;oq>l-OywZv6+xhvB*l?G@Z)lteePZC1c}ZfVDJDC7XXKEBJWi zhG<3sgf<&!Ld!j2SZxvp`}LW~*(-=%GUqJv&>$?!AnG python -m pip install yahoofinancials + + 2. Installation using github (Mac/Linux): + + .. code-block:: bash + + $ git clone https://github.com/JECSand/yahoofinancials.git + $ cd yahoofinancials + $ python setup.py install + + 3. Demo using the included demo script: + + .. code-block:: bash + + $ cd yahoofinancials + $ python demo.py -h + $ python demo.py + $ python demo.py WFC C BAC + + 4. Test using the included unit testing script: + + .. code-block:: bash + + $ cd yahoofinancials + $ python test/test_yahoofinancials.py + + Module Methods + -------------- + - The financial data from all methods is returned as JSON. + - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. + - YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). + + Featured Methods + ^^^^^^^^^^^^^^^^ + 1. get_financial_stmts(frequency, statement_type, reformat=True) + + - frequency can be either 'annual' or 'quarterly'. + - statement_type can be 'income', 'balance', 'cash' or a list of several. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. + 2. get_stock_price_data(reformat=True) + + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. + 3. get_stock_earnings_data(reformat=True) + + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. + 4. get_summary_data(reformat=True) + + - Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. + 5. get_stock_quote_type_data() + + 6. get_historical_price_data(start_date, end_date, time_interval) + + - This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. + - start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. + - end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. + - time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. + - Data response includes relevant pricing event data such as dividends and stock splits. + 7. get_num_shares_outstanding(price_type='current') + + - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. + + Methods Removed in V1.0 + ^^^^^^^^^^^^^^^^^^^^^^^ + - get_stock_summary_data(): + - get_historical_stock_data(): + + Additional Module Methods + ^^^^^^^^^^^^^^^^^^^^^^^^^ + - get_interest_expense(): + - get_operating_income(): + - get_total_operating_expense(): + - get_total_revenue(): + - get_cost_of_revenue(): + - get_income_before_tax(): + - get_income_tax_expense(): + - get_gross_profit(): + - get_net_income_from_continuing_ops(): + - get_research_and_development(): + - get_current_price(): + - get_current_change(): + - get_current_percent_change(): + - get_current_volume(): + - get_prev_close_price(): + - get_open_price(): + - get_ten_day_avg_daily_volume(): + - get_three_month_avg_daily_volume(): + - get_stock_exchange(): + - get_market_cap(): + - get_daily_low(): + - get_daily_high(): + - get_currency(): + - get_yearly_high(): + - get_yearly_low(): + - get_dividend_yield(): + - get_annual_avg_div_yield(): + - get_five_yr_avg_div_yield(): + - get_dividend_rate(): + - get_annual_avg_div_rate(): + - get_50day_moving_avg(): + - get_200day_moving_avg(): + - get_beta(): + - get_payout_ratio(): + - get_pe_ratio(): + - get_price_to_sales(): + - get_exdividend_date(): + - get_book_value(): + - get_ebit(): + - get_net_income(): + - get_earnings_per_share(): + + Usage Examples + -------------- + - The class constructor can take either a single ticker or a list of tickers as it's parameter. + - This makes it easy to initiate multiple classes for different groupings of financial assets. + - Quarterly statement data returns the last 4 periods of data, while annual returns the last 3. + + Single Ticker Example + ^^^^^^^^^^^^^^^^^^^^^ + + .. code-block:: python + + from yahoofinancials import YahooFinancials + + ticker = 'AAPL' + yahoo_financials = YahooFinancials(ticker) + + balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') + income_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'income') + all_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', ['income', 'cash', 'balance']) + apple_earnings_data = yahoo_financials.get_stock_earnings_data() + apple_net_income = yahoo_financials.get_net_income() + historical_stock_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') + + Lists of Tickers Example + ^^^^^^^^^^^^^^^^^^^^^^^^ + + .. code-block:: python + + from yahoofinancials import YahooFinancials + + tech_stocks = ['AAPL', 'MSFT', 'INTC'] + bank_stocks = ['WFC', 'BAC', 'C'] + commodity_futures = ['GC=F', 'SI=F', 'CL=F'] + cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] + currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] + mutual_funds = ['PRLAX', 'QASGX', 'HISFX'] + us_treasuries = ['^TNX', '^IRX', '^TYX'] + + yahoo_financials_tech = YahooFinancials(tech_stocks) + yahoo_financials_banks = YahooFinancials(bank_stocks) + yahoo_financials_commodities = YahooFinancials(commodity_futures) + yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) + yahoo_financials_currencies = YahooFinancials(currencies) + yahoo_financials_mutualfunds = YahooFinancials(mutual_funds) + yahoo_financials_treasuries = YahooFinancials(us_treasuries) + + tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') + bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') + + banks_net_ebit = yahoo_financials_banks.get_ebit() + tech_stock_price_data = yahoo_financials_tech.get_stock_price_data() + daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_mutualfund_prices = yahoo_financials_mutualfunds.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + daily_treasury_prices = yahoo_financials_treasuries.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') + + Examples of Returned JSON Data + ------------------------------ + + 1. Annual Income Statement Data for Apple: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_financial_stmts('annual', 'income')) + + + .. code-block:: javascript + + { + "incomeStatementHistory": { + "AAPL": [ + { + "2016-09-24": { + "minorityInterest": null, + "otherOperatingExpenses": null, + "netIncomeFromContinuingOps": 45687000000, + "totalRevenue": 215639000000, + "totalOtherIncomeExpenseNet": 1348000000, + "discontinuedOperations": null, + "incomeTaxExpense": 15685000000, + "extraordinaryItems": null, + "grossProfit": 84263000000, + "netIncome": 45687000000, + "sellingGeneralAdministrative": 14194000000, + "interestExpense": null, + "costOfRevenue": 131376000000, + "researchDevelopment": 10045000000, + "netIncomeApplicableToCommonShares": 45687000000, + "effectOfAccountingCharges": null, + "incomeBeforeTax": 61372000000, + "otherItems": null, + "operatingIncome": 60024000000, + "ebit": 61372000000, + "nonRecurring": null, + "totalOperatingExpenses": 0 + } + } + ] + } + } + + 2. Annual Balance Sheet Data for Apple: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_financial_stmts('annual', 'balance')) + + + .. code-block:: javascript + + { + "balanceSheetHistory": { + "AAPL": [ + { + "2016-09-24": { + "otherCurrentLiab": 8080000000, + "otherCurrentAssets": 8283000000, + "goodWill": 5414000000, + "shortTermInvestments": 46671000000, + "longTermInvestments": 170430000000, + "cash": 20484000000, + "netTangibleAssets": 119629000000, + "totalAssets": 321686000000, + "otherLiab": 36074000000, + "totalStockholderEquity": 128249000000, + "inventory": 2132000000, + "retainedEarnings": 96364000000, + "intangibleAssets": 3206000000, + "totalCurrentAssets": 106869000000, + "otherStockholderEquity": 634000000, + "shortLongTermDebt": 11605000000, + "propertyPlantEquipment": 27010000000, + "deferredLongTermLiab": 2930000000, + "netReceivables": 29299000000, + "otherAssets": 8757000000, + "longTermDebt": 75427000000, + "totalLiab": 193437000000, + "commonStock": 31251000000, + "accountsPayable": 59321000000, + "totalCurrentLiabilities": 79006000000 + } + } + ] + } + } + + 3. Quarterly Cash Flow Statement Data for Citigroup: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('C') + print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) + + + .. code-block:: javascript + + { + "cashflowStatementHistoryQuarterly": { + "C": [ + { + "2017-06-30": { + "totalCashFromOperatingActivities": -18505000000, + "effectOfExchangeRate": -117000000, + "totalCashFromFinancingActivities": 39798000000, + "netIncome": 3872000000, + "dividendsPaid": -760000000, + "salePurchaseOfStock": -1781000000, + "capitalExpenditures": -861000000, + "changeToLiabilities": -7626000000, + "otherCashflowsFromInvestingActivities": 82000000, + "totalCashflowsFromInvestingActivities": -22508000000, + "netBorrowings": 33586000000, + "depreciation": 901000000, + "changeInCash": -1332000000, + "changeToNetincome": 1444000000, + "otherCashflowsFromFinancingActivities": 8753000000, + "changeToOperatingActivities": -17096000000, + "investments": -23224000000 + } + } + ] + } + } + + 4. Monthly Historical Stock Price Data for Wells Fargo: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('WFC') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + + + .. code-block:: javascript + + { + "WFC": { + "currency": "USD", + "eventsData": { + "dividends": { + "2018-08-01": { + "amount": 0.43, + "date": 1533821400, + "formatted_date": "2018-08-09" + } + } + }, + "firstTradeDate": { + "date": 76233600, + "formatted_date": "1972-06-01" + }, + "instrumentType": "EQUITY", + "prices": [ + { + "adjclose": 57.19147872924805, + "close": 57.61000061035156, + "date": 1533096000, + "formatted_date": "2018-08-01", + "high": 59.5, + "low": 57.08000183105469, + "open": 57.959999084472656, + "volume": 138922900 + } + ], + "timeZone": { + "gmtOffset": -14400 + } + } + } + + 5. Monthly Historical Price Data for EURUSD: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('EURUSD=X') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + + + .. code-block:: javascript + + { + "EURUSD=X": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1070236800, + "formatted_date": "2003-12-01" + }, + "instrumentType": "CURRENCY", + "prices": [ + { + "adjclose": 1.1394712924957275, + "close": 1.1394712924957275, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 1.169864296913147, + "low": 1.1365960836410522, + "open": 1.168961763381958, + "volume": 0 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } + } + + 6. Monthly Historical Price Data for BTC-USD: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('BTC-USD') + print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) + + + .. code-block:: javascript + + { + "BTC-USD": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1279321200, + "formatted_date": "2010-07-16" + }, + "instrumentType": "CRYPTOCURRENCY", + "prices": [ + { + "adjclose": 6285.02001953125, + "close": 6285.02001953125, + "date": 1533078000, + "formatted_date": "2018-07-31", + "high": 7760.740234375, + "low": 6133.02978515625, + "open": 7736.25, + "volume": 4334347882 + } + ], + "timeZone": { + "gmtOffset": 3600 + } + } + } + + 7. Weekly Historical Price Data for Crude Oil Futures: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('CL=F') + print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) + + + .. code-block:: javascript + + { + "CL=F": { + "currency": "USD", + "eventsData": {}, + "firstTradeDate": { + "date": 1522555200, + "formatted_date": "2018-04-01" + }, + "instrumentType": "FUTURE", + "prices": [ + { + "adjclose": 68.58999633789062, + "close": 68.58999633789062, + "date": 1532923200, + "formatted_date": "2018-07-30", + "high": 69.3499984741211, + "low": 66.91999816894531, + "open": 68.37000274658203, + "volume": 683048039 + }, + { + "adjclose": 67.75, + "close": 67.75, + "date": 1533528000, + "formatted_date": "2018-08-06", + "high": 69.91999816894531, + "low": 66.13999938964844, + "open": 68.76000213623047, + "volume": 1102357981 + } + ], + "timeZone": { + "gmtOffset": -14400 + } + } + } + + 8. Apple Stock Quote Data: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_stock_quote_type_data()) + + + .. code-block:: javascripts + + { + "AAPL": { + "underlyingExchangeSymbol": null, + "exchangeTimezoneName": "America/New_York", + "underlyingSymbol": null, + "headSymbol": null, + "shortName": "Apple Inc.", + "symbol": "AAPL", + "uuid": "8b10e4ae-9eeb-3684-921a-9ab27e4d87aa", + "gmtOffSetMilliseconds": "-14400000", + "exchange": "NMS", + "exchangeTimezoneShortName": "EDT", + "messageBoardId": "finmb_24937", + "longName": "Apple Inc.", + "market": "us_market", + "quoteType": "EQUITY" + } + } + + 9. U.S. Treasury Current Pricing Data: + + + .. code-block:: python + + yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) + print(yahoo_financials.get_current_price()) + + + .. code-block:: javascript + + { + "^IRX": 2.033, + "^TNX": 2.895, + "^TYX": 3.062 + } + + 10. BTC-USD Summary Data: + + + .. code-block:: python + + yahoo_financials = YahooFinancials('BTC-USD') + print(yahoo_financials.get_summary_data()) + + + .. code-block:: javascript + + { + "BTC-USD": { + "algorithm": "SHA256", + "ask": null, + "askSize": null, + "averageDailyVolume10Day": 545573809, + "averageVolume": 496761640, + "averageVolume10days": 545573809, + "beta": null, + "bid": null, + "bidSize": null, + "circulatingSupply": 17209812, + "currency": "USD", + "dayHigh": 6266.5, + "dayLow": 5891.87, + "dividendRate": null, + "dividendYield": null, + "exDividendDate": "-", + "expireDate": "-", + "fiftyDayAverage": 6989.074, + "fiftyTwoWeekHigh": 19870.62, + "fiftyTwoWeekLow": 2979.88, + "fiveYearAvgDividendYield": null, + "forwardPE": null, + "fromCurrency": "BTC", + "lastMarket": "CCCAGG", + "marketCap": 106325663744, + "maxAge": 1, + "maxSupply": 21000000, + "navPrice": null, + "open": 6263.2, + "openInterest": null, + "payoutRatio": null, + "previousClose": 6263.2, + "priceHint": 2, + "priceToSalesTrailing12Months": null, + "regularMarketDayHigh": 6266.5, + "regularMarketDayLow": 5891.87, + "regularMarketOpen": 6263.2, + "regularMarketPreviousClose": 6263.2, + "regularMarketVolume": 755834368, + "startDate": "2009-01-03", + "strikePrice": null, + "totalAssets": null, + "tradeable": false, + "trailingAnnualDividendRate": null, + "trailingAnnualDividendYield": null, + "twoHundredDayAverage": 8165.154, + "volume": 755834368, + "volume24Hr": 750196480, + "volumeAllCurrencies": 2673437184, + "yield": null, + "ytdReturn": null + } + } + +Keywords: finance data,stocks,commodities,cryptocurrencies,currencies,forex,yahoo finance +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: Financial and Insurance Industry +Classifier: Topic :: Office/Business :: Financial :: Investment +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Operating System :: OS Independent +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 diff --git a/yahoofinancials.egg-info/SOURCES.txt b/yahoofinancials.egg-info/SOURCES.txt new file mode 100644 index 0000000..8470410 --- /dev/null +++ b/yahoofinancials.egg-info/SOURCES.txt @@ -0,0 +1,11 @@ +MANIFEST.in +README.rst +setup.py +test/test_yahoofinancials.py +yahoofinancials/__init__.py +yahoofinancials.egg-info/PKG-INFO +yahoofinancials.egg-info/SOURCES.txt +yahoofinancials.egg-info/dependency_links.txt +yahoofinancials.egg-info/not-zip-safe +yahoofinancials.egg-info/requires.txt +yahoofinancials.egg-info/top_level.txt \ No newline at end of file diff --git a/yahoofinancials.egg-info/dependency_links.txt b/yahoofinancials.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/yahoofinancials.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/yahoofinancials.egg-info/not-zip-safe b/yahoofinancials.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/yahoofinancials.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/yahoofinancials.egg-info/requires.txt b/yahoofinancials.egg-info/requires.txt new file mode 100644 index 0000000..fc15e64 --- /dev/null +++ b/yahoofinancials.egg-info/requires.txt @@ -0,0 +1,2 @@ +beautifulsoup4 +pytz diff --git a/yahoofinancials.egg-info/top_level.txt b/yahoofinancials.egg-info/top_level.txt new file mode 100644 index 0000000..cef6e42 --- /dev/null +++ b/yahoofinancials.egg-info/top_level.txt @@ -0,0 +1 @@ +yahoofinancials From 4cad5ccd38a2cf189575e5b93abe61d51fc6aacc Mon Sep 17 00:00:00 2001 From: alt Date: Thu, 23 Aug 2018 19:07:48 -0500 Subject: [PATCH 037/108] update unit test with #10 bug fix test --- yahoofinancials.egg-info/PKG-INFO | 651 ------------------ yahoofinancials.egg-info/SOURCES.txt | 11 - yahoofinancials.egg-info/dependency_links.txt | 1 - yahoofinancials.egg-info/not-zip-safe | 1 - yahoofinancials.egg-info/requires.txt | 2 - yahoofinancials.egg-info/top_level.txt | 1 - 6 files changed, 667 deletions(-) delete mode 100644 yahoofinancials.egg-info/PKG-INFO delete mode 100644 yahoofinancials.egg-info/SOURCES.txt delete mode 100644 yahoofinancials.egg-info/dependency_links.txt delete mode 100644 yahoofinancials.egg-info/not-zip-safe delete mode 100644 yahoofinancials.egg-info/requires.txt delete mode 100644 yahoofinancials.egg-info/top_level.txt diff --git a/yahoofinancials.egg-info/PKG-INFO b/yahoofinancials.egg-info/PKG-INFO deleted file mode 100644 index b17fb6b..0000000 --- a/yahoofinancials.egg-info/PKG-INFO +++ /dev/null @@ -1,651 +0,0 @@ -Metadata-Version: 1.1 -Name: yahoofinancials -Version: 1.1 -Summary: A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance -Home-page: https://github.com/JECSand/yahoofinancials -Author: Connor Sanders -Author-email: connor@exceleri.com -License: MIT -Download-URL: https://github.com/JECSand/yahoofinancials/archive/1.1.tar.gz -Description-Content-Type: UNKNOWN -Description: =============== - yahoofinancials - =============== - - A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, ETF, and US Treasury financial data from Yahoo Finance. - - .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master - :target: https://travis-ci.org/JECSand/yahoofinancials - - Current Version: v1.1 - - Version Released: 08/23/2018 - - Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues - - Overview - -------- - A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. - - - As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. - - Installation - ------------ - - yahoofinancials runs fine on most versions of python 2 and 3. - - It was built and tested using versions 2.7 and 3.5 - - The package depends on beautifulsoup4 and pytz to work. - - 1. Installation using pip: - - - Linux/Mac: - - .. code-block:: bash - - $ pip install yahoofinancials - - - Windows (If python doesn't work for you in cmd, try running the following command with just py): - - .. code-block:: - - > python -m pip install yahoofinancials - - 2. Installation using github (Mac/Linux): - - .. code-block:: bash - - $ git clone https://github.com/JECSand/yahoofinancials.git - $ cd yahoofinancials - $ python setup.py install - - 3. Demo using the included demo script: - - .. code-block:: bash - - $ cd yahoofinancials - $ python demo.py -h - $ python demo.py - $ python demo.py WFC C BAC - - 4. Test using the included unit testing script: - - .. code-block:: bash - - $ cd yahoofinancials - $ python test/test_yahoofinancials.py - - Module Methods - -------------- - - The financial data from all methods is returned as JSON. - - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. - - YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). - - Featured Methods - ^^^^^^^^^^^^^^^^ - 1. get_financial_stmts(frequency, statement_type, reformat=True) - - - frequency can be either 'annual' or 'quarterly'. - - statement_type can be 'income', 'balance', 'cash' or a list of several. - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. - 2. get_stock_price_data(reformat=True) - - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. - 3. get_stock_earnings_data(reformat=True) - - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. - 4. get_summary_data(reformat=True) - - - Returns financial summary data for cryptocurrencies, stocks, currencies, ETFs, mutual funds, U.S. Treasuries, commodity futures, and indexes. - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. - 5. get_stock_quote_type_data() - - 6. get_historical_price_data(start_date, end_date, time_interval) - - - This method will pull historical pricing data for stocks, currencies, ETFs, mutual funds, U.S. Treasuries, cryptocurrencies, commodities, and indexes. - - start_date should be entered in the 'YYYY-MM-DD' format and is the first day that data will be pulled for. - - end_date should be entered in the 'YYYY-MM-DD' format and is the last day that data will be pulled for. - - time_interval can be either 'daily', 'weekly', or 'monthly'. This variable determines the time period interval for your pull. - - Data response includes relevant pricing event data such as dividends and stock splits. - 7. get_num_shares_outstanding(price_type='current') - - - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. - - Methods Removed in V1.0 - ^^^^^^^^^^^^^^^^^^^^^^^ - - get_stock_summary_data(): - - get_historical_stock_data(): - - Additional Module Methods - ^^^^^^^^^^^^^^^^^^^^^^^^^ - - get_interest_expense(): - - get_operating_income(): - - get_total_operating_expense(): - - get_total_revenue(): - - get_cost_of_revenue(): - - get_income_before_tax(): - - get_income_tax_expense(): - - get_gross_profit(): - - get_net_income_from_continuing_ops(): - - get_research_and_development(): - - get_current_price(): - - get_current_change(): - - get_current_percent_change(): - - get_current_volume(): - - get_prev_close_price(): - - get_open_price(): - - get_ten_day_avg_daily_volume(): - - get_three_month_avg_daily_volume(): - - get_stock_exchange(): - - get_market_cap(): - - get_daily_low(): - - get_daily_high(): - - get_currency(): - - get_yearly_high(): - - get_yearly_low(): - - get_dividend_yield(): - - get_annual_avg_div_yield(): - - get_five_yr_avg_div_yield(): - - get_dividend_rate(): - - get_annual_avg_div_rate(): - - get_50day_moving_avg(): - - get_200day_moving_avg(): - - get_beta(): - - get_payout_ratio(): - - get_pe_ratio(): - - get_price_to_sales(): - - get_exdividend_date(): - - get_book_value(): - - get_ebit(): - - get_net_income(): - - get_earnings_per_share(): - - Usage Examples - -------------- - - The class constructor can take either a single ticker or a list of tickers as it's parameter. - - This makes it easy to initiate multiple classes for different groupings of financial assets. - - Quarterly statement data returns the last 4 periods of data, while annual returns the last 3. - - Single Ticker Example - ^^^^^^^^^^^^^^^^^^^^^ - - .. code-block:: python - - from yahoofinancials import YahooFinancials - - ticker = 'AAPL' - yahoo_financials = YahooFinancials(ticker) - - balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') - income_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'income') - all_statement_data_qt = yahoo_financials.get_financial_stmts('quarterly', ['income', 'cash', 'balance']) - apple_earnings_data = yahoo_financials.get_stock_earnings_data() - apple_net_income = yahoo_financials.get_net_income() - historical_stock_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - - Lists of Tickers Example - ^^^^^^^^^^^^^^^^^^^^^^^^ - - .. code-block:: python - - from yahoofinancials import YahooFinancials - - tech_stocks = ['AAPL', 'MSFT', 'INTC'] - bank_stocks = ['WFC', 'BAC', 'C'] - commodity_futures = ['GC=F', 'SI=F', 'CL=F'] - cryptocurrencies = ['BTC-USD', 'ETH-USD', 'XRP-USD'] - currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] - mutual_funds = ['PRLAX', 'QASGX', 'HISFX'] - us_treasuries = ['^TNX', '^IRX', '^TYX'] - - yahoo_financials_tech = YahooFinancials(tech_stocks) - yahoo_financials_banks = YahooFinancials(bank_stocks) - yahoo_financials_commodities = YahooFinancials(commodity_futures) - yahoo_financials_cryptocurrencies = YahooFinancials(cryptocurrencies) - yahoo_financials_currencies = YahooFinancials(currencies) - yahoo_financials_mutualfunds = YahooFinancials(mutual_funds) - yahoo_financials_treasuries = YahooFinancials(us_treasuries) - - tech_cash_flow_data_an = yahoo_financials_tech.get_financial_stmts('annual', 'cash') - bank_cash_flow_data_an = yahoo_financials_banks.get_financial_stmts('annual', 'cash') - - banks_net_ebit = yahoo_financials_banks.get_ebit() - tech_stock_price_data = yahoo_financials_tech.get_stock_price_data() - daily_bank_stock_prices = yahoo_financials_banks.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - daily_commodity_prices = yahoo_financials_commodities.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - daily_crypto_prices = yahoo_financials_cryptocurrencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - daily_currency_prices = yahoo_financials_currencies.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - daily_mutualfund_prices = yahoo_financials_mutualfunds.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - daily_treasury_prices = yahoo_financials_treasuries.get_historical_price_data('2008-09-15', '2017-09-15', 'daily') - - Examples of Returned JSON Data - ------------------------------ - - 1. Annual Income Statement Data for Apple: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('AAPL') - print(yahoo_financials.get_financial_stmts('annual', 'income')) - - - .. code-block:: javascript - - { - "incomeStatementHistory": { - "AAPL": [ - { - "2016-09-24": { - "minorityInterest": null, - "otherOperatingExpenses": null, - "netIncomeFromContinuingOps": 45687000000, - "totalRevenue": 215639000000, - "totalOtherIncomeExpenseNet": 1348000000, - "discontinuedOperations": null, - "incomeTaxExpense": 15685000000, - "extraordinaryItems": null, - "grossProfit": 84263000000, - "netIncome": 45687000000, - "sellingGeneralAdministrative": 14194000000, - "interestExpense": null, - "costOfRevenue": 131376000000, - "researchDevelopment": 10045000000, - "netIncomeApplicableToCommonShares": 45687000000, - "effectOfAccountingCharges": null, - "incomeBeforeTax": 61372000000, - "otherItems": null, - "operatingIncome": 60024000000, - "ebit": 61372000000, - "nonRecurring": null, - "totalOperatingExpenses": 0 - } - } - ] - } - } - - 2. Annual Balance Sheet Data for Apple: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('AAPL') - print(yahoo_financials.get_financial_stmts('annual', 'balance')) - - - .. code-block:: javascript - - { - "balanceSheetHistory": { - "AAPL": [ - { - "2016-09-24": { - "otherCurrentLiab": 8080000000, - "otherCurrentAssets": 8283000000, - "goodWill": 5414000000, - "shortTermInvestments": 46671000000, - "longTermInvestments": 170430000000, - "cash": 20484000000, - "netTangibleAssets": 119629000000, - "totalAssets": 321686000000, - "otherLiab": 36074000000, - "totalStockholderEquity": 128249000000, - "inventory": 2132000000, - "retainedEarnings": 96364000000, - "intangibleAssets": 3206000000, - "totalCurrentAssets": 106869000000, - "otherStockholderEquity": 634000000, - "shortLongTermDebt": 11605000000, - "propertyPlantEquipment": 27010000000, - "deferredLongTermLiab": 2930000000, - "netReceivables": 29299000000, - "otherAssets": 8757000000, - "longTermDebt": 75427000000, - "totalLiab": 193437000000, - "commonStock": 31251000000, - "accountsPayable": 59321000000, - "totalCurrentLiabilities": 79006000000 - } - } - ] - } - } - - 3. Quarterly Cash Flow Statement Data for Citigroup: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('C') - print(yahoo_financials.get_financial_stmts('quarterly', 'cash')) - - - .. code-block:: javascript - - { - "cashflowStatementHistoryQuarterly": { - "C": [ - { - "2017-06-30": { - "totalCashFromOperatingActivities": -18505000000, - "effectOfExchangeRate": -117000000, - "totalCashFromFinancingActivities": 39798000000, - "netIncome": 3872000000, - "dividendsPaid": -760000000, - "salePurchaseOfStock": -1781000000, - "capitalExpenditures": -861000000, - "changeToLiabilities": -7626000000, - "otherCashflowsFromInvestingActivities": 82000000, - "totalCashflowsFromInvestingActivities": -22508000000, - "netBorrowings": 33586000000, - "depreciation": 901000000, - "changeInCash": -1332000000, - "changeToNetincome": 1444000000, - "otherCashflowsFromFinancingActivities": 8753000000, - "changeToOperatingActivities": -17096000000, - "investments": -23224000000 - } - } - ] - } - } - - 4. Monthly Historical Stock Price Data for Wells Fargo: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('WFC') - print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) - - - .. code-block:: javascript - - { - "WFC": { - "currency": "USD", - "eventsData": { - "dividends": { - "2018-08-01": { - "amount": 0.43, - "date": 1533821400, - "formatted_date": "2018-08-09" - } - } - }, - "firstTradeDate": { - "date": 76233600, - "formatted_date": "1972-06-01" - }, - "instrumentType": "EQUITY", - "prices": [ - { - "adjclose": 57.19147872924805, - "close": 57.61000061035156, - "date": 1533096000, - "formatted_date": "2018-08-01", - "high": 59.5, - "low": 57.08000183105469, - "open": 57.959999084472656, - "volume": 138922900 - } - ], - "timeZone": { - "gmtOffset": -14400 - } - } - } - - 5. Monthly Historical Price Data for EURUSD: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('EURUSD=X') - print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) - - - .. code-block:: javascript - - { - "EURUSD=X": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1070236800, - "formatted_date": "2003-12-01" - }, - "instrumentType": "CURRENCY", - "prices": [ - { - "adjclose": 1.1394712924957275, - "close": 1.1394712924957275, - "date": 1533078000, - "formatted_date": "2018-07-31", - "high": 1.169864296913147, - "low": 1.1365960836410522, - "open": 1.168961763381958, - "volume": 0 - } - ], - "timeZone": { - "gmtOffset": 3600 - } - } - } - - 6. Monthly Historical Price Data for BTC-USD: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('BTC-USD') - print(yahoo_financials.get_historical_price_data("2018-07-10", "2018-08-10", "monthly")) - - - .. code-block:: javascript - - { - "BTC-USD": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1279321200, - "formatted_date": "2010-07-16" - }, - "instrumentType": "CRYPTOCURRENCY", - "prices": [ - { - "adjclose": 6285.02001953125, - "close": 6285.02001953125, - "date": 1533078000, - "formatted_date": "2018-07-31", - "high": 7760.740234375, - "low": 6133.02978515625, - "open": 7736.25, - "volume": 4334347882 - } - ], - "timeZone": { - "gmtOffset": 3600 - } - } - } - - 7. Weekly Historical Price Data for Crude Oil Futures: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('CL=F') - print(yahoo_financials.get_historical_price_data("2018-08-01", "2018-08-10", "weekly")) - - - .. code-block:: javascript - - { - "CL=F": { - "currency": "USD", - "eventsData": {}, - "firstTradeDate": { - "date": 1522555200, - "formatted_date": "2018-04-01" - }, - "instrumentType": "FUTURE", - "prices": [ - { - "adjclose": 68.58999633789062, - "close": 68.58999633789062, - "date": 1532923200, - "formatted_date": "2018-07-30", - "high": 69.3499984741211, - "low": 66.91999816894531, - "open": 68.37000274658203, - "volume": 683048039 - }, - { - "adjclose": 67.75, - "close": 67.75, - "date": 1533528000, - "formatted_date": "2018-08-06", - "high": 69.91999816894531, - "low": 66.13999938964844, - "open": 68.76000213623047, - "volume": 1102357981 - } - ], - "timeZone": { - "gmtOffset": -14400 - } - } - } - - 8. Apple Stock Quote Data: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('AAPL') - print(yahoo_financials.get_stock_quote_type_data()) - - - .. code-block:: javascripts - - { - "AAPL": { - "underlyingExchangeSymbol": null, - "exchangeTimezoneName": "America/New_York", - "underlyingSymbol": null, - "headSymbol": null, - "shortName": "Apple Inc.", - "symbol": "AAPL", - "uuid": "8b10e4ae-9eeb-3684-921a-9ab27e4d87aa", - "gmtOffSetMilliseconds": "-14400000", - "exchange": "NMS", - "exchangeTimezoneShortName": "EDT", - "messageBoardId": "finmb_24937", - "longName": "Apple Inc.", - "market": "us_market", - "quoteType": "EQUITY" - } - } - - 9. U.S. Treasury Current Pricing Data: - - - .. code-block:: python - - yahoo_financials = YahooFinancials(['^TNX', '^IRX', '^TYX']) - print(yahoo_financials.get_current_price()) - - - .. code-block:: javascript - - { - "^IRX": 2.033, - "^TNX": 2.895, - "^TYX": 3.062 - } - - 10. BTC-USD Summary Data: - - - .. code-block:: python - - yahoo_financials = YahooFinancials('BTC-USD') - print(yahoo_financials.get_summary_data()) - - - .. code-block:: javascript - - { - "BTC-USD": { - "algorithm": "SHA256", - "ask": null, - "askSize": null, - "averageDailyVolume10Day": 545573809, - "averageVolume": 496761640, - "averageVolume10days": 545573809, - "beta": null, - "bid": null, - "bidSize": null, - "circulatingSupply": 17209812, - "currency": "USD", - "dayHigh": 6266.5, - "dayLow": 5891.87, - "dividendRate": null, - "dividendYield": null, - "exDividendDate": "-", - "expireDate": "-", - "fiftyDayAverage": 6989.074, - "fiftyTwoWeekHigh": 19870.62, - "fiftyTwoWeekLow": 2979.88, - "fiveYearAvgDividendYield": null, - "forwardPE": null, - "fromCurrency": "BTC", - "lastMarket": "CCCAGG", - "marketCap": 106325663744, - "maxAge": 1, - "maxSupply": 21000000, - "navPrice": null, - "open": 6263.2, - "openInterest": null, - "payoutRatio": null, - "previousClose": 6263.2, - "priceHint": 2, - "priceToSalesTrailing12Months": null, - "regularMarketDayHigh": 6266.5, - "regularMarketDayLow": 5891.87, - "regularMarketOpen": 6263.2, - "regularMarketPreviousClose": 6263.2, - "regularMarketVolume": 755834368, - "startDate": "2009-01-03", - "strikePrice": null, - "totalAssets": null, - "tradeable": false, - "trailingAnnualDividendRate": null, - "trailingAnnualDividendYield": null, - "twoHundredDayAverage": 8165.154, - "volume": 755834368, - "volume24Hr": 750196480, - "volumeAllCurrencies": 2673437184, - "yield": null, - "ytdReturn": null - } - } - -Keywords: finance data,stocks,commodities,cryptocurrencies,currencies,forex,yahoo finance -Platform: UNKNOWN -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Financial and Insurance Industry -Classifier: Topic :: Office/Business :: Financial :: Investment -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Operating System :: OS Independent -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.3 -Classifier: Programming Language :: Python :: 3.4 -Classifier: Programming Language :: Python :: 3.5 -Classifier: Programming Language :: Python :: 3.6 diff --git a/yahoofinancials.egg-info/SOURCES.txt b/yahoofinancials.egg-info/SOURCES.txt deleted file mode 100644 index 8470410..0000000 --- a/yahoofinancials.egg-info/SOURCES.txt +++ /dev/null @@ -1,11 +0,0 @@ -MANIFEST.in -README.rst -setup.py -test/test_yahoofinancials.py -yahoofinancials/__init__.py -yahoofinancials.egg-info/PKG-INFO -yahoofinancials.egg-info/SOURCES.txt -yahoofinancials.egg-info/dependency_links.txt -yahoofinancials.egg-info/not-zip-safe -yahoofinancials.egg-info/requires.txt -yahoofinancials.egg-info/top_level.txt \ No newline at end of file diff --git a/yahoofinancials.egg-info/dependency_links.txt b/yahoofinancials.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/yahoofinancials.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/yahoofinancials.egg-info/not-zip-safe b/yahoofinancials.egg-info/not-zip-safe deleted file mode 100644 index 8b13789..0000000 --- a/yahoofinancials.egg-info/not-zip-safe +++ /dev/null @@ -1 +0,0 @@ - diff --git a/yahoofinancials.egg-info/requires.txt b/yahoofinancials.egg-info/requires.txt deleted file mode 100644 index fc15e64..0000000 --- a/yahoofinancials.egg-info/requires.txt +++ /dev/null @@ -1,2 +0,0 @@ -beautifulsoup4 -pytz diff --git a/yahoofinancials.egg-info/top_level.txt b/yahoofinancials.egg-info/top_level.txt deleted file mode 100644 index cef6e42..0000000 --- a/yahoofinancials.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -yahoofinancials From 8a1297721db6f4d74de0f96598925c0b01732b4e Mon Sep 17 00:00:00 2001 From: alt Date: Thu, 23 Aug 2018 19:09:21 -0500 Subject: [PATCH 038/108] update unit test with #10 bug fix test --- dist/yahoofinancials-1.1.tar.gz | Bin 23437 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dist/yahoofinancials-1.1.tar.gz diff --git a/dist/yahoofinancials-1.1.tar.gz b/dist/yahoofinancials-1.1.tar.gz deleted file mode 100644 index f2b4b8a0855ee23934d4ad3a5aee280184b643d7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23437 zcmb5#W0Nod*Cgh#=N{X(ZQHhO+qP}nwr$(C?S1lAHk;i|rIMd;zI9#Q2QLZ&;!(^` z3+T_-!pWJ=!@%6m&eX!zz}CpZz}kt1ftG>R*}xItrpHTfOT3Yu+wV*1oZf8ho))!J zBatJlNeYJeIy4y2%+#q7<1d(oKgh;|7~?R5Z11P0(st*LU)=TIZjQdx&dSb;%Z=2^ zW~b|pl$ikyX)Mc=S0nkI_uDugB(G>+oS#V%9m!v4-kq_E3an1h-|-!>MiR;gSHa&7 zlDDd_zDMAF4huBp)JhK=IO6XQ+ebC@>7Vi6%)`RLhqyCp9P3|aY06w*VdF;{Y)Rb} z{hv=yoi~~I+|%C(->e~{mW<88wf*1zvHbIQ{?pFW+23qg_#e}p@YiMK88z)r4*0zv z)kmFpXf5~DpPlWm+}}f^+xO}DP0^%Z58c_RUc&xd)3;yXTm+7~5>>q?-)8Pzdz20! zdH&OA=2n+r=s(U1!n3N6?Z?%w!QOi3JL@|~7No*Ppg`o)TW6N0kQ6()5UDk!)T)hX z4Q)s?TYDPqT2v)-69j$>D+UxGFUKBxQiC0-xg6-b2H-&iffla1?T#HSO||)^D}43O zE-o%LS^J(&c(PmXv%9b%C;udp-PvEtcGn&qgL7Kcd2njrq5U|o*KJ4BuEAj4-P{Qg-hGebIW^Xp^xWe)p0RHn7e*WH{R+okW=6Pel^EiZ6+QlNqewLX`B)_#W^ zQvmWCLAs#+LA&NX!Ku4FA7fcnN{_d9nR`8v&tnQ}TUC9!?)O)o zcD5e@HzN2CP0&jACg3yyr(JBs!ytmg2yNqt9fsd6_F&2;fN0lHi5*4dT#>|CAB$$M zi!VU@dz$U)Iv;~0n)Ye0Kpdl*5V3c4)(4xfnhAgZRH)DgwzRgoT&}y$1>*b?jUS1< zb?Co%4EE5rry%?i$)&a`G}&1V4t7PHy>0+bJeOw?fA!tez2^IX1Z5W5h9uVH5ZJ#x z$#SCzaFhy+LBT;L$i4%Mh+7|^Lmj&Tei*Yf#-}*6%kFok9GmxR^!4wzP{TET_}fh& z2#)PZRm*);r29Gk1#>~4o1IJp@q+h5Dy7p4OMwU>!0NW;m2-EthLo^$0G=Y1vd-@< zkDbz4TcNP?!}NA(S}MbF-e7(v&>8*3e!gIRV_$|Pis|)j5xq;$` zOkf-90zAT$hNLqBNb7VqWq5`4#gD^Nx%DRv(&?)Jbk}?iAjci~4bDEF-iQ_>6Xr}n z8X(Pg35D-^;->=U(`R(>XUvEsK!`BvlP)bHJZu=N-v>;j9XjG16712aApg7Wm>P>> z8p7G4fM?(~`+N>4vfb3NM)2C|s`b{>`6w$YphwodhI{H+;3o$ZLxXi~@y`%&9b=Q@ zG`DsJ|HCjl<6&4lpMq2((D%{pQtFeZFBK+|EFt6q$}pVu_YXbZ1h@w{yt_NI+aoSH z^wK|qBv;hegMVu3*AN8%e)%2*Re(l~hwcn7LSDHs?0}Lt*UU8wAbc&y?Mdfsn?-H` zsnV7ogX4Bjw#0Twl-D8G{suG5`6b~2n6t0fM{q~Y+F^ls>7awk(hj$Q?%&&zPP9dV zI40T>ykVO$_&J*|xv zRDD$?ka-C%=bae~p;^c0ww62Bi~`+niCVw??QG+);8>mT_HfR}(OIiK-IU{7NIk;wD zc|wGfyRX|>v6f_n0XH>5SsdSayT8cEe*X8b6DUtK@SLe@;ej7pMxZJFO(v+|0Eify z;i;POCbYcUt%|A2r>H_>3hKJz#NQuGz{_ll?HN|&ZCeLqjl6xsdRH^oNM}0U7H|^# z8)FqKnA)GyN&A}5&L3-gx?My9q=EV}rZ=+3d9jbk|)R6aN6s+x^$lF=iCt z!urvK&`#z)jfqTrx@JJ!vmrC?#|nBAqpJ$w6chRP&@w53?QvK)Z9#gThUl=JngFJK zQKUKmBMN(~MIKmZL+2Zim`7vAUJ-$$T8ChP_KEBs=RkAcMFR;`GWW_A-pc7M6=5{4 zkv!${vwHH5WSCzvom4wiH}ys1!Meen#2}Je1gLuW_Uq%!^oNj90Bh;Dd3dD5yN>Y7 zd>SUuJE9*HjZmQ$8JX&wfTadXSFYa?t1ImUR2_U2jeH~kvs$i1WM9XFt(JT3O`tPS z0_S*-){vZhw?GXVeFX~ql=jElqdzysD$xi!1k%~(ly)JV5*?4ok#ucJPp-CRL}kRQ z|EXA_L(8p|GP9Os-wmd>)GqA>1Z`q+7xW$qWOu!@2NMZ6r4>ortCfdzVI`lrErK^C z2I~D34BU+7=??uk!%OY;aGkdF1 zF>SVnN7u_`q;S;#?s~H|68eQCTRPrad{3TPPcXqUN_$=8(Apwk6S7`sjK$}*R(93k_f-%UCOCGyyfmoi~UZWX(GKlJwv zWEvc=O|rj7M+S&FTA%}TRmK?3zc-}SL24jlyZC_w{Jg52t z9KRtRpmh^7rML;;CxDajajSw(9%3e>!pXy8$&ovJas^Q=?q&px2UUjhGxuKTTzhnv zxCw3WlPi zKfjCazqi=>=MRQ2E^k;rCKRS!`V}t*%>U#NJpGiRg{u22zB$if5)x9h1Xw6bf1PD-ccvEy>3lclD$FXe-x`s+5z50*z6)Mj7c<{JAU|Qfxc=)J@F-zW0L{D|| z%IuJ=q;@8Ws0Qmxys>FluDI>=AbTBeyqhkdwL9HVIpD0k4aN%!fNz?> z8BJz~zuPTuQ3kQlFj%E&0^Rqxb!ru)($wo8n90 ziF0wrNYzcz%I|Fr01U7zquz7K7SyT@>#p9?sNMH1`;e#j8tTuPs~|)JXBUvyTY60e z?=E3!FAM}q5k0uEwR4tnZNPp%Mld)b?$qle(bdR(iZlGYJam7~!U zS9eR3EQ0dWr-Xl#E@gZiO&H5pR*p`u&viO<=o`D=?hep}m?k6$a zti&=lvB1wx3!3rFN>)UrVylOffDhOQGO6I5i;KJAlvR<{S)N@HM@+Ssi3KfIuh==q zR&J~*xjkGLnsHwP>6Z$?-jFkPn!J=V?;*L(J>6>Y+ z`9_$pAxNo=<^!Vr2>F~gXQ(o$d6rZ`sSq(D@e6F4Qh>1nMRgQ3K484M9{HA*PnAff zDFg`kkdu;q9mkF745xXTR>NqLbAaLgUJLZEyBQ0Oi2v~HXFU$AwD2m^*T1m?(@|ng zZ*>+v2jD&Buqx#&B4P0K!-`#R)+H~{T7~KnwAW^8353yU+ayEbm?fffi{F$hG9(7c zkvOA%y^WFYBIIqhK~ zWN<2IGUL#MObcF(TS78WW*4xy>;(l=SY*8VK~SDx&n;VA>Lx*g2wyi)OrT-=AbjTOC_1P`@-;2P>^Ea{zh zGjIZglEWF#02mqOo0l%vo@|IVGj5TPq2>1cuO)N7DEm%49$7(J?^>N7CFO=5Ek{3X9n?3g3 z7KXOgo$Al(el|BPCn=D*WwPj0xqpU4k`+0wJ5-b{vn~ME@C8mnf93$m)24M``}mQ<6=%IU}x-tU)WZ5f+Z>|WI`%h&tRW776j=Kz^t{zB9w1lf&viBM+ zK!%F;?e&G7gGgJ(>AT_fVD)fud1EG6MEW3U%ho5uIN@JNDrIam>QXZgNUNlTLC91T z%rAo%RtliW%=GV{J2Ev?L;+(^6@XR*aGH~{0pc{DRHA3@vY|-WPztyxr7$Zgf}a36 ze=(G9z+-(O$xM+3P#HGdYe<;|)xA>5X?) z%P;x>dwzgxL^6oDcE-|u{rNlCieve9jonBRK5omEeZ-_-^0BBDE~}yoyS0`krC5KR zXB!r%pat=8IDZ5Ctayhn5j zE<1FmhnwWw3&q2TnCR#%h(3mT5J`c=toEJ(i9!OXqKV zjHoYwRVt`L%7L(qF-GMDn5gl>l`5zvs}nV`&FeD43@iJWyC5IFC4-}Q8d4~@rnNL| zj|ba*46@R@BF?2nBx>r@SnsQ!w{V=J_4>xu4U8rR~?O`7S0T z_r6-jvUCkbiL5ytr$4EEN%l=!Z`6NhSzbVb&gSf0X9`lAuK?~QekDXhYjD=DI`eky zu_-HIDzAMrJT8ED7EE=LvQW?&k)!tsP@yA~dV-)4&sIS%9~m6FBVR+atV0>&<2*@g;j-$3Z)Ao6R@b~7cg*sE6C5+ zT4OpNL*^zeg+7W+b=a=+U6s@0JY)O{L~y|bR7oxppHXEOrG=6YfUbh z^uCp|TVUpLD(p5v^~y8iy;wiJ6_)yBX&QUzo=^C@SUCFU=n^z6tH5&|4OsE&pNC>9 z0@~h3V?%Td+m;?2nR>1(SNRS6G_`B|p&)LzRQB=QUYUvErPXgD9eShFZ88XRP?Vcg zaZpl>wEW^JA#Y~i%-_K|#LCXVYTqVJ)l65P3bjELU7~YQENstz26cDZc9(aL{rap? zrz1sUq(ptEhl8ZOjojqMMp zAzV5#_b`|Lx&aa;SA+~&`{QrFsw#|({M?B2)B8(XuNAQWtQkhC0jT0aeY(?Y{pnUI zAc)*9r0Y@Lv-_#J=;+Hi+C41zyVjN6dz;#SJG?ObX`7UeLeJ`Qpyvw3Y3`4bN7&5< z_?Ltt{uMx-A@5~>I>+XeY9nvBdL-hiUZSJ9aYK1^N?CXfu2oR_zf#R*&NalW?f=%& zBkFEbyq7l9&|%V8mD@~3-B_8|SUqd4&F3Y7{_0Ksp*`*!^MK*F97PS3?K3?3Njj4L zY3V7JoPFELJ$RVzoBi5N|3R~z|C~~L^}1N_nf~r!{~fhVnf>~m{+-RSjr{$Gk#b<+ z=CCS%M!)yw&iqzd&c5s9lKxm`Uv?#q0pOTwT_~44O_F3{$IvUhxM|(fwpzXK?dJX@zPU?()tak+)xJKN{`f}7a@LxHIC%18 z0?mH@$`k8y|C&DaZlpGu{`^8d{7!$&e)kMU_l_Sg6E@3F{AA>w`nK7aPcr`U4xa#n zztBUz#$@$%Vi?TcmDiEZ!A~%^k&0)Vl;8PA{tCigcarY=VhJ`Izw5~7uKeVV{Z9_dLG&yhG1zc zsj5P|g?)grl%}Z9P@QW$RX^qYsMz0lb`8$!Q0K#`gSE|V9Od$~ zWpB`uvIM31HSx(@XWVc=)DOMIS`c+O?eEf<*0S96aCz;Ne&9s&+-z@h`7wo8Ch4PL;XS@gjGo&aF zs?@aY|3!~kGIs9S;cuz)@#;_hqsP3a&|^$IY78(U%Pb7FOb=+OemdTmzkCS2`}L+F znyj~wd;`>-ZoTy$*aXg(o4n*~p=IECKnO!lrE+-em!&%+s10pBSCelmDSn)oP(lYI zzxDKq2N3L?=Ji#e)6)HiR=hX)RBVxs^-_=iYNvTLjYRPjr`#tUM@}{X?f?&A8mWnj z4?Oh@q}8CWF;C--Ti(3@sPR!@L>Q6^JG7}(-G@}_xn?yCNi?fPXwGMV|7F17aX56f ziZNw%@Co)0Sm7?8Y0v&$`2(U9?D;O-co$KFq-cg)K=*H~NyS>AKp*3+@?SDmO?6}< z?h3^xQrqB$t_In_D0ii{Lm>523tmKaCTL`pSC9%mnQ>9%KKAJnKn*$Jr_`lZWmzx+ z#EIph8svlI^B2s+y)kA*W*GSfeSm%3N<1dUV7C7g=pNDlA3)6cCV8sUq~>xp_%w%d zU)AnTD+m|R`{)vF^O%UO7>g}c825GbT6Wh>M7hTN&yM@3bR_5n%xD)$80Mt)NQG#^ z$FbJ2bsA#OFd)V z@eKYqy&YE!QNl+~nxkB@4WIOClZ2eqV$IBKsqQ!RfIE_&*&^=r zSZ}NyH+@=t#O~e&aE6vYb+7aH42t(~bs@0)?`G?7V*Zv!i~DiL3P<^q;bRrRhMNdB zvS={$q?TmG3|2@#(9lNuK9!Lq4Q)rCs9Sw%l(*;^v!epw1PkTo&@wTO^j9{fdN;u~FL9#;o3y@e!Rlr&h+(JdXY4QuwOQa3mO>0%iKjF0V&(KB| zjliGYH(8{DmduC+R6f3xX^A)*$nq(kPp!dm%S_Qu`t|DstaBajpV z%f{J0jiKmN)g!v7tRqDSX|}t^y9}P$`Gf7$ltHe!qZTzaYD<{+*;<>bc6mIz=ON-9 zLLZ?8Z%C0!n!vUn==11W&^QVP^8u}{XV|2dTl6P?WcutM3e5Lrdug;=~B+S3~Xv`vNw>8|6we>P;F1-(Jl31P#stS11t}?WsF~Y6dzXU$2L%*2xRD|%$IYyyC&J%{91qHg-)HqC2BJ?LMST2jdK)@=vU8 zu zahiWn%mhp$R|Kg;R#70c?i^T~4G>2ZfXQ3FHrXpKH>eE6k~@#3%STN()LXnpMikNh zFidfzo9&9i!ap=y6Rai6En?P;UXrHkXb@%~9(Euib%0(o^Pf%)H|y2nBH~b~ASNSF zR1g>HS;JPA-=ynJX35%bqn!gdsHt3GynX@c?CTn-koD?6F;uBJ;}gK)_JM1IDHGtM zBE_wEKa)Jy&nt64vynNNC89D(eb-)>-CKjs5zHa#qMq3OyvY#ZX>RMW%Mx0VW_7)6 zNf7h7p`w>M?2uP$aH3;+9w`v#2(>nzv(@2|D2~7+4VjBP+$1gzyty=)B(5>F3QIoh ze|By%;(cjh!(PGRO=3Y=kyT}G5RO#MrHS59j74{u2Ox@46I~(Q0gXVf&7fj9@qy-^ zcq-w0+4)3sz0F{ik?bpJ0*=P(W)@bxNk(qjUV3N6zVtMswHx&EfqH_V3QX@b4wpRU zs|30e9-{lej>S%0+tC?8C8}t!JfmjZYVANijgu;fbH$i&l`I{}U-l1Nx2Jf#~8OL0cC$`D2_K0Mk!8R)?Swq2bM z>SS-27x8z^bWIYA1wedXdYGq(yY=Z;aSw9P2LB)+yES9Kg|M*Mh26i-RM?~zW!zXA zFxn=s8<<|5a^!0KkQ0_vq|b=_rd-MTIhry>dQ-ER$wdb9HTHZsP)0PBUWq<|Y@na< z{1T{R=;XuxsR=Pf2|hRn|~3-m%W$$(d`flnPy~)sU2M zxS1_&b$h>py5^nqGNs(iS9!bvJp&W5O-SEv0-gd~JC3hIGg5T^AJS=(`3|!%g_!R* z|L;#keR(C7`#y8&KEOv{HLzN0rY58$%#u+(FEARa9!<6Ha&W3-C`5Kyx z+GIX3#-E7aWpkD$lNveDB_BqUkQr4#bkCdylpQRlx3u{gv*61z=gNEwh9`Gz1t+(~1ATnZRjv!p)!2EOt-wj8|Av6I-i$j) zvch@}h2t+$IO)Mod$e^{tPc!@%Yfx*CZFb^ai&EvhcY+HsO%YN_I$lCB6D-C(;cC` z%q{>J$7M1&o7;lurF%djQYXh`q;SzA78e6C91C9^G}BSia!{2l_$FBJy&@OWG-Y0( z*PhNVQ4k_G?F=epZc5C8&ih&LnO%xo&3ScUGD0?!R!~h<*~RRx`ynA!R+(=Ak(8&| z@~c)>dnl1$LYe2aGV&WX_LP)KwDI0IGv%nHiDwW8Oco1&jpz7za5Q#PzLF>5++P16 zE{YAX$ z0|jRcII$h;|ALB+HoPXJVh+Zhc*R{6mzd&;A{M0L4J=OiQ=$J{L%7egWKPc4lJd;4 zxl^B)v&7|mWnZ#p0Y71-<~E}X^mP6|pkfwjx1HJzRk+;HJjud5KbB?X!Zg^5iAsn7 zEDTc2#ioG6gavvf?J8>QaY?CQSpOWP$3dbjBlSG+yZ3!GVZ`Fw;S%x}X?wOl z1;#1=VoE7vlSz-dd0<8*B^*MgqEJBvoUlp|ZDwY{L?ksEQdTE>d;_9pJZ5>2sUy<7 zfL3i}!G(`f!g491GMkD76N!sFEq@!HQ6_9d1K&ciwIKK_%9B79o#)eqH4D>%cc2>x z-pcTEl?g>A^l1`nGqOEDaymiLgJg7c!8`2v37#>UNcoXF1A>&2WEeh1AF5QU|tNSu+ae<*%=n>*%5a{ z#88le#i0Av*gYVf_EZ+`5`BQ_8Jdv@R=KE!sHdC53ii|!x!cd=*#ProB%+n^E7+z~ zXh1p?d`qn0ckW}ppd2;H;^La_OoOSKd&9YBEVk3HrQSta1`SrKpArpEL4U`S zmW-koZu2F8-NcXhXb3fqx)o>M_8nFw1#G3&FZ#zh@Xox6PEuxa8Uu3lUOq}Rgc466 zw3((>J}w;6SdqS7`G6`u@I`oCfw;at#TJ|+oG-def!^|NlPt`rj+ZkCa6Pxm1u?&;0WIhQsc=Z5Q2*uUfzzndRsH03eW9) zjDoX@GUX(5Cwa#J5Kqp*L0x8GuFq9Fl^=RdwQ94Rd_S)@_96^|`|n+m8h8~D^pj0v}*JvJ5C>tn_%EFnAG zVXq=!7#yOCl2Oln4mMX{L@B-Ra?0>1yKnb*l29!gIHN)~PP0I((BZ z^d@WP9u23VzkJs(Iae}2I=JZ5rVHKf&+Ep=YueiD!)mGEA=cgSvS-N5oV1W5A6lX4 zIny1^)^=**+T0~duT**fO|^8-{Y~~|Rgk#?I{|3?iWQ#Z)2Qf+K7K4N*4KBoATUVl z%c(q(M|m`*=_vXq3-xhmODOjvoS+8Bv^kKgKygUvCTE~vEZ}T(u;yr4P;+jmYkY2W z!1E@qmfQE=Elvj-$Q7Wz*^TID)ad4(AK2ib>NN_n#lOWb>QojSS2VJV!WBRFw` zxwnd&x%V!WQ29TTO{Y%QM=b4qRMEj`Zcx1x|KUW@DNM_)N5d~n%`42GRhH+oX?wr# z?>erygg@A4DUaVV?O!$_hT^;6U#H(TJHuZjiyb_^zdY&*sA-NoKMl2m4#KL?&+NZD z3SKNMtel=nIdG%CQ%5_7og7(6b>K2;ls$r}fwQJOYLvY1HY#t)i1$a0hwat^@RMzJ*dt4} zi8caP&I|edt{hed+djA;PqBc)KDQp(Z~A}sQ$?$|C4&Q@0mclB*YSjltg$pVb}o+& z#dw&)pMVJ|P1sK&gZM>_xAztIYrgL*O&i5Ohe1HcbMzWE$^d4$C>{4L^}|v0MMoW7 znxP~K-A9WDbOaBq)DataTmfySU$)BLCo`tABI@rDqCLKDub674Hg!QAEb&|Co3H1+jxFre2> z0NQ_IX3_ZYKqV~T`^OKB%B28MS|hOy{f`!^H}&stcib>kpgvpDZYg3LXBFG=1hU(E z{v9;ry~yEnDenS|*I`sV)`9Ukf@1)2A{AyrclfwyR)vPm=KTHM&G|e#1-@WVe{Bren{x3MVsHXpgRP=Q|E|BaR~3<~cK9Fm&K1%7PGq9acO*O$LtxomVCpLz+@j z6fLgrZO@HeLKai8u2VoI*A?;FfjxTW#x{Xg@HoQe)RuL-2z{@?hzuo2B%>;>oL-B@ zgR$C$_H`C!B&w&5IV_QGkwV86RjfmPK$+pdwNo!3Pt|5z0)Uz$kuD9Zi9Z1;quDMA zRZ<^6mMEK~7oY_TSo4ce+{XY=?MsZKjz~}~9BOn8Ktc=x5wO0nu@%*wwX=N49o7gC zyNh&)6iQA?2{j5XtbQ3Yyaf+fRuzQ-!_DSvtHXr!?;3ylMJ$8!p~Or}X;Lv&Jj*po zl}ggN3W%VZTk7_Z`rHl^1=3@Dk0~G>m-zHz12sIcbwYC+$=xCWF~!E#C!Y7QZbz4O znIZWU2Z9x05fyWJfHdG}5AamA6ZJJ4quEOtsHKvZfsveT94e{l%DJd9e zXmcB;28`jmkpd3JLM#5@Pis(*uqsd!z$GRZlxw>c?a)rjxWlovF$sqSb%urP)rob3 zY{59F2#yI2A}krymtdcO&HLAuxjJ2XIxl2}zF>|r-MVojQ|os|_f783%P;foml@^v z+44<}tjmw{jXf+%D|7VqR*om((iMR?*}YsR$70$^mn)^)pamL#;#37X=Ku2Em<0+e!Eoz z->VfUNtaDxdUa(Ql!h#LRX*xl@to8dNB&` zhs}<}BIf$Em1*O6&f2RZgYMrx>ymsz&v6ZQGFXoc`lvi-j_FVq0}|Tq$m*S{sPI#D zw3mYe!XNxCRFq;R=QhrE6mA50FjG9k9yhXt2UV5^dOz^mfHx^E4IIdqC>RhNroL__EdyMS-lTrHFyYrEe zpi1%Q6P{iLA6_A?clEsCO;)7c%ECwD^?8S|@k!Glty!k96bpM;D!^#;jOuJUjubD< z91s+7zc;JB-@I$aw6&FiTvf6rh|e_Oleg3J9Xjk$^T^^$br)?bIvTv)wF_kz&O}Om z76hC<8rmN}ekFWvP1Gk@RJU@5W5|PiSFHW;40~dXbe2CuU>~AUY&D#xLV3A@fk+jS zacqcwYh|+GAOye*M=lscRvB!fGnyk*x(clr3Z1B~|9%}UqAo(M$zWfB-XL1LAb3DI zjPDpkknK1(sZVr^2)Q213Zr&Y+O!MCqusz>jYe)%NKf)=5QYbmxY4 z>f?cGR&25U8COa*No}gJxEX=XiG370+5khd02is0T?r}eb#TvajHStx-4-mwt%CTuuDEp5wNXJsI59+vd37VwJ;||l9ZTk0 z2=ub?fn$Ya2}tjRbYk*NCnM;s840D|Hh=1=I0!sKs@N1VP1|)K&I^~xVhdurj&==z(#qf^m66~}L?)(`yo6AG9hlX0=Pu(N$V@kso+|HE8{LC1Q z$(P~a>3^bV!CA*{tG^q@#g(ei`i^efhkS$V0KMH%#_-PZaO4TiUiSnK=u+$c1&RBh zfIweW$Kluc^S_0gu8<(N{l}c=wcc^I_0ieGMU-Eyb>%iti>S&HuSP?D&9{>DH$3Y( zmbQaL?=@{^{(wyq1*|!Lv55|YUI()QUHnYS9Whno>B|S69tTg*EFlcDR?r6JX3dSA zA)Z{miV3eIZ}f}H6GqzYub4IfBEf>(MO;b8D4&xV#jW%$XG)Xn-$d=smqUyh>#pw* zKH5Qm+7=hL_rVdpjSeV}x+oVQyCk-!0x59iPP~TaK1St*AEv63W2*p+*-Pg??0+dx zBA|n1J8{bAix~rVJiA$ZVqJ8~6kE1{@!5D6C*L zveNXxc$(84#~L%q>Ak@%ayhf*<=q*1%Zv&!oL$^dJxckD7bojI?l5x> zD1`Wd+oN^Fci3}Q30^XL^>!~{nwk#6DA3?3*3qe$5c1+MU7$570G6U*)9)FgqjBs`clI0qiMp<=mn{ZF#&%&*toaSlotxpXL~6u+ zKqEiWyvS}pMhJw3oCFf8g0nOe#c4*8@)?yo!;KZKp_m_JBEAx{@A_j{ zuX1q`dxWWaxUs=ZuBv5|pZc6Tnw-Vc8MmG9pfKGh>~72#gO4{1OJ`UI9+2k!T2qF< z>%%n|xh3lv`BB|SVXy}^faws2dWT>Ry?>L3{r;chy&p~u6!r zm{rgW+^|YwWYMJ%Mbny-M=X!)Nm9%m9^$U-cKG*U36ByPviQ<`W%ZqzpzQ^ytltGv z+N(eB1yj1NN?2(IA(fz@yS}Dt09Oqmeu43|7~iT5Hu5~UU~&Hd?$P%9zJEQgo>W=l zo#RI#=EG?116O{=0ZgF+9KD+MYCv&Bfhhnndu$)mFu~;!XN5W#4rPvR)$6_mg zGu>1#1l&kBmq3m$7z)L17oSJ7<1ol3itR!jiakiF4IRBP=r~$21L^47qNxxOZq6+3 zfu9)-eTwsTVRTyQW{sP&Njc11HBD^HEND7GWUIR-DX(_B3gdNQ0964Tlu2KpTlTgG zwI*uQ#G;Fv^8tQ6sg?<=q*OljcQ$M&w3YD0xbbCVSVLYoom(BuLe?U2;#sHn&)Rl~ znVSF78Xn4PyX&rIZfzy}q|+lOhQ&&ujwrs~%Cc#bQgQ$LBiKPXc5KW`GYPGO5Atgc zk_owb&dp&BPdtMxT2Rd)t9~vi6u1tBbx>mn_=}q^YazVB>X~r2wT{EzCM?I4w45u^ z=)-hAJP_;1)d#~3EU|>~sbwEaHcJ~6IR~*5#Z-a}4!!RVi!t zb&W|)v*aKs*)ofp%Zy_#79FCrt)iGQu{@1k3tqv*c^v$TH8R7@xrC}ikNd#LZq}NW za&k%Rn#!WauSLeQ7d+P;UGu2?tF)`@W1T?9pr#ULj&3=w9R$9+EDD zdlRvrg05(wgkb8rR1alJE@M96h_azjjBnDJt9uvQ)}W+g`T&4vRcFzBGh(1g=wwp2CG%nM z%+-AQ>iehNF_N=;C8<41+o_UX&(L0@w9gUw_KK->je??!O&l5P%v)AQ`Bvf%ul`Wb z)3;}vW@vdJkq3ui8;UVvB+{yvCWUIa` zetFB^7J`)10h93T!sO7fDD)kFzB|VF)~1lek=FJOG3X--kE`d~*?VTnu8Ao7s8_ju zCU&szdMqC?51s(}T6a7xiK3ge-}*VfN8<_gQ`6l!2!xP;sA(7)xdJ?ukc*D^l5 zMX4nldFVP^fT&a|XWSUFv}x@}5SqoI*Pl7w?wp~7pb^wtJZIiS)_ZodPuy)7RjIpn zigC!`rD-)2`g=Lq*7Kc^2L;WDWfLRgcf_CJftcS;2kDFonn$tJoKXtJU0l3PPPj{I zti@*ril!QYs=l8i0!H?tMH0f3!**qy(3PZh(~IUqkAW=aMQ!hu_&}Typu+fd!ky(1 z$w-=|er(vFvvs@L3h!H`cUA$qib_|}RSQ~9(S7_$ydH54DJ=heM8Gb|ZxK;CuRdo) zfx-Vmu`q9*F zVe@5PVBc1)3NHpI4e-95YJ5(wh@oXtv~6lH&n2J`I8Zqu@mj%tZ*vJECKA6huU-k; zqSb_%sVh4pc5$M`KQDWr@lp;9d8jKL6&d;y1szIdUHIqBBe&2$Ygr|{U17*03id@(THiqzbYQFL znH?e8s#t4th*dvZy?au%M829=}%Ec z>};VJX@mP^c#l%T!ydM-(aRIWt0T#ChW%WK4Ffr@g-1C4wk<%eZ7;d40|WO=5`kRg z)aXQWUvrMWOnO@XIehni2Oe;yT@BPVkn<-z)ZW*rvFKT=6Wf}2h$mQ&$2YwG=T0Zq ziXfIkdx)~1BHn>r-~#QEaus5-KHaRbZk~iP2=YC+B7i?!E`BkQeZ3I@w7op7myw4D zh48OEg`3hpDZ%0-3JjSj4w1zP@Tja!M{%Q)iu$?x^~Tguo>cpZ+Bx<1n_xc-qyXl3 z@s%`gW0oRFZ55k~|1aKDeRqc1TiTmb7W>|B-J94k&=f1uj)1!uvYRkH1bh8_xo&Lr zW+lDQV-HS)B%;cxM(002ex>?TaeIZ5kcLe%k0?cQZ-g!Nn^;tQAJka)|5=r20mUpr zm=Q2qPj7dHNi$06Yr$aFEpo%Nk}lJjHcfProS!4=y0+kF)bz?j81CPm4oPJA?O0o0 zUF)-7n>NY+>oou89=pjtTh4%djGrF<*i5R)r;RRjNF!QCLp54k0t(Sni$*Q|tZ30V?R{x(t%McuwT8rL)E8?!gqTt#E4Ab2(bV$blLw7gQJ#-`8 z4bt5$4T3Zx(j5cH3>{K}NDnE9pfsEB|M&j(;W}O?>w4Go-p3;>#AgOII(0ZW+0VZ0 zh_|9?_uoo1+|Dmna*5{@e~LuCA+03S^qjRGEkK@JoO>1cFzFLuM`<|}-uRR36iL9F z8UXx>&PBB@wXj0eDiSul8KXhdk~eX`7bn^{C=pjteSD#Cm3z4MPMNHoyz2LX%I}ll zL+cN*OJwIO6fV_$!o+04MBHWN(yI&Wli%=o5z4E0P#;B_rQYhi9v24H)@5GTb?D6% zv2II;B0tvI4*_Sd-}bF~-YPOP3HrSLC_tEFn}VwqX<0mm6*OmlXW1CwcCE$aoL-(i zQk(Aa4nU0!$yhLt=*E0agAsVZ4c}fn21Q>s!=k&|L!Sr6$rXq%&M#_1TxDIpEuwr!isY>Efs`2h;xbR}NqL5aB)6Fo4$_xB6o)!h@K z>5gn-Tr@ikdmF+b>oL8*`zs3DnaFJmLFclzeV3AQ3-8Wq4Od1H@P667Dd4Cs3&C~Z zN4x{^Hgj!p>}M*I(`IYxjcWF&{m4^M``cc(;jA`}U%oFODST$tegUMu((FmCSES63 zb(1n7`@AD=R(H_zHi`6;E~ec`&W@UkH*(M1kNXTpQi~|+9Ev8@ryg$!P73zw)MOqy zb{4>r(9U%puKLv^gQ^dYFJ}6JGb_C>u^M@s-{;HN@8<78S3G8OzKZDQ{##>XQZ1sVnjsy6L>k7~A)Oyr??)CSu2F8&&+nuRTB@>u>%R zjYCc&Q*P`c7rTL8F87G0sqD33KP}J#wpOuVyd%F?W@6t2@tEF3*F=cYE2*V2nkbOY ztVp574%3#{rn{w_XE|L9GV&N6f>+1w;GGdDH@Hl!7b`Hzi1Yt6^RX+-JI=96CL4?| z%{(VJ8$_yVEQ9j!(L7E^IaDG zk?6fvUfJ1T&CO|*bP0~6Z_By(eLjq~pwv{sS5-p&PJSZTRHg3Vx8^Ff;J@q5@D zB{V;;pZZGp9{{BMsduwTFD@j@yjOy@lHxx#kcrdkWz#pKhC@zP14?M0 zeM=`GvBpQ3_c>G?)PqbD3;y*ae{!=H$(T|h*Bq{;!|_if1nSEuOhlJ@F=&IEiEtYg zEqylQ;bHmPn^?tGMN{IQuqjUIWI3uUB06$?w3Vn%6K!7<(^u6O(Q}jK!=UWMGkwBi zI&=t4pdk5#;8xd_S0$EPLqiw$DnXwn9Y<=@*_;(1e`(j&#|fC@mL74NNJcnZF%X#v zt;nFp@vuMQ4DgDw5KmsJDIs=gQB#jeGd{OLQ$f(|X?}3&ko{E#mTHTOrCxiXJrI(A zmVjJom61z(ItCBXzTy@RwD&^V7hu$ZbR*k{-x8p|`d#yT+!eRp<+Cluq{REk}v zL}`_aI*QHtaPD)E&4jVK15n_o=BD^IQLw+sg_Pd9q5lK|AB+9B+FqQG=#Gd=7V9<< zLCh33w+u;{VQ8j#cvC#af7Lb;8t+)yxiZ_riz}1|CxL zYgvg~p{9sgk!Ci88np&49(G?Lf&e?NeA^<_<9MlL2tDl+5wyViZ$FuYI|~U!;-Jb| zn=m=rl4NI(UF<_r6+P>KNtpmI-PZeN{x=)iDjV|Z3dvUfax-jJ&XOPh7ExdYWk(ZD z^A&AlCf0%sNBI#p4s8q+BGEphhy*Jm(5F24!Xvh17@HS3dZpcc7IVNqfDj)EoZ&%# zqgHkpj###66$$9oM(_FdqPXZ04YhBOoI)vO{u)KTFXr8Vg9SGJBmyIfiiobm?d2ai z{qMc)1xtOOzq;C?GJN-zjr9MouEA(7EH?~wID4tIU)NJA1~~MWNAz;8m{AA3qq3WV z!p+O&7j^svn?`gfW=>6A2o?0S>Y7(QAC~qgbL|3K*3&;aMrKV7Z&27QZRbI3xr)h& zdw(PlY*CVZ-myI-Hp1|D?Z=K{61bVH2(I>zQ4pUhwWDxBd?(jMtfmpMu-k(-I$6Ao zWgQisqsU%!F9%o!Vtj%Gdc7M|YqnvqLp902*Zbf_L_!g=4=-z2pDgi9Gnp(AN!!dR zxt}J%NbgokZ-rY{lHK~$t{K4=%jbK)W=aYM&5l%GUg<}+cx&Sig=PbRcV8TS zN)C>YNzR(5s%s82+&5m4+L%=wA6V&g_^g3~OS>A#B$GYX?P=%RTc!|LPi|B%{}}(V zrZl^Q7+v*Uv>0A>+&CN1vVAw;h%N{#=nWw0hy5T=vX~bv3zW(Z@Y|{J|EYRcp?)P1 z(!lJ_GT=l+Ho#W>sF3*JrB6a}9!g>uU)CrZNW&a9&6GlrxOT^aNGX^9fvQ#AOA_gl z(rUH><$`@AU-Gw3p}6b{Szg|oX`SqZk~vNU?HgSLngONvClLW~?JnpXU4@iYql)_M zmYK!3HfprSK@;5#en(cJjb=8Egi-?YA{a_;LS8I`TfLWtQaSi)wAbZL+_+DM<|RO% z;3wSmE2EHe+L(=XQGjNF?NZ|3&elok&}i}9`p;O@^C5Bp;m=os>vZq$c2Oew%|d5* za`AKFPdM7qZod;A2X_!ZaTY_MU>^N9{r=M(`zZfgZ1+R^V=Z5&@d8G%iG;ommakbvz5a7$M9YL zKLJ;?*knnFSRzJD*fu#H03!u;GS$pcoSd)fB4()9K0bPA%+k!F+64^u864JMmmC%W zGlm_(-h65Z9~?lY(t^FFnEo>F1qMcai<#h!iiq&VJ<>_Bfz_Aq^N;yKl!(~v02uPMcI6*?T%i}rW;<}-EQv;X5%kQB~?4)|HuQ;xd( zU?|EGHX{m;1|J<(cL|FQ48sg|Ez8&P0YFO4i<8~+oPeJ ze-PYBy8gqO=1x)y&nWbrhpn(q+=TV>)=~V1^{*e;>FFLrIfo+5NqNz^9Kr#lJpIFW z`Jhr)tDTm$#)qiAxa3XUpF7@f`!9hx+D0cQ)x&9V`VlId8gZ|aHddBoTTMJZuQa~h z+!q$YGc*)zC53$bb-R!(xzC8M+!6*|19q>{;8Aje#dV`9I~PB_ro#pBu-D@g4j`BrzX>*b14%L-VAy%AHFP8@mq z3@V%6Bq-^WCDNI@&yx#{k7bVp>Mx#w21Wb|p zakSJ6WE|)3kC)P3`f=0>&l&V*euGsI6)qN~@~Gvspd3;QjJzTA_id+79XM$7x3bLm zljT=u5yB-&ax#2JVkZ;f7B`{A%hX+}Q1hn>ccdiF%_mA54HrtqMyk+%wox&0RrjJz zB-G~brw+{LV~aYZ3{(rW(ccE{Mb_6{8?dOK>@qRHpsxolnO^)Ro zV@?!}ErAdj3voQtdL_@O>rxJb1|^}9If|=j2F5qxNTj|ZOE0~3OqlJ+WyX1^&4p|| zm#|%QGg+uNU%&Q?EaAE@SL=YR$0LU8h1_NdM)STP}Ve*(;0eyW)Dr0T;thC zyYoL9vVx@)i?tah`)Tyc8bD5<8a@nHV5eHu_~OG*1^!mj7Du7ngKPuj`Wm*kb@Ms> zo|5TlB#R0Sk7#+rUoC=#nwjnfYQ3wf55-?{>#@b`CA4G7QPCT?rV6;F+Uc)MAg_6} zyAIXgiWFH~r*dIFH=EzhHMOq~jU@9>R{mine2HFIiMa&nEZ#Alm}ivIUJ!L8{75Q} zV4SAGgx8`kIH6+URJAL<^Z!r@H@7yx`CM5a0zO&b_VkzD-_fa8W^QxFTy(tpY%R7r z1|0L1Giu)LUprq+iE?b9voqiwF9r8sgC+gB)H*gQxW-=l&L)n~Peto8evAs@NXFU1 zZ|IG&3{0PeUlAL4l4ISO&S?c%vDE?*U2@OFea(<5F!*Jxrbp;DY0@{}ux4?+>6~9) zz9JK=iHR$qpFlR|mkGipna;TzWBZ|!^QIp8$>84G62xAX|C)vK^@M?XWuJ~NdOCfk z27BH$sokp+AUn$cM%`HpffC68x-0aHOmz)@l73&;Qw-9NFC!b&l6kSU+yY*SfZG>L zBBr|Oz-rm-u1#(|Tjx(elB{x6vn3k_KSPF_i*(#ovJiBTnULK7IMVw9QBEcgM^(qK z^*uR)t;`v0LUMHI;``d#cG31O7Lsw!QltZv4GrzBJnu1+>@Jq7j#d7dV&uHui2~x4 zB3*t@HO^IJW0Xik4)&@~KfW$Z-w&55xF0wXO9lwHW07b{>(CaIxTfMTMxQ{I@z$^a#_t z5Uo@hNALsYXe6RVyQAZ)RKca%4nkkg$?`?y5e3`ownZ}{^0@lss#u1n6@5o?R}Q_~ z`lum-*s{Y=t5|CC;D$P`K0gNtp*ZZC&BZK`?c91NVlCYGh2jwJ$U|E#`^1JNGkkUb zNLqoQ)cAdbTjamRcNfFZOYxmQT7LTpCOWAFp@d(fVYxSpK0^%3PY5vM5IOAB!($~|N}23i^%l3W8y zKzffdusR*6#CQJ?n8v(w!7)6Bxd#uDJm+oF$0dq}2A5;eW2v2a=;h>IHI(4|L& zy}DSOeus3Y4)1B-=kqV;=I1QH#T~=?a?6+6`&jafM@eI$ks8Ee)4?l^G8|WYHjzYn zDLjR3+#)#n6W8^=0^PlQnToAti3&`XWbLB2*xcx9t|SHb_uQFsNb}G#>)7ZRcEpsT zV#XMe^U$0TcD@KIrQY5iW-|?QzYlzS80!+WXiJGUH}PPDLtEl(hrOF-E7`Hy&(GO< zdCT19n#txAY@-33Wk=@q;|=7Bq4(9iwO>@;DQs!IX%EfmHe6j`8UErK4$k=+^JjHv zV9wTOPD?T@J$+bzDVIR@CG~bUd&xDj?|hG_n$q{Hj9Y;c`9YG(Sfb(WpRQVu9Y!ua zlDa7^ip2-lm_f_ zd>=E^;fj|$pu@bhH}3{!m{J*iM*nHK0RZ(6N*{a(tLYPsC;b?+k22Ey$&u#;eD;%w z9{~6OoJR4C{+f8^PruqcZ3ZH&y>-_vqq9{G@Zv`v-+_|tFXv2JD#baPvzsW58L^Yz zeq*(R-I5U+0O)pCUlQ&;W72d+fprpdKyFk(E_$;tQPBBoEGA-DUc0=_-AlNQ1n38< z8X9fb>156Em9ncCnoP)apSIhNu+M&2uT^52O$qy)fH!*b_N1bz=VB2XyTDGdt;eC4WQj- zL<`8W^L6|Hk-O&)`{o4kUj}?10sz;hgnGEoAI0)pPM9D3W-@{|ZpujPP6%z)E7%a# z^Txt!JPRbASRYtzAG8Q$eqcVrZ^|s0pIi(w+Q-wMIVMf0pDmYs%ZU}yzF$*6U$aa7 zfJaUBkCeY9pPwDSsnUgqycdb6gCOOHfll@I{+iR%kkVSDfUVSOVY>>fwkBct`|n)v z^_~CIC!^${XC|zEKZ@x_Sq{4a7&UrcS1%rO^!g<7D9}oKoN{uTZx6<5*qicn5EbC? z1k-y*5+N0u&LAP%5x8unyKO&wG2Y?nbZV9#(qz>P%8FXRQ;Dy{=k-1^`?#9QxvfmQ z2Rql_fQ(OGbA9R+9_+NhPfvrGEI$FQpHJ!myJ##RKUUvC?JOr^H@Ek-c8Fm$@ z*EApq*gLY8UL{GpblJaR(%G&F1@cuwP(C#*p6kXm5rmT1G~NC}-uWcaia)Cq z&3jb8sASDfdS48H;lhm7`Q$kUIE)4B!*kypCXJh-|2c|tou(B@PEV?EKZ`sP=t8$e zEr6q}mE}jHKU3OF72_puM!z5k)7nI?XJ1s2h6lnrrw(fDVB;#rmT`nI@K8}w^bs?F zJwr>j-;N~FYzgeCFQ9JofmX&Coo}8DYgizSmXHN6U6F7kGzLy!BM06mY7YZlG|O>* zGbdQCai1^pGTtj)pm3pqu+D=O1elBwXN@PTy-&Zu@ zGf7M>2rnVhS`C4DzUDQ`qAm$EH1TJ^C?FFwpUHZFzf zM^NDP2X0|{$4L3E;oq>l-OywZv6+xhvB*l?G@Z)lteePZC1c}ZfVDJDC7XXKEBJWi zhG<3sgf<&!Ld!j2SZxvp`}LW~*(-=%GUqJv&>$?!AnG Date: Wed, 29 Aug 2018 01:08:26 -0500 Subject: [PATCH 039/108] v1.2 testing --- CHANGES | 2 ++ README.rst | 4 ++-- setup.py | 4 ++-- test/test_yahoofinancials.py | 15 +++++---------- yahoofinancials/__init__.py | 15 +++++++-------- 5 files changed, 18 insertions(+), 22 deletions(-) diff --git a/CHANGES b/CHANGES index 2321154..e94806f 100644 --- a/CHANGES +++ b/CHANGES @@ -18,3 +18,5 @@ 1.0 08/22/2018 -- Removed the requests dependency and replaced it with urllib. 1.0 08/22/2018 -- Updated README.md to README.rst 1.1 08/23/2018 -- Fixed net income python 3 error +1.2 08/29/2018 -- Fixed timezone error in python testing. Issue #11. +1.2 08/29/2018 -- Fixed unicode formatted_date string for consistency. diff --git a/README.rst b/README.rst index 95fe4c2..5fec5b7 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Current Version: v1.1 +Current Version: v1.2 -Version Released: 08/23/2018 +Version Released: 08/29/2018 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.py b/setup.py index 48301a0..731bcb2 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.1', + version='1.2', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.1.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.2.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 6ea56d9..b2adaf3 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,5 +1,5 @@ -# YahooFinancials Unit Tests v1.0 -# Version Released: 08/22/2018 +# YahooFinancials Unit Tests v1.2 +# Version Released: 08/29/2018 # Author: Connor Sanders # Tested on Python 2.7 and 3.5 # Copyright (c) 2018 Connor Sanders @@ -82,14 +82,9 @@ def test_yf_fundamentals(self): # Historical Price Test def test_yf_historical_price(self): single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - if sys.version_info < (3, 0): - expect_dict = {'high': 49.099998474121094, 'volume': 125737200, u'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.959999084472656} - else: - expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.959999084472656} + expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.060001373291016} self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) # Extra Module Methods Test diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 37a532d..7e63441 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.1 +Version: 1.2 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/23/2018 +Version Released: 8/29/2018 Tested on Python 2.7 and 3.5 Copyright (c) 2018 Connor Sanders @@ -264,17 +264,17 @@ def _clean_historical_data(self, hist_data): for date_key, date_obj in type_obj.items(): formatted_date_key = self.format_date(int(date_key)) cleaned_date = self.format_date(int(date_obj['date'])) - date_obj.update({u'' + 'formatted_date': cleaned_date}) + date_obj.update({'formatted_date': cleaned_date}) formatted_type_obj.update({formatted_date_key: date_obj}) event_obj.update({type_key: formatted_type_obj}) dict_ent = {k: event_obj} elif 'date' in k.lower(): cleaned_date = self.format_date(v) - dict_ent = {k: {u'' + 'formatted_date': cleaned_date, 'date': v}} + dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} elif isinstance(v, list): sub_dict_list = [] for sub_dict in v: - sub_dict[u'' + 'formatted_date'] = self.format_date(sub_dict['date']) + sub_dict['formatted_date'] = self.format_date(sub_dict['date']) sub_dict_list.append(sub_dict) dict_ent = {k: sub_dict_list} else: @@ -288,7 +288,7 @@ def _build_api_url(hist_obj, up_ticker): base_url = "https://query1.finance.yahoo.com/v8/finance/chart/" api_url = base_url + up_ticker + '?symbol= ' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' +\ str(hist_obj['end']) + '&interval=' + hist_obj['interval'] - api_url += '&events=div|split|earn' + api_url += '&events=div|split|earn&lang=en-US®ion=US' return api_url # Private Static Method to get financial data via API Call @@ -546,8 +546,7 @@ def get_historical_price_data(self, start_date, end_date, time_interval): start = self.format_date(start_date) end = self.format_date(end_date) hist_obj = {'start': start, 'end': end, 'interval': interval_code} - data = self.get_stock_data('history', hist_obj=hist_obj) - return data + return self.get_stock_data('history', hist_obj=hist_obj) # Private Method for Functions needing stock_price_data def _stock_price_data(self, data_field): From 744711176ae8b4e515d41c49411720d76984bfb6 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 01:23:28 -0500 Subject: [PATCH 040/108] v1.2 testing2 --- yahoofinancials/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 7e63441..1b181d1 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -289,6 +289,7 @@ def _build_api_url(hist_obj, up_ticker): api_url = base_url + up_ticker + '?symbol= ' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' +\ str(hist_obj['end']) + '&interval=' + hist_obj['interval'] api_url += '&events=div|split|earn&lang=en-US®ion=US' + print(api_url) return api_url # Private Static Method to get financial data via API Call From abe3fceada86c6fd349a90c5c420cd19677a01a5 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 01:33:35 -0500 Subject: [PATCH 041/108] v1.2 testing3 --- yahoofinancials/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 1b181d1..95d7f98 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -544,6 +544,8 @@ def get_stock_quote_type_data(self): # Public Method for user to get historical price data with def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) + print(end_date) + print(start_date) start = self.format_date(start_date) end = self.format_date(end_date) hist_obj = {'start': start, 'end': end, 'interval': interval_code} From 9cfa8bc366488874f0488592f74356e26de587d3 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 01:42:28 -0500 Subject: [PATCH 042/108] v1.2 testing4 --- yahoofinancials/__init__.py | 109 +++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 95d7f98..0284968 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -105,12 +105,14 @@ def get_report_type(frequency): # Public static method to format date serial string to readable format and vice versa @staticmethod def format_date(in_date): + print(in_date) if isinstance(in_date, str): year, month, day = in_date.split()[0].split('-') d = date(int(year), int(month), int(day)) form_date = int(time.mktime(d.timetuple())) else: form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) + print(in_date) return form_date # Private Static Method to Convert Eastern Time to UTC @@ -290,7 +292,7 @@ def _build_api_url(hist_obj, up_ticker): str(hist_obj['end']) + '&interval=' + hist_obj['interval'] api_url += '&events=div|split|earn&lang=en-US®ion=US' print(api_url) - return api_url + return "https://query1.finance.yahoo.com/v8/finance/chart/C?symbol= C&period1=1421301600&period2=1508043600&interval=1wk&events=div|split|earn&lang=en-US®ion=US" # Private Static Method to get financial data via API Call @staticmethod @@ -544,8 +546,6 @@ def get_stock_quote_type_data(self): # Public Method for user to get historical price data with def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) - print(end_date) - print(start_date) start = self.format_date(start_date) end = self.format_date(end_date) hist_obj = {'start': start, 'end': end, 'interval': interval_code} @@ -789,3 +789,106 @@ def get_num_shares_outstanding(self, price_type='current'): else: ret_obj.update({tick: None}) return ret_obj + + +import sys +import datetime + +if sys.version_info < (2, 7): + from unittest2 import main as test_main, SkipTest, TestCase +else: + from unittest import main as test_main, SkipTest, TestCase + + +# Test Configuration Variables +stocks = ['AAPL', 'MSFT', 'C'] +currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] +us_treasuries = ['^TNX', '^IRX', '^TYX'] + + +# Global function to check Fundamental Test results +def check_fundamental(test_data, test_type): + if test_type == 'bal': + if 'balanceSheetHistoryQuarterly' in test_data and test_data['balanceSheetHistoryQuarterly']['C'] is not None: + return True + else: + return False + elif test_type == 'inc': + if 'incomeStatementHistoryQuarterly' in test_data and \ + test_data['incomeStatementHistoryQuarterly']['C'] is not None: + return True + else: + return False + elif test_type == 'all': + if 'balanceSheetHistoryQuarterly' in test_data and 'incomeStatementHistoryQuarterly' in test_data and \ + 'cashflowStatementHistoryQuarterly' in test_data: + return True + else: + return False + + +# Main Test Module Class +class TestModule(TestCase): + + def setUp(self): + self.test_yf_stock_single = YahooFinancials('C') + self.test_yf_stock_multi = YahooFinancials(stocks) + self.test_yf_treasuries_single = YahooFinancials('^IRX') + self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) + self.test_yf_currencies = YahooFinancials(currencies) + + # Fundamentals Test + def test_yf_fundamentals(self): + # Single stock test + single_balance_sheet_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'balance') + single_income_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'income') + single_all_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Multi stock test + multi_balance_sheet_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'balance') + multi_income_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'income') + multi_all_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Single stock check + result = check_fundamental(single_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(single_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(single_all_statement_data_qt, 'all') + self.assertEqual(result, True) + # Multi stock check + result = check_fundamental(multi_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(multi_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(multi_all_statement_data_qt, 'all') + self.assertEqual(result, True) + + # Historical Price Test + def test_yf_historical_price(self): + single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') + expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.060001373291016} + self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) + + # Extra Module Methods Test + def test_yf_module_methods(self): + # Stocks + if isinstance(self.test_yf_stock_single.get_current_price(), float): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + if isinstance(self.test_yf_stock_single.get_net_income(), int): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + # Treasuries + if isinstance(self.test_yf_treasuries_single.get_current_price(), float): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + +if __name__ == "__main__": + test_main() From c72375be2115feb73e4bb25ccc823014c1197987 Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 01:45:22 -0500 Subject: [PATCH 043/108] v1.2 testing5 --- yahoofinancials/__init__.py | 103 ------------------------------------ 1 file changed, 103 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 0284968..28bcb5a 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -789,106 +789,3 @@ def get_num_shares_outstanding(self, price_type='current'): else: ret_obj.update({tick: None}) return ret_obj - - -import sys -import datetime - -if sys.version_info < (2, 7): - from unittest2 import main as test_main, SkipTest, TestCase -else: - from unittest import main as test_main, SkipTest, TestCase - - -# Test Configuration Variables -stocks = ['AAPL', 'MSFT', 'C'] -currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] -us_treasuries = ['^TNX', '^IRX', '^TYX'] - - -# Global function to check Fundamental Test results -def check_fundamental(test_data, test_type): - if test_type == 'bal': - if 'balanceSheetHistoryQuarterly' in test_data and test_data['balanceSheetHistoryQuarterly']['C'] is not None: - return True - else: - return False - elif test_type == 'inc': - if 'incomeStatementHistoryQuarterly' in test_data and \ - test_data['incomeStatementHistoryQuarterly']['C'] is not None: - return True - else: - return False - elif test_type == 'all': - if 'balanceSheetHistoryQuarterly' in test_data and 'incomeStatementHistoryQuarterly' in test_data and \ - 'cashflowStatementHistoryQuarterly' in test_data: - return True - else: - return False - - -# Main Test Module Class -class TestModule(TestCase): - - def setUp(self): - self.test_yf_stock_single = YahooFinancials('C') - self.test_yf_stock_multi = YahooFinancials(stocks) - self.test_yf_treasuries_single = YahooFinancials('^IRX') - self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) - self.test_yf_currencies = YahooFinancials(currencies) - - # Fundamentals Test - def test_yf_fundamentals(self): - # Single stock test - single_balance_sheet_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'balance') - single_income_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', 'income') - single_all_statement_data_qt = self.test_yf_stock_single.get_financial_stmts('quarterly', - ['income', 'cash', 'balance']) - # Multi stock test - multi_balance_sheet_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'balance') - multi_income_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', 'income') - multi_all_statement_data_qt = self.test_yf_stock_multi.get_financial_stmts('quarterly', - ['income', 'cash', 'balance']) - # Single stock check - result = check_fundamental(single_balance_sheet_data_qt, 'bal') - self.assertEqual(result, True) - result = check_fundamental(single_income_statement_data_qt, 'inc') - self.assertEqual(result, True) - result = check_fundamental(single_all_statement_data_qt, 'all') - self.assertEqual(result, True) - # Multi stock check - result = check_fundamental(multi_balance_sheet_data_qt, 'bal') - self.assertEqual(result, True) - result = check_fundamental(multi_income_statement_data_qt, 'inc') - self.assertEqual(result, True) - result = check_fundamental(multi_all_statement_data_qt, 'all') - self.assertEqual(result, True) - - # Historical Price Test - def test_yf_historical_price(self): - single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.060001373291016} - self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) - - # Extra Module Methods Test - def test_yf_module_methods(self): - # Stocks - if isinstance(self.test_yf_stock_single.get_current_price(), float): - self.assertEqual(True, True) - else: - self.assertEqual(False, True) - if isinstance(self.test_yf_stock_single.get_net_income(), int): - self.assertEqual(True, True) - else: - self.assertEqual(False, True) - # Treasuries - if isinstance(self.test_yf_treasuries_single.get_current_price(), float): - self.assertEqual(True, True) - else: - self.assertEqual(False, True) - - -if __name__ == "__main__": - test_main() From 28d2bda75ac6618b3517d453a7980adf37188b5e Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 02:09:05 -0500 Subject: [PATCH 044/108] v1.2 testing6 --- test/test_yahoofinancials.py | 4 ++-- yahoofinancials/__init__.py | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index b2adaf3..b52a878 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -82,9 +82,9 @@ def test_yf_fundamentals(self): # Historical Price Test def test_yf_historical_price(self): single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - expect_dict = {'high': 48.2400016784668, 'volume': 81106400, 'formatted_date': '2015-01-12', + expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', 'low': 46.599998474121094, 'adjclose': 45.669029235839844, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.060001373291016} + 'close': 47.61000061035156, 'open': 48.959999084472656} self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) # Extra Module Methods Test diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 28bcb5a..42a2049 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -43,6 +43,7 @@ """ import sys +import calendar import re from json import loads import time @@ -109,10 +110,11 @@ def format_date(in_date): if isinstance(in_date, str): year, month, day = in_date.split()[0].split('-') d = date(int(year), int(month), int(day)) - form_date = int(time.mktime(d.timetuple())) + #form_date = int(time.mktime(d.timetuple())) + form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) else: form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) - print(in_date) + print(form_date) return form_date # Private Static Method to Convert Eastern Time to UTC @@ -292,7 +294,7 @@ def _build_api_url(hist_obj, up_ticker): str(hist_obj['end']) + '&interval=' + hist_obj['interval'] api_url += '&events=div|split|earn&lang=en-US®ion=US' print(api_url) - return "https://query1.finance.yahoo.com/v8/finance/chart/C?symbol= C&period1=1421301600&period2=1508043600&interval=1wk&events=div|split|earn&lang=en-US®ion=US" + return api_url # Private Static Method to get financial data via API Call @staticmethod From 4d47223725f585e714a325d979671b1775170ddc Mon Sep 17 00:00:00 2001 From: alt Date: Wed, 29 Aug 2018 02:22:15 -0500 Subject: [PATCH 045/108] #11 bug fix test --- yahoofinancials/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 42a2049..f7250a4 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -49,7 +49,6 @@ import time from bs4 import BeautifulSoup import datetime -from datetime import date import pytz try: from urllib import FancyURLopener @@ -106,15 +105,10 @@ def get_report_type(frequency): # Public static method to format date serial string to readable format and vice versa @staticmethod def format_date(in_date): - print(in_date) if isinstance(in_date, str): - year, month, day = in_date.split()[0].split('-') - d = date(int(year), int(month), int(day)) - #form_date = int(time.mktime(d.timetuple())) form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) else: form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) - print(form_date) return form_date # Private Static Method to Convert Eastern Time to UTC @@ -293,7 +287,6 @@ def _build_api_url(hist_obj, up_ticker): api_url = base_url + up_ticker + '?symbol= ' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' +\ str(hist_obj['end']) + '&interval=' + hist_obj['interval'] api_url += '&events=div|split|earn&lang=en-US®ion=US' - print(api_url) return api_url # Private Static Method to get financial data via API Call From 7a0645db401fb031ff8c6b1e134a8283558eecdf Mon Sep 17 00:00:00 2001 From: alt Date: Thu, 25 Oct 2018 19:50:08 -0500 Subject: [PATCH 046/108] Issue #14 patch --- CHANGES | 1 + README.rst | 10 +++++----- setup.py | 6 +++--- test/test_yahoofinancials.py | 7 ++++--- yahoofinancials/__init__.py | 35 +++++++++++++++++++++++++++++------ 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/CHANGES b/CHANGES index e94806f..5d6a214 100644 --- a/CHANGES +++ b/CHANGES @@ -20,3 +20,4 @@ 1.1 08/23/2018 -- Fixed net income python 3 error 1.2 08/29/2018 -- Fixed timezone error in python testing. Issue #11. 1.2 08/29/2018 -- Fixed unicode formatted_date string for consistency. +1.3 10/25/2018 -- Added patch for reported bug described in Issue #14. diff --git a/README.rst b/README.rst index 5fec5b7..e3e25e8 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Current Version: v1.2 +Current Version: v1.3 -Version Released: 08/29/2018 +Version Released: 10/25/2018 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -21,8 +21,8 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------ -- yahoofinancials runs fine on most versions of python 2 and 3. -- It was built and tested using versions 2.7 and 3.5 +- yahoofinancials runs on Python 2.7, 3.3, 3.4, 3.5, and 3.6. +- Python 3.7 support is coming soon. - The package depends on beautifulsoup4 and pytz to work. 1. Installation using pip: @@ -67,7 +67,7 @@ Module Methods -------------- - The financial data from all methods is returned as JSON. - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -- YahooFinancials works on most versions of python 2 and 3 and runs on all operating systems. (Windows, Mac, Linux). +- YahooFinancials works with Python 2.7, 3.3, 3.4, 3.5, and 3.6 and runs on all operating systems. (Windows, Mac, Linux). Featured Methods ^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index 731bcb2..5f43629 100644 --- a/setup.py +++ b/setup.py @@ -10,15 +10,15 @@ setup( name='yahoofinancials', - version='1.2', + version='1.3', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.2.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.3.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', - keywords = ['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], + keywords=['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], packages=['yahoofinancials'], install_requires=[ "beautifulsoup4", diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index b52a878..ec16fea 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,7 +1,7 @@ -# YahooFinancials Unit Tests v1.2 -# Version Released: 08/29/2018 +# YahooFinancials Unit Tests v1.3 +# Version Released: 10/25/2018 # Author: Connor Sanders -# Tested on Python 2.7 and 3.5 +# Tested on Python 2.7, 3.3, 3.4, 3.5, and 3.6 # Copyright (c) 2018 Connor Sanders # MIT License @@ -71,6 +71,7 @@ def test_yf_fundamentals(self): self.assertEqual(result, True) result = check_fundamental(single_all_statement_data_qt, 'all') self.assertEqual(result, True) + # Multi stock check result = check_fundamental(multi_balance_sheet_data_qt, 'bal') self.assertEqual(result, True) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index f7250a4..0861987 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,13 +1,13 @@ """ ============================== The Yahoo Financials Module -Version: 1.2 +Version: 1.3 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 8/29/2018 -Tested on Python 2.7 and 3.5 +Version Released: 10/25/2018 +Tested on Python 2.7, 3.3, 3.4, 3.5, and 3.6 Copyright (c) 2018 Connor Sanders MIT License @@ -249,7 +249,7 @@ def _build_historical_url(self, ticker, hist_oj): return url # Private Method to clean the dates of the newly returns historical stock data into readable format - def _clean_historical_data(self, hist_data): + def _clean_historical_data(self, hist_data, last_attempt=False): data = {} for k, v in hist_data.items(): if k == 'eventsData': @@ -267,8 +267,14 @@ def _clean_historical_data(self, hist_data): event_obj.update({type_key: formatted_type_obj}) dict_ent = {k: event_obj} elif 'date' in k.lower(): - cleaned_date = self.format_date(v) - dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} + if v is not None: + cleaned_date = self.format_date(v) + dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} + else: + if last_attempt is False: + return None + else: + dict_ent = {k: {'formatted_date': None, 'date': v}} elif isinstance(v, list): sub_dict_list = [] for sub_dict in v: @@ -339,6 +345,20 @@ def _clean_api_data(self, api_url): ret_obj.update({'prices': prices_list}) return ret_obj + # Private Method to Handle Recursive API Request + def _recursive_api_request(self, hist_obj, up_ticker, i=0): + api_url = self._build_api_url(hist_obj, up_ticker) + re_data = self._clean_api_data(api_url) + cleaned_re_data = self._clean_historical_data(re_data) + if cleaned_re_data is not None: + return cleaned_re_data + else: + if i < 3: + i += 1 + return self._recursive_api_request(hist_obj, up_ticker, i) + else: + return self._clean_historical_data(re_data, True) + # Private Method to take scrapped data and build a data dictionary with def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ @@ -359,9 +379,12 @@ def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hi else: YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) try: + cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) + ''' api_url = self._build_api_url(hist_obj, up_ticker) re_data = self._clean_api_data(api_url) cleaned_re_data = self._clean_historical_data(re_data) + ''' except KeyError: try: re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) From 1f2b95aceb087292e71c128e9f0f9eff7d856dbd Mon Sep 17 00:00:00 2001 From: Usernameles Date: Wed, 9 Jan 2019 00:08:53 +0100 Subject: [PATCH 047/108] Issue #15 idea --- yahoofinancials/__init__.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 0861987..ae9f9dd 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -50,6 +50,7 @@ from bs4 import BeautifulSoup import datetime import pytz +import random try: from urllib import FancyURLopener except: @@ -59,6 +60,9 @@ # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 +#Custom Exception class to handle custom error +class ManagedException(Exception): + pass # Class used to open urls for financial data class UrlOpener(FancyURLopener): @@ -132,6 +136,20 @@ def _scrape_data(self, url, tech_type, statement_type): _lastget = now urlopener = UrlOpener() response = urlopener.open(url) + #Try to open the URL up to 10 times sleeping random time if something goes wrong + maxRetry = 10 + for i in range(0,maxRetry): + response = urlopener.open(url) + if response.getcode() != 200: + #Sleep a random time between 10 to 20 seconds + time.sleep(random.randrange(10,20)) + else: + #break the loop if HTTP status equals 200 + break + if (i == maxRetry-1): + #raise a custom exception if we can't get the web page within maxRetry attempts + raise ManagedException("Server replied with HTTP "+str(response.getcode())+" code while opening the url: "+str(url)) + response_content = response.read() soup = BeautifulSoup(response_content, "html.parser") script = soup.find("script", text=re.compile("root.App.main")).text @@ -448,8 +466,13 @@ def get_stock_data(self, statement_type='income', tech_type='', report_name='', data.update(dict_ent) else: for tick in self.ticker: - dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) + try: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + except ManagedException: + print("Warning! Ticker: "+str(tick)+" error - "+ManagedException) + print("The process is still running...") + continue return data # Public Method to get technical stock data From 93f9f5ed2c9d261f6ca1bf88b440fa3125e7db1c Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 12 Jan 2019 14:03:28 -0600 Subject: [PATCH 048/108] issue #15 fix with Usernameles contributions --- .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/vcs.xml | 6 + .idea/workspace.xml | 309 +++++++++++++++++++++++++++++++++++ .idea/yahoofinancials.iml | 11 ++ LICENSE | 2 +- README.rst | 6 +- setup.py | 4 +- test/test_yahoofinancials.py | 6 +- yahoofinancials/__init__.py | 37 ++--- 10 files changed, 364 insertions(+), 29 deletions(-) create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml create mode 100644 .idea/yahoofinancials.iml diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..381406d --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..6085843 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..c157669 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,309 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ManagedException + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - 1547321328746 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/yahoofinancials.iml b/.idea/yahoofinancials.iml deleted file mode 100644 index 6711606..0000000 --- a/.idea/yahoofinancials.iml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - \ No newline at end of file From 9d31f52118fa2ecb60a19bb142e4633facbf594b Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 12 Jan 2019 17:51:20 -0600 Subject: [PATCH 050/108] issue #15 fix with Usernameles contributions --- .idea/workspace.xml | 400 +++++++++++++++++++++++++++++++++++ README.rst | 14 +- test/test_yahoofinancials.py | 2 +- yahoofinancials/__init__.py | 50 +++-- 4 files changed, 438 insertions(+), 28 deletions(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..c9ac1cd --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,400 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ManagedException + .getcode + api + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1547321328746 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 36072ae202873c9d4c44e73813afa9bb66404a7d Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 12 Jan 2019 18:04:58 -0600 Subject: [PATCH 052/108] python3.7 test --- .idea/workspace.xml | 399 ++++++++++++++++++++++++++++++++++++++++++++ setup.py | 3 +- 2 files changed, 401 insertions(+), 1 deletion(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..4651832 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,399 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ManagedException + .getcode + api + pytz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1547321328746 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 2f3fab97bac617920e7f7f955f8ec510cb17ffd4 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 12 Jan 2019 18:08:13 -0600 Subject: [PATCH 054/108] python3.7 test --- .idea/workspace.xml | 418 ++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 1 + 2 files changed, 419 insertions(+) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..e7103da --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,418 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ManagedException + .getcode + api + pytz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1547321328746 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 596f8dbaf0144a9437a1b09a97d2a5c9f22e7b58 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 12 Jan 2019 18:16:08 -0600 Subject: [PATCH 056/108] python3.7 test --- .idea/workspace.xml | 418 ++++++++++++++++++++++++++++++++++++++++++++ .travis.yml | 7 +- 2 files changed, 423 insertions(+), 2 deletions(-) create mode 100644 .idea/workspace.xml diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..126cfb4 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,418 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ManagedException + .getcode + api + pytz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 1547321328746 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 1d6bce98cbac64dec1b45c0007b1d64150d2b845 Mon Sep 17 00:00:00 2001 From: alt Date: Sun, 13 Jan 2019 04:01:04 -0600 Subject: [PATCH 058/108] added feature requested in #18 --- CHANGES | 3 ++ README.rst | 78 ++++++++++++++++++++++++++++++++---- test/test_yahoofinancials.py | 6 +-- yahoofinancials/__init__.py | 17 ++++++-- 4 files changed, 91 insertions(+), 13 deletions(-) diff --git a/CHANGES b/CHANGES index 5d6a214..b188d25 100644 --- a/CHANGES +++ b/CHANGES @@ -21,3 +21,6 @@ 1.2 08/29/2018 -- Fixed timezone error in python testing. Issue #11. 1.2 08/29/2018 -- Fixed unicode formatted_date string for consistency. 1.3 10/25/2018 -- Added patch for reported bug described in Issue #14. +1.4 01/13/2019 -- Python3.7 support added. +1.4 01/13/2019 -- Added patch for reported bug described in Issue #15. +1.4 01/13/2019 -- Added method for get_stock_earnings_data(). diff --git a/README.rst b/README.rst index acdb490..b3ac861 100644 --- a/README.rst +++ b/README.rst @@ -9,7 +9,7 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit Current Version: v1.4 -Version Released: 01/12/2019 +Version Released: 01/13/2019 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -21,8 +21,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------ -- yahoofinancials runs on Python 2.7, 3.3, 3.4, 3.5, and 3.6. -- Python 3.7 support is coming soon. +- yahoofinancials runs on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7. - The package depends on beautifulsoup4 and pytz to work. 1. Installation using pip: @@ -67,7 +66,7 @@ Module Methods -------------- - The financial data from all methods is returned as JSON. - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -- YahooFinancials works with Python 2.7, 3.3, 3.4, 3.5, and 3.6 and runs on all operating systems. (Windows, Mac, Linux). +- YahooFinancials works with Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 and runs on all operating systems. (Windows, Mac, Linux). Featured Methods ^^^^^^^^^^^^^^^^ @@ -99,10 +98,9 @@ Featured Methods - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -Methods Removed in V1.0 +Methods Added in V1.4 ^^^^^^^^^^^^^^^^^^^^^^^ -- get_stock_summary_data(): -- get_historical_stock_data(): +- get_key_statistics_data(): Additional Module Methods ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -622,3 +620,69 @@ Examples of Returned JSON Data "ytdReturn": null } } +11. Apple Key Statistics Data: + + +.. code-block:: python + + yahoo_financials = YahooFinancials('AAPL') + print(yahoo_financials.get_key_statistics_data()) + + +.. code-block:: javascript + + { + "AAPL": { + "annualHoldingsTurnover": null, + "enterpriseToRevenue": 2.973, + "beta3Year": null, + "profitMargins": 0.22413999, + "enterpriseToEbitda": 9.652, + "52WeekChange": -0.12707871, + "morningStarRiskRating": null, + "forwardEps": 13.49, + "revenueQuarterlyGrowth": null, + "sharesOutstanding": 4729800192, + "fundInceptionDate": "-", + "annualReportExpenseRatio": null, + "totalAssets": null, + "bookValue": 22.534, + "sharesShort": 44915125, + "sharesPercentSharesOut": 0.0095, + "fundFamily": null, + "lastFiscalYearEnd": 1538179200, + "heldPercentInstitutions": 0.61208, + "netIncomeToCommon": 59531001856, + "trailingEps": 11.91, + "lastDividendValue": null, + "SandP52WeekChange": -0.06475246, + "priceToBook": 6.7582316, + "heldPercentInsiders": 0.00072999997, + "nextFiscalYearEnd": 1601337600, + "yield": null, + "mostRecentQuarter": 1538179200, + "shortRatio": 1, + "sharesShortPreviousMonthDate": "2018-10-31", + "floatShares": 4489763410, + "beta": 1.127094, + "enterpriseValue": 789555511296, + "priceHint": 2, + "threeYearAverageReturn": null, + "lastSplitDate": "2014-06-09", + "lastSplitFactor": "1/7", + "legalType": null, + "morningStarOverallRating": null, + "earningsQuarterlyGrowth": 0.318, + "priceToSalesTrailing12Months": null, + "dateShortInterest": 1543536000, + "pegRatio": 0.98, + "ytdReturn": null, + "forwardPE": 11.289103, + "maxAge": 1, + "lastCapGain": null, + "shortPercentOfFloat": 0.0088, + "sharesShortPriorMonth": 36469092, + "category": null, + "fiveYearAverageReturn": null + } + } diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index c7df5a6..1bb971b 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,7 +1,7 @@ # YahooFinancials Unit Tests v1.4 -# Version Released: 01/12/2019 +# Version Released: 01/13/2019 # Author: Connor Sanders -# Tested on Python 2.7, 3.3, 3.4, 3.5, and 3.6 +# Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 # Copyright (c) 2019 Connor Sanders # MIT License @@ -16,7 +16,7 @@ # Test Configuration Variables -stocks = ['AAPL', 'MSFT', 'C'] +stocks = ['AAPL', 'MSFT', 'C', 'IL&FSTRANS.NS'] currencies = ['EURUSD=X', 'JPY=X', 'GBPUSD=X'] us_treasuries = ['^TNX', '^IRX', '^TYX'] diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 82cfc58..38abdb8 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -6,8 +6,8 @@ Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/12/2018 -Tested on Python 2.7, 3.3, 3.4, 3.5, and 3.6 +Version Released: 01/13/2019 +Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 Copyright (c) 2019 Connor Sanders MIT License @@ -86,6 +86,7 @@ def __init__(self, ticker): 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], + 'keystats': ['key-statistics'], 'history': ['history'] } @@ -484,7 +485,10 @@ def get_stock_data(self, statement_type='income', tech_type='', report_name='', # Public Method to get technical stock data def get_stock_tech_data(self, tech_type): - return self.get_stock_data(tech_type=tech_type) + if tech_type == 'defaultKeyStatistics': + return self.get_stock_data(statement_type='keystats', tech_type=tech_type) + else: + return self.get_stock_data(tech_type=tech_type) # Public Method to get reformatted statement data def get_reformatted_stmt_data(self, raw_data, statement_type): @@ -567,6 +571,13 @@ def get_stock_price_data(self, reformat=True): else: return self.get_stock_tech_data('price') + # Public Method for the user to return key-statistics data + def get_key_statistics_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('defaultKeyStatistics'), 'defaultKeyStatistics') + else: + return self.get_stock_tech_data('defaultKeyStatistics') + # Public Method for the user to get stock earnings data def get_stock_earnings_data(self, reformat=True): if reformat: From 0e1e1b0cb3169d8510f822cd683588438926ebf3 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 26 Jan 2019 22:28:05 -0600 Subject: [PATCH 059/108] get_daily_dividend_data() method development --- CHANGES | 1 + test/test_yahoofinancials.py | 12 +++++++++++ yahoofinancials/__init__.py | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/CHANGES b/CHANGES index b188d25..14ab5c8 100644 --- a/CHANGES +++ b/CHANGES @@ -24,3 +24,4 @@ 1.4 01/13/2019 -- Python3.7 support added. 1.4 01/13/2019 -- Added patch for reported bug described in Issue #15. 1.4 01/13/2019 -- Added method for get_stock_earnings_data(). +1.5 01/27/2019 -- Added get_daily_dividend_data() method as request in Issue #20. \ No newline at end of file diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 1bb971b..9bcd9b7 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -88,6 +88,18 @@ def test_yf_historical_price(self): 'close': 47.61000061035156, 'open': 48.959999084472656} self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) + # Historical Stock Daily Dividend Test + def test_yf_dividend_price(self): + single_stock_dividend = self.test_yf_stock_single.get_daily_dividend_data('1986-09-15', '1987-09-15') + expect_dict = {"C": [{"date": 533313000, "formatted_date": "1986-11-25", "amount": 0.02999}, + {"date": 541348200, "formatted_date": "1987-02-26", "amount": 0.02999}, + {"date": 544714200, "formatted_date": "1987-04-06", "amount": 0.332}, + {"date": 549120600, "formatted_date": "1987-05-27", "amount": 0.02999}, + {"date": 552576600, "formatted_date": "1987-07-06", "amount": 0.332}, + {"date": 557501400, "formatted_date": "1987-09-01", "amount": 0.02999}] + } + self.assertDictEqual(single_stock_dividend, expect_dict) + # Extra Module Methods Test def test_yf_module_methods(self): # Stocks diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 38abdb8..2d41a20 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -538,6 +538,40 @@ def get_clean_data(self, raw_report_data, report_type): cleaned_data_dict.update({tick: cleaned_data}) return cleaned_data_dict + # Private method to handle dividend data requests + def _handle_api_dividend_request(self, cur_ticker, start, end, interval): + re_dividends = [] + test_url = 'https://query1.finance.yahoo.com/v8/finance/chart/' + cur_ticker + \ + '?period1=' + str(start) + '&period2=' + str(end) + '&interval=' + interval + '&events=div' + div_dict = self._get_api_data(test_url)['chart']['result'][0]['events']['dividends'] + for div_time_key, div_obj in div_dict.items(): + dividend_obj = { + 'date': div_obj['date'], + 'formatted_date': self.format_date(int(div_obj['date'])), + 'amount': div_obj.get('amount', None) + } + re_dividends.append(dividend_obj) + return sorted(re_dividends, key=lambda div: div['date']) + + # Public method to get daily dividend data + def get_stock_dividend_data(self, start, end, interval): + interval_code = self.get_time_code(interval) + if isinstance(self.ticker, str): + try: + return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)} + except: + return {self.ticker: None} + else: + re_data = {} + for tick in self.ticker: + try: + div_data = self._handle_api_dividend_request(tick, start, end, interval_code) + re_data.update({tick: div_data}) + except: + re_data.update({tick: None}) + print(re_data) + return re_data + # Class containing methods to create stock data extracts class YahooFinancials(YahooFinanceETL): @@ -666,6 +700,12 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): data.update({tick: None}) return data + # Public method to get daily dividend data + def get_daily_dividend_data(self, start_date, end_date): + start = self.format_date(start_date) + end = self.format_date(end_date) + return self.get_stock_dividend_data(start, end, 'daily') + # Public Price Data Methods def get_current_price(self): return self._stock_price_data('regularMarketPrice') From 76fe4abe139edb20f5b9b1576645c7ad05b05031 Mon Sep 17 00:00:00 2001 From: alt Date: Sat, 26 Jan 2019 23:16:46 -0600 Subject: [PATCH 060/108] added daily dividend method as requested in Issue #20 --- CHANGES | 3 +- README.rst | 155 ++++++++++++++++++++++++----------- setup.py | 4 +- test/test_yahoofinancials.py | 5 +- yahoofinancials/__init__.py | 5 +- 5 files changed, 117 insertions(+), 55 deletions(-) diff --git a/CHANGES b/CHANGES index 14ab5c8..0b7d93d 100644 --- a/CHANGES +++ b/CHANGES @@ -24,4 +24,5 @@ 1.4 01/13/2019 -- Python3.7 support added. 1.4 01/13/2019 -- Added patch for reported bug described in Issue #15. 1.4 01/13/2019 -- Added method for get_stock_earnings_data(). -1.5 01/27/2019 -- Added get_daily_dividend_data() method as request in Issue #20. \ No newline at end of file +1.5 01/27/2019 -- Added get_daily_dividend_data() method as request in Issue #20. +1.5 01/27/2019 -- Added test_yf_dividend_price() unit testing method. \ No newline at end of file diff --git a/README.rst b/README.rst index b3ac861..bbd65c9 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Current Version: v1.4 +Current Version: v1.5 -Version Released: 01/13/2019 +Version Released: 01/27/2019 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -20,7 +20,7 @@ A powerful financial data module used for pulling both fundamental and technical - As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. Installation ------------- +------------- - yahoofinancials runs on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7. - The package depends on beautifulsoup4 and pytz to work. @@ -98,53 +98,54 @@ Featured Methods - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -Methods Added in V1.4 +Methods Added in V1.5 ^^^^^^^^^^^^^^^^^^^^^^^ -- get_key_statistics_data(): +- get_daily_dividend_data(start_date, end_date) Additional Module Methods ^^^^^^^^^^^^^^^^^^^^^^^^^ -- get_interest_expense(): -- get_operating_income(): -- get_total_operating_expense(): -- get_total_revenue(): -- get_cost_of_revenue(): -- get_income_before_tax(): -- get_income_tax_expense(): -- get_gross_profit(): -- get_net_income_from_continuing_ops(): -- get_research_and_development(): -- get_current_price(): -- get_current_change(): -- get_current_percent_change(): -- get_current_volume(): -- get_prev_close_price(): -- get_open_price(): -- get_ten_day_avg_daily_volume(): -- get_three_month_avg_daily_volume(): -- get_stock_exchange(): -- get_market_cap(): -- get_daily_low(): -- get_daily_high(): -- get_currency(): -- get_yearly_high(): -- get_yearly_low(): -- get_dividend_yield(): -- get_annual_avg_div_yield(): -- get_five_yr_avg_div_yield(): -- get_dividend_rate(): -- get_annual_avg_div_rate(): -- get_50day_moving_avg(): -- get_200day_moving_avg(): -- get_beta(): -- get_payout_ratio(): -- get_pe_ratio(): -- get_price_to_sales(): -- get_exdividend_date(): -- get_book_value(): -- get_ebit(): -- get_net_income(): -- get_earnings_per_share(): +- get_interest_expense() +- get_operating_income() +- get_total_operating_expense() +- get_total_revenue() +- get_cost_of_revenue() +- get_income_before_tax() +- get_income_tax_expense() +- get_gross_profit() +- get_net_income_from_continuing_ops() +- get_research_and_development() +- get_current_price() +- get_current_change() +- get_current_percent_change() +- get_current_volume() +- get_prev_close_price() +- get_open_price() +- get_ten_day_avg_daily_volume() +- get_three_month_avg_daily_volume() +- get_stock_exchange() +- get_market_cap() +- get_daily_low() +- get_daily_high() +- get_currency() +- get_yearly_high() +- get_yearly_low() +- get_dividend_yield() +- get_annual_avg_div_yield() +- get_five_yr_avg_div_yield() +- get_dividend_rate() +- get_annual_avg_div_rate() +- get_50day_moving_avg() +- get_200day_moving_avg() +- get_beta() +- get_payout_ratio() +- get_pe_ratio() +- get_price_to_sales() +- get_exdividend_date() +- get_book_value() +- get_ebit() +- get_net_income() +- get_earnings_per_share() +- get_key_statistics_data() Usage Examples -------------- @@ -620,6 +621,7 @@ Examples of Returned JSON Data "ytdReturn": null } } + 11. Apple Key Statistics Data: @@ -686,3 +688,64 @@ Examples of Returned JSON Data "fiveYearAverageReturn": null } } + +12. Apple and Wells Fargo Daily Dividend Data: + + +.. code-block:: python + + start_date = '1987-09-15' + end_date = '1988-09-15' + yahoo_financials = YahooFinancials(['AAPL', 'WFC']) + print(yahoo_financials.get_daily_dividend_data(start_date, end_date)) + + +.. code-block:: javascript + + { + "AAPL": [ + { + "date": 564157800, + "formatted_date": "1987-11-17", + "amount": 0.08 + }, + { + "date": 571674600, + "formatted_date": "1988-02-12", + "amount": 0.08 + }, + { + "date": 579792600, + "formatted_date": "1988-05-16", + "amount": 0.08 + }, + { + "date": 587655000, + "formatted_date": "1988-08-15", + "amount": 0.08 + } + ], + "WFC": [ + { + "date": 562861800, + "formatted_date": "1987-11-02", + "amount": 0.3008 + }, + { + "date": 570724200, + "formatted_date": "1988-02-01", + "amount": 0.3008 + }, + { + "date": 578583000, + "formatted_date": "1988-05-02", + "amount": 0.3344 + }, + { + "date": 586445400, + "formatted_date": "1988-08-01", + "amount": 0.3344 + } + ] + } + diff --git a/setup.py b/setup.py index 970cf83..0e22442 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.4', + version='1.5', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.4.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.5.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 9bcd9b7..2b79bc6 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,12 +1,11 @@ -# YahooFinancials Unit Tests v1.4 -# Version Released: 01/13/2019 +# YahooFinancials Unit Tests v1.5 +# Version Released: 01/27/2019 # Author: Connor Sanders # Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 # Copyright (c) 2019 Connor Sanders # MIT License import sys -import datetime from yahoofinancials import YahooFinancials if sys.version_info < (2, 7): diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 2d41a20..294392b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.4 +Version: 1.5 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/13/2019 +Version Released: 01/27/2019 Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 Copyright (c) 2019 Connor Sanders @@ -569,7 +569,6 @@ def get_stock_dividend_data(self, start, end, interval): re_data.update({tick: div_data}) except: re_data.update({tick: None}) - print(re_data) return re_data From d63a260a178da6e71344a7c00d3561b7bd0f9f80 Mon Sep 17 00:00:00 2001 From: Alex McFarlane Date: Sat, 23 Nov 2019 23:15:41 +0000 Subject: [PATCH 061/108] added simple method to pull financial data like debt and debt-to-equity ratios --- yahoofinancials/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 294392b..068acac 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -618,6 +618,13 @@ def get_stock_earnings_data(self, reformat=True): else: return self.get_stock_tech_data('earnings') + # Public Method for the user to return financial data + def get_financial_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data(statement_type='keystats', tech_type='financialData'), 'financialData') + else: + return self.get_stock_tech_data(statement_type='keystats', tech_type='financialData') + # Public Method for the user to get stock summary data def get_summary_data(self, reformat=True): if reformat: From 56e4ace9cf044f0ee500c26b83aab59be3197478 Mon Sep 17 00:00:00 2001 From: Alex McFarlane Date: Sat, 23 Nov 2019 23:21:07 +0000 Subject: [PATCH 062/108] added fix for annoying typo --- yahoofinancials/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 068acac..bc45b5d 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -621,9 +621,9 @@ def get_stock_earnings_data(self, reformat=True): # Public Method for the user to return financial data def get_financial_data(self, reformat=True): if reformat: - return self.get_clean_data(self.get_stock_tech_data(statement_type='keystats', tech_type='financialData'), 'financialData') + return self.get_clean_data(self.get_stock_data(statement_type='keystats', tech_type='financialData'), 'financialData') else: - return self.get_stock_tech_data(statement_type='keystats', tech_type='financialData') + return self.get_stock_data(statement_type='keystats', tech_type='financialData') # Public Method for the user to get stock summary data def get_summary_data(self, reformat=True): From d3c65ac36532de7d82479bf57c217435e46e2b0d Mon Sep 17 00:00:00 2001 From: Alex McFarlane Date: Sat, 23 Nov 2019 23:29:08 +0000 Subject: [PATCH 063/108] added docs --- README.rst | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/README.rst b/README.rst index bbd65c9..f8a4f44 100644 --- a/README.rst +++ b/README.rst @@ -749,3 +749,48 @@ Examples of Returned JSON Data ] } +13. Apple key Financial Data: + + +.. code-block:: python + + yahoo_financials = YahooFinancials("AAPL") + print(yahoo_financials.get_financial_data()) + + +.. code-block:: javascript + + { + 'AAPL': { + 'ebitdaMargins': 0.29395, + 'profitMargins': 0.21238, + 'grossMargins': 0.37818, + 'operatingCashflow': 69390999552, + 'revenueGrowth': 0.018, + 'operatingMargins': 0.24572, + 'ebitda': 76476997632, + 'targetLowPrice': 150, + 'recommendationKey': 'buy', + 'grossProfits': 98392000000, + 'freeCashflow': 42914250752, + 'targetMedianPrice': 270, + 'currentPrice': 261.78, + 'earningsGrowth': 0.039, + 'currentRatio': 1.54, + 'returnOnAssets': 0.11347, + 'numberOfAnalystOpinions': 40, + 'targetMeanPrice': 255.51, + 'debtToEquity': 119.405, + 'returnOnEquity': 0.55917, + 'targetHighPrice': 300, + 'totalCash': 100556996608, + 'totalDebt': 108046999552, + 'totalRevenue': 260174004224, + 'totalCashPerShare': 22.631, + 'financialCurrency': 'USD', + 'maxAge': 86400, + 'revenuePerShare': 56.341, + 'quickRatio': 1.384, + 'recommendationMean': 2.2 + } + } From 3df7243a44ce2f148b1cb1a3c5135acd3b0b3525 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 6 Mar 2020 19:55:52 -0700 Subject: [PATCH 064/108] yahoo issue was the client version --- yahoofinancials/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 294392b..797a89b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -68,7 +68,7 @@ class ManagedException(Exception): # Class used to open urls for financial data class UrlOpener(FancyURLopener): - version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11' + version = 'w3m/0.5.3+git20180125' # Class containing Yahoo Finance ETL Functionality From c504fcfbd0b71f1148ebd17953531d8b915e4d42 Mon Sep 17 00:00:00 2001 From: Shaun Patterson Date: Thu, 9 Apr 2020 01:31:26 -0400 Subject: [PATCH 065/108] adding profile data --- yahoofinancials/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 294392b..301972c 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -68,7 +68,7 @@ class ManagedException(Exception): # Class used to open urls for financial data class UrlOpener(FancyURLopener): - version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11' + version = 'w3m/0.5.3+git20180125' # Class containing Yahoo Finance ETL Functionality @@ -87,7 +87,8 @@ def __init__(self, ticker): 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], 'keystats': ['key-statistics'], - 'history': ['history'] + 'history': ['history'], + 'profile': ['profile'] } # Interval value translation dictionary @@ -611,6 +612,14 @@ def get_key_statistics_data(self, reformat=True): else: return self.get_stock_tech_data('defaultKeyStatistics') + # Public Method for the user to get company profile data + def get_stock_profile_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') + else: + return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') + + # Public Method for the user to get stock earnings data def get_stock_earnings_data(self, reformat=True): if reformat: From bfba2e492937a1ab3e0a5393aa540979c4616b54 Mon Sep 17 00:00:00 2001 From: Shaun Patterson Date: Thu, 9 Apr 2020 01:32:28 -0400 Subject: [PATCH 066/108] Readme update --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index bbd65c9..35c2cd0 100644 --- a/README.rst +++ b/README.rst @@ -146,6 +146,7 @@ Additional Module Methods - get_net_income() - get_earnings_per_share() - get_key_statistics_data() +- get_stock_profile_data() Usage Examples -------------- From 5805c54f96c9222e587603d49130a7c0e710480b Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 6 Mar 2020 19:55:52 -0700 Subject: [PATCH 067/108] yahoo issue was the client version --- yahoofinancials/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 294392b..797a89b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -68,7 +68,7 @@ class ManagedException(Exception): # Class used to open urls for financial data class UrlOpener(FancyURLopener): - version = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11' + version = 'w3m/0.5.3+git20180125' # Class containing Yahoo Finance ETL Functionality From 5a5566d5de0328e4e799d443b315a5d829a09fe8 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 24 Apr 2020 09:38:39 -0600 Subject: [PATCH 068/108] fix for bs4 4.9.0 script change preserves compatibility with prior versions --- yahoofinancials/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 797a89b..bd0f885 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -150,6 +150,9 @@ def _scrape_data(self, url, tech_type, statement_type): re_script = soup.find("script", text=re.compile("root.App.main")) if re_script is not None: script = re_script.text + # bs4 4.9.0 changed so text from scripts is no longer considered text + if not script: + script = re_script.string self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) response.close() break From e36587b928c00b69f3876187aa15ab48818bccc8 Mon Sep 17 00:00:00 2001 From: Sylvan Butler Date: Fri, 6 Mar 2020 20:16:51 -0700 Subject: [PATCH 069/108] fix test for 2020 adjclose --- test/test_yahoofinancials.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 2b79bc6..bcb6c73 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -83,8 +83,10 @@ def test_yf_fundamentals(self): def test_yf_historical_price(self): single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'adjclose': 45.35684585571289, 'date': 1421038800, + 'low': 46.599998474121094, 'date': 1421038800, 'close': 47.61000061035156, 'open': 48.959999084472656} + # ignore adjclose as it will change with every dividend paid in the future + del single_stock_prices['C']['prices'][0]['adjclose'] self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) # Historical Stock Daily Dividend Test From 585abfb4039135f6643688aa7c164896d8c9e083 Mon Sep 17 00:00:00 2001 From: alt Date: Sun, 18 Oct 2020 18:39:42 -0500 Subject: [PATCH 070/108] v1.6 released --- CHANGES | 3 ++- LICENSE | 2 +- README.rst | 4 ++-- yahoofinancials/__init__.py | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 0b7d93d..88413ff 100644 --- a/CHANGES +++ b/CHANGES @@ -25,4 +25,5 @@ 1.4 01/13/2019 -- Added patch for reported bug described in Issue #15. 1.4 01/13/2019 -- Added method for get_stock_earnings_data(). 1.5 01/27/2019 -- Added get_daily_dividend_data() method as request in Issue #20. -1.5 01/27/2019 -- Added test_yf_dividend_price() unit testing method. \ No newline at end of file +1.5 01/27/2019 -- Added test_yf_dividend_price() unit testing method. +1.6 10/18/2020 -- Merged in two pull requests with bug fixes from sylvandb. diff --git a/LICENSE b/LICENSE index 8255e9d..e658826 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2019 Connor Sanders +Copyright (c) 2020 Connor Sanders Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index bbd65c9..3b73f68 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master :target: https://travis-ci.org/JECSand/yahoofinancials -Current Version: v1.5 +Current Version: v1.6 -Version Released: 01/27/2019 +Version Released: 10/18/2020 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index bd0f885..f62affb 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,15 +1,15 @@ """ ============================== The Yahoo Financials Module -Version: 1.5 +Version: 1.6 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/27/2019 +Version Released: 10/18/2020 Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 -Copyright (c) 2019 Connor Sanders +Copyright (c) 2020 Connor Sanders MIT License List of Included Functions: From 9cc2bd1faa7380f6f451f7de9b5d228160627fd3 Mon Sep 17 00:00:00 2001 From: alt Date: Sun, 18 Oct 2020 18:43:29 -0500 Subject: [PATCH 071/108] version 1.6 release --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 0e22442..970eafb 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.5', + version='1.6', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.5.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.6.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', From 62119be23fe81f89fd5fb1bd450fc2088667c8df Mon Sep 17 00:00:00 2001 From: sedwards2000 Date: Mon, 19 Dec 2022 09:44:05 +0000 Subject: [PATCH 072/108] fixed encryption of incoming stock data --- yahoofinancials/__init__.py | 80 +++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index f62affb..ef5deec 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -56,6 +56,11 @@ except: from urllib.request import FancyURLopener +import hashlib +from base64 import b64decode +from Crypto.Cipher import AES +from Crypto.Util.Padding import unpad +import json # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -163,14 +168,83 @@ def _scrape_data(self, url, tech_type, statement_type): raise ManagedException("Server replied with HTTP " + str(response.getcode()) + " code while opening the url: " + str(url)) data = self._cache[url] + data = self._decryptData(data) if tech_type == '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] + stores = data["QuoteSummaryStore"] elif tech_type != '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"][tech_type] + stores = data["QuoteSummaryStore"][tech_type] else: - stores = data["context"]["dispatcher"]["stores"]["HistoricalPriceStore"] + stores = data["HistoricalPriceStore"] return stores + def _decryptData(self,data): + #function taken from another package at https://github.com/ranaroussi/yfinance/pull/1253/commits/8e5f0984af347afda6be74b27a989422e49a975b + encrypted_stores = data['context']['dispatcher']['stores'] + _cs = data["_cs"] + _cr = data["_cr"] + + _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) + password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + + encrypted_stores = b64decode(encrypted_stores) + assert encrypted_stores[0:8] == b"Salted__" + salt = encrypted_stores[8:16] + encrypted_stores = encrypted_stores[16:] + + def EVPKDF( + password, + salt, + keySize=32, + ivSize=16, + iterations=1, + hashAlgorithm="md5", + ) -> tuple: + """OpenSSL EVP Key Derivation Function + Args: + password (Union[str, bytes, bytearray]): Password to generate key from. + salt (Union[bytes, bytearray]): Salt to use. + keySize (int, optional): Output key length in bytes. Defaults to 32. + ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. + iterations (int, optional): Number of iterations to perform. Defaults to 1. + hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. + Returns: + key, iv: Derived key and Initialization Vector (IV) bytes. + Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 + OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 + """ + + assert iterations > 0, "Iterations can not be less than 1." + + if isinstance(password, str): + password = password.encode("utf-8") + + final_length = keySize + ivSize + key_iv = b"" + block = None + + while len(key_iv) < final_length: + hasher = hashlib.new(hashAlgorithm) + if block: + hasher.update(block) + hasher.update(password) + hasher.update(salt) + block = hasher.digest() + for _ in range(1, iterations): + block = hashlib.new(hashAlgorithm, block).digest() + key_iv += block + + key, iv = key_iv[:keySize], key_iv[keySize:final_length] + return key, iv + + key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + + cipher = AES.new(key, AES.MODE_CBC, iv=iv) + plaintext = cipher.decrypt(encrypted_stores) + plaintext = unpad(plaintext, 16, style="pkcs7") + decoded_stores = json.loads(plaintext) + + return decoded_stores + # Private static method to determine if a numerical value is in the data object being cleaned @staticmethod def _determine_numeric_value(value_dict): From 4823f6160fc20756ed98d9855a4e137c90cb80c2 Mon Sep 17 00:00:00 2001 From: Shaun Patterson Date: Fri, 23 Dec 2022 19:57:35 -0500 Subject: [PATCH 073/108] Merging in Crypto change --- yahoofinancials/__init__.py | 1617 ++++++++++++++++++----------------- 1 file changed, 845 insertions(+), 772 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 2ed802b..1fd96a3 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,15 +1,15 @@ """ ============================== The Yahoo Financials Module -Version: 1.5 +Version: 1.6 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/27/2019 +Version Released: 10/18/2020 Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 -Copyright (c) 2019 Connor Sanders +Copyright (c) 2020 Connor Sanders MIT License List of Included Functions: @@ -52,10 +52,15 @@ import pytz import random try: - from urllib import FancyURLopener + from urllib import FancyURLopener except: - from urllib.request import FancyURLopener + from urllib.request import FancyURLopener +import hashlib +from base64 import b64decode +from Crypto.Cipher import AES +from Crypto.Util.Padding import unpad +import json # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -63,839 +68,907 @@ # Custom Exception class to handle custom error class ManagedException(Exception): - pass + pass # Class used to open urls for financial data class UrlOpener(FancyURLopener): - version = 'w3m/0.5.3+git20180125' + version = 'w3m/0.5.3+git20180125' # Class containing Yahoo Finance ETL Functionality class YahooFinanceETL(object): - def __init__(self, ticker): - self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] - self._cache = {} - - # Minimum interval between Yahoo Finance requests for this instance - _MIN_INTERVAL = 7 - - # Meta-data dictionaries for the classes to use - YAHOO_FINANCIAL_TYPES = { - 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], - 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], - 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], - 'keystats': ['key-statistics'], - 'history': ['history'], - 'profile': ['profile'] - } - - # Interval value translation dictionary - _INTERVAL_DICT = { - 'daily': '1d', - 'weekly': '1wk', - 'monthly': '1mo' - } - - # Base Yahoo Finance URL for the class to build on - _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' - - # private static method to get the appropriate report type identifier - @staticmethod - def get_report_type(frequency): - if frequency == 'annual': - report_num = 1 + def __init__(self, ticker): + self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] + self._cache = {} + + # Minimum interval between Yahoo Finance requests for this instance + _MIN_INTERVAL = 7 + + # Meta-data dictionaries for the classes to use + YAHOO_FINANCIAL_TYPES = { + 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], + 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], + 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], + 'keystats': ['key-statistics'], + 'history': ['history'], + 'profile': ['profile'] + } + + # Interval value translation dictionary + _INTERVAL_DICT = { + 'daily': '1d', + 'weekly': '1wk', + 'monthly': '1mo' + } + + # Base Yahoo Finance URL for the class to build on + _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' + + # private static method to get the appropriate report type identifier + @staticmethod + def get_report_type(frequency): + if frequency == 'annual': + report_num = 1 + else: + report_num = 2 + return report_num + + # Public static method to format date serial string to readable format and vice versa + @staticmethod + def format_date(in_date): + if isinstance(in_date, str): + form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) + else: + form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) + return form_date + + # Private Static Method to Convert Eastern Time to UTC + @staticmethod + def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): + utc = pytz.utc + eastern = pytz.timezone('US/Eastern') + date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) + date_eastern = eastern.localize(date_, is_dst=None) + date_utc = date_eastern.astimezone(utc) + return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') + + # Private method to scrape data from yahoo finance + def _scrape_data(self, url, tech_type, statement_type): + global _lastget + if not self._cache.get(url): + now = int(time.time()) + if _lastget and now - _lastget < self._MIN_INTERVAL: + time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) + now = int(time.time()) + _lastget = now + urlopener = UrlOpener() + # Try to open the URL up to 10 times sleeping random time if something goes wrong + max_retry = 10 + for i in range(0, max_retry): + response = urlopener.open(url) + if response.getcode() != 200: + time.sleep(random.randrange(10, 20)) else: - report_num = 2 - return report_num - - # Public static method to format date serial string to readable format and vice versa - @staticmethod - def format_date(in_date): - if isinstance(in_date, str): - form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) - else: - form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) - return form_date - - # Private Static Method to Convert Eastern Time to UTC - @staticmethod - def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): - utc = pytz.utc - eastern = pytz.timezone('US/Eastern') - date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) - date_eastern = eastern.localize(date_, is_dst=None) - date_utc = date_eastern.astimezone(utc) - return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') - - # Private method to scrape data from yahoo finance - def _scrape_data(self, url, tech_type, statement_type): - global _lastget - if not self._cache.get(url): - now = int(time.time()) - if _lastget and now - _lastget < self._MIN_INTERVAL: - time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) - now = int(time.time()) - _lastget = now - urlopener = UrlOpener() - # Try to open the URL up to 10 times sleeping random time if something goes wrong - max_retry = 10 - for i in range(0, max_retry): - response = urlopener.open(url) - if response.getcode() != 200: - time.sleep(random.randrange(10, 20)) - else: - response_content = response.read() - soup = BeautifulSoup(response_content, "html.parser") - re_script = soup.find("script", text=re.compile("root.App.main")) - if re_script is not None: - script = re_script.text - # bs4 4.9.0 changed so text from scripts is no longer considered text - if not script: - script = re_script.string - self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) - response.close() - break - else: - time.sleep(random.randrange(10, 20)) - if i == max_retry - 1: - # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with HTTP " + str(response.getcode()) + - " code while opening the url: " + str(url)) - data = self._cache[url] - if tech_type == '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"] - elif tech_type != '' and statement_type != 'history': - stores = data["context"]["dispatcher"]["stores"]["QuoteSummaryStore"][tech_type] - else: - stores = data["context"]["dispatcher"]["stores"]["HistoricalPriceStore"] - return stores - - # Private static method to determine if a numerical value is in the data object being cleaned - @staticmethod - def _determine_numeric_value(value_dict): - if 'raw' in value_dict.keys(): - numerical_val = value_dict['raw'] - else: - numerical_val = None - return numerical_val - - # Private method to format date serial string to readable format and vice versa - def _format_time(self, in_time): - form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') - utc_dt = self._convert_to_utc(form_date_time) - return utc_dt - - # Private method to return the a sub dictionary entry for the earning report cleaning - def _get_cleaned_sub_dict_ent(self, key, val_list): - sub_list = [] - for rec in val_list: - sub_sub_dict = {} - for k, v in rec.items(): - if k == 'date': - sub_sub_dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - sub_sub_dict_ent = {k: numerical_val} - sub_sub_dict.update(sub_sub_dict_ent) - sub_list.append(sub_sub_dict) - sub_ent = {key: sub_list} - return sub_ent - - # Private method to process raw earnings data and clean - def _clean_earnings_data(self, raw_data): - cleaned_data = {} - earnings_key = 'earningsData' - financials_key = 'financialsData' - for k, v in raw_data.items(): - if k == 'earningsChart': - sub_dict = {} - for k2, v2 in v.items(): - if k2 == 'quarterly': - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - elif k2 == 'currentQuarterEstimate': - numerical_val = self._determine_numeric_value(v2) - sub_ent = {k2: numerical_val} - else: - sub_ent = {k2: v2} - sub_dict.update(sub_ent) - dict_ent = {earnings_key: sub_dict} - cleaned_data.update(dict_ent) - elif k == 'financialsChart': - sub_dict = {} - for k2, v2, in v.items(): - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - sub_dict.update(sub_ent) - dict_ent = {financials_key: sub_dict} - cleaned_data.update(dict_ent) - else: - if k != 'maxAge': - dict_ent = {k: v} - cleaned_data.update(dict_ent) - return cleaned_data - - # Private method to clean summary and price reports - def _clean_reports(self, raw_data): - cleaned_dict = {} - if raw_data is None: - return None - for k, v in raw_data.items(): - if 'Time' in k: - formatted_utc_time = self._format_time(v) - dict_ent = {k: formatted_utc_time} - elif 'Date' in k: - try: - formatted_date = v['fmt'] - except (KeyError, TypeError): - formatted_date = '-' - dict_ent = {k: formatted_date} - elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): - dict_ent = {k: v} - # Python 2 and Unicode - elif sys.version_info < (3, 0) and isinstance(v, unicode): - dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - dict_ent = {k: numerical_val} - cleaned_dict.update(dict_ent) - return cleaned_dict - - # Private Static Method to ensure ticker is URL encoded - @staticmethod - def _encode_ticker(ticker_str): - encoded_ticker = ticker_str.replace('=', '%3D') - return encoded_ticker - - # Private method to get time interval code - def _build_historical_url(self, ticker, hist_oj): - url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ - '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ - hist_oj['interval'] - return url - - # Private Method to clean the dates of the newly returns historical stock data into readable format - def _clean_historical_data(self, hist_data, last_attempt=False): - data = {} - for k, v in hist_data.items(): - if k == 'eventsData': - event_obj = {} - if isinstance(v, list): - dict_ent = {k: event_obj} - else: - for type_key, type_obj in v.items(): - formatted_type_obj = {} - for date_key, date_obj in type_obj.items(): - formatted_date_key = self.format_date(int(date_key)) - cleaned_date = self.format_date(int(date_obj['date'])) - date_obj.update({'formatted_date': cleaned_date}) - formatted_type_obj.update({formatted_date_key: date_obj}) - event_obj.update({type_key: formatted_type_obj}) - dict_ent = {k: event_obj} - elif 'date' in k.lower(): - if v is not None: - cleaned_date = self.format_date(v) - dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} - else: - if last_attempt is False: - return None - else: - dict_ent = {k: {'formatted_date': None, 'date': v}} - elif isinstance(v, list): - sub_dict_list = [] - for sub_dict in v: - sub_dict['formatted_date'] = self.format_date(sub_dict['date']) - sub_dict_list.append(sub_dict) - dict_ent = {k: sub_dict_list} - else: - dict_ent = {k: v} - data.update(dict_ent) - return data - - # Private Static Method to build API url for GET Request - @staticmethod - def _build_api_url(hist_obj, up_ticker): - base_url = "https://query1.finance.yahoo.com/v8/finance/chart/" - api_url = base_url + up_ticker + '?symbol=' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' + \ - str(hist_obj['end']) + '&interval=' + hist_obj['interval'] - api_url += '&events=div|split|earn&lang=en-US®ion=US' - return api_url - - # Private Method to get financial data via API Call - def _get_api_data(self, api_url, tries=0): - urlopener = UrlOpener() - response = urlopener.open(api_url) - if response.getcode() == 200: - res_content = response.read() + response_content = response.read() + soup = BeautifulSoup(response_content, "html.parser") + re_script = soup.find("script", text=re.compile("root.App.main")) + if re_script is not None: + script = re_script.text + # bs4 4.9.0 changed so text from scripts is no longer considered text + if not script: + script = re_script.string + self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) response.close() - if sys.version_info < (3, 0): - return loads(res_content) - return loads(res_content.decode('utf-8')) - else: - if tries < 5: - time.sleep(random.randrange(10, 20)) - tries += 1 - return self._get_api_data(api_url, tries) - else: - return None - - # Private Method to clean API data - def _clean_api_data(self, api_url): - raw_data = self._get_api_data(api_url) - ret_obj = {} - ret_obj.update({'eventsData': []}) - if raw_data is None: - return ret_obj - results = raw_data['chart']['result'] - if results is None: - return ret_obj - for result in results: - tz_sub_dict = {} - ret_obj.update({'eventsData': result.get('events', {})}) - ret_obj.update({'firstTradeDate': result['meta'].get('firstTradeDate', 'NA')}) - ret_obj.update({'currency': result['meta'].get('currency', 'NA')}) - ret_obj.update({'instrumentType': result['meta'].get('instrumentType', 'NA')}) - tz_sub_dict.update({'gmtOffset': result['meta']['gmtoffset']}) - ret_obj.update({'timeZone': tz_sub_dict}) - timestamp_list = result['timestamp'] - high_price_list = result['indicators']['quote'][0]['high'] - low_price_list = result['indicators']['quote'][0]['low'] - open_price_list = result['indicators']['quote'][0]['open'] - close_price_list = result['indicators']['quote'][0]['close'] - volume_list = result['indicators']['quote'][0]['volume'] - adj_close_list = result['indicators']['adjclose'][0]['adjclose'] - i = 0 - prices_list = [] - for timestamp in timestamp_list: - price_dict = {} - price_dict.update({'date': timestamp}) - price_dict.update({'high': high_price_list[i]}) - price_dict.update({'low': low_price_list[i]}) - price_dict.update({'open': open_price_list[i]}) - price_dict.update({'close': close_price_list[i]}) - price_dict.update({'volume': volume_list[i]}) - price_dict.update({'adjclose': adj_close_list[i]}) - prices_list.append(price_dict) - i += 1 - ret_obj.update({'prices': prices_list}) - return ret_obj - - # Private Method to Handle Recursive API Request - def _recursive_api_request(self, hist_obj, up_ticker, i=0): - api_url = self._build_api_url(hist_obj, up_ticker) - re_data = self._clean_api_data(api_url) - cleaned_re_data = self._clean_historical_data(re_data) - if cleaned_re_data is not None: - return cleaned_re_data + break + else: + time.sleep(random.randrange(10, 20)) + if i == max_retry - 1: + # Raise a custom exception if we can't get the web page within max_retry attempts + raise ManagedException("Server replied with HTTP " + str(response.getcode()) + + " code while opening the url: " + str(url)) + data = self._cache[url] + data = self._decryptData(data) + if tech_type == '' and statement_type != 'history': + stores = data["QuoteSummaryStore"] + elif tech_type != '' and statement_type != 'history': + stores = data["QuoteSummaryStore"][tech_type] + else: + stores = data["HistoricalPriceStore"] + return stores + + def _decryptData(self,data): + #function taken from another package at https://github.com/ranaroussi/yfinance/pull/1253/commits/8e5f0984af347afda6be74b27a989422e49a975b + encrypted_stores = data['context']['dispatcher']['stores'] + _cs = data["_cs"] + _cr = data["_cr"] + + _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) + password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + + encrypted_stores = b64decode(encrypted_stores) + assert encrypted_stores[0:8] == b"Salted__" + salt = encrypted_stores[8:16] + encrypted_stores = encrypted_stores[16:] + + def EVPKDF( + password, + salt, + keySize=32, + ivSize=16, + iterations=1, + hashAlgorithm="md5", + ) -> tuple: + """OpenSSL EVP Key Derivation Function + Args: + password (Union[str, bytes, bytearray]): Password to generate key from. + salt (Union[bytes, bytearray]): Salt to use. + keySize (int, optional): Output key length in bytes. Defaults to 32. + ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. + iterations (int, optional): Number of iterations to perform. Defaults to 1. + hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. + Returns: + key, iv: Derived key and Initialization Vector (IV) bytes. + Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 + OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 + """ + + assert iterations > 0, "Iterations can not be less than 1." + + if isinstance(password, str): + password = password.encode("utf-8") + + final_length = keySize + ivSize + key_iv = b"" + block = None + + while len(key_iv) < final_length: + hasher = hashlib.new(hashAlgorithm) + if block: + hasher.update(block) + hasher.update(password) + hasher.update(salt) + block = hasher.digest() + for _ in range(1, iterations): + block = hashlib.new(hashAlgorithm, block).digest() + key_iv += block + + key, iv = key_iv[:keySize], key_iv[keySize:final_length] + return key, iv + + key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + + cipher = AES.new(key, AES.MODE_CBC, iv=iv) + plaintext = cipher.decrypt(encrypted_stores) + plaintext = unpad(plaintext, 16, style="pkcs7") + decoded_stores = json.loads(plaintext) + + return decoded_stores + + # Private static method to determine if a numerical value is in the data object being cleaned + @staticmethod + def _determine_numeric_value(value_dict): + if 'raw' in value_dict.keys(): + numerical_val = value_dict['raw'] + else: + numerical_val = None + return numerical_val + + # Private method to format date serial string to readable format and vice versa + def _format_time(self, in_time): + form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') + utc_dt = self._convert_to_utc(form_date_time) + return utc_dt + + # Private method to return the a sub dictionary entry for the earning report cleaning + def _get_cleaned_sub_dict_ent(self, key, val_list): + sub_list = [] + for rec in val_list: + sub_sub_dict = {} + for k, v in rec.items(): + if k == 'date': + sub_sub_dict_ent = {k: v} else: - if i < 3: - i += 1 - return self._recursive_api_request(hist_obj, up_ticker, i) - else: - return self._clean_historical_data(re_data, True) - - # Private Method to take scrapped data and build a data dictionary with - def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): - YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ - up_ticker - if tech_type == '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data, 'dataType': report_name} - elif tech_type != '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data} - else: - YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) - try: - cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) - except KeyError: - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - cleaned_re_data = self._clean_historical_data(re_data) - except KeyError: - cleaned_re_data = None - dict_ent = {up_ticker: cleaned_re_data} - return dict_ent - - # Private method to return the stmt_id for the reformat_process - def _get_stmt_id(self, statement_type, raw_data): - stmt_id = '' - i = 0 - for key in raw_data.keys(): - if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: - stmt_id = key - i += 1 - if i != 1: - return None - return stmt_id - - # Private Method for the Reformat Process - def _reformat_stmt_data_process(self, raw_data, statement_type): - final_data_list = [] - if raw_data is not None: - stmt_id = self._get_stmt_id(statement_type, raw_data) - if stmt_id is None: - return final_data_list - hashed_data_list = raw_data[stmt_id] - for data_item in hashed_data_list: - data_date = '' - sub_data_dict = {} - for k, v in data_item.items(): - if k == 'endDate': - data_date = v['fmt'] - elif k != 'maxAge': - numerical_val = self._determine_numeric_value(v) - sub_dict_item = {k: numerical_val} - sub_data_dict.update(sub_dict_item) - dict_item = {data_date: sub_data_dict} - final_data_list.append(dict_item) - return final_data_list - else: - return raw_data - - # Private Method to return subdict entry for the statement reformat process - def _get_sub_dict_ent(self, ticker, raw_data, statement_type): - form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) - return {ticker: form_data_list} - - # Public method to get time interval code - def get_time_code(self, time_interval): - interval_code = self._INTERVAL_DICT[time_interval.lower()] - return interval_code - - # Public Method to get stock data - def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): - data = {} - if isinstance(self.ticker, str): - dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - else: - for tick in self.ticker: - try: - dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - except ManagedException: - print("Warning! Ticker: " + str(tick) + " error - " + str(ManagedException)) - print("The process is still running...") - continue - return data - - # Public Method to get technical stock data - def get_stock_tech_data(self, tech_type): - if tech_type == 'defaultKeyStatistics': - return self.get_stock_data(statement_type='keystats', tech_type=tech_type) - else: - return self.get_stock_data(tech_type=tech_type) - - # Public Method to get reformatted statement data - def get_reformatted_stmt_data(self, raw_data, statement_type): - data_dict = {} + numerical_val = self._determine_numeric_value(v) + sub_sub_dict_ent = {k: numerical_val} + sub_sub_dict.update(sub_sub_dict_ent) + sub_list.append(sub_sub_dict) + sub_ent = {key: sub_list} + return sub_ent + + # Private method to process raw earnings data and clean + def _clean_earnings_data(self, raw_data): + cleaned_data = {} + earnings_key = 'earningsData' + financials_key = 'financialsData' + for k, v in raw_data.items(): + if k == 'earningsChart': sub_dict = {} - data_type = raw_data['dataType'] - if isinstance(self.ticker, str): - sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {data_type: sub_dict} - data_dict.update(dict_ent) + for k2, v2 in v.items(): + if k2 == 'quarterly': + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + elif k2 == 'currentQuarterEstimate': + numerical_val = self._determine_numeric_value(v2) + sub_ent = {k2: numerical_val} + else: + sub_ent = {k2: v2} + sub_dict.update(sub_ent) + dict_ent = {earnings_key: sub_dict} + cleaned_data.update(dict_ent) + elif k == 'financialsChart': + sub_dict = {} + for k2, v2, in v.items(): + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + sub_dict.update(sub_ent) + dict_ent = {financials_key: sub_dict} + cleaned_data.update(dict_ent) + else: + if k != 'maxAge': + dict_ent = {k: v} + cleaned_data.update(dict_ent) + return cleaned_data + + # Private method to clean summary and price reports + def _clean_reports(self, raw_data): + cleaned_dict = {} + if raw_data is None: + return None + for k, v in raw_data.items(): + if 'Time' in k: + formatted_utc_time = self._format_time(v) + dict_ent = {k: formatted_utc_time} + elif 'Date' in k: + try: + formatted_date = v['fmt'] + except (KeyError, TypeError): + formatted_date = '-' + dict_ent = {k: formatted_date} + elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): + dict_ent = {k: v} + # Python 2 and Unicode + elif sys.version_info < (3, 0) and isinstance(v, unicode): + dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + dict_ent = {k: numerical_val} + cleaned_dict.update(dict_ent) + return cleaned_dict + + # Private Static Method to ensure ticker is URL encoded + @staticmethod + def _encode_ticker(ticker_str): + encoded_ticker = ticker_str.replace('=', '%3D') + return encoded_ticker + + # Private method to get time interval code + def _build_historical_url(self, ticker, hist_oj): + url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ + '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ + hist_oj['interval'] + return url + + # Private Method to clean the dates of the newly returns historical stock data into readable format + def _clean_historical_data(self, hist_data, last_attempt=False): + data = {} + for k, v in hist_data.items(): + if k == 'eventsData': + event_obj = {} + if isinstance(v, list): + dict_ent = {k: event_obj} else: - for tick in self.ticker: - sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {data_type: sub_dict} - data_dict.update(dict_ent) - return data_dict - - # Public method to get cleaned summary and price report data - def get_clean_data(self, raw_report_data, report_type): - cleaned_data_dict = {} - if isinstance(self.ticker, str): - if report_type == 'earnings': - try: - cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) - except: - cleaned_data = None - else: - try: - cleaned_data = self._clean_reports(raw_report_data[self.ticker]) - except: - cleaned_data = None - cleaned_data_dict.update({self.ticker: cleaned_data}) + for type_key, type_obj in v.items(): + formatted_type_obj = {} + for date_key, date_obj in type_obj.items(): + formatted_date_key = self.format_date(int(date_key)) + cleaned_date = self.format_date(int(date_obj['date'])) + date_obj.update({'formatted_date': cleaned_date}) + formatted_type_obj.update({formatted_date_key: date_obj}) + event_obj.update({type_key: formatted_type_obj}) + dict_ent = {k: event_obj} + elif 'date' in k.lower(): + if v is not None: + cleaned_date = self.format_date(v) + dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} else: - for tick in self.ticker: - if report_type == 'earnings': - try: - cleaned_data = self._clean_earnings_data(raw_report_data[tick]) - except: - cleaned_data = None - else: - try: - cleaned_data = self._clean_reports(raw_report_data[tick]) - except: - cleaned_data = None - cleaned_data_dict.update({tick: cleaned_data}) - return cleaned_data_dict - - # Private method to handle dividend data requests - def _handle_api_dividend_request(self, cur_ticker, start, end, interval): - re_dividends = [] - test_url = 'https://query1.finance.yahoo.com/v8/finance/chart/' + cur_ticker + \ - '?period1=' + str(start) + '&period2=' + str(end) + '&interval=' + interval + '&events=div' - div_dict = self._get_api_data(test_url)['chart']['result'][0]['events']['dividends'] - for div_time_key, div_obj in div_dict.items(): - dividend_obj = { - 'date': div_obj['date'], - 'formatted_date': self.format_date(int(div_obj['date'])), - 'amount': div_obj.get('amount', None) - } - re_dividends.append(dividend_obj) - return sorted(re_dividends, key=lambda div: div['date']) - - # Public method to get daily dividend data - def get_stock_dividend_data(self, start, end, interval): - interval_code = self.get_time_code(interval) - if isinstance(self.ticker, str): - try: - return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)} - except: - return {self.ticker: None} + if last_attempt is False: + return None + else: + dict_ent = {k: {'formatted_date': None, 'date': v}} + elif isinstance(v, list): + sub_dict_list = [] + for sub_dict in v: + sub_dict['formatted_date'] = self.format_date(sub_dict['date']) + sub_dict_list.append(sub_dict) + dict_ent = {k: sub_dict_list} + else: + dict_ent = {k: v} + data.update(dict_ent) + return data + + # Private Static Method to build API url for GET Request + @staticmethod + def _build_api_url(hist_obj, up_ticker): + base_url = "https://query1.finance.yahoo.com/v8/finance/chart/" + api_url = base_url + up_ticker + '?symbol=' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' + \ + str(hist_obj['end']) + '&interval=' + hist_obj['interval'] + api_url += '&events=div|split|earn&lang=en-US®ion=US' + return api_url + + # Private Method to get financial data via API Call + def _get_api_data(self, api_url, tries=0): + urlopener = UrlOpener() + response = urlopener.open(api_url) + if response.getcode() == 200: + res_content = response.read() + response.close() + if sys.version_info < (3, 0): + return loads(res_content) + return loads(res_content.decode('utf-8')) + else: + if tries < 5: + time.sleep(random.randrange(10, 20)) + tries += 1 + return self._get_api_data(api_url, tries) + else: + return None + + # Private Method to clean API data + def _clean_api_data(self, api_url): + raw_data = self._get_api_data(api_url) + ret_obj = {} + ret_obj.update({'eventsData': []}) + if raw_data is None: + return ret_obj + results = raw_data['chart']['result'] + if results is None: + return ret_obj + for result in results: + tz_sub_dict = {} + ret_obj.update({'eventsData': result.get('events', {})}) + ret_obj.update({'firstTradeDate': result['meta'].get('firstTradeDate', 'NA')}) + ret_obj.update({'currency': result['meta'].get('currency', 'NA')}) + ret_obj.update({'instrumentType': result['meta'].get('instrumentType', 'NA')}) + tz_sub_dict.update({'gmtOffset': result['meta']['gmtoffset']}) + ret_obj.update({'timeZone': tz_sub_dict}) + timestamp_list = result['timestamp'] + high_price_list = result['indicators']['quote'][0]['high'] + low_price_list = result['indicators']['quote'][0]['low'] + open_price_list = result['indicators']['quote'][0]['open'] + close_price_list = result['indicators']['quote'][0]['close'] + volume_list = result['indicators']['quote'][0]['volume'] + adj_close_list = result['indicators']['adjclose'][0]['adjclose'] + i = 0 + prices_list = [] + for timestamp in timestamp_list: + price_dict = {} + price_dict.update({'date': timestamp}) + price_dict.update({'high': high_price_list[i]}) + price_dict.update({'low': low_price_list[i]}) + price_dict.update({'open': open_price_list[i]}) + price_dict.update({'close': close_price_list[i]}) + price_dict.update({'volume': volume_list[i]}) + price_dict.update({'adjclose': adj_close_list[i]}) + prices_list.append(price_dict) + i += 1 + ret_obj.update({'prices': prices_list}) + return ret_obj + + # Private Method to Handle Recursive API Request + def _recursive_api_request(self, hist_obj, up_ticker, i=0): + api_url = self._build_api_url(hist_obj, up_ticker) + re_data = self._clean_api_data(api_url) + cleaned_re_data = self._clean_historical_data(re_data) + if cleaned_re_data is not None: + return cleaned_re_data + else: + if i < 3: + i += 1 + return self._recursive_api_request(hist_obj, up_ticker, i) + else: + return self._clean_historical_data(re_data, True) + + # Private Method to take scrapped data and build a data dictionary with + def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): + YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' + \ + up_ticker + if tech_type == '' and statement_type != 'history': + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} + except KeyError: + re_data = None + dict_ent = {up_ticker: re_data, 'dataType': report_name} + elif tech_type != '' and statement_type != 'history': + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + except KeyError: + re_data = None + dict_ent = {up_ticker: re_data} + else: + YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) + try: + cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) + except KeyError: + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + cleaned_re_data = self._clean_historical_data(re_data) + except KeyError: + cleaned_re_data = None + dict_ent = {up_ticker: cleaned_re_data} + return dict_ent + + # Private method to return the stmt_id for the reformat_process + def _get_stmt_id(self, statement_type, raw_data): + stmt_id = '' + i = 0 + for key in raw_data.keys(): + if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: + stmt_id = key + i += 1 + if i != 1: + return None + return stmt_id + + # Private Method for the Reformat Process + def _reformat_stmt_data_process(self, raw_data, statement_type): + final_data_list = [] + if raw_data is not None: + stmt_id = self._get_stmt_id(statement_type, raw_data) + if stmt_id is None: + return final_data_list + hashed_data_list = raw_data[stmt_id] + for data_item in hashed_data_list: + data_date = '' + sub_data_dict = {} + for k, v in data_item.items(): + if k == 'endDate': + data_date = v['fmt'] + elif k != 'maxAge': + numerical_val = self._determine_numeric_value(v) + sub_dict_item = {k: numerical_val} + sub_data_dict.update(sub_dict_item) + dict_item = {data_date: sub_data_dict} + final_data_list.append(dict_item) + return final_data_list + else: + return raw_data + + # Private Method to return subdict entry for the statement reformat process + def _get_sub_dict_ent(self, ticker, raw_data, statement_type): + form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) + return {ticker: form_data_list} + + # Public method to get time interval code + def get_time_code(self, time_interval): + interval_code = self._INTERVAL_DICT[time_interval.lower()] + return interval_code + + # Public Method to get stock data + def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): + data = {} + if isinstance(self.ticker, str): + dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + else: + for tick in self.ticker: + try: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + except ManagedException: + print("Warning! Ticker: " + str(tick) + " error - " + str(ManagedException)) + print("The process is still running...") + continue + return data + + # Public Method to get technical stock data + def get_stock_tech_data(self, tech_type): + if tech_type == 'defaultKeyStatistics': + return self.get_stock_data(statement_type='keystats', tech_type=tech_type) + else: + return self.get_stock_data(tech_type=tech_type) + + # Public Method to get reformatted statement data + def get_reformatted_stmt_data(self, raw_data, statement_type): + data_dict = {} + sub_dict = {} + data_type = raw_data['dataType'] + if isinstance(self.ticker, str): + sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {data_type: sub_dict} + data_dict.update(dict_ent) + else: + for tick in self.ticker: + sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {data_type: sub_dict} + data_dict.update(dict_ent) + return data_dict + + # Public method to get cleaned summary and price report data + def get_clean_data(self, raw_report_data, report_type): + cleaned_data_dict = {} + if isinstance(self.ticker, str): + if report_type == 'earnings': + try: + cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) + except: + cleaned_data = None + else: + try: + cleaned_data = self._clean_reports(raw_report_data[self.ticker]) + except: + cleaned_data = None + cleaned_data_dict.update({self.ticker: cleaned_data}) + else: + for tick in self.ticker: + if report_type == 'earnings': + try: + cleaned_data = self._clean_earnings_data(raw_report_data[tick]) + except: + cleaned_data = None else: - re_data = {} - for tick in self.ticker: - try: - div_data = self._handle_api_dividend_request(tick, start, end, interval_code) - re_data.update({tick: div_data}) - except: - re_data.update({tick: None}) - return re_data + try: + cleaned_data = self._clean_reports(raw_report_data[tick]) + except: + cleaned_data = None + cleaned_data_dict.update({tick: cleaned_data}) + return cleaned_data_dict + + # Private method to handle dividend data requests + def _handle_api_dividend_request(self, cur_ticker, start, end, interval): + re_dividends = [] + test_url = 'https://query1.finance.yahoo.com/v8/finance/chart/' + cur_ticker + \ + '?period1=' + str(start) + '&period2=' + str(end) + '&interval=' + interval + '&events=div' + div_dict = self._get_api_data(test_url)['chart']['result'][0]['events']['dividends'] + for div_time_key, div_obj in div_dict.items(): + dividend_obj = { + 'date': div_obj['date'], + 'formatted_date': self.format_date(int(div_obj['date'])), + 'amount': div_obj.get('amount', None) + } + re_dividends.append(dividend_obj) + return sorted(re_dividends, key=lambda div: div['date']) + + # Public method to get daily dividend data + def get_stock_dividend_data(self, start, end, interval): + interval_code = self.get_time_code(interval) + if isinstance(self.ticker, str): + try: + return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)} + except: + return {self.ticker: None} + else: + re_data = {} + for tick in self.ticker: + try: + div_data = self._handle_api_dividend_request(tick, start, end, interval_code) + re_data.update({tick: div_data}) + except: + re_data.update({tick: None}) + return re_data # Class containing methods to create stock data extracts class YahooFinancials(YahooFinanceETL): - # Private method that handles financial statement extraction - def _run_financial_stmt(self, statement_type, report_num, reformat): - report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] - if reformat: - raw_data = self.get_stock_data(statement_type, report_name=report_name) - data = self.get_reformatted_stmt_data(raw_data, statement_type) - else: - data = self.get_stock_data(statement_type, report_name=report_name) - return data - - # Public Method for the user to get financial statement data - def get_financial_stmts(self, frequency, statement_type, reformat=True): - report_num = self.get_report_type(frequency) - if isinstance(statement_type, str): - data = self._run_financial_stmt(statement_type, report_num, reformat) - else: - data = {} - for stmt_type in statement_type: - re_data = self._run_financial_stmt(stmt_type, report_num, reformat) - data.update(re_data) - return data - - # Public Method for the user to get stock price data - def get_stock_price_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('price'), 'price') - else: - return self.get_stock_tech_data('price') - - # Public Method for the user to return key-statistics data - def get_key_statistics_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('defaultKeyStatistics'), 'defaultKeyStatistics') - else: - return self.get_stock_tech_data('defaultKeyStatistics') - - # Public Method for the user to get company profile data - def get_stock_profile_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') - else: - return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') - - - # Public Method for the user to get stock earnings data - def get_stock_earnings_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') - else: - return self.get_stock_tech_data('earnings') - - # Public Method for the user to get stock summary data - def get_summary_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') + # Private method that handles financial statement extraction + def _run_financial_stmt(self, statement_type, report_num, reformat): + report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] + if reformat: + raw_data = self.get_stock_data(statement_type, report_name=report_name) + data = self.get_reformatted_stmt_data(raw_data, statement_type) + else: + data = self.get_stock_data(statement_type, report_name=report_name) + return data + + # Public Method for the user to get financial statement data + def get_financial_stmts(self, frequency, statement_type, reformat=True): + report_num = self.get_report_type(frequency) + if isinstance(statement_type, str): + data = self._run_financial_stmt(statement_type, report_num, reformat) + else: + data = {} + for stmt_type in statement_type: + re_data = self._run_financial_stmt(stmt_type, report_num, reformat) + data.update(re_data) + return data + + # Public Method for the user to get stock price data + def get_stock_price_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('price'), 'price') + else: + return self.get_stock_tech_data('price') + + # Public Method for the user to return key-statistics data + def get_key_statistics_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('defaultKeyStatistics'), 'defaultKeyStatistics') + else: + return self.get_stock_tech_data('defaultKeyStatistics') + + # Public Method for the user to get company profile data + def get_stock_profile_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') + else: + return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') + + # Public Method for the user to get stock earnings data + def get_stock_earnings_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') + else: + return self.get_stock_tech_data('earnings') + + # Public Method for the user to get stock summary data + def get_summary_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') + else: + return self.get_stock_tech_data('summaryDetail') + + # Public Method for the user to get the yahoo summary url + def get_stock_summary_url(self): + if isinstance(self.ticker, str): + return self._BASE_YAHOO_URL + self.ticker + return {t: self._BASE_YAHOO_URL + t for t in self.ticker} + + # Public Method for the user to get stock quote data + def get_stock_quote_type_data(self): + return self.get_stock_tech_data('quoteType') + + # Public Method for user to get historical price data with + def get_historical_price_data(self, start_date, end_date, time_interval): + interval_code = self.get_time_code(time_interval) + start = self.format_date(start_date) + end = self.format_date(end_date) + hist_obj = {'start': start, 'end': end, 'interval': interval_code} + return self.get_stock_data('history', hist_obj=hist_obj) + + # Private Method for Functions needing stock_price_data + def _stock_price_data(self, data_field): + if isinstance(self.ticker, str): + if self.get_stock_price_data()[self.ticker] is None: + return None + return self.get_stock_price_data()[self.ticker].get(data_field, None) + else: + ret_obj = {} + for tick in self.ticker: + if self.get_stock_price_data()[tick] is None: + ret_obj.update({tick: None}) else: - return self.get_stock_tech_data('summaryDetail') - - # Public Method for the user to get the yahoo summary url - def get_stock_summary_url(self): - if isinstance(self.ticker, str): - return self._BASE_YAHOO_URL + self.ticker - return {t: self._BASE_YAHOO_URL + t for t in self.ticker} - - # Public Method for the user to get stock quote data - def get_stock_quote_type_data(self): - return self.get_stock_tech_data('quoteType') - - # Public Method for user to get historical price data with - def get_historical_price_data(self, start_date, end_date, time_interval): - interval_code = self.get_time_code(time_interval) - start = self.format_date(start_date) - end = self.format_date(end_date) - hist_obj = {'start': start, 'end': end, 'interval': interval_code} - return self.get_stock_data('history', hist_obj=hist_obj) - - # Private Method for Functions needing stock_price_data - def _stock_price_data(self, data_field): - if isinstance(self.ticker, str): - if self.get_stock_price_data()[self.ticker] is None: - return None - return self.get_stock_price_data()[self.ticker].get(data_field, None) + ret_obj.update({tick: self.get_stock_price_data()[tick].get(data_field, None)}) + return ret_obj + + # Private Method for Functions needing stock_price_data + def _stock_summary_data(self, data_field): + if isinstance(self.ticker, str): + if self.get_summary_data()[self.ticker] is None: + return None + return self.get_summary_data()[self.ticker].get(data_field, None) + else: + ret_obj = {} + for tick in self.ticker: + if self.get_summary_data()[tick] is None: + ret_obj.update({tick: None}) else: - ret_obj = {} - for tick in self.ticker: - if self.get_stock_price_data()[tick] is None: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: self.get_stock_price_data()[tick].get(data_field, None)}) - return ret_obj - - # Private Method for Functions needing stock_price_data - def _stock_summary_data(self, data_field): - if isinstance(self.ticker, str): - if self.get_summary_data()[self.ticker] is None: - return None - return self.get_summary_data()[self.ticker].get(data_field, None) + ret_obj.update({tick: self.get_summary_data()[tick].get(data_field, None)}) + return ret_obj + + # Private Method for Functions needing financial statement data + def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): + re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] + if isinstance(self.ticker, str): + try: + date_key = re_data[self.ticker][0].keys()[0] + except (IndexError, AttributeError, TypeError): + date_key = list(re_data[self.ticker][0])[0] + data = re_data[self.ticker][0][date_key][field_name] + else: + data = {} + for tick in self.ticker: + try: + date_key = re_data[tick][0].keys()[0] + except: + try: + date_key = list(re_data[tick][0].keys())[0] + except: + date_key = None + if date_key is not None: + sub_data = re_data[tick][0][date_key][field_name] + data.update({tick: sub_data}) else: - ret_obj = {} - for tick in self.ticker: - if self.get_summary_data()[tick] is None: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: self.get_summary_data()[tick].get(data_field, None)}) - return ret_obj - - # Private Method for Functions needing financial statement data - def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): - re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] - if isinstance(self.ticker, str): - try: - date_key = re_data[self.ticker][0].keys()[0] - except (IndexError, AttributeError, TypeError): - date_key = list(re_data[self.ticker][0])[0] - data = re_data[self.ticker][0][date_key][field_name] - else: - data = {} - for tick in self.ticker: - try: - date_key = re_data[tick][0].keys()[0] - except: - try: - date_key = list(re_data[tick][0].keys())[0] - except: - date_key = None - if date_key is not None: - sub_data = re_data[tick][0][date_key][field_name] - data.update({tick: sub_data}) - else: - data.update({tick: None}) - return data + data.update({tick: None}) + return data - # Public method to get daily dividend data - def get_daily_dividend_data(self, start_date, end_date): - start = self.format_date(start_date) - end = self.format_date(end_date) - return self.get_stock_dividend_data(start, end, 'daily') + # Public method to get daily dividend data + def get_daily_dividend_data(self, start_date, end_date): + start = self.format_date(start_date) + end = self.format_date(end_date) + return self.get_stock_dividend_data(start, end, 'daily') - # Public Price Data Methods - def get_current_price(self): - return self._stock_price_data('regularMarketPrice') + # Public Price Data Methods + def get_current_price(self): + return self._stock_price_data('regularMarketPrice') - def get_current_change(self): - return self._stock_price_data('regularMarketChange') + def get_current_change(self): + return self._stock_price_data('regularMarketChange') - def get_current_percent_change(self): - return self._stock_price_data('regularMarketChangePercent') + def get_current_percent_change(self): + return self._stock_price_data('regularMarketChangePercent') - def get_current_volume(self): - return self._stock_price_data('regularMarketVolume') + def get_current_volume(self): + return self._stock_price_data('regularMarketVolume') - def get_prev_close_price(self): - return self._stock_price_data('regularMarketPreviousClose') + def get_prev_close_price(self): + return self._stock_price_data('regularMarketPreviousClose') - def get_open_price(self): - return self._stock_price_data('regularMarketOpen') + def get_open_price(self): + return self._stock_price_data('regularMarketOpen') - def get_ten_day_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume10Day') + def get_ten_day_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume10Day') - def get_three_month_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume3Month') + def get_three_month_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume3Month') - def get_stock_exchange(self): - return self._stock_price_data('exchangeName') + def get_stock_exchange(self): + return self._stock_price_data('exchangeName') - def get_market_cap(self): - return self._stock_price_data('marketCap') + def get_market_cap(self): + return self._stock_price_data('marketCap') - def get_daily_low(self): - return self._stock_price_data('regularMarketDayLow') + def get_daily_low(self): + return self._stock_price_data('regularMarketDayLow') - def get_daily_high(self): - return self._stock_price_data('regularMarketDayHigh') + def get_daily_high(self): + return self._stock_price_data('regularMarketDayHigh') - def get_currency(self): - return self._stock_price_data('currency') + def get_currency(self): + return self._stock_price_data('currency') - # Public Summary Data Methods - def get_yearly_high(self): - return self._stock_summary_data('fiftyTwoWeekHigh') + # Public Summary Data Methods + def get_yearly_high(self): + return self._stock_summary_data('fiftyTwoWeekHigh') - def get_yearly_low(self): - return self._stock_summary_data('fiftyTwoWeekLow') + def get_yearly_low(self): + return self._stock_summary_data('fiftyTwoWeekLow') - def get_dividend_yield(self): - return self._stock_summary_data('dividendYield') + def get_dividend_yield(self): + return self._stock_summary_data('dividendYield') - def get_annual_avg_div_yield(self): - return self._stock_summary_data('trailingAnnualDividendYield') + def get_annual_avg_div_yield(self): + return self._stock_summary_data('trailingAnnualDividendYield') - def get_five_yr_avg_div_yield(self): - return self._stock_summary_data('fiveYearAvgDividendYield') + def get_five_yr_avg_div_yield(self): + return self._stock_summary_data('fiveYearAvgDividendYield') - def get_dividend_rate(self): - return self._stock_summary_data('dividendRate') + def get_dividend_rate(self): + return self._stock_summary_data('dividendRate') - def get_annual_avg_div_rate(self): - return self._stock_summary_data('trailingAnnualDividendRate') + def get_annual_avg_div_rate(self): + return self._stock_summary_data('trailingAnnualDividendRate') - def get_50day_moving_avg(self): - return self._stock_summary_data('fiftyDayAverage') + def get_50day_moving_avg(self): + return self._stock_summary_data('fiftyDayAverage') - def get_200day_moving_avg(self): - return self._stock_summary_data('twoHundredDayAverage') + def get_200day_moving_avg(self): + return self._stock_summary_data('twoHundredDayAverage') - def get_beta(self): - return self._stock_summary_data('beta') + def get_beta(self): + return self._stock_summary_data('beta') - def get_payout_ratio(self): - return self._stock_summary_data('payoutRatio') + def get_payout_ratio(self): + return self._stock_summary_data('payoutRatio') - def get_pe_ratio(self): - return self._stock_summary_data('trailingPE') + def get_pe_ratio(self): + return self._stock_summary_data('trailingPE') - def get_price_to_sales(self): - return self._stock_summary_data('priceToSalesTrailing12Months') + def get_price_to_sales(self): + return self._stock_summary_data('priceToSalesTrailing12Months') - def get_exdividend_date(self): - return self._stock_summary_data('exDividendDate') + def get_exdividend_date(self): + return self._stock_summary_data('exDividendDate') - # Financial Statement Data Methods - def get_book_value(self): - return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', - 'totalStockholderEquity', 'quarterly') + # Financial Statement Data Methods + def get_book_value(self): + return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', + 'totalStockholderEquity', 'quarterly') - def get_ebit(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') + def get_ebit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') - def get_net_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') + def get_net_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') - def get_interest_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') + def get_interest_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') - def get_operating_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') + def get_operating_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') - def get_total_operating_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') + def get_total_operating_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') - def get_total_revenue(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'totalRevenue', 'annual') + def get_total_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalRevenue', 'annual') - def get_cost_of_revenue(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'costOfRevenue', 'annual') + def get_cost_of_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'costOfRevenue', 'annual') - def get_income_before_tax(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeBeforeTax', 'annual') + def get_income_before_tax(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeBeforeTax', 'annual') - def get_income_tax_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeTaxExpense', 'annual') + def get_income_tax_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeTaxExpense', 'annual') - def get_gross_profit(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') + def get_gross_profit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') - def get_net_income_from_continuing_ops(self): - return self._financial_statement_data('income', 'incomeStatementHistory', - 'netIncomeFromContinuingOps', 'annual') + def get_net_income_from_continuing_ops(self): + return self._financial_statement_data('income', 'incomeStatementHistory', + 'netIncomeFromContinuingOps', 'annual') - def get_research_and_development(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') + def get_research_and_development(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') - # Calculated Financial Methods - def get_earnings_per_share(self): - price_data = self.get_current_price() - pe_ratio = self.get_pe_ratio() - if isinstance(self.ticker, str): - if price_data is not None and pe_ratio is not None: - return price_data / pe_ratio - else: - return None + # Calculated Financial Methods + def get_earnings_per_share(self): + price_data = self.get_current_price() + pe_ratio = self.get_pe_ratio() + if isinstance(self.ticker, str): + if price_data is not None and pe_ratio is not None: + return price_data / pe_ratio + else: + return None + else: + ret_obj = {} + for tick in self.ticker: + if price_data[tick] is not None and pe_ratio[tick] is not None: + ret_obj.update({tick: price_data[tick] / pe_ratio[tick]}) else: - ret_obj = {} - for tick in self.ticker: - if price_data[tick] is not None and pe_ratio[tick] is not None: - ret_obj.update({tick: price_data[tick] / pe_ratio[tick]}) - else: - ret_obj.update({tick: None}) - return ret_obj - - def get_num_shares_outstanding(self, price_type='current'): - today_low = self._stock_summary_data('dayHigh') - today_high = self._stock_summary_data('dayLow') - cur_market_cap = self._stock_summary_data('marketCap') - if isinstance(self.ticker, str): - if cur_market_cap is not None: - if price_type == 'current': - current = self.get_current_price() - if current is not None: - today_average = current - else: - return None - else: - if today_high is not None and today_low is not None: - today_average = (today_high + today_low) / 2 - else: - return None - return cur_market_cap / today_average + ret_obj.update({tick: None}) + return ret_obj + + def get_num_shares_outstanding(self, price_type='current'): + today_low = self._stock_summary_data('dayHigh') + today_high = self._stock_summary_data('dayLow') + cur_market_cap = self._stock_summary_data('marketCap') + if isinstance(self.ticker, str): + if cur_market_cap is not None: + if price_type == 'current': + current = self.get_current_price() + if current is not None: + today_average = current + else: + return None + else: + if today_high is not None and today_low is not None: + today_average = (today_high + today_low) / 2 + else: + return None + return cur_market_cap / today_average + else: + return None + else: + ret_obj = {} + for tick in self.ticker: + if cur_market_cap[tick] is not None: + if price_type == 'current': + current = self.get_current_price() + if current[tick] is not None: + ret_obj.update({tick: cur_market_cap[tick] / current[tick]}) + else: + ret_obj.update({tick: None}) + else: + if today_low[tick] is not None and today_high[tick] is not None: + today_average = (today_high[tick] + today_low[tick]) / 2 + ret_obj.update({tick: cur_market_cap[tick] / today_average}) else: - return None + ret_obj.update({tick: None}) else: - ret_obj = {} - for tick in self.ticker: - if cur_market_cap[tick] is not None: - if price_type == 'current': - current = self.get_current_price() - if current[tick] is not None: - ret_obj.update({tick: cur_market_cap[tick] / current[tick]}) - else: - ret_obj.update({tick: None}) - else: - if today_low[tick] is not None and today_high[tick] is not None: - today_average = (today_high[tick] + today_low[tick]) / 2 - ret_obj.update({tick: cur_market_cap[tick] / today_average}) - else: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: None}) - return ret_obj + ret_obj.update({tick: None}) + return ret_obj From 5ccfc562cbff750e717f1ac65c3b023904469fcc Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:26:25 -0600 Subject: [PATCH 074/108] v1.7 removed travis-ci and adding github workflows --- .github/workflows/test.yml | 26 ++++++++++++++++++++++++++ .gitignore | 7 +++++++ .travis.yml | 25 ------------------------- CHANGES | 2 ++ LICENSE | 2 +- README.rst | 13 +++++-------- setup.py | 17 ++++++++--------- test/test_yahoofinancials.py | 17 +++++++---------- yahoofinancials/__init__.py | 8 ++++---- 9 files changed, 60 insertions(+), 57 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a680be6 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: Python package + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install pytest + python setup.py install + - name: Test with pytest + run: | + pytest \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7e77bd2..40f90b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ *.pyc yahoofinancials/__pycache__ +/yahoofinancials.egg-info/ +.idea/ +**/.DS_Store +/venv/ +/.pytest_cache/ +/dist/ +/build/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b33f960..0000000 --- a/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -language: python - -python: - - "2.7" - - "pypy" - - "pypy3" - - "3.4" - - "3.5" - - "3.6" -matrix: - include: - - python: 3.7 - dist: xenial - sudo: true - -install: - - if [ "$TRAVIS_PYTHON_VERSION" == "2.6" ]; then pip --quiet install argparse unittest2; fi - - python setup.py install - -script: - - nosetests --with-coverage --cover-package=yahoofinancials - -after_success: - - pip install --quiet coveralls - - coveralls diff --git a/CHANGES b/CHANGES index 88413ff..ccf2947 100644 --- a/CHANGES +++ b/CHANGES @@ -27,3 +27,5 @@ 1.5 01/27/2019 -- Added get_daily_dividend_data() method as request in Issue #20. 1.5 01/27/2019 -- Added test_yf_dividend_price() unit testing method. 1.6 10/18/2020 -- Merged in two pull requests with bug fixes from sylvandb. +1.7 01/01/2023 -- Merged in pull request with fixes from sedwards2000. +1.7 01/01/2023 -- Support for python 2.7 dropped. diff --git a/LICENSE b/LICENSE index e658826..733e1ec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2020 Connor Sanders +Copyright (c) 2023 Connor Sanders Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index 3b73f68..39af956 100644 --- a/README.rst +++ b/README.rst @@ -4,12 +4,9 @@ yahoofinancials A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, ETF, and US Treasury financial data from Yahoo Finance. -.. image:: https://travis-ci.org/JECSand/yahoofinancials.svg?branch=master - :target: https://travis-ci.org/JECSand/yahoofinancials +Current Version: v1.7 -Current Version: v1.6 - -Version Released: 10/18/2020 +Version Released: 01/01/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -21,8 +18,8 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- -- yahoofinancials runs on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7. -- The package depends on beautifulsoup4 and pytz to work. +- yahoofinancials runs on Python 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10. +- This package depends on beautifulsoup4, pytz, and pycryptodome to work. 1. Installation using pip: @@ -98,7 +95,7 @@ Featured Methods - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -Methods Added in V1.5 +Methods Added in v1.5 ^^^^^^^^^^^^^^^^^^^^^^^ - get_daily_dividend_data(start_date, end_date) diff --git a/setup.py b/setup.py index 970eafb..b7990a0 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.6', + version='1.7', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.6.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.7.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', @@ -22,7 +22,8 @@ packages=['yahoofinancials'], install_requires=[ "beautifulsoup4", - "pytz" + "pytz", + "pycryptodome" ], classifiers=[ 'Development Status :: 5 - Production/Stable', @@ -32,14 +33,12 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: OS Independent', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7' + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10' ], zip_safe=False ) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index bcb6c73..7afc5c0 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,8 +1,8 @@ -# YahooFinancials Unit Tests v1.5 -# Version Released: 01/27/2019 +# YahooFinancials Unit Tests v1.7 +# Version Released: 01/01/2023 # Author: Connor Sanders -# Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 -# Copyright (c) 2019 Connor Sanders +# Tested on Python 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10 +# Copyright (c) 2023 Connor Sanders # MIT License import sys @@ -92,12 +92,9 @@ def test_yf_historical_price(self): # Historical Stock Daily Dividend Test def test_yf_dividend_price(self): single_stock_dividend = self.test_yf_stock_single.get_daily_dividend_data('1986-09-15', '1987-09-15') - expect_dict = {"C": [{"date": 533313000, "formatted_date": "1986-11-25", "amount": 0.02999}, - {"date": 541348200, "formatted_date": "1987-02-26", "amount": 0.02999}, - {"date": 544714200, "formatted_date": "1987-04-06", "amount": 0.332}, - {"date": 549120600, "formatted_date": "1987-05-27", "amount": 0.02999}, - {"date": 552576600, "formatted_date": "1987-07-06", "amount": 0.332}, - {"date": 557501400, "formatted_date": "1987-09-01", "amount": 0.02999}] + print(single_stock_dividend) + expect_dict = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332}, + {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}] } self.assertDictEqual(single_stock_dividend, expect_dict) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index ef5deec..cb45bb9 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,15 +1,15 @@ """ ============================== The Yahoo Financials Module -Version: 1.6 +Version: 1.7 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 10/18/2020 -Tested on Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 +Version Released: 01/01/2023 +Tested on Python 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10 -Copyright (c) 2020 Connor Sanders +Copyright (c) 2023 Connor Sanders MIT License List of Included Functions: From 945703ecfd6fceb5c403bc1879928162116bc27b Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:36:17 -0600 Subject: [PATCH 075/108] v1.7 removed travis-ci and adding github workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a680be6..b402360 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: [push] jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] From f64d913feb353218d8a12e181f594add946dc7c0 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:41:34 -0600 Subject: [PATCH 076/108] v1.7 removed travis-ci and adding github workflows --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b402360..09d055d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v3 From 179d6adbf007dab1eb56651e6c1c7f8adf95f49f Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:47:45 -0600 Subject: [PATCH 077/108] v1.7 removed travis-ci and adding github workflows --- README.rst | 4 ++-- setup.py | 1 - test/test_yahoofinancials.py | 2 +- yahoofinancials/__init__.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index 39af956..d2dc22a 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- -- yahoofinancials runs on Python 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10. +- yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, and 3.10. - This package depends on beautifulsoup4, pytz, and pycryptodome to work. 1. Installation using pip: @@ -63,7 +63,7 @@ Module Methods -------------- - The financial data from all methods is returned as JSON. - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -- YahooFinancials works with Python 2.7, 3.3, 3.4, 3.5, 3.6, and 3.7 and runs on all operating systems. (Windows, Mac, Linux). +- YahooFinancials works with Python 3.6, 3.7, 3.8, 3.9, and 3.10 and runs on all operating systems. (Windows, Mac, Linux). Featured Methods ^^^^^^^^^^^^^^^^ diff --git a/setup.py b/setup.py index b7990a0..ff97e89 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: OS Independent', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 7afc5c0..92ba034 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,7 +1,7 @@ # YahooFinancials Unit Tests v1.7 # Version Released: 01/01/2023 # Author: Connor Sanders -# Tested on Python 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10 +# Tested on Python 3.6, 3.7, 3.8, 3.9, and 3.10 # Copyright (c) 2023 Connor Sanders # MIT License diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index cb45bb9..390148c 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -7,7 +7,7 @@ Author: Connor Sanders Email: sandersconnor1@gmail.com Version Released: 01/01/2023 -Tested on Python 2.7, 3.5, 3.6, 3.7, 3.8, 3.9, and 3.10 +Tested on Python 3.6, 3.7, 3.8, 3.9, and 3.10 Copyright (c) 2023 Connor Sanders MIT License From edf540e4ecdf16abc38b3fe563e7e9828be62d9f Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:51:16 -0600 Subject: [PATCH 078/108] v1.7 removed travis-ci and adding github workflows --- .github/workflows/test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09d055d..20ee768 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,15 +1,17 @@ -name: Python package +name: test -on: [push] +on: + push: + branches: + - master + - development jobs: build: - runs-on: ubuntu-20.04 strategy: matrix: python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From 282d463b685144bab4af91cb26614369f933ddec Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 20:56:25 -0600 Subject: [PATCH 079/108] v1.7 removed travis-ci and adding github workflows --- README.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.rst b/README.rst index d2dc22a..0107cd9 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,9 @@ yahoofinancials A python module that returns stock, cryptocurrency, forex, mutual fund, commodity futures, ETF, and US Treasury financial data from Yahoo Finance. +.. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master + :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml + Current Version: v1.7 Version Released: 01/01/2023 From 342d6b9186586e878d973d44347bd331e9271bbf Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sun, 1 Jan 2023 23:02:56 -0600 Subject: [PATCH 080/108] v1.7 removed travis-ci and adding github workflows --- test/test_yahoofinancials.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 92ba034..83b561f 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -92,7 +92,6 @@ def test_yf_historical_price(self): # Historical Stock Daily Dividend Test def test_yf_dividend_price(self): single_stock_dividend = self.test_yf_stock_single.get_daily_dividend_data('1986-09-15', '1987-09-15') - print(single_stock_dividend) expect_dict = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332}, {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}] } From 8c035e77ce22b8decc4ef94a466b1b583ffe1755 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Mon, 9 Jan 2023 08:54:52 -0600 Subject: [PATCH 081/108] Investigating replacing pycrptodome with cryptography for data store decryption --- setup.py | 4 +- yahoofinancials/__init__.py | 158 ++++++++++++++++++++---------------- 2 files changed, 88 insertions(+), 74 deletions(-) diff --git a/setup.py b/setup.py index ff97e89..097e102 100644 --- a/setup.py +++ b/setup.py @@ -21,9 +21,9 @@ keywords=['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], packages=['yahoofinancials'], install_requires=[ - "beautifulsoup4", + "beautifulsoup4>=4.11.1", "pytz", - "pycryptodome" + "cryptography>=3.3.2" ], classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 390148c..3c29b14 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -58,8 +58,16 @@ import hashlib from base64 import b64decode -from Crypto.Cipher import AES -from Crypto.Util.Padding import unpad +import hashlib +from base64 import b64decode +usePycryptodome = False # slightly faster +# usePycryptodome = True +if usePycryptodome: + from Crypto.Cipher import AES + from Crypto.Util.Padding import unpad +else: + from cryptography.hazmat.primitives import padding + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes import json # track the last get timestamp to add a minimum delay between gets - be nice! @@ -76,6 +84,76 @@ class UrlOpener(FancyURLopener): version = 'w3m/0.5.3+git20180125' +def decrypt_cryptojs_aes(data): + encrypted_stores = data['context']['dispatcher']['stores'] + _cs = data["_cs"] + _cr = data["_cr"] + + _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) + password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + + encrypted_stores = b64decode(encrypted_stores) + assert encrypted_stores[0:8] == b"Salted__" + salt = encrypted_stores[8:16] + encrypted_stores = encrypted_stores[16:] + + def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: + """OpenSSL EVP Key Derivation Function + Args: + password (Union[str, bytes, bytearray]): Password to generate key from. + salt (Union[bytes, bytearray]): Salt to use. + keySize (int, optional): Output key length in bytes. Defaults to 32. + ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. + iterations (int, optional): Number of iterations to perform. Defaults to 1. + hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. + Returns: + key, iv: Derived key and Initialization Vector (IV) bytes. + + Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 + OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 + """ + + assert iterations > 0, "Iterations can not be less than 1." + + if isinstance(password, str): + password = password.encode("utf-8") + + final_length = keySize + ivSize + key_iv = b"" + block = None + + while len(key_iv) < final_length: + hasher = hashlib.new(hashAlgorithm) + if block: + hasher.update(block) + hasher.update(password) + hasher.update(salt) + block = hasher.digest() + for _ in range(1, iterations): + block = hashlib.new(hashAlgorithm, block).digest() + key_iv += block + + key, iv = key_iv[:keySize], key_iv[keySize:final_length] + return key, iv + + key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + + if usePycryptodome: + cipher = AES.new(key, AES.MODE_CBC, iv=iv) + plaintext = cipher.decrypt(encrypted_stores) + plaintext = unpad(plaintext, 16, style="pkcs7") + else: + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(plaintext) + unpadder.finalize() + plaintext = plaintext.decode("utf-8") + + decoded_stores = json.loads(plaintext) + return decoded_stores + + # Class containing Yahoo Finance ETL Functionality class YahooFinanceETL(object): @@ -168,7 +246,11 @@ def _scrape_data(self, url, tech_type, statement_type): raise ManagedException("Server replied with HTTP " + str(response.getcode()) + " code while opening the url: " + str(url)) data = self._cache[url] - data = self._decryptData(data) + if "_cs" in data and "_cr" in data: + data = decrypt_cryptojs_aes(data) + if "context" in data and "dispatcher" in data["context"]: + # Keep old code, just in case + data = data['context']['dispatcher']['stores'] if tech_type == '' and statement_type != 'history': stores = data["QuoteSummaryStore"] elif tech_type != '' and statement_type != 'history': @@ -177,74 +259,6 @@ def _scrape_data(self, url, tech_type, statement_type): stores = data["HistoricalPriceStore"] return stores - def _decryptData(self,data): - #function taken from another package at https://github.com/ranaroussi/yfinance/pull/1253/commits/8e5f0984af347afda6be74b27a989422e49a975b - encrypted_stores = data['context']['dispatcher']['stores'] - _cs = data["_cs"] - _cr = data["_cr"] - - _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) - password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() - - encrypted_stores = b64decode(encrypted_stores) - assert encrypted_stores[0:8] == b"Salted__" - salt = encrypted_stores[8:16] - encrypted_stores = encrypted_stores[16:] - - def EVPKDF( - password, - salt, - keySize=32, - ivSize=16, - iterations=1, - hashAlgorithm="md5", - ) -> tuple: - """OpenSSL EVP Key Derivation Function - Args: - password (Union[str, bytes, bytearray]): Password to generate key from. - salt (Union[bytes, bytearray]): Salt to use. - keySize (int, optional): Output key length in bytes. Defaults to 32. - ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. - iterations (int, optional): Number of iterations to perform. Defaults to 1. - hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. - Returns: - key, iv: Derived key and Initialization Vector (IV) bytes. - Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 - OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 - """ - - assert iterations > 0, "Iterations can not be less than 1." - - if isinstance(password, str): - password = password.encode("utf-8") - - final_length = keySize + ivSize - key_iv = b"" - block = None - - while len(key_iv) < final_length: - hasher = hashlib.new(hashAlgorithm) - if block: - hasher.update(block) - hasher.update(password) - hasher.update(salt) - block = hasher.digest() - for _ in range(1, iterations): - block = hashlib.new(hashAlgorithm, block).digest() - key_iv += block - - key, iv = key_iv[:keySize], key_iv[keySize:final_length] - return key, iv - - key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - - cipher = AES.new(key, AES.MODE_CBC, iv=iv) - plaintext = cipher.decrypt(encrypted_stores) - plaintext = unpad(plaintext, 16, style="pkcs7") - decoded_stores = json.loads(plaintext) - - return decoded_stores - # Private static method to determine if a numerical value is in the data object being cleaned @staticmethod def _determine_numeric_value(value_dict): @@ -260,7 +274,7 @@ def _format_time(self, in_time): utc_dt = self._convert_to_utc(form_date_time) return utc_dt - # Private method to return the a sub dictionary entry for the earning report cleaning + # Private method to return a sub dictionary entry for the earning report cleaning def _get_cleaned_sub_dict_ent(self, key, val_list): sub_list = [] for rec in val_list: From a721d81d9d15475eb46c434ecb88e11f5a18d87a Mon Sep 17 00:00:00 2001 From: connorsanders Date: Mon, 9 Jan 2023 21:44:02 -0600 Subject: [PATCH 082/108] v1.8 --- CHANGES | 1 + README.rst | 6 ++--- setup.py | 4 ++-- yahoofinancials/__init__.py | 47 +++++++++---------------------------- 4 files changed, 17 insertions(+), 41 deletions(-) diff --git a/CHANGES b/CHANGES index ccf2947..19f4a8c 100644 --- a/CHANGES +++ b/CHANGES @@ -29,3 +29,4 @@ 1.6 10/18/2020 -- Merged in two pull requests with bug fixes from sylvandb. 1.7 01/01/2023 -- Merged in pull request with fixes from sedwards2000. 1.7 01/01/2023 -- Support for python 2.7 dropped. +1.8 01/09/2023 -- Improved decryption support diff --git a/README.rst b/README.rst index 0107cd9..65ec97c 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.7 +Current Version: v1.8 -Version Released: 01/01/2023 +Version Released: 01/09/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -22,7 +22,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- - yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, and 3.10. -- This package depends on beautifulsoup4, pytz, and pycryptodome to work. +- This package depends on beautifulsoup4, pytz, and cryptography to work. 1. Installation using pip: diff --git a/setup.py b/setup.py index 097e102..f00f221 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.7', + version='1.8', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.7.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.8.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 3c29b14..6d341d6 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.7 +Version: 1.8 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/01/2023 +Version Released: 01/09/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, and 3.10 Copyright (c) 2023 Connor Sanders @@ -55,21 +55,13 @@ from urllib import FancyURLopener except: from urllib.request import FancyURLopener - -import hashlib -from base64 import b64decode import hashlib from base64 import b64decode -usePycryptodome = False # slightly faster -# usePycryptodome = True -if usePycryptodome: - from Crypto.Cipher import AES - from Crypto.Util.Padding import unpad -else: - from cryptography.hazmat.primitives import padding - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes import json + # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -88,10 +80,8 @@ def decrypt_cryptojs_aes(data): encrypted_stores = data['context']['dispatcher']['stores'] _cs = data["_cs"] _cr = data["_cr"] - _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() - encrypted_stores = b64decode(encrypted_stores) assert encrypted_stores[0:8] == b"Salted__" salt = encrypted_stores[8:16] @@ -112,16 +102,12 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 """ - assert iterations > 0, "Iterations can not be less than 1." - if isinstance(password, str): password = password.encode("utf-8") - final_length = keySize + ivSize key_iv = b"" block = None - while len(key_iv) < final_length: hasher = hashlib.new(hashAlgorithm) if block: @@ -132,24 +118,16 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m for _ in range(1, iterations): block = hashlib.new(hashAlgorithm, block).digest() key_iv += block - key, iv = key_iv[:keySize], key_iv[keySize:final_length] return key, iv key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - - if usePycryptodome: - cipher = AES.new(key, AES.MODE_CBC, iv=iv) - plaintext = cipher.decrypt(encrypted_stores) - plaintext = unpad(plaintext, 16, style="pkcs7") - else: - cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() - unpadder = padding.PKCS7(128).unpadder() - plaintext = unpadder.update(plaintext) + unpadder.finalize() - plaintext = plaintext.decode("utf-8") - + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(plaintext) + unpadder.finalize() + plaintext = plaintext.decode("utf-8") decoded_stores = json.loads(plaintext) return decoded_stores @@ -339,9 +317,6 @@ def _clean_reports(self, raw_data): dict_ent = {k: formatted_date} elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): dict_ent = {k: v} - # Python 2 and Unicode - elif sys.version_info < (3, 0) and isinstance(v, unicode): - dict_ent = {k: v} else: numerical_val = self._determine_numeric_value(v) dict_ent = {k: numerical_val} From 06f071d1f3fe15252eb4701e0715f1ec67392e83 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 13 Jan 2023 04:16:04 -0600 Subject: [PATCH 083/108] added async execution, code enhancements, and region support --- .github/workflows/test.yml | 2 +- CHANGES | 10 +- README.rst | 19 +- demo.py | 13 +- setup.py | 5 +- test/test_yahoofinancials.py | 31 +- yahoofinancials/__init__.py | 683 ++--------------------------------- yahoofinancials/calcs.py | 22 ++ yahoofinancials/etl.py | 659 +++++++++++++++++++++++++++++++++ yahoofinancials/maps.py | 16 + 10 files changed, 789 insertions(+), 671 deletions(-) create mode 100644 yahoofinancials/calcs.py create mode 100644 yahoofinancials/etl.py create mode 100644 yahoofinancials/maps.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20ee768..ff76a5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/CHANGES b/CHANGES index 19f4a8c..55a341e 100644 --- a/CHANGES +++ b/CHANGES @@ -28,5 +28,11 @@ 1.5 01/27/2019 -- Added test_yf_dividend_price() unit testing method. 1.6 10/18/2020 -- Merged in two pull requests with bug fixes from sylvandb. 1.7 01/01/2023 -- Merged in pull request with fixes from sedwards2000. -1.7 01/01/2023 -- Support for python 2.7 dropped. -1.8 01/09/2023 -- Improved decryption support +1.7 01/01/2023 -- Support for Python 2.7 dropped. +1.8 01/09/2023 -- Improved decryption support. +1.9 01/14/2023 -- Added official Python 3.11 support. +1.9 01/14/2023 -- General code cleanup & quality improvements. +1.9 01/14/2023 -- Added new optional concurrent input to YahooFinancials(), if True extraction will run async. +1.9 01/14/2023 -- Added new optional meta input to YahooFinancials(), currently supports language & region. +1.9 01/14/2023 -- Updated beautifulsoup4 find to use string parameter instead deprecated text parameter. + diff --git a/README.rst b/README.rst index 65ec97c..b97c60f 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.8 +Current Version: v1.9 -Version Released: 01/09/2023 +Version Released: 01/14/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -17,11 +17,20 @@ Overview -------- A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. -- As of Version 0.10, Yahoo Financials now returns historical pricing data for commodity futures, cryptocurrencies, ETFs, mutual funds, U.S. Treasuries, currencies, indexes, and stocks. +- As of Version 1.9, YahooFinancials supports asynchronous execution and international requests. + +.. code-block:: python + + from yahoofinancials import YahooFinancials + tickers = ['AAPL', 'GOOG', 'C'] + yahoo_financials = YahooFinancials(tickers, concurrent=True) + balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') + print(balance_sheet_data_qt) + Installation ------------- -- yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, and 3.10. +- yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11. - This package depends on beautifulsoup4, pytz, and cryptography to work. 1. Installation using pip: @@ -66,7 +75,7 @@ Module Methods -------------- - The financial data from all methods is returned as JSON. - You can run multiple symbols at once using an inputted array or run an individual symbol using an inputted string. -- YahooFinancials works with Python 3.6, 3.7, 3.8, 3.9, and 3.10 and runs on all operating systems. (Windows, Mac, Linux). +- YahooFinancials works with Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 and runs on all operating systems. (Windows, Mac, Linux). Featured Methods ^^^^^^^^^^^^^^^^ diff --git a/demo.py b/demo.py index b1aac4b..20dcc77 100755 --- a/demo.py +++ b/demo.py @@ -23,7 +23,7 @@ mark = '-' * 64 -def defaultapi(ticker): +def default_api(ticker): tick = YF(ticker) print(tick.get_summary_data()) print(mark) @@ -43,14 +43,14 @@ def defaultapi(ticker): print(r) -def customapi(queries, ts): +def custom_api(queries, ts): yf = YF(ts[0] if 1 == len(ts) else ts) for q in queries: print('%s:' % (q,)) timeit(lambda: print(getattr(yf, q)())) -def helpapi(queries): +def help_api(queries): if len(queries) == 1: print(__doc__ % {'scriptname': sys.argv[0], 'defaultargs': ', '.join(DEFAULT_ARGS)}) else: @@ -71,7 +71,6 @@ def timeit(f, *args): print(et - st, 'seconds') - if __name__ == '__main__': api = set(s for s in dir(YF) if s.startswith('get_')) api.update(MODULE_ARGS) @@ -80,8 +79,8 @@ def timeit(f, *args): queries = [q for q in ts if q in api] ts = [t for t in ts if not t in queries] or DEFAULT_ARGS if [h for h in HELP_ARGS if h in queries]: - helpapi(queries) + help_api(queries) elif queries: - customapi(queries, ts) + custom_api(queries, ts) else: - timeit(defaultapi, ts[0] if 1 == len(ts) else ts) + timeit(default_api, ts[0] if 1 == len(ts) else ts) diff --git a/setup.py b/setup.py index f00f221..e31fe0a 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.8', + version='1.9', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.8.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.9.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', @@ -38,6 +38,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10' + 'Programming Language :: Python :: 3.11' ], zip_safe=False ) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 83b561f..2920cf2 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,17 +1,12 @@ -# YahooFinancials Unit Tests v1.7 -# Version Released: 01/01/2023 +# YahooFinancials Unit Tests v1.9 +# Version Released: 01/14/2023 # Author: Connor Sanders -# Tested on Python 3.6, 3.7, 3.8, 3.9, and 3.10 +# Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 # Copyright (c) 2023 Connor Sanders # MIT License -import sys from yahoofinancials import YahooFinancials - -if sys.version_info < (2, 7): - from unittest2 import main as test_main, SkipTest, TestCase -else: - from unittest import main as test_main, SkipTest, TestCase +from unittest import main as test_main, SkipTest, TestCase # Test Configuration Variables @@ -45,11 +40,12 @@ def check_fundamental(test_data, test_type): class TestModule(TestCase): def setUp(self): - self.test_yf_stock_single = YahooFinancials('C') + self.test_yf_stock_single = YahooFinancials('C', country='UK') self.test_yf_stock_multi = YahooFinancials(stocks) self.test_yf_treasuries_single = YahooFinancials('^IRX') self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) self.test_yf_currencies = YahooFinancials(currencies) + self.test_yf_concurrent = YahooFinancials(stocks, concurrent=True) # Fundamentals Test def test_yf_fundamentals(self): @@ -114,6 +110,21 @@ def test_yf_module_methods(self): else: self.assertEqual(False, True) + # Test concurrent functionality of module + def test_yf_concurrency(self): + # Multi stock test + multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance') + multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income') + multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Multi stock check + result = check_fundamental(multi_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(multi_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(multi_all_statement_data_qt, 'all') + self.assertEqual(result, True) + if __name__ == "__main__": test_main() diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 6d341d6..c0f5bdf 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,13 +1,13 @@ """ ============================== The Yahoo Financials Module -Version: 1.8 +Version: 1.9 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/09/2023 -Tested on Python 3.6, 3.7, 3.8, 3.9, and 3.10 +Version Released: 01/14/2023 +Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders MIT License @@ -42,605 +42,28 @@ historical_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') """ -import sys -import calendar -import re -from json import loads -import time -from bs4 import BeautifulSoup -import datetime -import pytz -import random -try: - from urllib import FancyURLopener -except: - from urllib.request import FancyURLopener -import hashlib -from base64 import b64decode -from cryptography.hazmat.primitives import padding -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -import json - - -# track the last get timestamp to add a minimum delay between gets - be nice! -_lastget = 0 - - -# Custom Exception class to handle custom error -class ManagedException(Exception): - pass - - -# Class used to open urls for financial data -class UrlOpener(FancyURLopener): - version = 'w3m/0.5.3+git20180125' - - -def decrypt_cryptojs_aes(data): - encrypted_stores = data['context']['dispatcher']['stores'] - _cs = data["_cs"] - _cr = data["_cr"] - _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) - password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() - encrypted_stores = b64decode(encrypted_stores) - assert encrypted_stores[0:8] == b"Salted__" - salt = encrypted_stores[8:16] - encrypted_stores = encrypted_stores[16:] - - def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: - """OpenSSL EVP Key Derivation Function - Args: - password (Union[str, bytes, bytearray]): Password to generate key from. - salt (Union[bytes, bytearray]): Salt to use. - keySize (int, optional): Output key length in bytes. Defaults to 32. - ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. - iterations (int, optional): Number of iterations to perform. Defaults to 1. - hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. - Returns: - key, iv: Derived key and Initialization Vector (IV) bytes. - - Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 - OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 - """ - assert iterations > 0, "Iterations can not be less than 1." - if isinstance(password, str): - password = password.encode("utf-8") - final_length = keySize + ivSize - key_iv = b"" - block = None - while len(key_iv) < final_length: - hasher = hashlib.new(hashAlgorithm) - if block: - hasher.update(block) - hasher.update(password) - hasher.update(salt) - block = hasher.digest() - for _ in range(1, iterations): - block = hashlib.new(hashAlgorithm, block).digest() - key_iv += block - key, iv = key_iv[:keySize], key_iv[keySize:final_length] - return key, iv - - key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() - unpadder = padding.PKCS7(128).unpadder() - plaintext = unpadder.update(plaintext) + unpadder.finalize() - plaintext = plaintext.decode("utf-8") - decoded_stores = json.loads(plaintext) - return decoded_stores - - -# Class containing Yahoo Finance ETL Functionality -class YahooFinanceETL(object): - - def __init__(self, ticker): - self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] - self._cache = {} - - # Minimum interval between Yahoo Finance requests for this instance - _MIN_INTERVAL = 7 - - # Meta-data dictionaries for the classes to use - YAHOO_FINANCIAL_TYPES = { - 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], - 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], - 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], - 'keystats': ['key-statistics'], - 'history': ['history'] - } - - # Interval value translation dictionary - _INTERVAL_DICT = { - 'daily': '1d', - 'weekly': '1wk', - 'monthly': '1mo' - } - - # Base Yahoo Finance URL for the class to build on - _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' - - # private static method to get the appropriate report type identifier - @staticmethod - def get_report_type(frequency): - if frequency == 'annual': - report_num = 1 - else: - report_num = 2 - return report_num - - # Public static method to format date serial string to readable format and vice versa - @staticmethod - def format_date(in_date): - if isinstance(in_date, str): - form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) - else: - form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) - return form_date - - # Private Static Method to Convert Eastern Time to UTC - @staticmethod - def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): - utc = pytz.utc - eastern = pytz.timezone('US/Eastern') - date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) - date_eastern = eastern.localize(date_, is_dst=None) - date_utc = date_eastern.astimezone(utc) - return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') - - # Private method to scrape data from yahoo finance - def _scrape_data(self, url, tech_type, statement_type): - global _lastget - if not self._cache.get(url): - now = int(time.time()) - if _lastget and now - _lastget < self._MIN_INTERVAL: - time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) - now = int(time.time()) - _lastget = now - urlopener = UrlOpener() - # Try to open the URL up to 10 times sleeping random time if something goes wrong - max_retry = 10 - for i in range(0, max_retry): - response = urlopener.open(url) - if response.getcode() != 200: - time.sleep(random.randrange(10, 20)) - else: - response_content = response.read() - soup = BeautifulSoup(response_content, "html.parser") - re_script = soup.find("script", text=re.compile("root.App.main")) - if re_script is not None: - script = re_script.text - # bs4 4.9.0 changed so text from scripts is no longer considered text - if not script: - script = re_script.string - self._cache[url] = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) - response.close() - break - else: - time.sleep(random.randrange(10, 20)) - if i == max_retry - 1: - # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with HTTP " + str(response.getcode()) + - " code while opening the url: " + str(url)) - data = self._cache[url] - if "_cs" in data and "_cr" in data: - data = decrypt_cryptojs_aes(data) - if "context" in data and "dispatcher" in data["context"]: - # Keep old code, just in case - data = data['context']['dispatcher']['stores'] - if tech_type == '' and statement_type != 'history': - stores = data["QuoteSummaryStore"] - elif tech_type != '' and statement_type != 'history': - stores = data["QuoteSummaryStore"][tech_type] - else: - stores = data["HistoricalPriceStore"] - return stores - - # Private static method to determine if a numerical value is in the data object being cleaned - @staticmethod - def _determine_numeric_value(value_dict): - if 'raw' in value_dict.keys(): - numerical_val = value_dict['raw'] - else: - numerical_val = None - return numerical_val - - # Private method to format date serial string to readable format and vice versa - def _format_time(self, in_time): - form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') - utc_dt = self._convert_to_utc(form_date_time) - return utc_dt - - # Private method to return a sub dictionary entry for the earning report cleaning - def _get_cleaned_sub_dict_ent(self, key, val_list): - sub_list = [] - for rec in val_list: - sub_sub_dict = {} - for k, v in rec.items(): - if k == 'date': - sub_sub_dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - sub_sub_dict_ent = {k: numerical_val} - sub_sub_dict.update(sub_sub_dict_ent) - sub_list.append(sub_sub_dict) - sub_ent = {key: sub_list} - return sub_ent - - # Private method to process raw earnings data and clean - def _clean_earnings_data(self, raw_data): - cleaned_data = {} - earnings_key = 'earningsData' - financials_key = 'financialsData' - for k, v in raw_data.items(): - if k == 'earningsChart': - sub_dict = {} - for k2, v2 in v.items(): - if k2 == 'quarterly': - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - elif k2 == 'currentQuarterEstimate': - numerical_val = self._determine_numeric_value(v2) - sub_ent = {k2: numerical_val} - else: - sub_ent = {k2: v2} - sub_dict.update(sub_ent) - dict_ent = {earnings_key: sub_dict} - cleaned_data.update(dict_ent) - elif k == 'financialsChart': - sub_dict = {} - for k2, v2, in v.items(): - sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) - sub_dict.update(sub_ent) - dict_ent = {financials_key: sub_dict} - cleaned_data.update(dict_ent) - else: - if k != 'maxAge': - dict_ent = {k: v} - cleaned_data.update(dict_ent) - return cleaned_data - - # Private method to clean summary and price reports - def _clean_reports(self, raw_data): - cleaned_dict = {} - if raw_data is None: - return None - for k, v in raw_data.items(): - if 'Time' in k: - formatted_utc_time = self._format_time(v) - dict_ent = {k: formatted_utc_time} - elif 'Date' in k: - try: - formatted_date = v['fmt'] - except (KeyError, TypeError): - formatted_date = '-' - dict_ent = {k: formatted_date} - elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): - dict_ent = {k: v} - else: - numerical_val = self._determine_numeric_value(v) - dict_ent = {k: numerical_val} - cleaned_dict.update(dict_ent) - return cleaned_dict - - # Private Static Method to ensure ticker is URL encoded - @staticmethod - def _encode_ticker(ticker_str): - encoded_ticker = ticker_str.replace('=', '%3D') - return encoded_ticker - - # Private method to get time interval code - def _build_historical_url(self, ticker, hist_oj): - url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ - '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ - hist_oj['interval'] - return url - - # Private Method to clean the dates of the newly returns historical stock data into readable format - def _clean_historical_data(self, hist_data, last_attempt=False): - data = {} - for k, v in hist_data.items(): - if k == 'eventsData': - event_obj = {} - if isinstance(v, list): - dict_ent = {k: event_obj} - else: - for type_key, type_obj in v.items(): - formatted_type_obj = {} - for date_key, date_obj in type_obj.items(): - formatted_date_key = self.format_date(int(date_key)) - cleaned_date = self.format_date(int(date_obj['date'])) - date_obj.update({'formatted_date': cleaned_date}) - formatted_type_obj.update({formatted_date_key: date_obj}) - event_obj.update({type_key: formatted_type_obj}) - dict_ent = {k: event_obj} - elif 'date' in k.lower(): - if v is not None: - cleaned_date = self.format_date(v) - dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} - else: - if last_attempt is False: - return None - else: - dict_ent = {k: {'formatted_date': None, 'date': v}} - elif isinstance(v, list): - sub_dict_list = [] - for sub_dict in v: - sub_dict['formatted_date'] = self.format_date(sub_dict['date']) - sub_dict_list.append(sub_dict) - dict_ent = {k: sub_dict_list} - else: - dict_ent = {k: v} - data.update(dict_ent) - return data - - # Private Static Method to build API url for GET Request - @staticmethod - def _build_api_url(hist_obj, up_ticker): - base_url = "https://query1.finance.yahoo.com/v8/finance/chart/" - api_url = base_url + up_ticker + '?symbol=' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' + \ - str(hist_obj['end']) + '&interval=' + hist_obj['interval'] - api_url += '&events=div|split|earn&lang=en-US®ion=US' - return api_url - - # Private Method to get financial data via API Call - def _get_api_data(self, api_url, tries=0): - urlopener = UrlOpener() - response = urlopener.open(api_url) - if response.getcode() == 200: - res_content = response.read() - response.close() - if sys.version_info < (3, 0): - return loads(res_content) - return loads(res_content.decode('utf-8')) - else: - if tries < 5: - time.sleep(random.randrange(10, 20)) - tries += 1 - return self._get_api_data(api_url, tries) - else: - return None - - # Private Method to clean API data - def _clean_api_data(self, api_url): - raw_data = self._get_api_data(api_url) - ret_obj = {} - ret_obj.update({'eventsData': []}) - if raw_data is None: - return ret_obj - results = raw_data['chart']['result'] - if results is None: - return ret_obj - for result in results: - tz_sub_dict = {} - ret_obj.update({'eventsData': result.get('events', {})}) - ret_obj.update({'firstTradeDate': result['meta'].get('firstTradeDate', 'NA')}) - ret_obj.update({'currency': result['meta'].get('currency', 'NA')}) - ret_obj.update({'instrumentType': result['meta'].get('instrumentType', 'NA')}) - tz_sub_dict.update({'gmtOffset': result['meta']['gmtoffset']}) - ret_obj.update({'timeZone': tz_sub_dict}) - timestamp_list = result['timestamp'] - high_price_list = result['indicators']['quote'][0]['high'] - low_price_list = result['indicators']['quote'][0]['low'] - open_price_list = result['indicators']['quote'][0]['open'] - close_price_list = result['indicators']['quote'][0]['close'] - volume_list = result['indicators']['quote'][0]['volume'] - adj_close_list = result['indicators']['adjclose'][0]['adjclose'] - i = 0 - prices_list = [] - for timestamp in timestamp_list: - price_dict = {} - price_dict.update({'date': timestamp}) - price_dict.update({'high': high_price_list[i]}) - price_dict.update({'low': low_price_list[i]}) - price_dict.update({'open': open_price_list[i]}) - price_dict.update({'close': close_price_list[i]}) - price_dict.update({'volume': volume_list[i]}) - price_dict.update({'adjclose': adj_close_list[i]}) - prices_list.append(price_dict) - i += 1 - ret_obj.update({'prices': prices_list}) - return ret_obj - - # Private Method to Handle Recursive API Request - def _recursive_api_request(self, hist_obj, up_ticker, i=0): - api_url = self._build_api_url(hist_obj, up_ticker) - re_data = self._clean_api_data(api_url) - cleaned_re_data = self._clean_historical_data(re_data) - if cleaned_re_data is not None: - return cleaned_re_data - else: - if i < 3: - i += 1 - return self._recursive_api_request(hist_obj, up_ticker, i) - else: - return self._clean_historical_data(re_data, True) - - # Private Method to take scrapped data and build a data dictionary with - def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): - YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' +\ - up_ticker - if tech_type == '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data, 'dataType': report_name} - elif tech_type != '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data} - else: - YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) - try: - cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) - except KeyError: - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - cleaned_re_data = self._clean_historical_data(re_data) - except KeyError: - cleaned_re_data = None - dict_ent = {up_ticker: cleaned_re_data} - return dict_ent - - # Private method to return the stmt_id for the reformat_process - def _get_stmt_id(self, statement_type, raw_data): - stmt_id = '' - i = 0 - for key in raw_data.keys(): - if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: - stmt_id = key - i += 1 - if i != 1: - return None - return stmt_id - - # Private Method for the Reformat Process - def _reformat_stmt_data_process(self, raw_data, statement_type): - final_data_list = [] - if raw_data is not None: - stmt_id = self._get_stmt_id(statement_type, raw_data) - if stmt_id is None: - return final_data_list - hashed_data_list = raw_data[stmt_id] - for data_item in hashed_data_list: - data_date = '' - sub_data_dict = {} - for k, v in data_item.items(): - if k == 'endDate': - data_date = v['fmt'] - elif k != 'maxAge': - numerical_val = self._determine_numeric_value(v) - sub_dict_item = {k: numerical_val} - sub_data_dict.update(sub_dict_item) - dict_item = {data_date: sub_data_dict} - final_data_list.append(dict_item) - return final_data_list - else: - return raw_data - - # Private Method to return subdict entry for the statement reformat process - def _get_sub_dict_ent(self, ticker, raw_data, statement_type): - form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) - return {ticker: form_data_list} - - # Public method to get time interval code - def get_time_code(self, time_interval): - interval_code = self._INTERVAL_DICT[time_interval.lower()] - return interval_code - - # Public Method to get stock data - def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): - data = {} - if isinstance(self.ticker, str): - dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - else: - for tick in self.ticker: - try: - dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - except ManagedException: - print("Warning! Ticker: " + str(tick) + " error - " + str(ManagedException)) - print("The process is still running...") - continue - return data - - # Public Method to get technical stock data - def get_stock_tech_data(self, tech_type): - if tech_type == 'defaultKeyStatistics': - return self.get_stock_data(statement_type='keystats', tech_type=tech_type) - else: - return self.get_stock_data(tech_type=tech_type) - - # Public Method to get reformatted statement data - def get_reformatted_stmt_data(self, raw_data, statement_type): - data_dict = {} - sub_dict = {} - data_type = raw_data['dataType'] - if isinstance(self.ticker, str): - sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {data_type: sub_dict} - data_dict.update(dict_ent) - else: - for tick in self.ticker: - sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) - sub_dict.update(sub_dict_ent) - dict_ent = {data_type: sub_dict} - data_dict.update(dict_ent) - return data_dict - - # Public method to get cleaned summary and price report data - def get_clean_data(self, raw_report_data, report_type): - cleaned_data_dict = {} - if isinstance(self.ticker, str): - if report_type == 'earnings': - try: - cleaned_data = self._clean_earnings_data(raw_report_data[self.ticker]) - except: - cleaned_data = None - else: - try: - cleaned_data = self._clean_reports(raw_report_data[self.ticker]) - except: - cleaned_data = None - cleaned_data_dict.update({self.ticker: cleaned_data}) - else: - for tick in self.ticker: - if report_type == 'earnings': - try: - cleaned_data = self._clean_earnings_data(raw_report_data[tick]) - except: - cleaned_data = None - else: - try: - cleaned_data = self._clean_reports(raw_report_data[tick]) - except: - cleaned_data = None - cleaned_data_dict.update({tick: cleaned_data}) - return cleaned_data_dict - - # Private method to handle dividend data requests - def _handle_api_dividend_request(self, cur_ticker, start, end, interval): - re_dividends = [] - test_url = 'https://query1.finance.yahoo.com/v8/finance/chart/' + cur_ticker + \ - '?period1=' + str(start) + '&period2=' + str(end) + '&interval=' + interval + '&events=div' - div_dict = self._get_api_data(test_url)['chart']['result'][0]['events']['dividends'] - for div_time_key, div_obj in div_dict.items(): - dividend_obj = { - 'date': div_obj['date'], - 'formatted_date': self.format_date(int(div_obj['date'])), - 'amount': div_obj.get('amount', None) - } - re_dividends.append(dividend_obj) - return sorted(re_dividends, key=lambda div: div['date']) - - # Public method to get daily dividend data - def get_stock_dividend_data(self, start, end, interval): - interval_code = self.get_time_code(interval) - if isinstance(self.ticker, str): - try: - return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)} - except: - return {self.ticker: None} - else: - re_data = {} - for tick in self.ticker: - try: - div_data = self._handle_api_dividend_request(tick, start, end, interval_code) - re_data.update({tick: div_data}) - except: - re_data.update({tick: None}) - return re_data +from yahoofinancials.maps import COUNTRY_MAP +from yahoofinancials.etl import YahooFinanceETL +from yahoofinancials.calcs import num_shares_outstanding, eps # Class containing methods to create stock data extracts class YahooFinancials(YahooFinanceETL): - + """ + Arguments + ---------- + tickers: str or list + Ticker or listed collection of tickers + Keyword Arguments + ----------------- + concurrent: bool, default False, optional + Defines whether the requests are made synchronously or asynchronously. + country: str, default 'US', optional + This allows you to alter the region, lang, corsDomain parameter sent with each request based on selected country + max_workers: int, default 8, optional + Defines the number of workers used to make concurrent requests. + Only relevant if concurrent=True + """ # Private method that handles financial statement extraction def _run_financial_stmt(self, statement_type, report_num, reformat): report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] @@ -711,32 +134,34 @@ def get_historical_price_data(self, start_date, end_date, time_interval): # Private Method for Functions needing stock_price_data def _stock_price_data(self, data_field): + price_data = self.get_stock_price_data() if isinstance(self.ticker, str): - if self.get_stock_price_data()[self.ticker] is None: + if price_data[self.ticker] is None: return None - return self.get_stock_price_data()[self.ticker].get(data_field, None) + return price_data[self.ticker].get(data_field) else: ret_obj = {} for tick in self.ticker: - if self.get_stock_price_data()[tick] is None: + if price_data[tick] is None: ret_obj.update({tick: None}) else: - ret_obj.update({tick: self.get_stock_price_data()[tick].get(data_field, None)}) + ret_obj.update({tick: price_data[tick].get(data_field)}) return ret_obj # Private Method for Functions needing stock_price_data def _stock_summary_data(self, data_field): + sum_data = self.get_summary_data() if isinstance(self.ticker, str): - if self.get_summary_data()[self.ticker] is None: + if sum_data[self.ticker] is None: return None - return self.get_summary_data()[self.ticker].get(data_field, None) + return sum_data[self.ticker].get(data_field, None) else: ret_obj = {} for tick in self.ticker: - if self.get_summary_data()[tick] is None: + if sum_data[tick] is None: ret_obj.update({tick: None}) else: - ret_obj.update({tick: self.get_summary_data()[tick].get(data_field, None)}) + ret_obj.update({tick: sum_data[tick].get(data_field, None)}) return ret_obj # Private Method for Functions needing financial statement data @@ -901,55 +326,25 @@ def get_earnings_per_share(self): price_data = self.get_current_price() pe_ratio = self.get_pe_ratio() if isinstance(self.ticker, str): - if price_data is not None and pe_ratio is not None: - return price_data / pe_ratio - else: - return None + return eps(price_data, pe_ratio) else: ret_obj = {} for tick in self.ticker: - if price_data[tick] is not None and pe_ratio[tick] is not None: - ret_obj.update({tick: price_data[tick] / pe_ratio[tick]}) - else: - ret_obj.update({tick: None}) + re_val = eps(price_data[tick], pe_ratio[tick]) + ret_obj.update({tick: re_val}) return ret_obj def get_num_shares_outstanding(self, price_type='current'): today_low = self._stock_summary_data('dayHigh') today_high = self._stock_summary_data('dayLow') cur_market_cap = self._stock_summary_data('marketCap') + current = self.get_current_price() if isinstance(self.ticker, str): - if cur_market_cap is not None: - if price_type == 'current': - current = self.get_current_price() - if current is not None: - today_average = current - else: - return None - else: - if today_high is not None and today_low is not None: - today_average = (today_high + today_low) / 2 - else: - return None - return cur_market_cap / today_average - else: - return None + return num_shares_outstanding(cur_market_cap, today_low, today_high, price_type) else: ret_obj = {} for tick in self.ticker: - if cur_market_cap[tick] is not None: - if price_type == 'current': - current = self.get_current_price() - if current[tick] is not None: - ret_obj.update({tick: cur_market_cap[tick] / current[tick]}) - else: - ret_obj.update({tick: None}) - else: - if today_low[tick] is not None and today_high[tick] is not None: - today_average = (today_high[tick] + today_low[tick]) / 2 - ret_obj.update({tick: cur_market_cap[tick] / today_average}) - else: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: None}) + re_data = num_shares_outstanding(cur_market_cap[tick], today_low[tick], + today_high[tick], price_type, current[tick]) + ret_obj.update({tick: re_data}) return ret_obj diff --git a/yahoofinancials/calcs.py b/yahoofinancials/calcs.py new file mode 100644 index 0000000..6fc8168 --- /dev/null +++ b/yahoofinancials/calcs.py @@ -0,0 +1,22 @@ +def eps(price_data, pe_ratio): + if price_data is not None and pe_ratio is not None: + return price_data / pe_ratio + else: + return None + + +def num_shares_outstanding(cur_market_cap, today_low, today_high, price_type, current): + if cur_market_cap is not None: + if price_type == 'current': + if current is not None: + today_average = current + else: + return None + else: + if today_high is not None and today_low is not None: + today_average = (today_high + today_low) / 2 + else: + return None + return cur_market_cap / today_average + else: + return None diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py new file mode 100644 index 0000000..d3a4f9a --- /dev/null +++ b/yahoofinancials/etl.py @@ -0,0 +1,659 @@ +import calendar +import re +from json import loads +import time +from bs4 import BeautifulSoup +import datetime +import pytz +import random +import logging +from urllib.request import FancyURLopener +import hashlib +from base64 import b64decode +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +import json +from functools import partial +from multiprocessing import Pool +from yahoofinancials.maps import COUNTRY_MAP + +# track the last get timestamp to add a minimum delay between gets - be nice! +_lastget = 0 +# logger = log_to_stderr(logging.DEBUG) + + +# Custom Exception class to handle custom error +class ManagedException(Exception): + pass + + +# Class used to open urls for financial data +class UrlOpener(FancyURLopener): + version = 'w3m/0.5.3+git20180125' + + +def decrypt_cryptojs_aes(data): + encrypted_stores = data['context']['dispatcher']['stores'] + _cs = data["_cs"] + _cr = data["_cr"] + _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) + password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + encrypted_stores = b64decode(encrypted_stores) + assert encrypted_stores[0:8] == b"Salted__" + salt = encrypted_stores[8:16] + encrypted_stores = encrypted_stores[16:] + + def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: + """OpenSSL EVP Key Derivation Function + Args: + password (Union[str, bytes, bytearray]): Password to generate key from. + salt (Union[bytes, bytearray]): Salt to use. + keySize (int, optional): Output key length in bytes. Defaults to 32. + ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. + iterations (int, optional): Number of iterations to perform. Defaults to 1. + hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. + Returns: + key, iv: Derived key and Initialization Vector (IV) bytes. + + Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 + OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 + """ + assert iterations > 0, "Iterations can not be less than 1." + if isinstance(password, str): + password = password.encode("utf-8") + final_length = keySize + ivSize + key_iv = b"" + block = None + while len(key_iv) < final_length: + hasher = hashlib.new(hashAlgorithm) + if block: + hasher.update(block) + hasher.update(password) + hasher.update(salt) + block = hasher.digest() + for _ in range(1, iterations): + block = hashlib.new(hashAlgorithm, block).digest() + key_iv += block + key, iv = key_iv[:keySize], key_iv[keySize:final_length] + return key, iv + + key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(plaintext) + unpadder.finalize() + plaintext = plaintext.decode("utf-8") + decoded_stores = json.loads(plaintext) + return decoded_stores + + +class YahooFinanceETL(object): + + def __init__(self, ticker, **kwargs): + self.ticker = ticker.upper() if isinstance(ticker, str) else [t.upper() for t in ticker] + self.country = kwargs.get("country", "US") + if self.country.upper() not in COUNTRY_MAP.keys(): + raise ReferenceError("invalid country: " + self.country) + self.concurrent = kwargs.get("concurrent", False) + self.max_workers = kwargs.get("max_workers", 8) + self._cache = {} + + # Minimum interval between Yahoo Finance requests for this instance + _MIN_INTERVAL = 7 + + # Meta-data dictionaries for the classes to use + YAHOO_FINANCIAL_TYPES = { + 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], + 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], + 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], + 'keystats': ['key-statistics'], + 'history': ['history'] + } + + # Interval value translation dictionary + _INTERVAL_DICT = { + 'daily': '1d', + 'weekly': '1wk', + 'monthly': '1mo' + } + + # Base Yahoo Finance URL for the class to build on + _BASE_YAHOO_URL = 'https://finance.yahoo.com/quote/' + + # private static method to get the appropriate report type identifier + @staticmethod + def get_report_type(frequency): + if frequency == 'annual': + report_num = 1 + else: + report_num = 2 + return report_num + + # Public static method to format date serial string to readable format and vice versa + @staticmethod + def format_date(in_date): + if isinstance(in_date, str): + form_date = int(calendar.timegm(time.strptime(in_date, '%Y-%m-%d'))) + else: + form_date = str((datetime.datetime(1970, 1, 1) + datetime.timedelta(seconds=in_date)).date()) + return form_date + + # Private Static Method to Convert Eastern Time to UTC + @staticmethod + def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): + utc = pytz.utc + eastern = pytz.timezone('US/Eastern') + date_ = datetime.datetime.strptime(date.replace(" 0:", " 12:"), mask) + date_eastern = eastern.localize(date_, is_dst=None) + date_utc = date_eastern.astimezone(utc) + return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') + + # Private method that determines number of workers to use in a process + def _get_worker_count(self): + workers = self.max_workers + if len(self.ticker) < workers: + workers = len(self.ticker) + return workers + + # Private method to execute a web scrape request and decrypt the return + def _request_handler(self, url): + urlopener = UrlOpener() + # Try to open the URL up to 10 times sleeping random time if something goes wrong + max_retry = 10 + for i in range(0, max_retry): + response = urlopener.open(url) + if response.getcode() != 200: + time.sleep(random.randrange(10, 20)) + response.close() + else: + response_content = response.read() + response.close() + soup = BeautifulSoup(response_content, "html.parser") + re_script = soup.find("script", string=re.compile("root.App.main")) + if re_script is not None: + script = re_script.string + re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) + if "_cs" in re_data and "_cr" in re_data: + re_data = decrypt_cryptojs_aes(re_data) + data = re_data + if "context" in re_data and "dispatcher" in re_data["context"]: # Keep old code, just in case + data = re_data['context']['dispatcher']['stores'] + try: + if data.get("QuoteSummaryStore"): + self._cache[url] = re_data + break + except AttributeError: + continue + else: + time.sleep(random.randrange(10, 20)) + if i == max_retry - 1: + # Raise a custom exception if we can't get the web page within max_retry attempts + raise ManagedException("Server replied with HTTP " + str(response.getcode()) + + " code while opening the url: " + str(url)) + + # Private method to scrape data from yahoo finance + def _scrape_data(self, url, tech_type, statement_type): + global _lastget + country_ent = COUNTRY_MAP.get(self.country.upper()) + meta_str = '&lang=' + country_ent.get("lang", "en-US") + '®ion=' + country_ent.get("region", "US") + url += meta_str + if not self._cache.get(url): + now = int(time.time()) + if _lastget and now - _lastget < self._MIN_INTERVAL: + time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) + now = int(time.time()) + _lastget = now + self._request_handler(url) + data = self._cache[url] + if "context" in data and "dispatcher" in data["context"]: # Keep old code, just in case + data = data['context']['dispatcher']['stores'] + if tech_type == '' and statement_type != 'history': + stores = data["QuoteSummaryStore"] + elif tech_type != '' and statement_type != 'history': + stores = data["QuoteSummaryStore"][tech_type] + else: + stores = data["HistoricalPriceStore"] + return stores + + # Private static method to determine if a numerical value is in the data object being cleaned + @staticmethod + def _determine_numeric_value(value_dict): + if 'raw' in value_dict.keys(): + numerical_val = value_dict['raw'] + else: + numerical_val = None + return numerical_val + + # Private method to format date serial string to readable format and vice versa + def _format_time(self, in_time): + form_date_time = datetime.datetime.fromtimestamp(int(in_time)).strftime('%Y-%m-%d %H:%M:%S') + return self._convert_to_utc(form_date_time) + + # Private method to return a sub dictionary entry for the earning report cleaning + def _get_cleaned_sub_dict_ent(self, key, val_list): + sub_list = [] + for rec in val_list: + sub_sub_dict = {} + for k, v in rec.items(): + if k == 'date': + sub_sub_dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + sub_sub_dict_ent = {k: numerical_val} + sub_sub_dict.update(sub_sub_dict_ent) + sub_list.append(sub_sub_dict) + return {key: sub_list} + + # Private method to process raw earnings data and clean + def _clean_earnings_data(self, raw_data): + cleaned_data = {} + earnings_key = 'earningsData' + financials_key = 'financialsData' + for k, v in raw_data.items(): + if k == 'earningsChart': + sub_dict = {} + for k2, v2 in v.items(): + if k2 == 'quarterly': + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + elif k2 == 'currentQuarterEstimate': + numerical_val = self._determine_numeric_value(v2) + sub_ent = {k2: numerical_val} + else: + sub_ent = {k2: v2} + sub_dict.update(sub_ent) + dict_ent = {earnings_key: sub_dict} + cleaned_data.update(dict_ent) + elif k == 'financialsChart': + sub_dict = {} + for k2, v2, in v.items(): + sub_ent = self._get_cleaned_sub_dict_ent(k2, v2) + sub_dict.update(sub_ent) + dict_ent = {financials_key: sub_dict} + cleaned_data.update(dict_ent) + else: + if k != 'maxAge': + dict_ent = {k: v} + cleaned_data.update(dict_ent) + return cleaned_data + + # Private method to clean summary and price reports + def _clean_reports(self, raw_data): + cleaned_dict = {} + if raw_data is None: + return None + for k, v in raw_data.items(): + if 'Time' in k: + formatted_utc_time = self._format_time(v) + dict_ent = {k: formatted_utc_time} + elif 'Date' in k: + try: + formatted_date = v['fmt'] + except (KeyError, TypeError): + formatted_date = '-' + dict_ent = {k: formatted_date} + elif v is None or isinstance(v, str) or isinstance(v, int) or isinstance(v, float): + dict_ent = {k: v} + else: + numerical_val = self._determine_numeric_value(v) + dict_ent = {k: numerical_val} + cleaned_dict.update(dict_ent) + return cleaned_dict + + # Private Static Method to ensure ticker is URL encoded + @staticmethod + def _encode_ticker(ticker_str): + encoded_ticker = ticker_str.replace('=', '%3D') + return encoded_ticker + + # Private method to get time interval code + def _build_historical_url(self, ticker, hist_oj): + url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ + '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ + hist_oj['interval'] + return url + + # Private Method to clean the dates of the newly returns historical stock data into readable format + def _clean_historical_data(self, hist_data, last_attempt=False): + data = {} + for k, v in hist_data.items(): + if k == 'eventsData': + event_obj = {} + if isinstance(v, list): + dict_ent = {k: event_obj} + else: + for type_key, type_obj in v.items(): + formatted_type_obj = {} + for date_key, date_obj in type_obj.items(): + formatted_date_key = self.format_date(int(date_key)) + cleaned_date = self.format_date(int(date_obj['date'])) + date_obj.update({'formatted_date': cleaned_date}) + formatted_type_obj.update({formatted_date_key: date_obj}) + event_obj.update({type_key: formatted_type_obj}) + dict_ent = {k: event_obj} + elif 'date' in k.lower(): + if v is not None: + cleaned_date = self.format_date(v) + dict_ent = {k: {'formatted_date': cleaned_date, 'date': v}} + else: + if last_attempt is False: + return None + else: + dict_ent = {k: {'formatted_date': None, 'date': v}} + elif isinstance(v, list): + sub_dict_list = [] + for sub_dict in v: + sub_dict['formatted_date'] = self.format_date(sub_dict['date']) + sub_dict_list.append(sub_dict) + dict_ent = {k: sub_dict_list} + else: + dict_ent = {k: v} + data.update(dict_ent) + return data + + # Private Static Method to build API url for GET Request + def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): + if events is None: + events = [" div", "split", "earn"] + event_str = '' + for idx, s in enumerate(events, start=1): + if idx < len(events): + event_str += s + "|" + elif idx == len(events): + event_str += s + base_url = "https://query" + v + ".finance.yahoo.com/v8/finance/chart/" + api_url = base_url + up_ticker + '?symbol=' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' + \ + str(hist_obj['end']) + '&interval=' + hist_obj['interval'] + country_ent = COUNTRY_MAP.get(self.country.upper()) + meta_str = '&lang=' + country_ent.get("lang", "en-US") + '®ion=' + country_ent.get("region", "US") + api_url += '&events=' + event_str + meta_str + return api_url + + # Private Method to get financial data via API Call + def _get_api_data(self, api_url, tries=0): + urlopener = UrlOpener() + response = urlopener.open(api_url) + if response.getcode() == 200: + res_content = response.read() + response.close() + return loads(res_content.decode('utf-8')) + else: + if tries < 5: + time.sleep(random.randrange(10, 20)) + tries += 1 + return self._get_api_data(api_url, tries) + else: + return None + + # Private Method to clean API data + def _clean_api_data(self, api_url): + raw_data = self._get_api_data(api_url) + ret_obj = {} + ret_obj.update({'eventsData': []}) + if raw_data is None: + return ret_obj + results = raw_data['chart']['result'] + if results is None: + return ret_obj + for result in results: + tz_sub_dict = {} + ret_obj.update({'eventsData': result.get('events', {})}) + ret_obj.update({'firstTradeDate': result['meta'].get('firstTradeDate', 'NA')}) + ret_obj.update({'currency': result['meta'].get('currency', 'NA')}) + ret_obj.update({'instrumentType': result['meta'].get('instrumentType', 'NA')}) + tz_sub_dict.update({'gmtOffset': result['meta']['gmtoffset']}) + ret_obj.update({'timeZone': tz_sub_dict}) + timestamp_list = result['timestamp'] + high_price_list = result['indicators']['quote'][0]['high'] + low_price_list = result['indicators']['quote'][0]['low'] + open_price_list = result['indicators']['quote'][0]['open'] + close_price_list = result['indicators']['quote'][0]['close'] + volume_list = result['indicators']['quote'][0]['volume'] + adj_close_list = result['indicators']['adjclose'][0]['adjclose'] + i = 0 + prices_list = [] + for timestamp in timestamp_list: + price_dict = {} + price_dict.update({'date': timestamp}) + price_dict.update({'high': high_price_list[i]}) + price_dict.update({'low': low_price_list[i]}) + price_dict.update({'open': open_price_list[i]}) + price_dict.update({'close': close_price_list[i]}) + price_dict.update({'volume': volume_list[i]}) + price_dict.update({'adjclose': adj_close_list[i]}) + prices_list.append(price_dict) + i += 1 + ret_obj.update({'prices': prices_list}) + return ret_obj + + # Private Method to Handle Recursive API Request + def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): + v = "2" + if i == 3: # After 3 tries querying against 'query2.finance.yahoo.com' try 'query1.finance.yahoo.com' instead + v = "1" + if clean: + re_data = self._clean_api_data(self._build_api_url(hist_obj, up_ticker, v)) + cleaned_re_data = self._clean_historical_data(re_data) + if cleaned_re_data is not None: + return cleaned_re_data + else: + re_data = self._get_api_data(self._build_api_url(hist_obj, up_ticker, v)) + if re_data is not None: + return re_data + if i < 6: + i += 1 + return self._recursive_api_request(hist_obj, up_ticker, clean, i) + elif clean: + return self._clean_historical_data(re_data, True) + + # Private Method to take scrapped data and build a data dictionary with, used by get_stock_data() + def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): + YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' + \ + up_ticker + if tech_type == '' and statement_type != 'history': + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} + except KeyError: + re_data = None + dict_ent = {up_ticker: re_data, 'dataType': report_name} + elif tech_type != '' and statement_type != 'history': + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + except KeyError: + re_data = None + dict_ent = {up_ticker: re_data} + else: + YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) + try: + cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) + except KeyError: + try: + re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) + cleaned_re_data = self._clean_historical_data(re_data) + except KeyError: + cleaned_re_data = None + dict_ent = {up_ticker: cleaned_re_data} + return dict_ent + + # Private method to return the stmt_id for the reformat_process + def _get_stmt_id(self, statement_type, raw_data): + stmt_id = '' + i = 0 + for key in raw_data.keys(): + if key in self.YAHOO_FINANCIAL_TYPES[statement_type.lower()]: + stmt_id = key + i += 1 + if i != 1: + return None + return stmt_id + + # Private Method for the Reformat Process + def _reformat_stmt_data_process(self, raw_data, statement_type): + final_data_list = [] + if raw_data is not None: + stmt_id = self._get_stmt_id(statement_type, raw_data) + if stmt_id is None: + return final_data_list + hashed_data_list = raw_data[stmt_id] + for data_item in hashed_data_list: + data_date = '' + sub_data_dict = {} + for k, v in data_item.items(): + if k == 'endDate': + data_date = v['fmt'] + elif k != 'maxAge': + numerical_val = self._determine_numeric_value(v) + sub_dict_item = {k: numerical_val} + sub_data_dict.update(sub_dict_item) + dict_item = {data_date: sub_data_dict} + final_data_list.append(dict_item) + return final_data_list + else: + return raw_data + + # Private Method to return subdict entry for the statement reformat process + def _get_sub_dict_ent(self, ticker, raw_data, statement_type): + form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) + return {ticker: form_data_list} + + # Public method to get time interval code + def get_time_code(self, time_interval): + interval_code = self._INTERVAL_DICT[time_interval.lower()] + return interval_code + + # Public Method to get stock data + def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): + data = {} + if isinstance(self.ticker, str): + dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + else: + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + dict_ents = pool.map(partial(self._create_dict_ent, + statement_type=statement_type, + tech_type=tech_type, + report_name=report_name, + hist_obj=hist_obj), self.ticker) + for dict_ent in dict_ents: + data.update(dict_ent) + else: + for tick in self.ticker: + try: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + except ManagedException: + logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", + str(tick), statement_type, str(ManagedException)) + continue + return data + + # Public Method to get technical stock data + def get_stock_tech_data(self, tech_type): + if tech_type == 'defaultKeyStatistics': + return self.get_stock_data(statement_type='keystats', tech_type=tech_type) + else: + return self.get_stock_data(tech_type=tech_type) + + # Public Method to get reformatted statement data + def get_reformatted_stmt_data(self, raw_data, statement_type): + sub_dict, data_dict = {}, {} + data_type = raw_data['dataType'] + if isinstance(self.ticker, str): + sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {data_type: sub_dict} + data_dict.update(dict_ent) + else: + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + # _get_sub_dict_ent(self, ticker, raw_data, statement_type) + sub_dict_ents = pool.map(partial(self._get_sub_dict_ent, + raw_data=raw_data, + statement_type=statement_type), self.ticker) + for dict_ent in sub_dict_ents: + sub_dict.update(dict_ent) + else: + for tick in self.ticker: + sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) + sub_dict.update(sub_dict_ent) + dict_ent = {data_type: sub_dict} + data_dict.update(dict_ent) + return data_dict + + # Public method to get cleaned report data + def _clean_data_process(self, tick, report_type, raw_report_data): + if report_type == 'earnings': + try: + cleaned_data = self._clean_earnings_data(raw_report_data[tick]) + except: + cleaned_data = None + else: + try: + cleaned_data = self._clean_reports(raw_report_data[tick]) + except: + cleaned_data = None + return cleaned_data + + # Public method to get cleaned summary and price report data + def get_clean_data(self, raw_report_data, report_type): + cleaned_data_dict = {} + if isinstance(self.ticker, str): + cleaned_data = self._clean_data_process(self.ticker, report_type, raw_report_data) + cleaned_data_dict.update({self.ticker: cleaned_data}) + else: + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + cleaned_data_list = pool.map(partial(self._clean_data_process, + report_type=report_type, + raw_report_data=raw_report_data), self.ticker) + for idx, cleaned_data in enumerate(cleaned_data_list): + cleaned_data_dict.update({self.ticker[idx]: cleaned_data}) + else: + for tick in self.ticker: + cleaned_data = self._clean_data_process(tick, report_type, raw_report_data) + cleaned_data_dict.update({tick: cleaned_data}) + return cleaned_data_dict + + # Private method to handle dividend data requests + def _handle_api_dividend_request(self, cur_ticker, start, end, interval): + re_dividends = [] + hist_obj = {"start": start, "end": end, "interval": interval} + div_dict = self._recursive_api_request(hist_obj, cur_ticker, False)['chart']['result'][0]['events']['dividends'] + for div_time_key, div_obj in div_dict.items(): + dividend_obj = { + 'date': div_obj['date'], + 'formatted_date': self.format_date(int(div_obj['date'])), + 'amount': div_obj.get('amount', None) + } + re_dividends.append(dividend_obj) + return sorted(re_dividends, key=lambda div: div['date']) + + # Public method to get daily dividend data + def get_stock_dividend_data(self, start, end, interval): + interval_code = self.get_time_code(interval) + if isinstance(self.ticker, str): + try: + return {self.ticker: self._handle_api_dividend_request(self.ticker, start, end, interval_code)} + except: + return {self.ticker: None} + else: + re_data = {} + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + div_data_list = pool.map(partial(self._handle_api_dividend_request, + start=start, + end=end, + interval=interval_code), self.ticker) + for idx, div_data in enumerate(div_data_list): + re_data.update({self.ticker[idx]: div_data}) + else: + for tick in self.ticker: + # TODO ASYNC + try: + div_data = self._handle_api_dividend_request(tick, start, end, interval_code) + re_data.update({tick: div_data}) + except: + re_data.update({tick: None}) + return re_data diff --git a/yahoofinancials/maps.py b/yahoofinancials/maps.py new file mode 100644 index 0000000..d6caef7 --- /dev/null +++ b/yahoofinancials/maps.py @@ -0,0 +1,16 @@ +COUNTRY_MAP = { + "FR": {"lang": "fr-FR", "region": "FR", "corsDomain": "fr.finance.yahoo.com"}, + "IN": {"lang": "en-IN", "region": "IN", "corsDomain": "in.finance.yahoo.com"}, + "HK": {"lang": "zh-Hant-HK", "region": "HK", "corsDomain": "hk.finance.yahoo.com"}, + "DE": {"lang": "de-DE", "region": "DE", "corsDomain": "de.finance.yahoo.com"}, + "CA": {"lang": "en-CA", "region": "CA", "corsDomain": "ca.finance.yahoo.com"}, + "ES": {"lang": "es-ES", "region": "ES", "corsDomain": "es.finance.yahoo.com"}, + "IT": {"lang": "it-IT", "region": "IT", "corsDomain": "it.finance.yahoo.com"}, + "US": {"lang": "en-US", "region": "US", "corsDomain": "finance.yahoo.com"}, + "AU": {"lang": "en-AU", "region": "AU", "corsDomain": "au.finance.yahoo.com"}, + "UK": {"lang": "en-GB", "region": "GB", "corsDomain": "uk.finance.yahoo.com"}, + "BR": {"lang": "pt-BR", "region": "BR", "corsDomain": "br.financas.yahoo.com"}, + "NZ": {"lang": "en-NZ", "region": "NZ", "corsDomain": "nz.finance.yahoo.com"}, + "SG": {"lang": "en-SG", "region": "SG", "corsDomain": "sg.finance.yahoo.com"}, + "TW": {"lang": "zh-tw", "region": "TW", "corsDomain": "tw.finance.yahoo.com"}, +} From e9d1fc10402dfe2caddd968ddc34300a8e0d2490 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 13 Jan 2023 21:39:28 -0600 Subject: [PATCH 084/108] fixed new data encrpytion bug, replace urllib with requests --- CHANGES | 2 + README.rst | 2 +- setup.py | 4 +- test/test_yahoofinancials.py | 2 +- yahoofinancials/__init__.py | 3 ++ yahoofinancials/etl.py | 99 ++++++++++++++++++++++-------------- 6 files changed, 72 insertions(+), 40 deletions(-) diff --git a/CHANGES b/CHANGES index 55a341e..3e6f833 100644 --- a/CHANGES +++ b/CHANGES @@ -35,4 +35,6 @@ 1.9 01/14/2023 -- Added new optional concurrent input to YahooFinancials(), if True extraction will run async. 1.9 01/14/2023 -- Added new optional meta input to YahooFinancials(), currently supports language & region. 1.9 01/14/2023 -- Updated beautifulsoup4 find to use string parameter instead deprecated text parameter. +1.9 01/14/2023 -- Replace urllib with Requests due to deprecation warning. +1.9 01/14/2023 -- Fixed new data encryption issue. diff --git a/README.rst b/README.rst index b97c60f..d7c55e8 100644 --- a/README.rst +++ b/README.rst @@ -23,7 +23,7 @@ A powerful financial data module used for pulling both fundamental and technical from yahoofinancials import YahooFinancials tickers = ['AAPL', 'GOOG', 'C'] - yahoo_financials = YahooFinancials(tickers, concurrent=True) + yahoo_financials = YahooFinancials(tickers, concurrent=True, max_workers=8, country="US") balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') print(balance_sheet_data_qt) diff --git a/setup.py b/setup.py index e31fe0a..677973a 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,9 @@ install_requires=[ "beautifulsoup4>=4.11.1", "pytz", - "cryptography>=3.3.2" + "cryptography>=3.3.2", + "requests>=2.26", + "pycryptodome>=3.6.6" ], classifiers=[ 'Development Status :: 5 - Production/Stable', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 2920cf2..75eff02 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -40,7 +40,7 @@ def check_fundamental(test_data, test_type): class TestModule(TestCase): def setUp(self): - self.test_yf_stock_single = YahooFinancials('C', country='UK') + self.test_yf_stock_single = YahooFinancials('C') self.test_yf_stock_multi = YahooFinancials(stocks) self.test_yf_treasuries_single = YahooFinancials('^IRX') self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index c0f5bdf..8757bc3 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -46,6 +46,9 @@ from yahoofinancials.etl import YahooFinanceETL from yahoofinancials.calcs import num_shares_outstanding, eps +__version__ = "1.9" +__author__ = "Connor Sanders" + # Class containing methods to create stock data extracts class YahooFinancials(YahooFinanceETL): diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index d3a4f9a..34d2dac 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -7,15 +7,21 @@ import pytz import random import logging -from urllib.request import FancyURLopener +import requests as requests import hashlib from base64 import b64decode -from cryptography.hazmat.primitives import padding -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes import json from functools import partial from multiprocessing import Pool from yahoofinancials.maps import COUNTRY_MAP +usePycryptodome = False +if usePycryptodome: + from Crypto.Cipher import AES + from Crypto.Util.Padding import unpad +else: + from cryptography.hazmat.primitives import padding + from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -27,17 +33,31 @@ class ManagedException(Exception): pass -# Class used to open urls for financial data -class UrlOpener(FancyURLopener): - version = 'w3m/0.5.3+git20180125' +# Class used to get data from urls +class UrlOpener: + + user_agent_headers = { + 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' + } + + def __init__(self, session=None): + self._session = session or requests + + def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30): + response = self._session.get( + url=url, + params=params, + proxies=proxy, + timeout=timeout, + headers=user_agent_headers or self.user_agent_headers + ) + return response def decrypt_cryptojs_aes(data): encrypted_stores = data['context']['dispatcher']['stores'] - _cs = data["_cs"] - _cr = data["_cr"] - _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) - password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + password_key = next(key for key in data.keys() if key not in ["context", "plugins"]) + password = data[password_key] encrypted_stores = b64decode(encrypted_stores) assert encrypted_stores[0:8] == b"Salted__" salt = encrypted_stores[8:16] @@ -54,7 +74,6 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. Returns: key, iv: Derived key and Initialization Vector (IV) bytes. - Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 """ @@ -74,16 +93,22 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m for _ in range(1, iterations): block = hashlib.new(hashAlgorithm, block).digest() key_iv += block + key, iv = key_iv[:keySize], key_iv[keySize:final_length] return key, iv key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() - unpadder = padding.PKCS7(128).unpadder() - plaintext = unpadder.update(plaintext) + unpadder.finalize() - plaintext = plaintext.decode("utf-8") + if usePycryptodome: + cipher = AES.new(key, AES.MODE_CBC, iv=iv) + plaintext = cipher.decrypt(encrypted_stores) + plaintext = unpad(plaintext, 16, style="pkcs7") + else: + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(plaintext) + unpadder.finalize() + plaintext = plaintext.decode("utf-8") decoded_stores = json.loads(plaintext) return decoded_stores @@ -163,33 +188,33 @@ def _request_handler(self, url): max_retry = 10 for i in range(0, max_retry): response = urlopener.open(url) - if response.getcode() != 200: + if response.status_code != 200: time.sleep(random.randrange(10, 20)) response.close() else: - response_content = response.read() - response.close() - soup = BeautifulSoup(response_content, "html.parser") - re_script = soup.find("script", string=re.compile("root.App.main")) + # response_content = response.read() + soup = BeautifulSoup(response.content, "html.parser") + re_script = soup.find("script", string=re.compile("root.App.main")).text + # re_script = soup.find("script", string=re.compile("root.App.main")) + # response.close() if re_script is not None: - script = re_script.string - re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", script).group(1)) - if "_cs" in re_data and "_cr" in re_data: - re_data = decrypt_cryptojs_aes(re_data) - data = re_data + re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) if "context" in re_data and "dispatcher" in re_data["context"]: # Keep old code, just in case data = re_data['context']['dispatcher']['stores'] - try: - if data.get("QuoteSummaryStore"): - self._cache[url] = re_data - break - except AttributeError: - continue + if "QuoteSummaryStore" not in data: + data = decrypt_cryptojs_aes(re_data) + try: + if data.get("QuoteSummaryStore"): + self._cache[url] = data + break + except AttributeError: + time.sleep(random.randrange(10, 20)) + continue else: time.sleep(random.randrange(10, 20)) if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with HTTP " + str(response.getcode()) + + raise ManagedException("Server replied with HTTP " + str(response.status_code) + " code while opening the url: " + str(url)) # Private method to scrape data from yahoo finance @@ -373,10 +398,10 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): def _get_api_data(self, api_url, tries=0): urlopener = UrlOpener() response = urlopener.open(api_url) - if response.getcode() == 200: - res_content = response.read() + if response.status_code == 200: + res_content = response.text response.close() - return loads(res_content.decode('utf-8')) + return loads(res_content) else: if tries < 5: time.sleep(random.randrange(10, 20)) From 552b6c175cc3111490cde23a60e58a740db0b476 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 13 Jan 2023 21:43:04 -0600 Subject: [PATCH 085/108] fixed new data encrpytion bug, replace urllib with requests --- README.rst | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d7c55e8..cabea19 100644 --- a/README.rst +++ b/README.rst @@ -31,7 +31,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- - yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11. -- This package depends on beautifulsoup4, pytz, and cryptography to work. +- This package depends on beautifulsoup4, pytz, requests, and cryptography to work. 1. Installation using pip: diff --git a/setup.py b/setup.py index 677973a..75f04be 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,8 @@ "beautifulsoup4>=4.11.1", "pytz", "cryptography>=3.3.2", + # "pycryptodome>=3.6.6" "requests>=2.26", - "pycryptodome>=3.6.6" ], classifiers=[ 'Development Status :: 5 - Production/Stable', From 877e4d30556b5da169b66b6e3792c1a4e35afbfa Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 13 Jan 2023 21:57:06 -0600 Subject: [PATCH 086/108] fixed new data encrpytion bug, replace urllib with requests --- yahoofinancials/__init__.py | 3 +++ yahoofinancials/etl.py | 14 +++++--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 8757bc3..7f58b55 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -66,6 +66,9 @@ class YahooFinancials(YahooFinanceETL): max_workers: int, default 8, optional Defines the number of workers used to make concurrent requests. Only relevant if concurrent=True + timeout: int, default 30, optional + Defines how long a request will stay open. + Only relevant if concurrent=True """ # Private method that handles financial statement extraction def _run_financial_stmt(self, statement_type, report_num, reformat): diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 34d2dac..c0f6fab 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -122,6 +122,7 @@ def __init__(self, ticker, **kwargs): raise ReferenceError("invalid country: " + self.country) self.concurrent = kwargs.get("concurrent", False) self.max_workers = kwargs.get("max_workers", 8) + self.timeout = kwargs.get("timeout", 30) self._cache = {} # Minimum interval between Yahoo Finance requests for this instance @@ -187,19 +188,16 @@ def _request_handler(self, url): # Try to open the URL up to 10 times sleeping random time if something goes wrong max_retry = 10 for i in range(0, max_retry): - response = urlopener.open(url) + response = urlopener.open(url, timeout=self.timeout) if response.status_code != 200: time.sleep(random.randrange(10, 20)) response.close() else: - # response_content = response.read() soup = BeautifulSoup(response.content, "html.parser") re_script = soup.find("script", string=re.compile("root.App.main")).text - # re_script = soup.find("script", string=re.compile("root.App.main")) - # response.close() if re_script is not None: re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) - if "context" in re_data and "dispatcher" in re_data["context"]: # Keep old code, just in case + if "context" in re_data and "dispatcher" in re_data["context"]: data = re_data['context']['dispatcher']['stores'] if "QuoteSummaryStore" not in data: data = decrypt_cryptojs_aes(re_data) @@ -231,7 +229,7 @@ def _scrape_data(self, url, tech_type, statement_type): _lastget = now self._request_handler(url) data = self._cache[url] - if "context" in data and "dispatcher" in data["context"]: # Keep old code, just in case + if "context" in data and "dispatcher" in data["context"]: data = data['context']['dispatcher']['stores'] if tech_type == '' and statement_type != 'history': stores = data["QuoteSummaryStore"] @@ -397,7 +395,7 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): # Private Method to get financial data via API Call def _get_api_data(self, api_url, tries=0): urlopener = UrlOpener() - response = urlopener.open(api_url) + response = urlopener.open(api_url, timeout=self.timeout) if response.status_code == 200: res_content = response.text response.close() @@ -593,7 +591,6 @@ def get_reformatted_stmt_data(self, raw_data, statement_type): else: if self.concurrent: with Pool(self._get_worker_count()) as pool: - # _get_sub_dict_ent(self, ticker, raw_data, statement_type) sub_dict_ents = pool.map(partial(self._get_sub_dict_ent, raw_data=raw_data, statement_type=statement_type), self.ticker) @@ -675,7 +672,6 @@ def get_stock_dividend_data(self, start, end, interval): re_data.update({self.ticker[idx]: div_data}) else: for tick in self.ticker: - # TODO ASYNC try: div_data = self._handle_api_dividend_request(tick, start, end, interval_code) re_data.update({tick: div_data}) From 84e9b9c9cab10719907d8014840f07cdf2ba2e91 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 00:10:55 -0600 Subject: [PATCH 087/108] fixed new data encrpytion bug, replace urllib with requests --- CHANGES | 5 ++++- README.rst | 11 ++++++++++- yahoofinancials/__init__.py | 13 +++++++++++-- yahoofinancials/etl.py | 25 ++++++++++++++++++++----- 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/CHANGES b/CHANGES index 3e6f833..a10ba38 100644 --- a/CHANGES +++ b/CHANGES @@ -36,5 +36,8 @@ 1.9 01/14/2023 -- Added new optional meta input to YahooFinancials(), currently supports language & region. 1.9 01/14/2023 -- Updated beautifulsoup4 find to use string parameter instead deprecated text parameter. 1.9 01/14/2023 -- Replace urllib with Requests due to deprecation warning. -1.9 01/14/2023 -- Fixed new data encryption issue. +1.9 01/14/2023 -- Merged in pull request from shaunpatterson. +1.9 01/14/2023 -- Merged in pull request from flipdazed. +1.9 01/14/2023 -- Added basic proxy support. +1.9 01/14/2023 -- Fixed new data encryption issue & hardened. diff --git a/README.rst b/README.rst index cabea19..d663740 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,7 @@ Overview -------- A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance. -- As of Version 1.9, YahooFinancials supports asynchronous execution and international requests. +- As of Version 1.9, YahooFinancials supports optional parameters for asynchronous execution, proxies, and international requests. .. code-block:: python @@ -27,6 +27,15 @@ A powerful financial data module used for pulling both fundamental and technical balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') print(balance_sheet_data_qt) + proxy_addresses = [ "mysuperproxy.com:5000", "mysuperproxy.com:5001"] + yahoo_financials = YahooFinancials(tickers, concurrent=True, proxies=proxy_addresses) + balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') + print(balance_sheet_data_qt) + +- New methods in Version 1.9: + get_stock_profile_data() + get_financial_data() + Installation ------------- diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 7f58b55..459abba 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -68,7 +68,8 @@ class YahooFinancials(YahooFinanceETL): Only relevant if concurrent=True timeout: int, default 30, optional Defines how long a request will stay open. - Only relevant if concurrent=True + proxies: str or list, default None, optional + Defines any proxies to use during this instantiation. """ # Private method that handles financial statement extraction def _run_financial_stmt(self, statement_type, report_num, reformat): @@ -113,6 +114,14 @@ def get_stock_earnings_data(self, reformat=True): else: return self.get_stock_tech_data('earnings') + # Public Method for the user to return financial data + def get_financial_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_data(statement_type='keystats', tech_type='financialData'), + 'financialData') + else: + return self.get_stock_data(statement_type='keystats', tech_type='financialData') + # Public Method for the user to get stock summary data def get_summary_data(self, reformat=True): if reformat: @@ -346,7 +355,7 @@ def get_num_shares_outstanding(self, price_type='current'): cur_market_cap = self._stock_summary_data('marketCap') current = self.get_current_price() if isinstance(self.ticker, str): - return num_shares_outstanding(cur_market_cap, today_low, today_high, price_type) + return num_shares_outstanding(cur_market_cap, today_low, today_high, price_type, current) else: ret_obj = {} for tick in self.ticker: diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index c0f6fab..ed9646d 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -56,8 +56,14 @@ def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30 def decrypt_cryptojs_aes(data): encrypted_stores = data['context']['dispatcher']['stores'] - password_key = next(key for key in data.keys() if key not in ["context", "plugins"]) - password = data[password_key] + if "_cs" in data and "_cr" in data: + _cs = data["_cs"] + _cr = data["_cr"] + _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) + password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() + else: + password_key = next(key for key in data.keys() if key not in ["context", "plugins"]) + password = data[password_key] encrypted_stores = b64decode(encrypted_stores) assert encrypted_stores[0:8] == b"Salted__" salt = encrypted_stores[8:16] @@ -93,7 +99,6 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m for _ in range(1, iterations): block = hashlib.new(hashAlgorithm, block).digest() key_iv += block - key, iv = key_iv[:keySize], key_iv[keySize:final_length] return key, iv @@ -123,6 +128,7 @@ def __init__(self, ticker, **kwargs): self.concurrent = kwargs.get("concurrent", False) self.max_workers = kwargs.get("max_workers", 8) self.timeout = kwargs.get("timeout", 30) + self.proxies = kwargs.get("proxies") self._cache = {} # Minimum interval between Yahoo Finance requests for this instance @@ -175,6 +181,15 @@ def _convert_to_utc(date, mask='%Y-%m-%d %H:%M:%S'): date_utc = date_eastern.astimezone(utc) return date_utc.strftime('%Y-%m-%d %H:%M:%S %Z%z') + # _get_proxy randomly picks a proxy in the proxies list if not None + def _get_proxy(self): + if self.proxies: + proxy_str = self.proxies + if isinstance(self.proxies, list): + proxy_str = random.choice(self.proxies) + return {"https": proxy_str} + return None + # Private method that determines number of workers to use in a process def _get_worker_count(self): workers = self.max_workers @@ -188,7 +203,7 @@ def _request_handler(self, url): # Try to open the URL up to 10 times sleeping random time if something goes wrong max_retry = 10 for i in range(0, max_retry): - response = urlopener.open(url, timeout=self.timeout) + response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout) if response.status_code != 200: time.sleep(random.randrange(10, 20)) response.close() @@ -395,7 +410,7 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): # Private Method to get financial data via API Call def _get_api_data(self, api_url, tries=0): urlopener = UrlOpener() - response = urlopener.open(api_url, timeout=self.timeout) + response = urlopener.open(api_url, proxy=self._get_proxy(), timeout=self.timeout) if response.status_code == 200: res_content = response.text response.close() From 5cfbe98df01d2baac9b28ee76ab223478fd7a096 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 00:20:13 -0600 Subject: [PATCH 088/108] cleaned, further decryption hardening + proxies --- CHANGES | 4 ++-- README.rst | 1 + yahoofinancials/__init__.py | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index a10ba38..357bcf2 100644 --- a/CHANGES +++ b/CHANGES @@ -33,11 +33,11 @@ 1.9 01/14/2023 -- Added official Python 3.11 support. 1.9 01/14/2023 -- General code cleanup & quality improvements. 1.9 01/14/2023 -- Added new optional concurrent input to YahooFinancials(), if True extraction will run async. -1.9 01/14/2023 -- Added new optional meta input to YahooFinancials(), currently supports language & region. +1.9 01/14/2023 -- Added new optional country input to YahooFinancials(), currently supports language & region. +1.9 01/14/2023 -- Added new optional proxies input to YahooFinancials(), currently supports a list or string. 1.9 01/14/2023 -- Updated beautifulsoup4 find to use string parameter instead deprecated text parameter. 1.9 01/14/2023 -- Replace urllib with Requests due to deprecation warning. 1.9 01/14/2023 -- Merged in pull request from shaunpatterson. 1.9 01/14/2023 -- Merged in pull request from flipdazed. -1.9 01/14/2023 -- Added basic proxy support. 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. diff --git a/README.rst b/README.rst index 448f0f1..2373e2f 100644 --- a/README.rst +++ b/README.rst @@ -165,6 +165,7 @@ Additional Module Methods - get_earnings_per_share() - get_key_statistics_data() - get_stock_profile_data() +- get_financial_data() Usage Examples -------------- diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 459abba..ac966ff 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -107,6 +107,14 @@ def get_key_statistics_data(self, reformat=True): else: return self.get_stock_tech_data('defaultKeyStatistics') + # Public Method for the user to get company profile data + def get_stock_profile_data(self, reformat=True): + if reformat: + return self.get_clean_data( + self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') + else: + return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') + # Public Method for the user to get stock earnings data def get_stock_earnings_data(self, reformat=True): if reformat: From 92983939784932377ed8695303489ec87d3a719b Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 00:22:50 -0600 Subject: [PATCH 089/108] cleaned, further decryption hardening + proxies --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2373e2f..2daa97a 100644 --- a/README.rst +++ b/README.rst @@ -33,8 +33,8 @@ A powerful financial data module used for pulling both fundamental and technical print(balance_sheet_data_qt) - New methods in Version 1.9: - get_stock_profile_data() - get_financial_data() + - get_stock_profile_data() + - get_financial_data() Installation From a6ecf2ca2756d777065ab747579cea8e16c94f8d Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 00:38:59 -0600 Subject: [PATCH 090/108] code cleanup --- demo.py | 3 ++- test/test_yahoofinancials.py | 10 +++++----- yahoofinancials/__init__.py | 5 +++-- yahoofinancials/etl.py | 27 +++++++++++++++------------ 4 files changed, 25 insertions(+), 20 deletions(-) diff --git a/demo.py b/demo.py index 20dcc77..0ae759d 100755 --- a/demo.py +++ b/demo.py @@ -11,10 +11,11 @@ """ from __future__ import print_function + import sys import time -from yahoofinancials import YahooFinancials as YF +from yahoofinancials import YahooFinancials as YF DEFAULT_ARGS = ('MMM', 'AAPL') MODULE_ARGS = ('yf', 'yahoofinancial', 'yahoofinancials') diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 75eff02..20ecd42 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -5,9 +5,9 @@ # Copyright (c) 2023 Connor Sanders # MIT License -from yahoofinancials import YahooFinancials -from unittest import main as test_main, SkipTest, TestCase +from unittest import main as test_main, TestCase +from yahoofinancials import YahooFinancials # Test Configuration Variables stocks = ['AAPL', 'MSFT', 'C', 'IL&FSTRANS.NS'] @@ -24,13 +24,13 @@ def check_fundamental(test_data, test_type): return False elif test_type == 'inc': if 'incomeStatementHistoryQuarterly' in test_data and \ - test_data['incomeStatementHistoryQuarterly']['C'] is not None: + test_data['incomeStatementHistoryQuarterly']['C'] is not None: return True else: return False elif test_type == 'all': if 'balanceSheetHistoryQuarterly' in test_data and 'incomeStatementHistoryQuarterly' in test_data and \ - 'cashflowStatementHistoryQuarterly' in test_data: + 'cashflowStatementHistoryQuarterly' in test_data: return True else: return False @@ -116,7 +116,7 @@ def test_yf_concurrency(self): multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance') multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income') multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', - ['income', 'cash', 'balance']) + ['income', 'cash', 'balance']) # Multi stock check result = check_fundamental(multi_balance_sheet_data_qt, 'bal') self.assertEqual(result, True) diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index ac966ff..f53327b 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -42,9 +42,9 @@ historical_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') """ -from yahoofinancials.maps import COUNTRY_MAP -from yahoofinancials.etl import YahooFinanceETL from yahoofinancials.calcs import num_shares_outstanding, eps +from yahoofinancials.etl import YahooFinanceETL +from yahoofinancials.maps import COUNTRY_MAP __version__ = "1.9" __author__ = "Connor Sanders" @@ -71,6 +71,7 @@ class YahooFinancials(YahooFinanceETL): proxies: str or list, default None, optional Defines any proxies to use during this instantiation. """ + # Private method that handles financial statement extraction def _run_financial_stmt(self, statement_type, report_num, reformat): report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index ed9646d..3a06e42 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -1,19 +1,22 @@ import calendar -import re -from json import loads -import time -from bs4 import BeautifulSoup import datetime -import pytz -import random -import logging -import requests as requests import hashlib -from base64 import b64decode import json +import logging +import random +import re +import time +from base64 import b64decode from functools import partial +from json import loads from multiprocessing import Pool + +import pytz +import requests as requests +from bs4 import BeautifulSoup + from yahoofinancials.maps import COUNTRY_MAP + usePycryptodome = False if usePycryptodome: from Crypto.Cipher import AES @@ -22,9 +25,10 @@ from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes - # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 + + # logger = log_to_stderr(logging.DEBUG) @@ -35,7 +39,6 @@ class ManagedException(Exception): # Class used to get data from urls class UrlOpener: - user_agent_headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' } @@ -401,7 +404,7 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): event_str += s base_url = "https://query" + v + ".finance.yahoo.com/v8/finance/chart/" api_url = base_url + up_ticker + '?symbol=' + up_ticker + '&period1=' + str(hist_obj['start']) + '&period2=' + \ - str(hist_obj['end']) + '&interval=' + hist_obj['interval'] + str(hist_obj['end']) + '&interval=' + hist_obj['interval'] country_ent = COUNTRY_MAP.get(self.country.upper()) meta_str = '&lang=' + country_ent.get("lang", "en-US") + '®ion=' + country_ent.get("region", "US") api_url += '&events=' + event_str + meta_str From f6e9b0e0dc51a168bfc737cf48926a2f4619b2bc Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 01:04:09 -0600 Subject: [PATCH 091/108] code cleanup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 75f04be..8f9744b 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10' + 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11' ], zip_safe=False From bf9e2ff629fdc6561b302387c86896039ea7c266 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 01:06:20 -0600 Subject: [PATCH 092/108] code cleanup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8f9744b..76a0c2f 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11' + 'Programming Language :: Python :: 3.11', ], zip_safe=False ) From d8fe59943f9eac55124accd4a23614b95751535d Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 01:37:33 -0600 Subject: [PATCH 093/108] concurrent runtime bug --- CHANGES | 1 + README.rst | 2 +- setup.py | 8 ++++---- test/test_yahoofinancials.py | 2 +- yahoofinancials/__init__.py | 4 ++-- yahoofinancials/etl.py | 8 ++++++++ 6 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGES b/CHANGES index 357bcf2..9e7cc29 100644 --- a/CHANGES +++ b/CHANGES @@ -40,4 +40,5 @@ 1.9 01/14/2023 -- Merged in pull request from shaunpatterson. 1.9 01/14/2023 -- Merged in pull request from flipdazed. 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. +1.10 01/14/2023 -- Fixed concurrent runtime bug. diff --git a/README.rst b/README.rst index 2daa97a..da1c4e7 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.9 +Current Version: v1.10 Version Released: 01/14/2023 diff --git a/setup.py b/setup.py index 75f04be..fc565e8 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.9', + version='1.10', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.9.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.10.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', @@ -39,8 +39,8 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10' - 'Programming Language :: Python :: 3.11' + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], zip_safe=False ) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 20ecd42..32f8f2b 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,4 +1,4 @@ -# YahooFinancials Unit Tests v1.9 +# YahooFinancials Unit Tests v1.10 # Version Released: 01/14/2023 # Author: Connor Sanders # Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index f53327b..6d0d60a 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,7 +1,7 @@ """ ============================== The Yahoo Financials Module -Version: 1.9 +Version: 1.10 ============================== Author: Connor Sanders @@ -46,7 +46,7 @@ from yahoofinancials.etl import YahooFinanceETL from yahoofinancials.maps import COUNTRY_MAP -__version__ = "1.9" +__version__ = "1.10" __author__ = "Connor Sanders" diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 3a06e42..315c0e2 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -579,6 +579,8 @@ def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj=hist_obj), self.ticker) for dict_ent in dict_ents: data.update(dict_ent) + pool.close() + pool.join() else: for tick in self.ticker: try: @@ -614,6 +616,8 @@ def get_reformatted_stmt_data(self, raw_data, statement_type): statement_type=statement_type), self.ticker) for dict_ent in sub_dict_ents: sub_dict.update(dict_ent) + pool.close() + pool.join() else: for tick in self.ticker: sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) @@ -650,6 +654,8 @@ def get_clean_data(self, raw_report_data, report_type): raw_report_data=raw_report_data), self.ticker) for idx, cleaned_data in enumerate(cleaned_data_list): cleaned_data_dict.update({self.ticker[idx]: cleaned_data}) + pool.close() + pool.join() else: for tick in self.ticker: cleaned_data = self._clean_data_process(tick, report_type, raw_report_data) @@ -688,6 +694,8 @@ def get_stock_dividend_data(self, start, end, interval): interval=interval_code), self.ticker) for idx, div_data in enumerate(div_data_list): re_data.update({self.ticker[idx]: div_data}) + pool.close() + pool.join() else: for tick in self.ticker: try: From f3c0fbe9bc9a6968d7ac63e10b1af8836b18030b Mon Sep 17 00:00:00 2001 From: connorsanders Date: Sat, 14 Jan 2023 02:19:35 -0600 Subject: [PATCH 094/108] concurrent runtime bug --- yahoofinancials/__init__.py | 379 +----------------------------------- yahoofinancials/yf.py | 373 +++++++++++++++++++++++++++++++++++ 2 files changed, 380 insertions(+), 372 deletions(-) create mode 100644 yahoofinancials/yf.py diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 6d0d60a..3ee9aad 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,374 +1,9 @@ -""" -============================== -The Yahoo Financials Module -Version: 1.10 -============================== +from multiprocessing import freeze_support, set_start_method -Author: Connor Sanders -Email: sandersconnor1@gmail.com -Version Released: 01/14/2023 -Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 -Copyright (c) 2023 Connor Sanders -MIT License - -List of Included Functions: - -1) get_financial_stmts(frequency, statement_type, reformat=True) - - frequency can be either 'annual' or 'quarterly'. - - statement_type can be 'income', 'balance', 'cash'. - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -2) get_stock_price_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -3) get_stock_earnings_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -4) get_summary_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -5) get_stock_quote_type_data() -6) get_historical_price_data(start_date, end_date, time_interval) - - Gets historical price data for currencies, stocks, indexes, cryptocurrencies, and commodity futures. - - start_date should be entered in the 'YYYY-MM-DD' format. First day that financial data will be pulled. - - end_date should be entered in the 'YYYY-MM-DD' format. Last day that financial data will be pulled. - - time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. - -Usage Examples: -from yahoofinancials import YahooFinancials -#tickers = 'AAPL' -#or -tickers = ['AAPL', 'WFC', 'F', 'JPY=X', 'XRP-USD', 'GC=F'] -yahoo_financials = YahooFinancials(tickers) -balance_sheet_data = yahoo_financials.get_financial_stmts('quarterly', 'balance') -earnings_data = yahoo_financials.get_stock_earnings_data() -historical_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') -""" - -from yahoofinancials.calcs import num_shares_outstanding, eps -from yahoofinancials.etl import YahooFinanceETL -from yahoofinancials.maps import COUNTRY_MAP - -__version__ = "1.10" -__author__ = "Connor Sanders" - - -# Class containing methods to create stock data extracts -class YahooFinancials(YahooFinanceETL): - """ - Arguments - ---------- - tickers: str or list - Ticker or listed collection of tickers - Keyword Arguments - ----------------- - concurrent: bool, default False, optional - Defines whether the requests are made synchronously or asynchronously. - country: str, default 'US', optional - This allows you to alter the region, lang, corsDomain parameter sent with each request based on selected country - max_workers: int, default 8, optional - Defines the number of workers used to make concurrent requests. - Only relevant if concurrent=True - timeout: int, default 30, optional - Defines how long a request will stay open. - proxies: str or list, default None, optional - Defines any proxies to use during this instantiation. - """ - - # Private method that handles financial statement extraction - def _run_financial_stmt(self, statement_type, report_num, reformat): - report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] - if reformat: - raw_data = self.get_stock_data(statement_type, report_name=report_name) - data = self.get_reformatted_stmt_data(raw_data, statement_type) - else: - data = self.get_stock_data(statement_type, report_name=report_name) - return data - - # Public Method for the user to get financial statement data - def get_financial_stmts(self, frequency, statement_type, reformat=True): - report_num = self.get_report_type(frequency) - if isinstance(statement_type, str): - data = self._run_financial_stmt(statement_type, report_num, reformat) - else: - data = {} - for stmt_type in statement_type: - re_data = self._run_financial_stmt(stmt_type, report_num, reformat) - data.update(re_data) - return data - - # Public Method for the user to get stock price data - def get_stock_price_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('price'), 'price') - else: - return self.get_stock_tech_data('price') - - # Public Method for the user to return key-statistics data - def get_key_statistics_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('defaultKeyStatistics'), 'defaultKeyStatistics') - else: - return self.get_stock_tech_data('defaultKeyStatistics') - - # Public Method for the user to get company profile data - def get_stock_profile_data(self, reformat=True): - if reformat: - return self.get_clean_data( - self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') - else: - return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') - - # Public Method for the user to get stock earnings data - def get_stock_earnings_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') - else: - return self.get_stock_tech_data('earnings') - - # Public Method for the user to return financial data - def get_financial_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_data(statement_type='keystats', tech_type='financialData'), - 'financialData') - else: - return self.get_stock_data(statement_type='keystats', tech_type='financialData') - - # Public Method for the user to get stock summary data - def get_summary_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') - else: - return self.get_stock_tech_data('summaryDetail') - - # Public Method for the user to get the yahoo summary url - def get_stock_summary_url(self): - if isinstance(self.ticker, str): - return self._BASE_YAHOO_URL + self.ticker - return {t: self._BASE_YAHOO_URL + t for t in self.ticker} - - # Public Method for the user to get stock quote data - def get_stock_quote_type_data(self): - return self.get_stock_tech_data('quoteType') - - # Public Method for user to get historical price data with - def get_historical_price_data(self, start_date, end_date, time_interval): - interval_code = self.get_time_code(time_interval) - start = self.format_date(start_date) - end = self.format_date(end_date) - hist_obj = {'start': start, 'end': end, 'interval': interval_code} - return self.get_stock_data('history', hist_obj=hist_obj) - - # Private Method for Functions needing stock_price_data - def _stock_price_data(self, data_field): - price_data = self.get_stock_price_data() - if isinstance(self.ticker, str): - if price_data[self.ticker] is None: - return None - return price_data[self.ticker].get(data_field) - else: - ret_obj = {} - for tick in self.ticker: - if price_data[tick] is None: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: price_data[tick].get(data_field)}) - return ret_obj - - # Private Method for Functions needing stock_price_data - def _stock_summary_data(self, data_field): - sum_data = self.get_summary_data() - if isinstance(self.ticker, str): - if sum_data[self.ticker] is None: - return None - return sum_data[self.ticker].get(data_field, None) - else: - ret_obj = {} - for tick in self.ticker: - if sum_data[tick] is None: - ret_obj.update({tick: None}) - else: - ret_obj.update({tick: sum_data[tick].get(data_field, None)}) - return ret_obj - - # Private Method for Functions needing financial statement data - def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): - re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] - if isinstance(self.ticker, str): - try: - date_key = re_data[self.ticker][0].keys()[0] - except (IndexError, AttributeError, TypeError): - date_key = list(re_data[self.ticker][0])[0] - data = re_data[self.ticker][0][date_key][field_name] - else: - data = {} - for tick in self.ticker: - try: - date_key = re_data[tick][0].keys()[0] - except: - try: - date_key = list(re_data[tick][0].keys())[0] - except: - date_key = None - if date_key is not None: - sub_data = re_data[tick][0][date_key][field_name] - data.update({tick: sub_data}) - else: - data.update({tick: None}) - return data - - # Public method to get daily dividend data - def get_daily_dividend_data(self, start_date, end_date): - start = self.format_date(start_date) - end = self.format_date(end_date) - return self.get_stock_dividend_data(start, end, 'daily') - - # Public Price Data Methods - def get_current_price(self): - return self._stock_price_data('regularMarketPrice') - - def get_current_change(self): - return self._stock_price_data('regularMarketChange') - - def get_current_percent_change(self): - return self._stock_price_data('regularMarketChangePercent') - - def get_current_volume(self): - return self._stock_price_data('regularMarketVolume') - - def get_prev_close_price(self): - return self._stock_price_data('regularMarketPreviousClose') - - def get_open_price(self): - return self._stock_price_data('regularMarketOpen') - - def get_ten_day_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume10Day') - - def get_three_month_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume3Month') - - def get_stock_exchange(self): - return self._stock_price_data('exchangeName') - - def get_market_cap(self): - return self._stock_price_data('marketCap') - - def get_daily_low(self): - return self._stock_price_data('regularMarketDayLow') - - def get_daily_high(self): - return self._stock_price_data('regularMarketDayHigh') - - def get_currency(self): - return self._stock_price_data('currency') - - # Public Summary Data Methods - def get_yearly_high(self): - return self._stock_summary_data('fiftyTwoWeekHigh') - - def get_yearly_low(self): - return self._stock_summary_data('fiftyTwoWeekLow') - - def get_dividend_yield(self): - return self._stock_summary_data('dividendYield') - - def get_annual_avg_div_yield(self): - return self._stock_summary_data('trailingAnnualDividendYield') - - def get_five_yr_avg_div_yield(self): - return self._stock_summary_data('fiveYearAvgDividendYield') - - def get_dividend_rate(self): - return self._stock_summary_data('dividendRate') - - def get_annual_avg_div_rate(self): - return self._stock_summary_data('trailingAnnualDividendRate') - - def get_50day_moving_avg(self): - return self._stock_summary_data('fiftyDayAverage') - - def get_200day_moving_avg(self): - return self._stock_summary_data('twoHundredDayAverage') - - def get_beta(self): - return self._stock_summary_data('beta') - - def get_payout_ratio(self): - return self._stock_summary_data('payoutRatio') - - def get_pe_ratio(self): - return self._stock_summary_data('trailingPE') - - def get_price_to_sales(self): - return self._stock_summary_data('priceToSalesTrailing12Months') - - def get_exdividend_date(self): - return self._stock_summary_data('exDividendDate') - - # Financial Statement Data Methods - def get_book_value(self): - return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', - 'totalStockholderEquity', 'quarterly') - - def get_ebit(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') - - def get_net_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') - - def get_interest_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') - - def get_operating_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') - - def get_total_operating_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') - - def get_total_revenue(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'totalRevenue', 'annual') - - def get_cost_of_revenue(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'costOfRevenue', 'annual') - - def get_income_before_tax(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeBeforeTax', 'annual') - - def get_income_tax_expense(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeTaxExpense', 'annual') - - def get_gross_profit(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') - - def get_net_income_from_continuing_ops(self): - return self._financial_statement_data('income', 'incomeStatementHistory', - 'netIncomeFromContinuingOps', 'annual') - - def get_research_and_development(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') - - # Calculated Financial Methods - def get_earnings_per_share(self): - price_data = self.get_current_price() - pe_ratio = self.get_pe_ratio() - if isinstance(self.ticker, str): - return eps(price_data, pe_ratio) - else: - ret_obj = {} - for tick in self.ticker: - re_val = eps(price_data[tick], pe_ratio[tick]) - ret_obj.update({tick: re_val}) - return ret_obj - - def get_num_shares_outstanding(self, price_type='current'): - today_low = self._stock_summary_data('dayHigh') - today_high = self._stock_summary_data('dayLow') - cur_market_cap = self._stock_summary_data('marketCap') - current = self.get_current_price() - if isinstance(self.ticker, str): - return num_shares_outstanding(cur_market_cap, today_low, today_high, price_type, current) - else: - ret_obj = {} - for tick in self.ticker: - re_data = num_shares_outstanding(cur_market_cap[tick], today_low[tick], - today_high[tick], price_type, current[tick]) - ret_obj.update({tick: re_data}) - return ret_obj +if __name__ == '__main__': + freeze_support() + set_start_method("spawn") + from yahoofinancials.yf import YahooFinancials +else: + from yahoofinancials.yf import YahooFinancials diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py new file mode 100644 index 0000000..66c731d --- /dev/null +++ b/yahoofinancials/yf.py @@ -0,0 +1,373 @@ +""" +============================== +The Yahoo Financials Module +Version: 1.10 +============================== + +Author: Connor Sanders +Email: sandersconnor1@gmail.com +Version Released: 01/14/2023 +Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 + +Copyright (c) 2023 Connor Sanders +MIT License + +List of Included Functions: + +1) get_financial_stmts(frequency, statement_type, reformat=True) + - frequency can be either 'annual' or 'quarterly'. + - statement_type can be 'income', 'balance', 'cash'. + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +2) get_stock_price_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +3) get_stock_earnings_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +4) get_summary_data(reformat=True) + - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. +5) get_stock_quote_type_data() +6) get_historical_price_data(start_date, end_date, time_interval) + - Gets historical price data for currencies, stocks, indexes, cryptocurrencies, and commodity futures. + - start_date should be entered in the 'YYYY-MM-DD' format. First day that financial data will be pulled. + - end_date should be entered in the 'YYYY-MM-DD' format. Last day that financial data will be pulled. + - time_interval can be either 'daily', 'weekly', or 'monthly'. Parameter determines the time period interval. + +Usage Examples: +from yahoofinancials import YahooFinancials +#tickers = 'AAPL' +#or +tickers = ['AAPL', 'WFC', 'F', 'JPY=X', 'XRP-USD', 'GC=F'] +yahoo_financials = YahooFinancials(tickers) +balance_sheet_data = yahoo_financials.get_financial_stmts('quarterly', 'balance') +earnings_data = yahoo_financials.get_stock_earnings_data() +historical_prices = yahoo_financials.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') +""" + +from yahoofinancials.calcs import num_shares_outstanding, eps +from yahoofinancials.etl import YahooFinanceETL + +__version__ = "1.10" +__author__ = "Connor Sanders" + + +# Class containing methods to create stock data extracts +class YahooFinancials(YahooFinanceETL): + """ + Arguments + ---------- + tickers: str or list + Ticker or listed collection of tickers + Keyword Arguments + ----------------- + concurrent: bool, default False, optional + Defines whether the requests are made synchronously or asynchronously. + country: str, default 'US', optional + This allows you to alter the region, lang, corsDomain parameter sent with each request based on selected country + max_workers: int, default 8, optional + Defines the number of workers used to make concurrent requests. + Only relevant if concurrent=True + timeout: int, default 30, optional + Defines how long a request will stay open. + proxies: str or list, default None, optional + Defines any proxies to use during this instantiation. + """ + + # Private method that handles financial statement extraction + def _run_financial_stmt(self, statement_type, report_num, reformat): + report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] + if reformat: + raw_data = self.get_stock_data(statement_type, report_name=report_name) + data = self.get_reformatted_stmt_data(raw_data, statement_type) + else: + data = self.get_stock_data(statement_type, report_name=report_name) + return data + + # Public Method for the user to get financial statement data + def get_financial_stmts(self, frequency, statement_type, reformat=True): + report_num = self.get_report_type(frequency) + if isinstance(statement_type, str): + data = self._run_financial_stmt(statement_type, report_num, reformat) + else: + data = {} + for stmt_type in statement_type: + re_data = self._run_financial_stmt(stmt_type, report_num, reformat) + data.update(re_data) + return data + + # Public Method for the user to get stock price data + def get_stock_price_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('price'), 'price') + else: + return self.get_stock_tech_data('price') + + # Public Method for the user to return key-statistics data + def get_key_statistics_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('defaultKeyStatistics'), 'defaultKeyStatistics') + else: + return self.get_stock_tech_data('defaultKeyStatistics') + + # Public Method for the user to get company profile data + def get_stock_profile_data(self, reformat=True): + if reformat: + return self.get_clean_data( + self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') + else: + return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') + + # Public Method for the user to get stock earnings data + def get_stock_earnings_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') + else: + return self.get_stock_tech_data('earnings') + + # Public Method for the user to return financial data + def get_financial_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_data(statement_type='keystats', tech_type='financialData'), + 'financialData') + else: + return self.get_stock_data(statement_type='keystats', tech_type='financialData') + + # Public Method for the user to get stock summary data + def get_summary_data(self, reformat=True): + if reformat: + return self.get_clean_data(self.get_stock_tech_data('summaryDetail'), 'summaryDetail') + else: + return self.get_stock_tech_data('summaryDetail') + + # Public Method for the user to get the yahoo summary url + def get_stock_summary_url(self): + if isinstance(self.ticker, str): + return self._BASE_YAHOO_URL + self.ticker + return {t: self._BASE_YAHOO_URL + t for t in self.ticker} + + # Public Method for the user to get stock quote data + def get_stock_quote_type_data(self): + return self.get_stock_tech_data('quoteType') + + # Public Method for user to get historical price data with + def get_historical_price_data(self, start_date, end_date, time_interval): + interval_code = self.get_time_code(time_interval) + start = self.format_date(start_date) + end = self.format_date(end_date) + hist_obj = {'start': start, 'end': end, 'interval': interval_code} + return self.get_stock_data('history', hist_obj=hist_obj) + + # Private Method for Functions needing stock_price_data + def _stock_price_data(self, data_field): + price_data = self.get_stock_price_data() + if isinstance(self.ticker, str): + if price_data[self.ticker] is None: + return None + return price_data[self.ticker].get(data_field) + else: + ret_obj = {} + for tick in self.ticker: + if price_data[tick] is None: + ret_obj.update({tick: None}) + else: + ret_obj.update({tick: price_data[tick].get(data_field)}) + return ret_obj + + # Private Method for Functions needing stock_price_data + def _stock_summary_data(self, data_field): + sum_data = self.get_summary_data() + if isinstance(self.ticker, str): + if sum_data[self.ticker] is None: + return None + return sum_data[self.ticker].get(data_field, None) + else: + ret_obj = {} + for tick in self.ticker: + if sum_data[tick] is None: + ret_obj.update({tick: None}) + else: + ret_obj.update({tick: sum_data[tick].get(data_field, None)}) + return ret_obj + + # Private Method for Functions needing financial statement data + def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): + re_data = self.get_financial_stmts(freq, stmt_type)[stmt_code] + if isinstance(self.ticker, str): + try: + date_key = re_data[self.ticker][0].keys()[0] + except (IndexError, AttributeError, TypeError): + date_key = list(re_data[self.ticker][0])[0] + data = re_data[self.ticker][0][date_key][field_name] + else: + data = {} + for tick in self.ticker: + try: + date_key = re_data[tick][0].keys()[0] + except: + try: + date_key = list(re_data[tick][0].keys())[0] + except: + date_key = None + if date_key is not None: + sub_data = re_data[tick][0][date_key][field_name] + data.update({tick: sub_data}) + else: + data.update({tick: None}) + return data + + # Public method to get daily dividend data + def get_daily_dividend_data(self, start_date, end_date): + start = self.format_date(start_date) + end = self.format_date(end_date) + return self.get_stock_dividend_data(start, end, 'daily') + + # Public Price Data Methods + def get_current_price(self): + return self._stock_price_data('regularMarketPrice') + + def get_current_change(self): + return self._stock_price_data('regularMarketChange') + + def get_current_percent_change(self): + return self._stock_price_data('regularMarketChangePercent') + + def get_current_volume(self): + return self._stock_price_data('regularMarketVolume') + + def get_prev_close_price(self): + return self._stock_price_data('regularMarketPreviousClose') + + def get_open_price(self): + return self._stock_price_data('regularMarketOpen') + + def get_ten_day_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume10Day') + + def get_three_month_avg_daily_volume(self): + return self._stock_price_data('averageDailyVolume3Month') + + def get_stock_exchange(self): + return self._stock_price_data('exchangeName') + + def get_market_cap(self): + return self._stock_price_data('marketCap') + + def get_daily_low(self): + return self._stock_price_data('regularMarketDayLow') + + def get_daily_high(self): + return self._stock_price_data('regularMarketDayHigh') + + def get_currency(self): + return self._stock_price_data('currency') + + # Public Summary Data Methods + def get_yearly_high(self): + return self._stock_summary_data('fiftyTwoWeekHigh') + + def get_yearly_low(self): + return self._stock_summary_data('fiftyTwoWeekLow') + + def get_dividend_yield(self): + return self._stock_summary_data('dividendYield') + + def get_annual_avg_div_yield(self): + return self._stock_summary_data('trailingAnnualDividendYield') + + def get_five_yr_avg_div_yield(self): + return self._stock_summary_data('fiveYearAvgDividendYield') + + def get_dividend_rate(self): + return self._stock_summary_data('dividendRate') + + def get_annual_avg_div_rate(self): + return self._stock_summary_data('trailingAnnualDividendRate') + + def get_50day_moving_avg(self): + return self._stock_summary_data('fiftyDayAverage') + + def get_200day_moving_avg(self): + return self._stock_summary_data('twoHundredDayAverage') + + def get_beta(self): + return self._stock_summary_data('beta') + + def get_payout_ratio(self): + return self._stock_summary_data('payoutRatio') + + def get_pe_ratio(self): + return self._stock_summary_data('trailingPE') + + def get_price_to_sales(self): + return self._stock_summary_data('priceToSalesTrailing12Months') + + def get_exdividend_date(self): + return self._stock_summary_data('exDividendDate') + + # Financial Statement Data Methods + def get_book_value(self): + return self._financial_statement_data('balance', 'balanceSheetHistoryQuarterly', + 'totalStockholderEquity', 'quarterly') + + def get_ebit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'ebit', 'annual') + + def get_net_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncome', 'annual') + + def get_interest_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') + + def get_operating_income(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') + + def get_total_operating_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') + + def get_total_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'totalRevenue', 'annual') + + def get_cost_of_revenue(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'costOfRevenue', 'annual') + + def get_income_before_tax(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeBeforeTax', 'annual') + + def get_income_tax_expense(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'incomeTaxExpense', 'annual') + + def get_gross_profit(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'grossProfit', 'annual') + + def get_net_income_from_continuing_ops(self): + return self._financial_statement_data('income', 'incomeStatementHistory', + 'netIncomeFromContinuingOps', 'annual') + + def get_research_and_development(self): + return self._financial_statement_data('income', 'incomeStatementHistory', 'researchDevelopment', 'annual') + + # Calculated Financial Methods + def get_earnings_per_share(self): + price_data = self.get_current_price() + pe_ratio = self.get_pe_ratio() + if isinstance(self.ticker, str): + return eps(price_data, pe_ratio) + else: + ret_obj = {} + for tick in self.ticker: + re_val = eps(price_data[tick], pe_ratio[tick]) + ret_obj.update({tick: re_val}) + return ret_obj + + def get_num_shares_outstanding(self, price_type='current'): + today_low = self._stock_summary_data('dayHigh') + today_high = self._stock_summary_data('dayLow') + cur_market_cap = self._stock_summary_data('marketCap') + current = self.get_current_price() + if isinstance(self.ticker, str): + return num_shares_outstanding(cur_market_cap, today_low, today_high, price_type, current) + else: + ret_obj = {} + for tick in self.ticker: + re_data = num_shares_outstanding(cur_market_cap[tick], today_low[tick], + today_high[tick], price_type, current[tick]) + ret_obj.update({tick: re_data}) + return ret_obj From c5dcc0f75d07bbb3be864c43b32f34c0fa3d74bf Mon Sep 17 00:00:00 2001 From: connorsanders Date: Wed, 25 Jan 2023 12:37:55 -0600 Subject: [PATCH 095/108] fix for new decryption issue --- CHANGES | 2 +- README.rst | 4 +-- setup.py | 4 +-- yahoofinancials/__init__.py | 10 +----- yahoofinancials/etl.py | 69 +++++++++++++++++++++++++++++-------- yahoofinancials/yf.py | 4 +-- 6 files changed, 63 insertions(+), 30 deletions(-) diff --git a/CHANGES b/CHANGES index 9e7cc29..db6050c 100644 --- a/CHANGES +++ b/CHANGES @@ -41,4 +41,4 @@ 1.9 01/14/2023 -- Merged in pull request from flipdazed. 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. 1.10 01/14/2023 -- Fixed concurrent runtime bug. - +1.11 01/25/2023 -- Fixed new decryption issue. diff --git a/README.rst b/README.rst index da1c4e7..2bdf8b7 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.10 +Current Version: v1.11 -Version Released: 01/14/2023 +Version Released: 01/25/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.py b/setup.py index fc565e8..e23726a 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.10', + version='1.11', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.10.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.11.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/__init__.py b/yahoofinancials/__init__.py index 3ee9aad..721c598 100644 --- a/yahoofinancials/__init__.py +++ b/yahoofinancials/__init__.py @@ -1,9 +1 @@ -from multiprocessing import freeze_support, set_start_method - - -if __name__ == '__main__': - freeze_support() - set_start_method("spawn") - from yahoofinancials.yf import YahooFinancials -else: - from yahoofinancials.yf import YahooFinancials +from yahoofinancials.yf import YahooFinancials diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 315c0e2..bde5619 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -59,20 +59,41 @@ def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30 def decrypt_cryptojs_aes(data): encrypted_stores = data['context']['dispatcher']['stores'] + password = None + candidate_passwords = [] if "_cs" in data and "_cr" in data: _cs = data["_cs"] _cr = data["_cr"] _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() else: - password_key = next(key for key in data.keys() if key not in ["context", "plugins"]) - password = data[password_key] + # Currently assume one extra key in dict, which is password. Print error if + # more extra keys detected. + new_keys = [k for k in data.keys() if k not in ["context", "plugins"]] + new_keys_values = set([data[k] for k in new_keys]) + # Maybe multiple keys have same value - keep one of each + new_keys2 = [] + new_keys2_values = set() + for k in new_keys: + v = data[k] + if not v in new_keys2_values: + new_keys2.append(k) + new_keys2_values.add(v) + l = len(new_keys) + if l == 0: + return None + elif l == 1 and isinstance(data[new_keys[0]], str): + password_key = new_keys[0] + candidate_passwords += ["ad4d90b3c9f2e1d156ef98eadfa0ff93e4042f6960e54aa2a13f06f528e6b50ba4265a26a1fd5b9cd3db0d268a9c34e1d080592424309429a58bce4adc893c87", \ + "e9a8ab8e5620b712ebc2fb4f33d5c8b9c80c0d07e8c371911c785cf674789f1747d76a909510158a7b7419e86857f2d7abbd777813ff64840e4cbc514d12bcae", + "6ae2523aeafa283dad746556540145bf603f44edbf37ad404d3766a8420bb5eb1d3738f52a227b88283cca9cae44060d5f0bba84b6a495082589f5fe7acbdc9e", + "3365117c2a368ffa5df7313a4a84988f73926a86358e8eea9497c5ff799ce27d104b68e5f2fbffa6f8f92c1fef41765a7066fa6bcf050810a9c4c7872fd3ebf0"] encrypted_stores = b64decode(encrypted_stores) assert encrypted_stores[0:8] == b"Salted__" salt = encrypted_stores[8:16] encrypted_stores = encrypted_stores[16:] - def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: + def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: """OpenSSL EVP Key Derivation Function Args: password (Union[str, bytes, bytearray]): Password to generate key from. @@ -105,18 +126,38 @@ def EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="m key, iv = key_iv[:keySize], key_iv[keySize:final_length] return key, iv - key, iv = EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - if usePycryptodome: - cipher = AES.new(key, AES.MODE_CBC, iv=iv) - plaintext = cipher.decrypt(encrypted_stores) - plaintext = unpad(plaintext, 16, style="pkcs7") + def _decrypt(encrypted_stores, password, key, iv): + if usePycryptodome: + cipher = AES.new(key, AES.MODE_CBC, iv=iv) + plaintext = cipher.decrypt(encrypted_stores) + plaintext = unpad(plaintext, 16, style="pkcs7") + else: + cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) + decryptor = cipher.decryptor() + plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() + unpadder = padding.PKCS7(128).unpadder() + plaintext = unpadder.update(plaintext) + unpadder.finalize() + plaintext = plaintext.decode("utf-8") + return plaintext + if not password is None: + try: + key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + except: + raise Exception("error decrypting Yahoo datastore") + plaintext = _decrypt(encrypted_stores, password, key, iv) else: - cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() - unpadder = padding.PKCS7(128).unpadder() - plaintext = unpadder.update(plaintext) + unpadder.finalize() - plaintext = plaintext.decode("utf-8") + success = False + for i in range(len(candidate_passwords)): + password = candidate_passwords[i] + try: + key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") + plaintext = _decrypt(encrypted_stores, password, key, iv) + success = True + break + except: + pass + if not success: + raise Exception("error decrypting Yahoo datastore with hardcoded keys") decoded_stores = json.loads(plaintext) return decoded_stores diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 66c731d..5f0e610 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.10 +Version: 1.11 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/14/2023 +Version Released: 01/25/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders From 3750f247c88fe2b36f5a4eff5d16d7d7f2724bf6 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Wed, 25 Jan 2023 12:40:37 -0600 Subject: [PATCH 096/108] fix for new decryption issue #122 --- CHANGES | 3 +-- README.rst | 2 +- setup.py | 4 ++-- yahoofinancials/yf.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index db6050c..e9daf4c 100644 --- a/CHANGES +++ b/CHANGES @@ -40,5 +40,4 @@ 1.9 01/14/2023 -- Merged in pull request from shaunpatterson. 1.9 01/14/2023 -- Merged in pull request from flipdazed. 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. -1.10 01/14/2023 -- Fixed concurrent runtime bug. -1.11 01/25/2023 -- Fixed new decryption issue. +1.10 01/25/2023 -- Fixed new decryption issue. diff --git a/README.rst b/README.rst index 2bdf8b7..30b7eb7 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.11 +Current Version: v1.10 Version Released: 01/25/2023 diff --git a/setup.py b/setup.py index e23726a..fc565e8 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.11', + version='1.10', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.11.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.10.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 5f0e610..b93f35d 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,7 +1,7 @@ """ ============================== The Yahoo Financials Module -Version: 1.11 +Version: 1.10 ============================== Author: Connor Sanders From 55aa61edb2f54621d4fa53612f33fa46b5094f3d Mon Sep 17 00:00:00 2001 From: connorsanders Date: Thu, 26 Jan 2023 00:16:03 -0600 Subject: [PATCH 097/108] dynamic fix for new decryption issue #124 --- CHANGES | 1 + README.rst | 4 +- setup.py | 4 +- yahoofinancials/etl.py | 118 +++++++++++++++++++++++++++-------------- yahoofinancials/yf.py | 6 +-- 5 files changed, 87 insertions(+), 46 deletions(-) diff --git a/CHANGES b/CHANGES index e9daf4c..40d2c8b 100644 --- a/CHANGES +++ b/CHANGES @@ -41,3 +41,4 @@ 1.9 01/14/2023 -- Merged in pull request from flipdazed. 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. 1.10 01/25/2023 -- Fixed new decryption issue. +1.11 01/26/2023 -- Added a dynamic fix for the decryption issue. diff --git a/README.rst b/README.rst index 30b7eb7..2d766bf 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.10 +Current Version: v1.11 -Version Released: 01/25/2023 +Version Released: 01/26/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.py b/setup.py index fc565e8..e23726a 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.10', + version='1.11', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.10.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.11.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index bde5619..202300c 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -57,9 +57,9 @@ def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30 return response -def decrypt_cryptojs_aes(data): +def decrypt_cryptojs_aes(data, key_obj): encrypted_stores = data['context']['dispatcher']['stores'] - password = None + password = "" candidate_passwords = [] if "_cs" in data and "_cr" in data: _cs = data["_cs"] @@ -67,27 +67,8 @@ def decrypt_cryptojs_aes(data): _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() else: - # Currently assume one extra key in dict, which is password. Print error if - # more extra keys detected. - new_keys = [k for k in data.keys() if k not in ["context", "plugins"]] - new_keys_values = set([data[k] for k in new_keys]) - # Maybe multiple keys have same value - keep one of each - new_keys2 = [] - new_keys2_values = set() - for k in new_keys: - v = data[k] - if not v in new_keys2_values: - new_keys2.append(k) - new_keys2_values.add(v) - l = len(new_keys) - if l == 0: - return None - elif l == 1 and isinstance(data[new_keys[0]], str): - password_key = new_keys[0] - candidate_passwords += ["ad4d90b3c9f2e1d156ef98eadfa0ff93e4042f6960e54aa2a13f06f528e6b50ba4265a26a1fd5b9cd3db0d268a9c34e1d080592424309429a58bce4adc893c87", \ - "e9a8ab8e5620b712ebc2fb4f33d5c8b9c80c0d07e8c371911c785cf674789f1747d76a909510158a7b7419e86857f2d7abbd777813ff64840e4cbc514d12bcae", - "6ae2523aeafa283dad746556540145bf603f44edbf37ad404d3766a8420bb5eb1d3738f52a227b88283cca9cae44060d5f0bba84b6a495082589f5fe7acbdc9e", - "3365117c2a368ffa5df7313a4a84988f73926a86358e8eea9497c5ff799ce27d104b68e5f2fbffa6f8f92c1fef41765a7066fa6bcf050810a9c4c7872fd3ebf0"] + for v in list(key_obj.values()): + password += v encrypted_stores = b64decode(encrypted_stores) assert encrypted_stores[0:8] == b"Salted__" salt = encrypted_stores[8:16] @@ -123,6 +104,7 @@ def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm=" for _ in range(1, iterations): block = hashlib.new(hashAlgorithm, block).digest() key_iv += block + key, iv = key_iv[:keySize], key_iv[keySize:final_length] return key, iv @@ -143,7 +125,7 @@ def _decrypt(encrypted_stores, password, key, iv): try: key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") except: - raise Exception("error decrypting Yahoo datastore") + raise Exception("yahoofinancials failed to decrypt Yahoo data response") plaintext = _decrypt(encrypted_stores, password, key, iv) else: success = False @@ -157,11 +139,65 @@ def _decrypt(encrypted_stores, password, key, iv): except: pass if not success: - raise Exception("error decrypting Yahoo datastore with hardcoded keys") + raise Exception("yahoofinancials failed to decrypt Yahoo data response with hardcoded keys") decoded_stores = json.loads(plaintext) return decoded_stores +def _get_decryption_keys(soup): + key_count = 4 + re_script = soup.find("script", string=re.compile("root.App.main")).text + re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) + re_data.pop("context", None) + key_list = list(re_data.keys()) + if re_data.get("plugins"): # 1) attempt to get last 4 keys after plugins + ind = key_list.index("plugins") + if len(key_list) > ind+1: + sub_keys = key_list[ind+1:] + if len(sub_keys) == key_count: + re_obj = {} + missing_val = False + for k in sub_keys: + if not re_data.get(k): + missing_val = True + break + re_obj.update({k: re_data.get(k)}) + if not missing_val: + return re_obj + re_keys = [] # 2) attempt scan main.js file approach to get keys + prefix = "https://s.yimg.com/uc/finance/dd-site/js/main." + tags = [tag['src'] for tag in soup.find_all('script') if prefix in tag.get('src', '')] + for t in tags: + urlopener_js = UrlOpener() + response_js = urlopener_js.open(t) + if response_js.status_code != 200: + time.sleep(random.randrange(10, 20)) + response_js.close() + else: + r_data = response_js.content.decode("utf8") + re_list = [ + x.group() for x in re.finditer(r"context.dispatcher.stores=JSON.parse((?:.*?\r?\n?)*)toString", r_data) + ] + for rl in re_list: + re_sublist = [x.group() for x in re.finditer(r"t\[\"((?:.*?\r?\n?)*)\"\]", rl)] + if len(re_sublist) == key_count: + re_keys = [sl.replace('t["', '').replace('"]', '') for sl in re_sublist] + break + response_js.close() + if len(re_keys) == key_count: + break + re_obj = {} + missing_val = False + for k in re_keys: + if not re_data.get(k): + missing_val = True + break + re_obj.update({k: re_data.get(k)}) + if not missing_val: + return re_obj + return None + + class YahooFinanceETL(object): def __init__(self, ticker, **kwargs): @@ -253,20 +289,24 @@ def _request_handler(self, url): response.close() else: soup = BeautifulSoup(response.content, "html.parser") - re_script = soup.find("script", string=re.compile("root.App.main")).text - if re_script is not None: - re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) - if "context" in re_data and "dispatcher" in re_data["context"]: - data = re_data['context']['dispatcher']['stores'] - if "QuoteSummaryStore" not in data: - data = decrypt_cryptojs_aes(re_data) - try: - if data.get("QuoteSummaryStore"): - self._cache[url] = data - break - except AttributeError: - time.sleep(random.randrange(10, 20)) - continue + key_obj = _get_decryption_keys(soup) + if key_obj is not None: + re_script = soup.find("script", string=re.compile("root.App.main")).text + if re_script is not None: + re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) + if "context" in re_data and "dispatcher" in re_data["context"]: + data = re_data['context']['dispatcher']['stores'] + if "QuoteSummaryStore" not in data: + data = decrypt_cryptojs_aes(re_data, key_obj) + try: + if data.get("QuoteSummaryStore"): + self._cache[url] = data + break + except AttributeError: + time.sleep(random.randrange(10, 20)) + continue + else: + time.sleep(random.randrange(10, 20)) else: time.sleep(random.randrange(10, 20)) if i == max_retry - 1: diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index b93f35d..8bf00bf 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.10 +Version: 1.11 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/25/2023 +Version Released: 01/26/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders @@ -45,7 +45,7 @@ from yahoofinancials.calcs import num_shares_outstanding, eps from yahoofinancials.etl import YahooFinanceETL -__version__ = "1.10" +__version__ = "1.11" __author__ = "Connor Sanders" From 8d7a713be182ea253792e5bcb34832212cdda13e Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 27 Jan 2023 00:02:00 -0600 Subject: [PATCH 098/108] fix for #127 + expanded test coverage --- test/test_yahoofinancials.py | 4 ++++ yahoofinancials/etl.py | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 32f8f2b..691c0ee 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -104,6 +104,10 @@ def test_yf_module_methods(self): self.assertEqual(True, True) else: self.assertEqual(False, True) + if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) # Treasuries if isinstance(self.test_yf_treasuries_single.get_current_price(), float): self.assertEqual(True, True) diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 202300c..c4b2ad9 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -220,7 +220,8 @@ def __init__(self, ticker, **kwargs): 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], 'keystats': ['key-statistics'], - 'history': ['history'] + 'history': ['history'], + 'profile': ['profile'] } # Interval value translation dictionary From 76659881683251e674ae8f29c62cf29d42f56a48 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Fri, 27 Jan 2023 00:40:38 -0600 Subject: [PATCH 099/108] v1.12 push --- CHANGES | 1 + README.rst | 4 ++-- setup.py | 4 ++-- yahoofinancials/yf.py | 6 +++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 40d2c8b..a8ae929 100644 --- a/CHANGES +++ b/CHANGES @@ -42,3 +42,4 @@ 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. 1.10 01/25/2023 -- Fixed new decryption issue. 1.11 01/26/2023 -- Added a dynamic fix for the decryption issue. +1.12 01/27/2023 -- Fixed get profile function for #127 and added additional unit test diff --git a/README.rst b/README.rst index 2d766bf..58936cc 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.11 +Current Version: v1.12 -Version Released: 01/26/2023 +Version Released: 01/27/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.py b/setup.py index e23726a..e5a439d 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.11', + version='1.12', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.11.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.12.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 8bf00bf..24e143a 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.11 +Version: 1.12 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/26/2023 +Version Released: 01/27/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders @@ -45,7 +45,7 @@ from yahoofinancials.calcs import num_shares_outstanding, eps from yahoofinancials.etl import YahooFinanceETL -__version__ = "1.11" +__version__ = "1.12" __author__ = "Connor Sanders" From 1c12ddf63ee51c9f708ec79ca5e54e2da2fc4fa6 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Mon, 13 Feb 2023 22:56:31 -0600 Subject: [PATCH 100/108] fixes for #132 and #128 --- README.rst | 7 +- setup.py | 7 +- test/test_yahoofinancials.py | 6 +- yahoofinancials/etl.py | 368 ++---- yahoofinancials/maps.py | 2362 ++++++++++++++++++++++++++++++++++ yahoofinancials/utils.py | 19 + yahoofinancials/yf.py | 27 +- 7 files changed, 2534 insertions(+), 262 deletions(-) create mode 100644 yahoofinancials/utils.py diff --git a/README.rst b/README.rst index 58936cc..61b3ff9 100644 --- a/README.rst +++ b/README.rst @@ -7,9 +7,12 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml/badge.svg?branch=master :target: https://github.com/JECSand/yahoofinancials/actions/workflows/test.yml -Current Version: v1.12 +.. image:: https://static.pepy.tech/badge/yahoofinancials + :target: https://pepy.tech/project/yahoofinancials -Version Released: 01/27/2023 +Current Version: v1.13 + +Version Released: 02/14/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues diff --git a/setup.py b/setup.py index e5a439d..8a81d46 100644 --- a/setup.py +++ b/setup.py @@ -10,21 +10,18 @@ setup( name='yahoofinancials', - version='1.12', + version='1.13', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.12.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.13.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', keywords=['finance data', 'stocks', 'commodities', 'cryptocurrencies', 'currencies', 'forex', 'yahoo finance'], packages=['yahoofinancials'], install_requires=[ - "beautifulsoup4>=4.11.1", "pytz", - "cryptography>=3.3.2", - # "pycryptodome>=3.6.6" "requests>=2.26", ], classifiers=[ diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 691c0ee..b97b5a6 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,5 +1,5 @@ -# YahooFinancials Unit Tests v1.10 -# Version Released: 01/14/2023 +# YahooFinancials Unit Tests v1.13 +# Version Released: 02/14/2023 # Author: Connor Sanders # Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 # Copyright (c) 2023 Connor Sanders @@ -100,7 +100,7 @@ def test_yf_module_methods(self): self.assertEqual(True, True) else: self.assertEqual(False, True) - if isinstance(self.test_yf_stock_single.get_net_income(), int): + if isinstance(self.test_yf_stock_single.get_net_income(), float): self.assertEqual(True, True) else: self.assertEqual(False, True) diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index c4b2ad9..7dbd3fc 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -1,29 +1,17 @@ import calendar import datetime -import hashlib -import json import logging import random -import re import time -from base64 import b64decode from functools import partial from json import loads from multiprocessing import Pool - import pytz import requests as requests -from bs4 import BeautifulSoup -from yahoofinancials.maps import COUNTRY_MAP +from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP +from yahoofinancials.utils import remove_prefix, get_request_config, get_request_category -usePycryptodome = False -if usePycryptodome: - from Crypto.Cipher import AES - from Crypto.Util.Padding import unpad -else: - from cryptography.hazmat.primitives import padding - from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -57,147 +45,6 @@ def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30 return response -def decrypt_cryptojs_aes(data, key_obj): - encrypted_stores = data['context']['dispatcher']['stores'] - password = "" - candidate_passwords = [] - if "_cs" in data and "_cr" in data: - _cs = data["_cs"] - _cr = data["_cr"] - _cr = b"".join(int.to_bytes(i, length=4, byteorder="big", signed=True) for i in json.loads(_cr)["words"]) - password = hashlib.pbkdf2_hmac("sha1", _cs.encode("utf8"), _cr, 1, dklen=32).hex() - else: - for v in list(key_obj.values()): - password += v - encrypted_stores = b64decode(encrypted_stores) - assert encrypted_stores[0:8] == b"Salted__" - salt = encrypted_stores[8:16] - encrypted_stores = encrypted_stores[16:] - - def _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") -> tuple: - """OpenSSL EVP Key Derivation Function - Args: - password (Union[str, bytes, bytearray]): Password to generate key from. - salt (Union[bytes, bytearray]): Salt to use. - keySize (int, optional): Output key length in bytes. Defaults to 32. - ivSize (int, optional): Output Initialization Vector (IV) length in bytes. Defaults to 16. - iterations (int, optional): Number of iterations to perform. Defaults to 1. - hashAlgorithm (str, optional): Hash algorithm to use for the KDF. Defaults to 'md5'. - Returns: - key, iv: Derived key and Initialization Vector (IV) bytes. - Taken from: https://gist.github.com/rafiibrahim8/0cd0f8c46896cafef6486cb1a50a16d3 - OpenSSL original code: https://github.com/openssl/openssl/blob/master/crypto/evp/evp_key.c#L78 - """ - assert iterations > 0, "Iterations can not be less than 1." - if isinstance(password, str): - password = password.encode("utf-8") - final_length = keySize + ivSize - key_iv = b"" - block = None - while len(key_iv) < final_length: - hasher = hashlib.new(hashAlgorithm) - if block: - hasher.update(block) - hasher.update(password) - hasher.update(salt) - block = hasher.digest() - for _ in range(1, iterations): - block = hashlib.new(hashAlgorithm, block).digest() - key_iv += block - - key, iv = key_iv[:keySize], key_iv[keySize:final_length] - return key, iv - - def _decrypt(encrypted_stores, password, key, iv): - if usePycryptodome: - cipher = AES.new(key, AES.MODE_CBC, iv=iv) - plaintext = cipher.decrypt(encrypted_stores) - plaintext = unpad(plaintext, 16, style="pkcs7") - else: - cipher = Cipher(algorithms.AES(key), modes.CBC(iv)) - decryptor = cipher.decryptor() - plaintext = decryptor.update(encrypted_stores) + decryptor.finalize() - unpadder = padding.PKCS7(128).unpadder() - plaintext = unpadder.update(plaintext) + unpadder.finalize() - plaintext = plaintext.decode("utf-8") - return plaintext - if not password is None: - try: - key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - except: - raise Exception("yahoofinancials failed to decrypt Yahoo data response") - plaintext = _decrypt(encrypted_stores, password, key, iv) - else: - success = False - for i in range(len(candidate_passwords)): - password = candidate_passwords[i] - try: - key, iv = _EVPKDF(password, salt, keySize=32, ivSize=16, iterations=1, hashAlgorithm="md5") - plaintext = _decrypt(encrypted_stores, password, key, iv) - success = True - break - except: - pass - if not success: - raise Exception("yahoofinancials failed to decrypt Yahoo data response with hardcoded keys") - decoded_stores = json.loads(plaintext) - return decoded_stores - - -def _get_decryption_keys(soup): - key_count = 4 - re_script = soup.find("script", string=re.compile("root.App.main")).text - re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) - re_data.pop("context", None) - key_list = list(re_data.keys()) - if re_data.get("plugins"): # 1) attempt to get last 4 keys after plugins - ind = key_list.index("plugins") - if len(key_list) > ind+1: - sub_keys = key_list[ind+1:] - if len(sub_keys) == key_count: - re_obj = {} - missing_val = False - for k in sub_keys: - if not re_data.get(k): - missing_val = True - break - re_obj.update({k: re_data.get(k)}) - if not missing_val: - return re_obj - re_keys = [] # 2) attempt scan main.js file approach to get keys - prefix = "https://s.yimg.com/uc/finance/dd-site/js/main." - tags = [tag['src'] for tag in soup.find_all('script') if prefix in tag.get('src', '')] - for t in tags: - urlopener_js = UrlOpener() - response_js = urlopener_js.open(t) - if response_js.status_code != 200: - time.sleep(random.randrange(10, 20)) - response_js.close() - else: - r_data = response_js.content.decode("utf8") - re_list = [ - x.group() for x in re.finditer(r"context.dispatcher.stores=JSON.parse((?:.*?\r?\n?)*)toString", r_data) - ] - for rl in re_list: - re_sublist = [x.group() for x in re.finditer(r"t\[\"((?:.*?\r?\n?)*)\"\]", rl)] - if len(re_sublist) == key_count: - re_keys = [sl.replace('t["', '').replace('"]', '') for sl in re_sublist] - break - response_js.close() - if len(re_keys) == key_count: - break - re_obj = {} - missing_val = False - for k in re_keys: - if not re_data.get(k): - missing_val = True - break - re_obj.update({k: re_data.get(k)}) - if not missing_val: - return re_obj - return None - - class YahooFinanceETL(object): def __init__(self, ticker, **kwargs): @@ -216,12 +63,27 @@ def __init__(self, ticker, **kwargs): # Meta-data dictionaries for the classes to use YAHOO_FINANCIAL_TYPES = { - 'income': ['financials', 'incomeStatementHistory', 'incomeStatementHistoryQuarterly'], - 'balance': ['balance-sheet', 'balanceSheetHistory', 'balanceSheetHistoryQuarterly', 'balanceSheetStatements'], - 'cash': ['cash-flow', 'cashflowStatementHistory', 'cashflowStatementHistoryQuarterly', 'cashflowStatements'], + 'income': [ + 'income_statement', + 'incomeStatementHistory', + 'incomeStatementHistoryQuarterly', + 'incomeStatements' + ], + 'balance': [ + 'balance_sheet', + 'balanceSheetHistory', + 'balanceSheetHistoryQuarterly', + 'balanceSheetStatements', + ], + 'cash': [ + 'cash_flow', + 'cashflowStatementHistory', + 'cashflowStatementHistoryQuarterly', + 'cashflowStatements', + ], 'keystats': ['key-statistics'], 'history': ['history'], - 'profile': ['profile'] + 'profile': ['summaryProfile'] } # Interval value translation dictionary @@ -239,8 +101,10 @@ def __init__(self, ticker, **kwargs): def get_report_type(frequency): if frequency == 'annual': report_num = 1 - else: + elif frequency == 'quarterly': report_num = 2 + else: + report_num = 3 return report_num # Public static method to format date serial string to readable format and vice versa @@ -278,8 +142,35 @@ def _get_worker_count(self): workers = len(self.ticker) return workers + # Private method to construct historical data url + def _construct_url(self, symbol, config, params, freq, request_type): + url = config["path"].replace("{symbol}", symbol.lower()) + _default_query_params = COUNTRY_MAP.get(self.country.upper()) + for k, v in config['request'].items(): # request type defaults + if k == "type": + params.update({k: v['options'][request_type].get(freq)}) + elif k == "modules" and request_type in v['options']: + params.update({k: request_type}) + elif k not in params: + params.update({k: v['default']}) + for k, v in _default_query_params.items(): # general defaults + if k not in params: + params.update({k: v}) + if params.get("type"): + field_params = "%2C".join(params.get("type")) + url += "?type=" + field_params + for k, v in params.items(): + if k != "type": + url += "&" + k + "=" + str(v) + elif params.get("modules"): + url += "?modules=" + params.get("modules") + for k, v in params.items(): + if k != "modules": + url += "&" + k + "=" + str(v) + return url + # Private method to execute a web scrape request and decrypt the return - def _request_handler(self, url): + def _request_handler(self, url, res_field=""): urlopener = UrlOpener() # Try to open the URL up to 10 times sleeping random time if something goes wrong max_retry = 10 @@ -289,55 +180,58 @@ def _request_handler(self, url): time.sleep(random.randrange(10, 20)) response.close() else: - soup = BeautifulSoup(response.content, "html.parser") - key_obj = _get_decryption_keys(soup) - if key_obj is not None: - re_script = soup.find("script", string=re.compile("root.App.main")).text - if re_script is not None: - re_data = loads(re.search("root.App.main\s+=\s+(\{.*\})", re_script).group(1)) - if "context" in re_data and "dispatcher" in re_data["context"]: - data = re_data['context']['dispatcher']['stores'] - if "QuoteSummaryStore" not in data: - data = decrypt_cryptojs_aes(re_data, key_obj) - try: - if data.get("QuoteSummaryStore"): - self._cache[url] = data - break - except AttributeError: - time.sleep(random.randrange(10, 20)) - continue - else: - time.sleep(random.randrange(10, 20)) - else: - time.sleep(random.randrange(10, 20)) + res_content = response.text + response.close() + self._cache[url] = loads(res_content).get(res_field) + break if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts raise ManagedException("Server replied with HTTP " + str(response.status_code) + " code while opening the url: " + str(url)) - # Private method to scrape data from yahoo finance - def _scrape_data(self, url, tech_type, statement_type): + @staticmethod + def _format_raw_fundamental_data(raw_data): + data = {} + for i in raw_data.get("result"): + for k, v in i.items(): + if k not in ['meta', 'timestamp']: + cleaned_k = remove_prefix(remove_prefix(remove_prefix(k, "quarterly"), "annual"), "trailing") + if cleaned_k in ['EBIT']: + cleaned_k = cleaned_k.lower() + else: + cleaned_k = cleaned_k[0].lower() + cleaned_k[1:] + for rec in v: + if rec.get("asOfDate") in data: + data[rec.get("asOfDate")].update({cleaned_k: rec.get('reportedValue', {}).get('raw')}) + else: + data.update({rec.get("asOfDate"): {cleaned_k: rec.get('reportedValue', {}).get('raw')}}) + return data + + @staticmethod + def _format_raw_module_data(raw_data, tech_type): + data = {} + for i in raw_data.get("result", {}): + if i.get(tech_type): + for k, v in i.get(tech_type, {}).items(): + data.update({k: v}) + return data + + # Private method to _get_historical_data from yahoo finance + def _get_historical_data(self, url, config, tech_type, statement_type): global _lastget - country_ent = COUNTRY_MAP.get(self.country.upper()) - meta_str = '&lang=' + country_ent.get("lang", "en-US") + '®ion=' + country_ent.get("region", "US") - url += meta_str if not self._cache.get(url): now = int(time.time()) if _lastget and now - _lastget < self._MIN_INTERVAL: time.sleep(self._MIN_INTERVAL - (now - _lastget) + 1) now = int(time.time()) _lastget = now - self._request_handler(url) + self._request_handler(url, config.get("response_field")) data = self._cache[url] - if "context" in data and "dispatcher" in data["context"]: - data = data['context']['dispatcher']['stores'] - if tech_type == '' and statement_type != 'history': - stores = data["QuoteSummaryStore"] - elif tech_type != '' and statement_type != 'history': - stores = data["QuoteSummaryStore"][tech_type] + if tech_type == '' and statement_type in ["income", "balance", "cash"]: + data = self._format_raw_fundamental_data(data) else: - stores = data["HistoricalPriceStore"] - return stores + data = self._format_raw_module_data(data, tech_type) + return data # Private static method to determine if a numerical value is in the data object being cleaned @staticmethod @@ -571,33 +465,40 @@ def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): # Private Method to take scrapped data and build a data dictionary with, used by get_stock_data() def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): - YAHOO_URL = self._BASE_YAHOO_URL + up_ticker + '/' + self.YAHOO_FINANCIAL_TYPES[statement_type][0] + '?p=' + \ - up_ticker - if tech_type == '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - dict_ent = {up_ticker: re_data[u'' + report_name], 'dataType': report_name} - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data, 'dataType': report_name} - elif tech_type != '' and statement_type != 'history': - try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - except KeyError: - re_data = None - dict_ent = {up_ticker: re_data} - else: - YAHOO_URL = self._build_historical_url(up_ticker, hist_obj) + if statement_type == 'history': try: cleaned_re_data = self._recursive_api_request(hist_obj, up_ticker) except KeyError: + cleaned_re_data = None + return {up_ticker: cleaned_re_data} + else: + dict_ent = {} + params = {} + r_map = get_request_config(tech_type, REQUEST_MAP) + r_cat = get_request_category(tech_type, self.YAHOO_FINANCIAL_TYPES, statement_type) + YAHOO_URL = self._construct_url( + up_ticker.lower(), + r_map, + params, + hist_obj.get("interval"), + r_cat + ) + if tech_type == '' and statement_type != 'history': + try: + re_data = self._get_historical_data(YAHOO_URL, REQUEST_MAP['fundamentals'], tech_type, + statement_type) + dict_ent = {up_ticker: re_data, 'dataType': report_name} + except KeyError: + re_data = None + dict_ent = {up_ticker: re_data, 'dataType': report_name} + elif tech_type != '' and statement_type != 'history': + r_map = get_request_config(tech_type, REQUEST_MAP) try: - re_data = self._scrape_data(YAHOO_URL, tech_type, statement_type) - cleaned_re_data = self._clean_historical_data(re_data) + re_data = self._get_historical_data(YAHOO_URL, r_map, tech_type, statement_type) except KeyError: - cleaned_re_data = None - dict_ent = {up_ticker: cleaned_re_data} - return dict_ent + re_data = None + dict_ent = {up_ticker: re_data} + return dict_ent # Private method to return the stmt_id for the reformat_process def _get_stmt_id(self, statement_type, raw_data): @@ -612,32 +513,20 @@ def _get_stmt_id(self, statement_type, raw_data): return stmt_id # Private Method for the Reformat Process - def _reformat_stmt_data_process(self, raw_data, statement_type): + @staticmethod + def _reformat_stmt_data_process(raw_data): final_data_list = [] if raw_data is not None: - stmt_id = self._get_stmt_id(statement_type, raw_data) - if stmt_id is None: - return final_data_list - hashed_data_list = raw_data[stmt_id] - for data_item in hashed_data_list: - data_date = '' - sub_data_dict = {} - for k, v in data_item.items(): - if k == 'endDate': - data_date = v['fmt'] - elif k != 'maxAge': - numerical_val = self._determine_numeric_value(v) - sub_dict_item = {k: numerical_val} - sub_data_dict.update(sub_dict_item) - dict_item = {data_date: sub_data_dict} + for date_key, data_item in raw_data.items(): + dict_item = {date_key: data_item} final_data_list.append(dict_item) return final_data_list else: return raw_data # Private Method to return subdict entry for the statement reformat process - def _get_sub_dict_ent(self, ticker, raw_data, statement_type): - form_data_list = self._reformat_stmt_data_process(raw_data[ticker], statement_type) + def _get_sub_dict_ent(self, ticker, raw_data): + form_data_list = self._reformat_stmt_data_process(raw_data[ticker]) return {ticker: form_data_list} # Public method to get time interval code @@ -686,7 +575,7 @@ def get_reformatted_stmt_data(self, raw_data, statement_type): sub_dict, data_dict = {}, {} data_type = raw_data['dataType'] if isinstance(self.ticker, str): - sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data, statement_type) + sub_dict_ent = self._get_sub_dict_ent(self.ticker, raw_data) sub_dict.update(sub_dict_ent) dict_ent = {data_type: sub_dict} data_dict.update(dict_ent) @@ -694,15 +583,14 @@ def get_reformatted_stmt_data(self, raw_data, statement_type): if self.concurrent: with Pool(self._get_worker_count()) as pool: sub_dict_ents = pool.map(partial(self._get_sub_dict_ent, - raw_data=raw_data, - statement_type=statement_type), self.ticker) + raw_data=raw_data), self.ticker) for dict_ent in sub_dict_ents: sub_dict.update(dict_ent) pool.close() pool.join() else: for tick in self.ticker: - sub_dict_ent = self._get_sub_dict_ent(tick, raw_data, statement_type) + sub_dict_ent = self._get_sub_dict_ent(tick, raw_data) sub_dict.update(sub_dict_ent) dict_ent = {data_type: sub_dict} data_dict.update(dict_ent) diff --git a/yahoofinancials/maps.py b/yahoofinancials/maps.py index d6caef7..530a03e 100644 --- a/yahoofinancials/maps.py +++ b/yahoofinancials/maps.py @@ -1,3 +1,5 @@ +import time + COUNTRY_MAP = { "FR": {"lang": "fr-FR", "region": "FR", "corsDomain": "fr.finance.yahoo.com"}, "IN": {"lang": "en-IN", "region": "IN", "corsDomain": "in.finance.yahoo.com"}, @@ -14,3 +16,2363 @@ "SG": {"lang": "en-SG", "region": "SG", "corsDomain": "sg.finance.yahoo.com"}, "TW": {"lang": "zh-tw", "region": "TW", "corsDomain": "tw.finance.yahoo.com"}, } + +FUNDAMENTALS_MAP = { + "income_statement": { + "annual": [ + "annualAmortization", + "annualAmortizationOfIntangiblesIncomeStatement", + "annualAverageDilutionEarnings", + "annualBasicAccountingChange", + "annualBasicAverageShares", + "annualBasicContinuousOperations", + "annualBasicDiscontinuousOperations", + "annualBasicEPS", + "annualBasicEPSOtherGainsLosses", + "annualBasicExtraordinary", + "annualContinuingAndDiscontinuedBasicEPS", + "annualContinuingAndDiscontinuedDilutedEPS", + "annualCostOfRevenue", + "annualDepletionIncomeStatement", + "annualDepreciationAmortizationDepletionIncomeStatement", + "annualDepreciationAndAmortizationInIncomeStatement", + "annualDepreciationIncomeStatement", + "annualDilutedAccountingChange", + "annualDilutedAverageShares", + "annualDilutedContinuousOperations", + "annualDilutedDiscontinuousOperations", + "annualDilutedEPS", + "annualDilutedEPSOtherGainsLosses", + "annualDilutedExtraordinary", + "annualDilutedNIAvailtoComStockholders", + "annualDividendPerShare", + "annualEBIT", + "annualEBITDA", + "annualEarningsFromEquityInterest", + "annualEarningsFromEquityInterestNetOfTax", + "annualExciseTaxes", + "annualGainOnSaleOfBusiness", + "annualGainOnSaleOfPPE", + "annualGainOnSaleOfSecurity", + "annualGeneralAndAdministrativeExpense", + "annualGrossProfit", + "annualImpairmentOfCapitalAssets", + "annualInsuranceAndClaims", + "annualInterestExpense", + "annualInterestExpenseNonOperating", + "annualInterestIncome", + "annualInterestIncomeNonOperating", + "annualMinorityInterests", + "annualNetIncome", + "annualNetIncomeCommonStockholders", + "annualNetIncomeContinuousOperations", + "annualNetIncomeDiscontinuousOperations", + "annualNetIncomeExtraordinary", + "annualNetIncomeFromContinuingAndDiscontinuedOperation", + "annualNetIncomeFromContinuingOperationNetMinorityInterest", + "annualNetIncomeFromTaxLossCarryforward", + "annualNetIncomeIncludingNoncontrollingInterests", + "annualNetInterestIncome", + "annualNetNonOperatingInterestIncomeExpense", + "annualNormalizedBasicEPS", + "annualNormalizedDilutedEPS", + "annualNormalizedEBITDA", + "annualNormalizedIncome", + "annualOperatingExpense", + "annualOperatingIncome", + "annualOperatingRevenue", + "annualOtherGandA", + "annualOtherIncomeExpense", + "annualOtherNonOperatingIncomeExpenses", + "annualOtherOperatingExpenses", + "annualOtherSpecialCharges", + "annualOtherTaxes", + "annualOtherunderPreferredStockDividend", + "annualPreferredStockDividends", + "annualPretaxIncome", + "annualProvisionForDoubtfulAccounts", + "annualReconciledCostOfRevenue", + "annualReconciledDepreciation", + "annualRentAndLandingFees", + "annualRentExpenseSupplemental", + "annualReportedNormalizedBasicEPS", + "annualReportedNormalizedDilutedEPS", + "annualResearchAndDevelopment", + "annualRestructuringAndMergernAcquisition", + "annualSalariesAndWages", + "annualSecuritiesAmortization", + "annualSellingAndMarketingExpense", + "annualSellingGeneralAndAdministration", + "annualSpecialIncomeCharges", + "annualTaxEffectOfUnusualItems", + "annualTaxLossCarryforwardBasicEPS", + "annualTaxLossCarryforwardDilutedEPS", + "annualTaxProvision", + "annualTaxRateForCalcs", + "annualTotalExpenses", + "annualTotalOperatingIncomeAsReported", + "annualTotalOtherFinanceCost", + "annualTotalRevenue", + "annualTotalUnusualItems", + "annualTotalUnusualItemsExcludingGoodwill", + "annualWriteOff", + "trailingAmortization", + "trailingAmortizationOfIntangiblesIncomeStatement", + "trailingAverageDilutionEarnings", + "trailingBasicAccountingChange", + "trailingBasicAverageShares", + "trailingBasicContinuousOperations", + "trailingBasicDiscontinuousOperations", + "trailingBasicEPS", + "trailingBasicEPSOtherGainsLosses", + "trailingBasicExtraordinary", + "trailingContinuingAndDiscontinuedBasicEPS", + "trailingContinuingAndDiscontinuedDilutedEPS", + "trailingCostOfRevenue", + "trailingDepletionIncomeStatement", + "trailingDepreciationAmortizationDepletionIncomeStatement", + "trailingDepreciationAndAmortizationInIncomeStatement", + "trailingDepreciationIncomeStatement", + "trailingDilutedAccountingChange", + "trailingDilutedAverageShares", + "trailingDilutedContinuousOperations", + "trailingDilutedDiscontinuousOperations", + "trailingDilutedEPS", + "trailingDilutedEPSOtherGainsLosses", + "trailingDilutedExtraordinary", + "trailingDilutedNIAvailtoComStockholders", + "trailingDividendPerShare", + "trailingEBIT", + "trailingEBITDA", + "trailingEarningsFromEquityInterest", + "trailingEarningsFromEquityInterestNetOfTax", + "trailingExciseTaxes", + "trailingGainOnSaleOfBusiness", + "trailingGainOnSaleOfPPE", + "trailingGainOnSaleOfSecurity", + "trailingGeneralAndAdministrativeExpense", + "trailingGrossProfit", + "trailingImpairmentOfCapitalAssets", + "trailingInsuranceAndClaims", + "trailingInterestExpense", + "trailingInterestExpenseNonOperating", + "trailingInterestIncome", + "trailingInterestIncomeNonOperating", + "trailingMinorityInterests", + "trailingNetIncome", + "trailingNetIncomeCommonStockholders", + "trailingNetIncomeContinuousOperations", + "trailingNetIncomeDiscontinuousOperations", + "trailingNetIncomeExtraordinary", + "trailingNetIncomeFromContinuingAndDiscontinuedOperation", + "trailingNetIncomeFromContinuingOperationNetMinorityInterest", + "trailingNetIncomeFromTaxLossCarryforward", + "trailingNetIncomeIncludingNoncontrollingInterests", + "trailingNetInterestIncome", + "trailingNetNonOperatingInterestIncomeExpense", + "trailingNormalizedBasicEPS", + "trailingNormalizedDilutedEPS", + "trailingNormalizedEBITDA", + "trailingNormalizedIncome", + "trailingOperatingExpense", + "trailingOperatingIncome", + "trailingOperatingRevenue", + "trailingOtherGandA", + "trailingOtherIncomeExpense", + "trailingOtherNonOperatingIncomeExpenses", + "trailingOtherOperatingExpenses", + "trailingOtherSpecialCharges", + "trailingOtherTaxes", + "trailingOtherunderPreferredStockDividend", + "trailingPreferredStockDividends", + "trailingPretaxIncome", + "trailingProvisionForDoubtfulAccounts", + "trailingReconciledCostOfRevenue", + "trailingReconciledDepreciation", + "trailingRentAndLandingFees", + "trailingRentExpenseSupplemental", + "trailingReportedNormalizedBasicEPS", + "trailingReportedNormalizedDilutedEPS", + "trailingResearchAndDevelopment", + "trailingRestructuringAndMergernAcquisition", + "trailingSalariesAndWages", + "trailingSecuritiesAmortization", + "trailingSellingAndMarketingExpense", + "trailingSellingGeneralAndAdministration", + "trailingSpecialIncomeCharges", + "trailingTaxEffectOfUnusualItems", + "trailingTaxLossCarryforwardBasicEPS", + "trailingTaxLossCarryforwardDilutedEPS", + "trailingTaxProvision", + "trailingTaxRateForCalcs", + "trailingTotalExpenses", + "trailingTotalOperatingIncomeAsReported", + "trailingTotalOtherFinanceCost", + "trailingTotalRevenue", + "trailingTotalUnusualItems", + "trailingTotalUnusualItemsExcludingGoodwill", + "trailingWriteOff", + ], + "quarterly": [ + "quarterlyAmortization", + "quarterlyAmortizationOfIntangiblesIncomeStatement", + "quarterlyAverageDilutionEarnings", + "quarterlyBasicAccountingChange", + "quarterlyBasicAverageShares", + "quarterlyBasicContinuousOperations", + "quarterlyBasicDiscontinuousOperations", + "quarterlyBasicEPS", + "quarterlyBasicEPSOtherGainsLosses", + "quarterlyBasicExtraordinary", + "quarterlyContinuingAndDiscontinuedBasicEPS", + "quarterlyContinuingAndDiscontinuedDilutedEPS", + "quarterlyCostOfRevenue", + "quarterlyDepletionIncomeStatement", + "quarterlyDepreciationAmortizationDepletionIncomeStatement", + "quarterlyDepreciationAndAmortizationInIncomeStatement", + "quarterlyDepreciationIncomeStatement", + "quarterlyDilutedAccountingChange", + "quarterlyDilutedAverageShares", + "quarterlyDilutedContinuousOperations", + "quarterlyDilutedDiscontinuousOperations", + "quarterlyDilutedEPS", + "quarterlyDilutedEPSOtherGainsLosses", + "quarterlyDilutedExtraordinary", + "quarterlyDilutedNIAvailtoComStockholders", + "quarterlyDividendPerShare", + "quarterlyEBIT", + "quarterlyEBITDA", + "quarterlyEarningsFromEquityInterest", + "quarterlyEarningsFromEquityInterestNetOfTax", + "quarterlyExciseTaxes", + "quarterlyGainOnSaleOfBusiness", + "quarterlyGainOnSaleOfPPE", + "quarterlyGainOnSaleOfSecurity", + "quarterlyGeneralAndAdministrativeExpense", + "quarterlyGrossProfit", + "quarterlyImpairmentOfCapitalAssets", + "quarterlyInsuranceAndClaims", + "quarterlyInterestExpense", + "quarterlyInterestExpenseNonOperating", + "quarterlyInterestIncome", + "quarterlyInterestIncomeNonOperating", + "quarterlyMinorityInterests", + "quarterlyNetIncome", + "quarterlyNetIncomeCommonStockholders", + "quarterlyNetIncomeContinuousOperations", + "quarterlyNetIncomeDiscontinuousOperations", + "quarterlyNetIncomeExtraordinary", + "quarterlyNetIncomeFromContinuingAndDiscontinuedOperation", + "quarterlyNetIncomeFromContinuingOperationNetMinorityInterest", + "quarterlyNetIncomeFromTaxLossCarryforward", + "quarterlyNetIncomeIncludingNoncontrollingInterests", + "quarterlyNetInterestIncome", + "quarterlyNetNonOperatingInterestIncomeExpense", + "quarterlyNormalizedBasicEPS", + "quarterlyNormalizedDilutedEPS", + "quarterlyNormalizedEBITDA", + "quarterlyNormalizedIncome", + "quarterlyOperatingExpense", + "quarterlyOperatingIncome", + "quarterlyOperatingRevenue", + "quarterlyOtherGandA", + "quarterlyOtherIncomeExpense", + "quarterlyOtherNonOperatingIncomeExpenses", + "quarterlyOtherOperatingExpenses", + "quarterlyOtherSpecialCharges", + "quarterlyOtherTaxes", + "quarterlyOtherunderPreferredStockDividend", + "quarterlyPreferredStockDividends", + "quarterlyPretaxIncome", + "quarterlyProvisionForDoubtfulAccounts", + "quarterlyReconciledCostOfRevenue", + "quarterlyReconciledDepreciation", + "quarterlyRentAndLandingFees", + "quarterlyRentExpenseSupplemental", + "quarterlyReportedNormalizedBasicEPS", + "quarterlyReportedNormalizedDilutedEPS", + "quarterlyResearchAndDevelopment", + "quarterlyRestructuringAndMergernAcquisition", + "quarterlySalariesAndWages", + "quarterlySecuritiesAmortization", + "quarterlySellingAndMarketingExpense", + "quarterlySellingGeneralAndAdministration", + "quarterlySpecialIncomeCharges", + "quarterlyTaxEffectOfUnusualItems", + "quarterlyTaxLossCarryforwardBasicEPS", + "quarterlyTaxLossCarryforwardDilutedEPS", + "quarterlyTaxProvision", + "quarterlyTaxRateForCalcs", + "quarterlyTotalExpenses", + "quarterlyTotalOperatingIncomeAsReported", + "quarterlyTotalOtherFinanceCost", + "quarterlyTotalRevenue", + "quarterlyTotalUnusualItems", + "quarterlyTotalUnusualItemsExcludingGoodwill", + "quarterlyWriteOff", + "trailingAmortization", + "trailingAmortizationOfIntangiblesIncomeStatement", + "trailingAverageDilutionEarnings", + "trailingBasicAccountingChange", + "trailingBasicAverageShares", + "trailingBasicContinuousOperations", + "trailingBasicDiscontinuousOperations", + "trailingBasicEPS", + "trailingBasicEPSOtherGainsLosses", + "trailingBasicExtraordinary", + "trailingContinuingAndDiscontinuedBasicEPS", + "trailingContinuingAndDiscontinuedDilutedEPS", + "trailingCostOfRevenue", + "trailingDepletionIncomeStatement", + "trailingDepreciationAmortizationDepletionIncomeStatement", + "trailingDepreciationAndAmortizationInIncomeStatement", + "trailingDepreciationIncomeStatement", + "trailingDilutedAccountingChange", + "trailingDilutedAverageShares", + "trailingDilutedContinuousOperations", + "trailingDilutedDiscontinuousOperations", + "trailingDilutedEPS", + "trailingDilutedEPSOtherGainsLosses", + "trailingDilutedExtraordinary", + "trailingDilutedNIAvailtoComStockholders", + "trailingDividendPerShare", + "trailingEBIT", + "trailingEBITDA", + "trailingEarningsFromEquityInterest", + "trailingEarningsFromEquityInterestNetOfTax", + "trailingExciseTaxes", + "trailingGainOnSaleOfBusiness", + "trailingGainOnSaleOfPPE", + "trailingGainOnSaleOfSecurity", + "trailingGeneralAndAdministrativeExpense", + "trailingGrossProfit", + "trailingImpairmentOfCapitalAssets", + "trailingInsuranceAndClaims", + "trailingInterestExpense", + "trailingInterestExpenseNonOperating", + "trailingInterestIncome", + "trailingInterestIncomeNonOperating", + "trailingMinorityInterests", + "trailingNetIncome", + "trailingNetIncomeCommonStockholders", + "trailingNetIncomeContinuousOperations", + "trailingNetIncomeDiscontinuousOperations", + "trailingNetIncomeExtraordinary", + "trailingNetIncomeFromContinuingAndDiscontinuedOperation", + "trailingNetIncomeFromContinuingOperationNetMinorityInterest", + "trailingNetIncomeFromTaxLossCarryforward", + "trailingNetIncomeIncludingNoncontrollingInterests", + "trailingNetInterestIncome", + "trailingNetNonOperatingInterestIncomeExpense", + "trailingNormalizedBasicEPS", + "trailingNormalizedDilutedEPS", + "trailingNormalizedEBITDA", + "trailingNormalizedIncome", + "trailingOperatingExpense", + "trailingOperatingIncome", + "trailingOperatingRevenue", + "trailingOtherGandA", + "trailingOtherIncomeExpense", + "trailingOtherNonOperatingIncomeExpenses", + "trailingOtherOperatingExpenses", + "trailingOtherSpecialCharges", + "trailingOtherTaxes", + "trailingOtherunderPreferredStockDividend", + "trailingPreferredStockDividends", + "trailingPretaxIncome", + "trailingProvisionForDoubtfulAccounts", + "trailingReconciledCostOfRevenue", + "trailingReconciledDepreciation", + "trailingRentAndLandingFees", + "trailingRentExpenseSupplemental", + "trailingReportedNormalizedBasicEPS", + "trailingReportedNormalizedDilutedEPS", + "trailingResearchAndDevelopment", + "trailingRestructuringAndMergernAcquisition", + "trailingSalariesAndWages", + "trailingSecuritiesAmortization", + "trailingSellingAndMarketingExpense", + "trailingSellingGeneralAndAdministration", + "trailingSpecialIncomeCharges", + "trailingTaxEffectOfUnusualItems", + "trailingTaxLossCarryforwardBasicEPS", + "trailingTaxLossCarryforwardDilutedEPS", + "trailingTaxProvision", + "trailingTaxRateForCalcs", + "trailingTotalExpenses", + "trailingTotalOperatingIncomeAsReported", + "trailingTotalOtherFinanceCost", + "trailingTotalRevenue", + "trailingTotalUnusualItems", + "trailingTotalUnusualItemsExcludingGoodwill", + "trailingWriteOff" + ], + "monthly": [ + "monthlyAmortization", + "monthlyAmortizationOfIntangiblesIncomeStatement", + "monthlyAverageDilutionEarnings", + "monthlyBasicAccountingChange", + "monthlyBasicAverageShares", + "monthlyBasicContinuousOperations", + "monthlyBasicDiscontinuousOperations", + "monthlyBasicEPS", + "monthlyBasicEPSOtherGainsLosses", + "monthlyBasicExtraordinary", + "monthlyContinuingAndDiscontinuedBasicEPS", + "monthlyContinuingAndDiscontinuedDilutedEPS", + "monthlyCostOfRevenue", + "monthlyDepletionIncomeStatement", + "monthlyDepreciationAmortizationDepletionIncomeStatement", + "monthlyDepreciationAndAmortizationInIncomeStatement", + "monthlyDepreciationIncomeStatement", + "monthlyDilutedAccountingChange", + "monthlyDilutedAverageShares", + "monthlyDilutedContinuousOperations", + "monthlyDilutedDiscontinuousOperations", + "monthlyDilutedEPS", + "monthlyDilutedEPSOtherGainsLosses", + "monthlyDilutedExtraordinary", + "monthlyDilutedNIAvailtoComStockholders", + "monthlyDividendPerShare", + "monthlyEBIT", + "monthlyEBITDA", + "monthlyEarningsFromEquityInterest", + "monthlyEarningsFromEquityInterestNetOfTax", + "monthlyExciseTaxes", + "monthlyGainOnSaleOfBusiness", + "monthlyGainOnSaleOfPPE", + "monthlyGainOnSaleOfSecurity", + "monthlyGeneralAndAdministrativeExpense", + "monthlyGrossProfit", + "monthlyImpairmentOfCapitalAssets", + "monthlyInsuranceAndClaims", + "monthlyInterestExpense", + "monthlyInterestExpenseNonOperating", + "monthlyInterestIncome", + "monthlyInterestIncomeNonOperating", + "monthlyMinorityInterests", + "monthlyNetIncome", + "monthlyNetIncomeCommonStockholders", + "monthlyNetIncomeContinuousOperations", + "monthlyNetIncomeDiscontinuousOperations", + "monthlyNetIncomeExtraordinary", + "monthlyNetIncomeFromContinuingAndDiscontinuedOperation", + "monthlyNetIncomeFromContinuingOperationNetMinorityInterest", + "monthlyNetIncomeFromTaxLossCarryforward", + "monthlyNetIncomeIncludingNoncontrollingInterests", + "monthlyNetInterestIncome", + "monthlyNetNonOperatingInterestIncomeExpense", + "monthlyNormalizedBasicEPS", + "monthlyNormalizedDilutedEPS", + "monthlyNormalizedEBITDA", + "monthlyNormalizedIncome", + "monthlyOperatingExpense", + "monthlyOperatingIncome", + "monthlyOperatingRevenue", + "monthlyOtherGandA", + "monthlyOtherIncomeExpense", + "monthlyOtherNonOperatingIncomeExpenses", + "monthlyOtherOperatingExpenses", + "monthlyOtherSpecialCharges", + "monthlyOtherTaxes", + "monthlyOtherunderPreferredStockDividend", + "monthlyPreferredStockDividends", + "monthlyPretaxIncome", + "monthlyProvisionForDoubtfulAccounts", + "monthlyReconciledCostOfRevenue", + "monthlyReconciledDepreciation", + "monthlyRentAndLandingFees", + "monthlyRentExpenseSupplemental", + "monthlyReportedNormalizedBasicEPS", + "monthlyReportedNormalizedDilutedEPS", + "monthlyResearchAndDevelopment", + "monthlyRestructuringAndMergernAcquisition", + "monthlySalariesAndWages", + "monthlySecuritiesAmortization", + "monthlySellingAndMarketingExpense", + "monthlySellingGeneralAndAdministration", + "monthlySpecialIncomeCharges", + "monthlyTaxEffectOfUnusualItems", + "monthlyTaxLossCarryforwardBasicEPS", + "monthlyTaxLossCarryforwardDilutedEPS", + "monthlyTaxProvision", + "monthlyTaxRateForCalcs", + "monthlyTotalExpenses", + "monthlyTotalOperatingIncomeAsReported", + "monthlyTotalOtherFinanceCost", + "monthlyTotalRevenue", + "monthlyTotalUnusualItems", + "monthlyTotalUnusualItemsExcludingGoodwill", + "monthlyWriteOff", + "trailingAmortization", + "trailingAmortizationOfIntangiblesIncomeStatement", + "trailingAverageDilutionEarnings", + "trailingBasicAccountingChange", + "trailingBasicAverageShares", + "trailingBasicContinuousOperations", + "trailingBasicDiscontinuousOperations", + "trailingBasicEPS", + "trailingBasicEPSOtherGainsLosses", + "trailingBasicExtraordinary", + "trailingContinuingAndDiscontinuedBasicEPS", + "trailingContinuingAndDiscontinuedDilutedEPS", + "trailingCostOfRevenue", + "trailingDepletionIncomeStatement", + "trailingDepreciationAmortizationDepletionIncomeStatement", + "trailingDepreciationAndAmortizationInIncomeStatement", + "trailingDepreciationIncomeStatement", + "trailingDilutedAccountingChange", + "trailingDilutedAverageShares", + "trailingDilutedContinuousOperations", + "trailingDilutedDiscontinuousOperations", + "trailingDilutedEPS", + "trailingDilutedEPSOtherGainsLosses", + "trailingDilutedExtraordinary", + "trailingDilutedNIAvailtoComStockholders", + "trailingDividendPerShare", + "trailingEBIT", + "trailingEBITDA", + "trailingEarningsFromEquityInterest", + "trailingEarningsFromEquityInterestNetOfTax", + "trailingExciseTaxes", + "trailingGainOnSaleOfBusiness", + "trailingGainOnSaleOfPPE", + "trailingGainOnSaleOfSecurity", + "trailingGeneralAndAdministrativeExpense", + "trailingGrossProfit", + "trailingImpairmentOfCapitalAssets", + "trailingInsuranceAndClaims", + "trailingInterestExpense", + "trailingInterestExpenseNonOperating", + "trailingInterestIncome", + "trailingInterestIncomeNonOperating", + "trailingMinorityInterests", + "trailingNetIncome", + "trailingNetIncomeCommonStockholders", + "trailingNetIncomeContinuousOperations", + "trailingNetIncomeDiscontinuousOperations", + "trailingNetIncomeExtraordinary", + "trailingNetIncomeFromContinuingAndDiscontinuedOperation", + "trailingNetIncomeFromContinuingOperationNetMinorityInterest", + "trailingNetIncomeFromTaxLossCarryforward", + "trailingNetIncomeIncludingNoncontrollingInterests", + "trailingNetInterestIncome", + "trailingNetNonOperatingInterestIncomeExpense", + "trailingNormalizedBasicEPS", + "trailingNormalizedDilutedEPS", + "trailingNormalizedEBITDA", + "trailingNormalizedIncome", + "trailingOperatingExpense", + "trailingOperatingIncome", + "trailingOperatingRevenue", + "trailingOtherGandA", + "trailingOtherIncomeExpense", + "trailingOtherNonOperatingIncomeExpenses", + "trailingOtherOperatingExpenses", + "trailingOtherSpecialCharges", + "trailingOtherTaxes", + "trailingOtherunderPreferredStockDividend", + "trailingPreferredStockDividends", + "trailingPretaxIncome", + "trailingProvisionForDoubtfulAccounts", + "trailingReconciledCostOfRevenue", + "trailingReconciledDepreciation", + "trailingRentAndLandingFees", + "trailingRentExpenseSupplemental", + "trailingReportedNormalizedBasicEPS", + "trailingReportedNormalizedDilutedEPS", + "trailingResearchAndDevelopment", + "trailingRestructuringAndMergernAcquisition", + "trailingSalariesAndWages", + "trailingSecuritiesAmortization", + "trailingSellingAndMarketingExpense", + "trailingSellingGeneralAndAdministration", + "trailingSpecialIncomeCharges", + "trailingTaxEffectOfUnusualItems", + "trailingTaxLossCarryforwardBasicEPS", + "trailingTaxLossCarryforwardDilutedEPS", + "trailingTaxProvision", + "trailingTaxRateForCalcs", + "trailingTotalExpenses", + "trailingTotalOperatingIncomeAsReported", + "trailingTotalOtherFinanceCost", + "trailingTotalRevenue", + "trailingTotalUnusualItems", + "trailingTotalUnusualItemsExcludingGoodwill", + "trailingWriteOff", + ] + }, + "balance_sheet": { + "annual": [ + "annualAccountsPayable", + "annualAccountsReceivable", + "annualAccruedInterestReceivable", + "annualAccumulatedDepreciation", + "annualAdditionalPaidInCapital", + "annualAllowanceForDoubtfulAccountsReceivable", + "annualAssetsHeldForSaleCurrent", + "annualAvailableForSaleSecurities", + "annualBuildingsAndImprovements", + "annualCapitalLeaseObligations", + "annualCapitalStock", + "annualCashAndCashEquivalents", + "annualCashCashEquivalentsAndShortTermInvestments", + "annualCashEquivalents", + "annualCashFinancial", + "annualCommercialPaper", + "annualCommonStock", + "annualCommonStockEquity", + "annualConstructionInProgress", + "annualCurrentAccruedExpenses", + "annualCurrentAssets", + "annualCurrentCapitalLeaseObligation", + "annualCurrentDebt", + "annualCurrentDebtAndCapitalLeaseObligation", + "annualCurrentDeferredAssets", + "annualCurrentDeferredLiabilities", + "annualCurrentDeferredRevenue", + "annualCurrentDeferredTaxesAssets", + "annualCurrentDeferredTaxesLiabilities", + "annualCurrentLiabilities", + "annualCurrentNotesPayable", + "annualCurrentProvisions", + "annualDefinedPensionBenefit", + "annualDerivativeProductLiabilities", + "annualDividendsPayable", + "annualDuefromRelatedPartiesCurrent", + "annualDuefromRelatedPartiesNonCurrent", + "annualDuetoRelatedPartiesCurrent", + "annualDuetoRelatedPartiesNonCurrent", + "annualEmployeeBenefits", + "annualFinancialAssets", + "annualFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "annualFinishedGoods", + "annualFixedAssetsRevaluationReserve", + "annualForeignCurrencyTranslationAdjustments", + "annualGainsLossesNotAffectingRetainedEarnings", + "annualGeneralPartnershipCapital", + "annualGoodwill", + "annualGoodwillAndOtherIntangibleAssets", + "annualGrossAccountsReceivable", + "annualGrossPPE", + "annualHedgingAssetsCurrent", + "annualHeldToMaturitySecurities", + "annualIncomeTaxPayable", + "annualInterestPayable", + "annualInventoriesAdjustmentsAllowances", + "annualInventory", + "annualInvestedCapital", + "annualInvestmentProperties", + "annualInvestmentinFinancialAssets", + "annualInvestmentsAndAdvances", + "annualInvestmentsInOtherVenturesUnderEquityMethod", + "annualInvestmentsinAssociatesatCost", + "annualInvestmentsinJointVenturesatCost", + "annualInvestmentsinSubsidiariesatCost", + "annualLandAndImprovements", + "annualLeases", + "annualLiabilitiesHeldforSaleNonCurrent", + "annualLimitedPartnershipCapital", + "annualLineOfCredit", + "annualLoansReceivable", + "annualLongTermCapitalLeaseObligation", + "annualLongTermDebt", + "annualLongTermDebtAndCapitalLeaseObligation", + "annualLongTermEquityInvestment", + "annualLongTermProvisions", + "annualMachineryFurnitureEquipment", + "annualMinimumPensionLiabilities", + "annualMinorityInterest", + "annualNetDebt", + "annualNetPPE", + "annualNetTangibleAssets", + "annualNonCurrentAccountsReceivable", + "annualNonCurrentAccruedExpenses", + "annualNonCurrentDeferredAssets", + "annualNonCurrentDeferredLiabilities", + "annualNonCurrentDeferredRevenue", + "annualNonCurrentDeferredTaxesAssets", + "annualNonCurrentDeferredTaxesLiabilities", + "annualNonCurrentNoteReceivables", + "annualNonCurrentPensionAndOtherPostretirementBenefitPlans", + "annualNonCurrentPrepaidAssets", + "annualNotesReceivable", + "annualOrdinarySharesNumber", + "annualOtherCapitalStock", + "annualOtherCurrentAssets", + "annualOtherCurrentBorrowings", + "annualOtherCurrentLiabilities", + "annualOtherEquityAdjustments", + "annualOtherEquityInterest", + "annualOtherIntangibleAssets", + "annualOtherInventories", + "annualOtherInvestments", + "annualOtherNonCurrentAssets", + "annualOtherNonCurrentLiabilities", + "annualOtherPayable", + "annualOtherProperties", + "annualOtherReceivables", + "annualOtherShortTermInvestments", + "annualPayables", + "annualPayablesAndAccruedExpenses", + "annualPensionandOtherPostRetirementBenefitPlansCurrent", + "annualPreferredSecuritiesOutsideStockEquity", + "annualPreferredSharesNumber", + "annualPreferredStock", + "annualPreferredStockEquity", + "annualPrepaidAssets", + "annualProperties", + "annualRawMaterials", + "annualReceivables", + "annualReceivablesAdjustmentsAllowances", + "annualRestrictedCash", + "annualRestrictedCommonStock", + "annualRetainedEarnings", + "annualShareIssued", + "annualStockholdersEquity", + "annualTangibleBookValue", + "annualTaxesReceivable", + "annualTotalAssets", + "annualTotalCapitalization", + "annualTotalDebt", + "annualTotalEquityGrossMinorityInterest", + "annualTotalLiabilitiesNetMinorityInterest", + "annualTotalNonCurrentAssets", + "annualTotalNonCurrentLiabilitiesNetMinorityInterest", + "annualTotalPartnershipCapital", + "annualTotalTaxPayable", + "annualTradeandOtherPayablesNonCurrent", + "annualTradingSecurities", + "annualTreasurySharesNumber", + "annualTreasuryStock", + "annualUnrealizedGainLoss", + "annualWorkInProcess", + "annualWorkingCapital", + "trailingAccountsPayable", + "trailingAccountsReceivable", + "trailingAccruedInterestReceivable", + "trailingAccumulatedDepreciation", + "trailingAdditionalPaidInCapital", + "trailingAllowanceForDoubtfulAccountsReceivable", + "trailingAssetsHeldForSaleCurrent", + "trailingAvailableForSaleSecurities", + "trailingBuildingsAndImprovements", + "trailingCapitalLeaseObligations", + "trailingCapitalStock", + "trailingCashAndCashEquivalents", + "trailingCashCashEquivalentsAndShortTermInvestments", + "trailingCashEquivalents", + "trailingCashFinancial", + "trailingCommercialPaper", + "trailingCommonStock", + "trailingCommonStockEquity", + "trailingConstructionInProgress", + "trailingCurrentAccruedExpenses", + "trailingCurrentAssets", + "trailingCurrentCapitalLeaseObligation", + "trailingCurrentDebt", + "trailingCurrentDebtAndCapitalLeaseObligation", + "trailingCurrentDeferredAssets", + "trailingCurrentDeferredLiabilities", + "trailingCurrentDeferredRevenue", + "trailingCurrentDeferredTaxesAssets", + "trailingCurrentDeferredTaxesLiabilities", + "trailingCurrentLiabilities", + "trailingCurrentNotesPayable", + "trailingCurrentProvisions", + "trailingDefinedPensionBenefit", + "trailingDerivativeProductLiabilities", + "trailingDividendsPayable", + "trailingDuefromRelatedPartiesCurrent", + "trailingDuefromRelatedPartiesNonCurrent", + "trailingDuetoRelatedPartiesCurrent", + "trailingDuetoRelatedPartiesNonCurrent", + "trailingEmployeeBenefits", + "trailingFinancialAssets", + "trailingFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "trailingFinishedGoods", + "trailingFixedAssetsRevaluationReserve", + "trailingForeignCurrencyTranslationAdjustments", + "trailingGainsLossesNotAffectingRetainedEarnings", + "trailingGeneralPartnershipCapital", + "trailingGoodwill", + "trailingGoodwillAndOtherIntangibleAssets", + "trailingGrossAccountsReceivable", + "trailingGrossPPE", + "trailingHedgingAssetsCurrent", + "trailingHeldToMaturitySecurities", + "trailingIncomeTaxPayable", + "trailingInterestPayable", + "trailingInventoriesAdjustmentsAllowances", + "trailingInventory", + "trailingInvestedCapital", + "trailingInvestmentProperties", + "trailingInvestmentinFinancialAssets", + "trailingInvestmentsAndAdvances", + "trailingInvestmentsInOtherVenturesUnderEquityMethod", + "trailingInvestmentsinAssociatesatCost", + "trailingInvestmentsinJointVenturesatCost", + "trailingInvestmentsinSubsidiariesatCost", + "trailingLandAndImprovements", + "trailingLeases", + "trailingLiabilitiesHeldforSaleNonCurrent", + "trailingLimitedPartnershipCapital", + "trailingLineOfCredit", + "trailingLoansReceivable", + "trailingLongTermCapitalLeaseObligation", + "trailingLongTermDebt", + "trailingLongTermDebtAndCapitalLeaseObligation", + "trailingLongTermEquityInvestment", + "trailingLongTermProvisions", + "trailingMachineryFurnitureEquipment", + "trailingMinimumPensionLiabilities", + "trailingMinorityInterest", + "trailingNetDebt", + "trailingNetPPE", + "trailingNetTangibleAssets", + "trailingNonCurrentAccountsReceivable", + "trailingNonCurrentAccruedExpenses", + "trailingNonCurrentDeferredAssets", + "trailingNonCurrentDeferredLiabilities", + "trailingNonCurrentDeferredRevenue", + "trailingNonCurrentDeferredTaxesAssets", + "trailingNonCurrentDeferredTaxesLiabilities", + "trailingNonCurrentNoteReceivables", + "trailingNonCurrentPensionAndOtherPostretirementBenefitPlans", + "trailingNonCurrentPrepaidAssets", + "trailingNotesReceivable", + "trailingOrdinarySharesNumber", + "trailingOtherCapitalStock", + "trailingOtherCurrentAssets", + "trailingOtherCurrentBorrowings", + "trailingOtherCurrentLiabilities", + "trailingOtherEquityAdjustments", + "trailingOtherEquityInterest", + "trailingOtherIntangibleAssets", + "trailingOtherInventories", + "trailingOtherInvestments", + "trailingOtherNonCurrentAssets", + "trailingOtherNonCurrentLiabilities", + "trailingOtherPayable", + "trailingOtherProperties", + "trailingOtherReceivables", + "trailingOtherShortTermInvestments", + "trailingPayables", + "trailingPayablesAndAccruedExpenses", + "trailingPensionandOtherPostRetirementBenefitPlansCurrent", + "trailingPreferredSecuritiesOutsideStockEquity", + "trailingPreferredSharesNumber", + "trailingPreferredStock", + "trailingPreferredStockEquity", + "trailingPrepaidAssets", + "trailingProperties", + "trailingRawMaterials", + "trailingReceivables", + "trailingReceivablesAdjustmentsAllowances", + "trailingRestrictedCash", + "trailingRestrictedCommonStock", + "trailingRetainedEarnings", + "trailingShareIssued", + "trailingStockholdersEquity", + "trailingTangibleBookValue", + "trailingTaxesReceivable", + "trailingTotalAssets", + "trailingTotalCapitalization", + "trailingTotalDebt", + "trailingTotalEquityGrossMinorityInterest", + "trailingTotalLiabilitiesNetMinorityInterest", + "trailingTotalNonCurrentAssets", + "trailingTotalNonCurrentLiabilitiesNetMinorityInterest", + "trailingTotalPartnershipCapital", + "trailingTotalTaxPayable", + "trailingTradeandOtherPayablesNonCurrent", + "trailingTradingSecurities", + "trailingTreasurySharesNumber", + "trailingTreasuryStock", + "trailingUnrealizedGainLoss", + "trailingWorkInProcess", + "trailingWorkingCapital", + ], + "quarterly": [ + "quarterlyAccountsPayable", + "quarterlyAccountsReceivable", + "quarterlyAccruedInterestReceivable", + "quarterlyAccumulatedDepreciation", + "quarterlyAdditionalPaidInCapital", + "quarterlyAllowanceForDoubtfulAccountsReceivable", + "quarterlyAssetsHeldForSaleCurrent", + "quarterlyAvailableForSaleSecurities", + "quarterlyBuildingsAndImprovements", + "quarterlyCapitalLeaseObligations", + "quarterlyCapitalStock", + "quarterlyCashAndCashEquivalents", + "quarterlyCashCashEquivalentsAndShortTermInvestments", + "quarterlyCashEquivalents", + "quarterlyCashFinancial", + "quarterlyCommercialPaper", + "quarterlyCommonStock", + "quarterlyCommonStockEquity", + "quarterlyConstructionInProgress", + "quarterlyCurrentAccruedExpenses", + "quarterlyCurrentAssets", + "quarterlyCurrentCapitalLeaseObligation", + "quarterlyCurrentDebt", + "quarterlyCurrentDebtAndCapitalLeaseObligation", + "quarterlyCurrentDeferredAssets", + "quarterlyCurrentDeferredLiabilities", + "quarterlyCurrentDeferredRevenue", + "quarterlyCurrentDeferredTaxesAssets", + "quarterlyCurrentDeferredTaxesLiabilities", + "quarterlyCurrentLiabilities", + "quarterlyCurrentNotesPayable", + "quarterlyCurrentProvisions", + "quarterlyDefinedPensionBenefit", + "quarterlyDerivativeProductLiabilities", + "quarterlyDividendsPayable", + "quarterlyDuefromRelatedPartiesCurrent", + "quarterlyDuefromRelatedPartiesNonCurrent", + "quarterlyDuetoRelatedPartiesCurrent", + "quarterlyDuetoRelatedPartiesNonCurrent", + "quarterlyEmployeeBenefits", + "quarterlyFinancialAssets", + "quarterlyFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "quarterlyFinishedGoods", + "quarterlyFixedAssetsRevaluationReserve", + "quarterlyForeignCurrencyTranslationAdjustments", + "quarterlyGainsLossesNotAffectingRetainedEarnings", + "quarterlyGeneralPartnershipCapital", + "quarterlyGoodwill", + "quarterlyGoodwillAndOtherIntangibleAssets", + "quarterlyGrossAccountsReceivable", + "quarterlyGrossPPE", + "quarterlyHedgingAssetsCurrent", + "quarterlyHeldToMaturitySecurities", + "quarterlyIncomeTaxPayable", + "quarterlyInterestPayable", + "quarterlyInventoriesAdjustmentsAllowances", + "quarterlyInventory", + "quarterlyInvestedCapital", + "quarterlyInvestmentProperties", + "quarterlyInvestmentinFinancialAssets", + "quarterlyInvestmentsAndAdvances", + "quarterlyInvestmentsInOtherVenturesUnderEquityMethod", + "quarterlyInvestmentsinAssociatesatCost", + "quarterlyInvestmentsinJointVenturesatCost", + "quarterlyInvestmentsinSubsidiariesatCost", + "quarterlyLandAndImprovements", + "quarterlyLeases", + "quarterlyLiabilitiesHeldforSaleNonCurrent", + "quarterlyLimitedPartnershipCapital", + "quarterlyLineOfCredit", + "quarterlyLoansReceivable", + "quarterlyLongTermCapitalLeaseObligation", + "quarterlyLongTermDebt", + "quarterlyLongTermDebtAndCapitalLeaseObligation", + "quarterlyLongTermEquityInvestment", + "quarterlyLongTermProvisions", + "quarterlyMachineryFurnitureEquipment", + "quarterlyMinimumPensionLiabilities", + "quarterlyMinorityInterest", + "quarterlyNetDebt", + "quarterlyNetPPE", + "quarterlyNetTangibleAssets", + "quarterlyNonCurrentAccountsReceivable", + "quarterlyNonCurrentAccruedExpenses", + "quarterlyNonCurrentDeferredAssets", + "quarterlyNonCurrentDeferredLiabilities", + "quarterlyNonCurrentDeferredRevenue", + "quarterlyNonCurrentDeferredTaxesAssets", + "quarterlyNonCurrentDeferredTaxesLiabilities", + "quarterlyNonCurrentNoteReceivables", + "quarterlyNonCurrentPensionAndOtherPostretirementBenefitPlans", + "quarterlyNonCurrentPrepaidAssets", + "quarterlyNotesReceivable", + "quarterlyOrdinarySharesNumber", + "quarterlyOtherCapitalStock", + "quarterlyOtherCurrentAssets", + "quarterlyOtherCurrentBorrowings", + "quarterlyOtherCurrentLiabilities", + "quarterlyOtherEquityAdjustments", + "quarterlyOtherEquityInterest", + "quarterlyOtherIntangibleAssets", + "quarterlyOtherInventories", + "quarterlyOtherInvestments", + "quarterlyOtherNonCurrentAssets", + "quarterlyOtherNonCurrentLiabilities", + "quarterlyOtherPayable", + "quarterlyOtherProperties", + "quarterlyOtherReceivables", + "quarterlyOtherShortTermInvestments", + "quarterlyPayables", + "quarterlyPayablesAndAccruedExpenses", + "quarterlyPensionandOtherPostRetirementBenefitPlansCurrent", + "quarterlyPreferredSecuritiesOutsideStockEquity", + "quarterlyPreferredSharesNumber", + "quarterlyPreferredStock", + "quarterlyPreferredStockEquity", + "quarterlyPrepaidAssets", + "quarterlyProperties", + "quarterlyRawMaterials", + "quarterlyReceivables", + "quarterlyReceivablesAdjustmentsAllowances", + "quarterlyRestrictedCash", + "quarterlyRestrictedCommonStock", + "quarterlyRetainedEarnings", + "quarterlyShareIssued", + "quarterlyStockholdersEquity", + "quarterlyTangibleBookValue", + "quarterlyTaxesReceivable", + "quarterlyTotalAssets", + "quarterlyTotalCapitalization", + "quarterlyTotalDebt", + "quarterlyTotalEquityGrossMinorityInterest", + "quarterlyTotalLiabilitiesNetMinorityInterest", + "quarterlyTotalNonCurrentAssets", + "quarterlyTotalNonCurrentLiabilitiesNetMinorityInterest", + "quarterlyTotalPartnershipCapital", + "quarterlyTotalTaxPayable", + "quarterlyTradeandOtherPayablesNonCurrent", + "quarterlyTradingSecurities", + "quarterlyTreasurySharesNumber", + "quarterlyTreasuryStock", + "quarterlyUnrealizedGainLoss", + "quarterlyWorkInProcess", + "quarterlyWorkingCapital", + "trailingAccountsPayable", + "trailingAccountsReceivable", + "trailingAccruedInterestReceivable", + "trailingAccumulatedDepreciation", + "trailingAdditionalPaidInCapital", + "trailingAllowanceForDoubtfulAccountsReceivable", + "trailingAssetsHeldForSaleCurrent", + "trailingAvailableForSaleSecurities", + "trailingBuildingsAndImprovements", + "trailingCapitalLeaseObligations", + "trailingCapitalStock", + "trailingCashAndCashEquivalents", + "trailingCashCashEquivalentsAndShortTermInvestments", + "trailingCashEquivalents", + "trailingCashFinancial", + "trailingCommercialPaper", + "trailingCommonStock", + "trailingCommonStockEquity", + "trailingConstructionInProgress", + "trailingCurrentAccruedExpenses", + "trailingCurrentAssets", + "trailingCurrentCapitalLeaseObligation", + "trailingCurrentDebt", + "trailingCurrentDebtAndCapitalLeaseObligation", + "trailingCurrentDeferredAssets", + "trailingCurrentDeferredLiabilities", + "trailingCurrentDeferredRevenue", + "trailingCurrentDeferredTaxesAssets", + "trailingCurrentDeferredTaxesLiabilities", + "trailingCurrentLiabilities", + "trailingCurrentNotesPayable", + "trailingCurrentProvisions", + "trailingDefinedPensionBenefit", + "trailingDerivativeProductLiabilities", + "trailingDividendsPayable", + "trailingDuefromRelatedPartiesCurrent", + "trailingDuefromRelatedPartiesNonCurrent", + "trailingDuetoRelatedPartiesCurrent", + "trailingDuetoRelatedPartiesNonCurrent", + "trailingEmployeeBenefits", + "trailingFinancialAssets", + "trailingFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "trailingFinishedGoods", + "trailingFixedAssetsRevaluationReserve", + "trailingForeignCurrencyTranslationAdjustments", + "trailingGainsLossesNotAffectingRetainedEarnings", + "trailingGeneralPartnershipCapital", + "trailingGoodwill", + "trailingGoodwillAndOtherIntangibleAssets", + "trailingGrossAccountsReceivable", + "trailingGrossPPE", + "trailingHedgingAssetsCurrent", + "trailingHeldToMaturitySecurities", + "trailingIncomeTaxPayable", + "trailingInterestPayable", + "trailingInventoriesAdjustmentsAllowances", + "trailingInventory", + "trailingInvestedCapital", + "trailingInvestmentProperties", + "trailingInvestmentinFinancialAssets", + "trailingInvestmentsAndAdvances", + "trailingInvestmentsInOtherVenturesUnderEquityMethod", + "trailingInvestmentsinAssociatesatCost", + "trailingInvestmentsinJointVenturesatCost", + "trailingInvestmentsinSubsidiariesatCost", + "trailingLandAndImprovements", + "trailingLeases", + "trailingLiabilitiesHeldforSaleNonCurrent", + "trailingLimitedPartnershipCapital", + "trailingLineOfCredit", + "trailingLoansReceivable", + "trailingLongTermCapitalLeaseObligation", + "trailingLongTermDebt", + "trailingLongTermDebtAndCapitalLeaseObligation", + "trailingLongTermEquityInvestment", + "trailingLongTermProvisions", + "trailingMachineryFurnitureEquipment", + "trailingMinimumPensionLiabilities", + "trailingMinorityInterest", + "trailingNetDebt", + "trailingNetPPE", + "trailingNetTangibleAssets", + "trailingNonCurrentAccountsReceivable", + "trailingNonCurrentAccruedExpenses", + "trailingNonCurrentDeferredAssets", + "trailingNonCurrentDeferredLiabilities", + "trailingNonCurrentDeferredRevenue", + "trailingNonCurrentDeferredTaxesAssets", + "trailingNonCurrentDeferredTaxesLiabilities", + "trailingNonCurrentNoteReceivables", + "trailingNonCurrentPensionAndOtherPostretirementBenefitPlans", + "trailingNonCurrentPrepaidAssets", + "trailingNotesReceivable", + "trailingOrdinarySharesNumber", + "trailingOtherCapitalStock", + "trailingOtherCurrentAssets", + "trailingOtherCurrentBorrowings", + "trailingOtherCurrentLiabilities", + "trailingOtherEquityAdjustments", + "trailingOtherEquityInterest", + "trailingOtherIntangibleAssets", + "trailingOtherInventories", + "trailingOtherInvestments", + "trailingOtherNonCurrentAssets", + "trailingOtherNonCurrentLiabilities", + "trailingOtherPayable", + "trailingOtherProperties", + "trailingOtherReceivables", + "trailingOtherShortTermInvestments", + "trailingPayables", + "trailingPayablesAndAccruedExpenses", + "trailingPensionandOtherPostRetirementBenefitPlansCurrent", + "trailingPreferredSecuritiesOutsideStockEquity", + "trailingPreferredSharesNumber", + "trailingPreferredStock", + "trailingPreferredStockEquity", + "trailingPrepaidAssets", + "trailingProperties", + "trailingRawMaterials", + "trailingReceivables", + "trailingReceivablesAdjustmentsAllowances", + "trailingRestrictedCash", + "trailingRestrictedCommonStock", + "trailingRetainedEarnings", + "trailingShareIssued", + "trailingStockholdersEquity", + "trailingTangibleBookValue", + "trailingTaxesReceivable", + "trailingTotalAssets", + "trailingTotalCapitalization", + "trailingTotalDebt", + "trailingTotalEquityGrossMinorityInterest", + "trailingTotalLiabilitiesNetMinorityInterest", + "trailingTotalNonCurrentAssets", + "trailingTotalNonCurrentLiabilitiesNetMinorityInterest", + "trailingTotalPartnershipCapital", + "trailingTotalTaxPayable", + "trailingTradeandOtherPayablesNonCurrent", + "trailingTradingSecurities", + "trailingTreasurySharesNumber", + "trailingTreasuryStock", + "trailingUnrealizedGainLoss", + "trailingWorkInProcess", + "trailingWorkingCapital", + ], + "monthly": [ + "monthlyAccountsPayable", + "monthlyAccountsReceivable", + "monthlyAccruedInterestReceivable", + "monthlyAccumulatedDepreciation", + "monthlyAdditionalPaidInCapital", + "monthlyAllowanceForDoubtfulAccountsReceivable", + "monthlyAssetsHeldForSaleCurrent", + "monthlyAvailableForSaleSecurities", + "monthlyBuildingsAndImprovements", + "monthlyCapitalLeaseObligations", + "monthlyCapitalStock", + "monthlyCashAndCashEquivalents", + "monthlyCashCashEquivalentsAndShortTermInvestments", + "monthlyCashEquivalents", + "monthlyCashFinancial", + "monthlyCommercialPaper", + "monthlyCommonStock", + "monthlyCommonStockEquity", + "monthlyConstructionInProgress", + "monthlyCurrentAccruedExpenses", + "monthlyCurrentAssets", + "monthlyCurrentCapitalLeaseObligation", + "monthlyCurrentDebt", + "monthlyCurrentDebtAndCapitalLeaseObligation", + "monthlyCurrentDeferredAssets", + "monthlyCurrentDeferredLiabilities", + "monthlyCurrentDeferredRevenue", + "monthlyCurrentDeferredTaxesAssets", + "monthlyCurrentDeferredTaxesLiabilities", + "monthlyCurrentLiabilities", + "monthlyCurrentNotesPayable", + "monthlyCurrentProvisions", + "monthlyDefinedPensionBenefit", + "monthlyDerivativeProductLiabilities", + "monthlyDividendsPayable", + "monthlyDuefromRelatedPartiesCurrent", + "monthlyDuefromRelatedPartiesNonCurrent", + "monthlyDuetoRelatedPartiesCurrent", + "monthlyDuetoRelatedPartiesNonCurrent", + "monthlyEmployeeBenefits", + "monthlyFinancialAssets", + "monthlyFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "monthlyFinishedGoods", + "monthlyFixedAssetsRevaluationReserve", + "monthlyForeignCurrencyTranslationAdjustments", + "monthlyGainsLossesNotAffectingRetainedEarnings", + "monthlyGeneralPartnershipCapital", + "monthlyGoodwill", + "monthlyGoodwillAndOtherIntangibleAssets", + "monthlyGrossAccountsReceivable", + "monthlyGrossPPE", + "monthlyHedgingAssetsCurrent", + "monthlyHeldToMaturitySecurities", + "monthlyIncomeTaxPayable", + "monthlyInterestPayable", + "monthlyInventoriesAdjustmentsAllowances", + "monthlyInventory", + "monthlyInvestedCapital", + "monthlyInvestmentProperties", + "monthlyInvestmentinFinancialAssets", + "monthlyInvestmentsAndAdvances", + "monthlyInvestmentsInOtherVenturesUnderEquityMethod", + "monthlyInvestmentsinAssociatesatCost", + "monthlyInvestmentsinJointVenturesatCost", + "monthlyInvestmentsinSubsidiariesatCost", + "monthlyLandAndImprovements", + "monthlyLeases", + "monthlyLiabilitiesHeldforSaleNonCurrent", + "monthlyLimitedPartnershipCapital", + "monthlyLineOfCredit", + "monthlyLoansReceivable", + "monthlyLongTermCapitalLeaseObligation", + "monthlyLongTermDebt", + "monthlyLongTermDebtAndCapitalLeaseObligation", + "monthlyLongTermEquityInvestment", + "monthlyLongTermProvisions", + "monthlyMachineryFurnitureEquipment", + "monthlyMinimumPensionLiabilities", + "monthlyMinorityInterest", + "monthlyNetDebt", + "monthlyNetPPE", + "monthlyNetTangibleAssets", + "monthlyNonCurrentAccountsReceivable", + "monthlyNonCurrentAccruedExpenses", + "monthlyNonCurrentDeferredAssets", + "monthlyNonCurrentDeferredLiabilities", + "monthlyNonCurrentDeferredRevenue", + "monthlyNonCurrentDeferredTaxesAssets", + "monthlyNonCurrentDeferredTaxesLiabilities", + "monthlyNonCurrentNoteReceivables", + "monthlyNonCurrentPensionAndOtherPostretirementBenefitPlans", + "monthlyNonCurrentPrepaidAssets", + "monthlyNotesReceivable", + "monthlyOrdinarySharesNumber", + "monthlyOtherCapitalStock", + "monthlyOtherCurrentAssets", + "monthlyOtherCurrentBorrowings", + "monthlyOtherCurrentLiabilities", + "monthlyOtherEquityAdjustments", + "monthlyOtherEquityInterest", + "monthlyOtherIntangibleAssets", + "monthlyOtherInventories", + "monthlyOtherInvestments", + "monthlyOtherNonCurrentAssets", + "monthlyOtherNonCurrentLiabilities", + "monthlyOtherPayable", + "monthlyOtherProperties", + "monthlyOtherReceivables", + "monthlyOtherShortTermInvestments", + "monthlyPayables", + "monthlyPayablesAndAccruedExpenses", + "monthlyPensionandOtherPostRetirementBenefitPlansCurrent", + "monthlyPreferredSecuritiesOutsideStockEquity", + "monthlyPreferredSharesNumber", + "monthlyPreferredStock", + "monthlyPreferredStockEquity", + "monthlyPrepaidAssets", + "monthlyProperties", + "monthlyRawMaterials", + "monthlyReceivables", + "monthlyReceivablesAdjustmentsAllowances", + "monthlyRestrictedCash", + "monthlyRestrictedCommonStock", + "monthlyRetainedEarnings", + "monthlyShareIssued", + "monthlyStockholdersEquity", + "monthlyTangibleBookValue", + "monthlyTaxesReceivable", + "monthlyTotalAssets", + "monthlyTotalCapitalization", + "monthlyTotalDebt", + "monthlyTotalEquityGrossMinorityInterest", + "monthlyTotalLiabilitiesNetMinorityInterest", + "monthlyTotalNonCurrentAssets", + "monthlyTotalNonCurrentLiabilitiesNetMinorityInterest", + "monthlyTotalPartnershipCapital", + "monthlyTotalTaxPayable", + "monthlyTradeandOtherPayablesNonCurrent", + "monthlyTradingSecurities", + "monthlyTreasurySharesNumber", + "monthlyTreasuryStock", + "monthlyUnrealizedGainLoss", + "monthlyWorkInProcess", + "monthlyWorkingCapital", + "trailingAccountsPayable", + "trailingAccountsReceivable", + "trailingAccruedInterestReceivable", + "trailingAccumulatedDepreciation", + "trailingAdditionalPaidInCapital", + "trailingAllowanceForDoubtfulAccountsReceivable", + "trailingAssetsHeldForSaleCurrent", + "trailingAvailableForSaleSecurities", + "trailingBuildingsAndImprovements", + "trailingCapitalLeaseObligations", + "trailingCapitalStock", + "trailingCashAndCashEquivalents", + "trailingCashCashEquivalentsAndShortTermInvestments", + "trailingCashEquivalents", + "trailingCashFinancial", + "trailingCommercialPaper", + "trailingCommonStock", + "trailingCommonStockEquity", + "trailingConstructionInProgress", + "trailingCurrentAccruedExpenses", + "trailingCurrentAssets", + "trailingCurrentCapitalLeaseObligation", + "trailingCurrentDebt", + "trailingCurrentDebtAndCapitalLeaseObligation", + "trailingCurrentDeferredAssets", + "trailingCurrentDeferredLiabilities", + "trailingCurrentDeferredRevenue", + "trailingCurrentDeferredTaxesAssets", + "trailingCurrentDeferredTaxesLiabilities", + "trailingCurrentLiabilities", + "trailingCurrentNotesPayable", + "trailingCurrentProvisions", + "trailingDefinedPensionBenefit", + "trailingDerivativeProductLiabilities", + "trailingDividendsPayable", + "trailingDuefromRelatedPartiesCurrent", + "trailingDuefromRelatedPartiesNonCurrent", + "trailingDuetoRelatedPartiesCurrent", + "trailingDuetoRelatedPartiesNonCurrent", + "trailingEmployeeBenefits", + "trailingFinancialAssets", + "trailingFinancialAssetsDesignatedasFairValueThroughProfitorLossTotal", + "trailingFinishedGoods", + "trailingFixedAssetsRevaluationReserve", + "trailingForeignCurrencyTranslationAdjustments", + "trailingGainsLossesNotAffectingRetainedEarnings", + "trailingGeneralPartnershipCapital", + "trailingGoodwill", + "trailingGoodwillAndOtherIntangibleAssets", + "trailingGrossAccountsReceivable", + "trailingGrossPPE", + "trailingHedgingAssetsCurrent", + "trailingHeldToMaturitySecurities", + "trailingIncomeTaxPayable", + "trailingInterestPayable", + "trailingInventoriesAdjustmentsAllowances", + "trailingInventory", + "trailingInvestedCapital", + "trailingInvestmentProperties", + "trailingInvestmentinFinancialAssets", + "trailingInvestmentsAndAdvances", + "trailingInvestmentsInOtherVenturesUnderEquityMethod", + "trailingInvestmentsinAssociatesatCost", + "trailingInvestmentsinJointVenturesatCost", + "trailingInvestmentsinSubsidiariesatCost", + "trailingLandAndImprovements", + "trailingLeases", + "trailingLiabilitiesHeldforSaleNonCurrent", + "trailingLimitedPartnershipCapital", + "trailingLineOfCredit", + "trailingLoansReceivable", + "trailingLongTermCapitalLeaseObligation", + "trailingLongTermDebt", + "trailingLongTermDebtAndCapitalLeaseObligation", + "trailingLongTermEquityInvestment", + "trailingLongTermProvisions", + "trailingMachineryFurnitureEquipment", + "trailingMinimumPensionLiabilities", + "trailingMinorityInterest", + "trailingNetDebt", + "trailingNetPPE", + "trailingNetTangibleAssets", + "trailingNonCurrentAccountsReceivable", + "trailingNonCurrentAccruedExpenses", + "trailingNonCurrentDeferredAssets", + "trailingNonCurrentDeferredLiabilities", + "trailingNonCurrentDeferredRevenue", + "trailingNonCurrentDeferredTaxesAssets", + "trailingNonCurrentDeferredTaxesLiabilities", + "trailingNonCurrentNoteReceivables", + "trailingNonCurrentPensionAndOtherPostretirementBenefitPlans", + "trailingNonCurrentPrepaidAssets", + "trailingNotesReceivable", + "trailingOrdinarySharesNumber", + "trailingOtherCapitalStock", + "trailingOtherCurrentAssets", + "trailingOtherCurrentBorrowings", + "trailingOtherCurrentLiabilities", + "trailingOtherEquityAdjustments", + "trailingOtherEquityInterest", + "trailingOtherIntangibleAssets", + "trailingOtherInventories", + "trailingOtherInvestments", + "trailingOtherNonCurrentAssets", + "trailingOtherNonCurrentLiabilities", + "trailingOtherPayable", + "trailingOtherProperties", + "trailingOtherReceivables", + "trailingOtherShortTermInvestments", + "trailingPayables", + "trailingPayablesAndAccruedExpenses", + "trailingPensionandOtherPostRetirementBenefitPlansCurrent", + "trailingPreferredSecuritiesOutsideStockEquity", + "trailingPreferredSharesNumber", + "trailingPreferredStock", + "trailingPreferredStockEquity", + "trailingPrepaidAssets", + "trailingProperties", + "trailingRawMaterials", + "trailingReceivables", + "trailingReceivablesAdjustmentsAllowances", + "trailingRestrictedCash", + "trailingRestrictedCommonStock", + "trailingRetainedEarnings", + "trailingShareIssued", + "trailingStockholdersEquity", + "trailingTangibleBookValue", + "trailingTaxesReceivable", + "trailingTotalAssets", + "trailingTotalCapitalization", + "trailingTotalDebt", + "trailingTotalEquityGrossMinorityInterest", + "trailingTotalLiabilitiesNetMinorityInterest", + "trailingTotalNonCurrentAssets", + "trailingTotalNonCurrentLiabilitiesNetMinorityInterest", + "trailingTotalPartnershipCapital", + "trailingTotalTaxPayable", + "trailingTradeandOtherPayablesNonCurrent", + "trailingTradingSecurities", + "trailingTreasurySharesNumber", + "trailingTreasuryStock", + "trailingUnrealizedGainLoss", + "trailingWorkInProcess", + "trailingWorkingCapital", + ] + }, + "cash_flow": { + "annual": [ + "annualAdjustedGeographySegmentData", + "annualAmortizationCashFlow", + "annualAmortizationOfIntangibles", + "annualAmortizationOfSecurities", + "annualAssetImpairmentCharge", + "annualBeginningCashPosition", + "annualCapitalExpenditure", + "annualCapitalExpenditureReported", + "annualCashDividendsPaid", + "annualCashFlowFromContinuingFinancingActivities", + "annualCashFlowFromContinuingInvestingActivities", + "annualCashFlowFromContinuingOperatingActivities", + "annualCashFlowFromDiscontinuedOperation", + "annualCashFlowsfromusedinOperatingActivitiesDirect", + "annualCashFromDiscontinuedFinancingActivities", + "annualCashFromDiscontinuedInvestingActivities", + "annualCashFromDiscontinuedOperatingActivities", + "annualChangeInAccountPayable", + "annualChangeInAccruedExpense", + "annualChangeInCashSupplementalAsReported", + "annualChangeInDividendPayable", + "annualChangeInIncomeTaxPayable", + "annualChangeInInterestPayable", + "annualChangeInInventory", + "annualChangeInOtherCurrentAssets", + "annualChangeInOtherCurrentLiabilities", + "annualChangeInOtherWorkingCapital", + "annualChangeInPayable", + "annualChangeInPayablesAndAccruedExpense", + "annualChangeInPrepaidAssets", + "annualChangeInReceivables", + "annualChangeInTaxPayable", + "annualChangeInWorkingCapital", + "annualChangesInAccountReceivables", + "annualChangesInCash", + "annualClassesofCashPayments", + "annualClassesofCashReceiptsfromOperatingActivities", + "annualCommonStockDividendPaid", + "annualCommonStockIssuance", + "annualCommonStockPayments", + "annualDeferredIncomeTax", + "annualDeferredTax", + "annualDepletion", + "annualDepreciation", + "annualDepreciationAmortizationDepletion", + "annualDepreciationAndAmortization", + "annualDividendPaidCFO", + "annualDividendReceivedCFO", + "annualDividendsPaidDirect", + "annualDividendsReceivedCFI", + "annualDividendsReceivedDirect", + "annualDomesticSales", + "annualEarningsLossesFromEquityInvestments", + "annualEffectOfExchangeRateChanges", + "annualEndCashPosition", + "annualExcessTaxBenefitFromStockBasedCompensation", + "annualFinancingCashFlow", + "annualForeignSales", + "annualFreeCashFlow", + "annualGainLossOnInvestmentSecurities", + "annualGainLossOnSaleOfBusiness", + "annualGainLossOnSaleOfPPE", + "annualIncomeTaxPaidSupplementalData", + "annualInterestPaidCFF", + "annualInterestPaidCFO", + "annualInterestPaidDirect", + "annualInterestPaidSupplementalData", + "annualInterestReceivedCFI", + "annualInterestReceivedCFO", + "annualInterestReceivedDirect", + "annualInvestingCashFlow", + "annualIssuanceOfCapitalStock", + "annualIssuanceOfDebt", + "annualLongTermDebtIssuance", + "annualLongTermDebtPayments", + "annualNetBusinessPurchaseAndSale", + "annualNetCommonStockIssuance", + "annualNetForeignCurrencyExchangeGainLoss", + "annualNetIncome", + "annualNetIncomeFromContinuingOperations", + "annualNetIntangiblesPurchaseAndSale", + "annualNetInvestmentPropertiesPurchaseAndSale", + "annualNetInvestmentPurchaseAndSale", + "annualNetIssuancePaymentsOfDebt", + "annualNetLongTermDebtIssuance", + "annualNetOtherFinancingCharges", + "annualNetOtherInvestingChanges", + "annualNetPPEPurchaseAndSale", + "annualNetPreferredStockIssuance", + "annualNetShortTermDebtIssuance", + "annualOperatingCashFlow", + "annualOperatingGainsLosses", + "annualOtherCashAdjustmentInsideChangeinCash", + "annualOtherCashAdjustmentOutsideChangeinCash", + "annualOtherCashPaymentsfromOperatingActivities", + "annualOtherCashReceiptsfromOperatingActivities", + "annualOtherNonCashItems", + "annualPaymentsonBehalfofEmployees", + "annualPaymentstoSuppliersforGoodsandServices", + "annualPensionAndEmployeeBenefitExpense", + "annualPreferredStockDividendPaid", + "annualPreferredStockIssuance", + "annualPreferredStockPayments", + "annualProceedsFromStockOptionExercised", + "annualProvisionandWriteOffofAssets", + "annualPurchaseOfBusiness", + "annualPurchaseOfIntangibles", + "annualPurchaseOfInvestment", + "annualPurchaseOfInvestmentProperties", + "annualPurchaseOfPPE", + "annualReceiptsfromCustomers", + "annualReceiptsfromGovernmentGrants", + "annualRepaymentOfDebt", + "annualRepurchaseOfCapitalStock", + "annualSaleOfBusiness", + "annualSaleOfIntangibles", + "annualSaleOfInvestment", + "annualSaleOfInvestmentProperties", + "annualSaleOfPPE", + "annualShortTermDebtIssuance", + "annualShortTermDebtPayments", + "annualStockBasedCompensation", + "annualTaxesRefundPaid", + "annualTaxesRefundPaidDirect", + "annualUnrealizedGainLossOnInvestmentSecurities", + "trailingAdjustedGeographySegmentData", + "trailingAmortizationCashFlow", + "trailingAmortizationOfIntangibles", + "trailingAmortizationOfSecurities", + "trailingAssetImpairmentCharge", + "trailingBeginningCashPosition", + "trailingCapitalExpenditure", + "trailingCapitalExpenditureReported", + "trailingCashDividendsPaid", + "trailingCashFlowFromContinuingFinancingActivities", + "trailingCashFlowFromContinuingInvestingActivities", + "trailingCashFlowFromContinuingOperatingActivities", + "trailingCashFlowFromDiscontinuedOperation", + "trailingCashFlowsfromusedinOperatingActivitiesDirect", + "trailingCashFromDiscontinuedFinancingActivities", + "trailingCashFromDiscontinuedInvestingActivities", + "trailingCashFromDiscontinuedOperatingActivities", + "trailingChangeInAccountPayable", + "trailingChangeInAccruedExpense", + "trailingChangeInCashSupplementalAsReported", + "trailingChangeInDividendPayable", + "trailingChangeInIncomeTaxPayable", + "trailingChangeInInterestPayable", + "trailingChangeInInventory", + "trailingChangeInOtherCurrentAssets", + "trailingChangeInOtherCurrentLiabilities", + "trailingChangeInOtherWorkingCapital", + "trailingChangeInPayable", + "trailingChangeInPayablesAndAccruedExpense", + "trailingChangeInPrepaidAssets", + "trailingChangeInReceivables", + "trailingChangeInTaxPayable", + "trailingChangeInWorkingCapital", + "trailingChangesInAccountReceivables", + "trailingChangesInCash", + "trailingClassesofCashPayments", + "trailingClassesofCashReceiptsfromOperatingActivities", + "trailingCommonStockDividendPaid", + "trailingCommonStockIssuance", + "trailingCommonStockPayments", + "trailingDeferredIncomeTax", + "trailingDeferredTax", + "trailingDepletion", + "trailingDepreciation", + "trailingDepreciationAmortizationDepletion", + "trailingDepreciationAndAmortization", + "trailingDividendPaidCFO", + "trailingDividendReceivedCFO", + "trailingDividendsPaidDirect", + "trailingDividendsReceivedCFI", + "trailingDividendsReceivedDirect", + "trailingDomesticSales", + "trailingEarningsLossesFromEquityInvestments", + "trailingEffectOfExchangeRateChanges", + "trailingEndCashPosition", + "trailingExcessTaxBenefitFromStockBasedCompensation", + "trailingFinancingCashFlow", + "trailingForeignSales", + "trailingFreeCashFlow", + "trailingGainLossOnInvestmentSecurities", + "trailingGainLossOnSaleOfBusiness", + "trailingGainLossOnSaleOfPPE", + "trailingIncomeTaxPaidSupplementalData", + "trailingInterestPaidCFF", + "trailingInterestPaidCFO", + "trailingInterestPaidDirect", + "trailingInterestPaidSupplementalData", + "trailingInterestReceivedCFI", + "trailingInterestReceivedCFO", + "trailingInterestReceivedDirect", + "trailingInvestingCashFlow", + "trailingIssuanceOfCapitalStock", + "trailingIssuanceOfDebt", + "trailingLongTermDebtIssuance", + "trailingLongTermDebtPayments", + "trailingNetBusinessPurchaseAndSale", + "trailingNetCommonStockIssuance", + "trailingNetForeignCurrencyExchangeGainLoss", + "trailingNetIncome", + "trailingNetIncomeFromContinuingOperations", + "trailingNetIntangiblesPurchaseAndSale", + "trailingNetInvestmentPropertiesPurchaseAndSale", + "trailingNetInvestmentPurchaseAndSale", + "trailingNetIssuancePaymentsOfDebt", + "trailingNetLongTermDebtIssuance", + "trailingNetOtherFinancingCharges", + "trailingNetOtherInvestingChanges", + "trailingNetPPEPurchaseAndSale", + "trailingNetPreferredStockIssuance", + "trailingNetShortTermDebtIssuance", + "trailingOperatingCashFlow", + "trailingOperatingGainsLosses", + "trailingOtherCashAdjustmentInsideChangeinCash", + "trailingOtherCashAdjustmentOutsideChangeinCash", + "trailingOtherCashPaymentsfromOperatingActivities", + "trailingOtherCashReceiptsfromOperatingActivities", + "trailingOtherNonCashItems", + "trailingPaymentsonBehalfofEmployees", + "trailingPaymentstoSuppliersforGoodsandServices", + "trailingPensionAndEmployeeBenefitExpense", + "trailingPreferredStockDividendPaid", + "trailingPreferredStockIssuance", + "trailingPreferredStockPayments", + "trailingProceedsFromStockOptionExercised", + "trailingProvisionandWriteOffofAssets", + "trailingPurchaseOfBusiness", + "trailingPurchaseOfIntangibles", + "trailingPurchaseOfInvestment", + "trailingPurchaseOfInvestmentProperties", + "trailingPurchaseOfPPE", + "trailingReceiptsfromCustomers", + "trailingReceiptsfromGovernmentGrants", + "trailingRepaymentOfDebt", + "trailingRepurchaseOfCapitalStock", + "trailingSaleOfBusiness", + "trailingSaleOfIntangibles", + "trailingSaleOfInvestment", + "trailingSaleOfInvestmentProperties", + "trailingSaleOfPPE", + "trailingShortTermDebtIssuance", + "trailingShortTermDebtPayments", + "trailingStockBasedCompensation", + "trailingTaxesRefundPaid", + "trailingTaxesRefundPaidDirect", + "trailingUnrealizedGainLossOnInvestmentSecurities", + ], + "quarterly": [ + "quarterlyAdjustedGeographySegmentData", + "quarterlyAmortizationCashFlow", + "quarterlyAmortizationOfIntangibles", + "quarterlyAmortizationOfSecurities", + "quarterlyAssetImpairmentCharge", + "quarterlyBeginningCashPosition", + "quarterlyCapitalExpenditure", + "quarterlyCapitalExpenditureReported", + "quarterlyCashDividendsPaid", + "quarterlyCashFlowFromContinuingFinancingActivities", + "quarterlyCashFlowFromContinuingInvestingActivities", + "quarterlyCashFlowFromContinuingOperatingActivities", + "quarterlyCashFlowFromDiscontinuedOperation", + "quarterlyCashFlowsfromusedinOperatingActivitiesDirect", + "quarterlyCashFromDiscontinuedFinancingActivities", + "quarterlyCashFromDiscontinuedInvestingActivities", + "quarterlyCashFromDiscontinuedOperatingActivities", + "quarterlyChangeInAccountPayable", + "quarterlyChangeInAccruedExpense", + "quarterlyChangeInCashSupplementalAsReported", + "quarterlyChangeInDividendPayable", + "quarterlyChangeInIncomeTaxPayable", + "quarterlyChangeInInterestPayable", + "quarterlyChangeInInventory", + "quarterlyChangeInOtherCurrentAssets", + "quarterlyChangeInOtherCurrentLiabilities", + "quarterlyChangeInOtherWorkingCapital", + "quarterlyChangeInPayable", + "quarterlyChangeInPayablesAndAccruedExpense", + "quarterlyChangeInPrepaidAssets", + "quarterlyChangeInReceivables", + "quarterlyChangeInTaxPayable", + "quarterlyChangeInWorkingCapital", + "quarterlyChangesInAccountReceivables", + "quarterlyChangesInCash", + "quarterlyClassesofCashPayments", + "quarterlyClassesofCashReceiptsfromOperatingActivities", + "quarterlyCommonStockDividendPaid", + "quarterlyCommonStockIssuance", + "quarterlyCommonStockPayments", + "quarterlyDeferredIncomeTax", + "quarterlyDeferredTax", + "quarterlyDepletion", + "quarterlyDepreciation", + "quarterlyDepreciationAmortizationDepletion", + "quarterlyDepreciationAndAmortization", + "quarterlyDividendPaidCFO", + "quarterlyDividendReceivedCFO", + "quarterlyDividendsPaidDirect", + "quarterlyDividendsReceivedCFI", + "quarterlyDividendsReceivedDirect", + "quarterlyDomesticSales", + "quarterlyEarningsLossesFromEquityInvestments", + "quarterlyEffectOfExchangeRateChanges", + "quarterlyEndCashPosition", + "quarterlyExcessTaxBenefitFromStockBasedCompensation", + "quarterlyFinancingCashFlow", + "quarterlyForeignSales", + "quarterlyFreeCashFlow", + "quarterlyGainLossOnInvestmentSecurities", + "quarterlyGainLossOnSaleOfBusiness", + "quarterlyGainLossOnSaleOfPPE", + "quarterlyIncomeTaxPaidSupplementalData", + "quarterlyInterestPaidCFF", + "quarterlyInterestPaidCFO", + "quarterlyInterestPaidDirect", + "quarterlyInterestPaidSupplementalData", + "quarterlyInterestReceivedCFI", + "quarterlyInterestReceivedCFO", + "quarterlyInterestReceivedDirect", + "quarterlyInvestingCashFlow", + "quarterlyIssuanceOfCapitalStock", + "quarterlyIssuanceOfDebt", + "quarterlyLongTermDebtIssuance", + "quarterlyLongTermDebtPayments", + "quarterlyNetBusinessPurchaseAndSale", + "quarterlyNetCommonStockIssuance", + "quarterlyNetForeignCurrencyExchangeGainLoss", + "quarterlyNetIncome", + "quarterlyNetIncomeFromContinuingOperations", + "quarterlyNetIntangiblesPurchaseAndSale", + "quarterlyNetInvestmentPropertiesPurchaseAndSale", + "quarterlyNetInvestmentPurchaseAndSale", + "quarterlyNetIssuancePaymentsOfDebt", + "quarterlyNetLongTermDebtIssuance", + "quarterlyNetOtherFinancingCharges", + "quarterlyNetOtherInvestingChanges", + "quarterlyNetPPEPurchaseAndSale", + "quarterlyNetPreferredStockIssuance", + "quarterlyNetShortTermDebtIssuance", + "quarterlyOperatingCashFlow", + "quarterlyOperatingGainsLosses", + "quarterlyOtherCashAdjustmentInsideChangeinCash", + "quarterlyOtherCashAdjustmentOutsideChangeinCash", + "quarterlyOtherCashPaymentsfromOperatingActivities", + "quarterlyOtherCashReceiptsfromOperatingActivities", + "quarterlyOtherNonCashItems", + "quarterlyPaymentsonBehalfofEmployees", + "quarterlyPaymentstoSuppliersforGoodsandServices", + "quarterlyPensionAndEmployeeBenefitExpense", + "quarterlyPreferredStockDividendPaid", + "quarterlyPreferredStockIssuance", + "quarterlyPreferredStockPayments", + "quarterlyProceedsFromStockOptionExercised", + "quarterlyProvisionandWriteOffofAssets", + "quarterlyPurchaseOfBusiness", + "quarterlyPurchaseOfIntangibles", + "quarterlyPurchaseOfInvestment", + "quarterlyPurchaseOfInvestmentProperties", + "quarterlyPurchaseOfPPE", + "quarterlyReceiptsfromCustomers", + "quarterlyReceiptsfromGovernmentGrants", + "quarterlyRepaymentOfDebt", + "quarterlyRepurchaseOfCapitalStock", + "quarterlySaleOfBusiness", + "quarterlySaleOfIntangibles", + "quarterlySaleOfInvestment", + "quarterlySaleOfInvestmentProperties", + "quarterlySaleOfPPE", + "quarterlyShortTermDebtIssuance", + "quarterlyShortTermDebtPayments", + "quarterlyStockBasedCompensation", + "quarterlyTaxesRefundPaid", + "quarterlyTaxesRefundPaidDirect", + "quarterlyUnrealizedGainLossOnInvestmentSecurities", + "trailingAdjustedGeographySegmentData", + "trailingAmortizationCashFlow", + "trailingAmortizationOfIntangibles", + "trailingAmortizationOfSecurities", + "trailingAssetImpairmentCharge", + "trailingBeginningCashPosition", + "trailingCapitalExpenditure", + "trailingCapitalExpenditureReported", + "trailingCashDividendsPaid", + "trailingCashFlowFromContinuingFinancingActivities", + "trailingCashFlowFromContinuingInvestingActivities", + "trailingCashFlowFromContinuingOperatingActivities", + "trailingCashFlowFromDiscontinuedOperation", + "trailingCashFlowsfromusedinOperatingActivitiesDirect", + "trailingCashFromDiscontinuedFinancingActivities", + "trailingCashFromDiscontinuedInvestingActivities", + "trailingCashFromDiscontinuedOperatingActivities", + "trailingChangeInAccountPayable", + "trailingChangeInAccruedExpense", + "trailingChangeInCashSupplementalAsReported", + "trailingChangeInDividendPayable", + "trailingChangeInIncomeTaxPayable", + "trailingChangeInInterestPayable", + "trailingChangeInInventory", + "trailingChangeInOtherCurrentAssets", + "trailingChangeInOtherCurrentLiabilities", + "trailingChangeInOtherWorkingCapital", + "trailingChangeInPayable", + "trailingChangeInPayablesAndAccruedExpense", + "trailingChangeInPrepaidAssets", + "trailingChangeInReceivables", + "trailingChangeInTaxPayable", + "trailingChangeInWorkingCapital", + "trailingChangesInAccountReceivables", + "trailingChangesInCash", + "trailingClassesofCashPayments", + "trailingClassesofCashReceiptsfromOperatingActivities", + "trailingCommonStockDividendPaid", + "trailingCommonStockIssuance", + "trailingCommonStockPayments", + "trailingDeferredIncomeTax", + "trailingDeferredTax", + "trailingDepletion", + "trailingDepreciation", + "trailingDepreciationAmortizationDepletion", + "trailingDepreciationAndAmortization", + "trailingDividendPaidCFO", + "trailingDividendReceivedCFO", + "trailingDividendsPaidDirect", + "trailingDividendsReceivedCFI", + "trailingDividendsReceivedDirect", + "trailingDomesticSales", + "trailingEarningsLossesFromEquityInvestments", + "trailingEffectOfExchangeRateChanges", + "trailingEndCashPosition", + "trailingExcessTaxBenefitFromStockBasedCompensation", + "trailingFinancingCashFlow", + "trailingForeignSales", + "trailingFreeCashFlow", + "trailingGainLossOnInvestmentSecurities", + "trailingGainLossOnSaleOfBusiness", + "trailingGainLossOnSaleOfPPE", + "trailingIncomeTaxPaidSupplementalData", + "trailingInterestPaidCFF", + "trailingInterestPaidCFO", + "trailingInterestPaidDirect", + "trailingInterestPaidSupplementalData", + "trailingInterestReceivedCFI", + "trailingInterestReceivedCFO", + "trailingInterestReceivedDirect", + "trailingInvestingCashFlow", + "trailingIssuanceOfCapitalStock", + "trailingIssuanceOfDebt", + "trailingLongTermDebtIssuance", + "trailingLongTermDebtPayments", + "trailingNetBusinessPurchaseAndSale", + "trailingNetCommonStockIssuance", + "trailingNetForeignCurrencyExchangeGainLoss", + "trailingNetIncome", + "trailingNetIncomeFromContinuingOperations", + "trailingNetIntangiblesPurchaseAndSale", + "trailingNetInvestmentPropertiesPurchaseAndSale", + "trailingNetInvestmentPurchaseAndSale", + "trailingNetIssuancePaymentsOfDebt", + "trailingNetLongTermDebtIssuance", + "trailingNetOtherFinancingCharges", + "trailingNetOtherInvestingChanges", + "trailingNetPPEPurchaseAndSale", + "trailingNetPreferredStockIssuance", + "trailingNetShortTermDebtIssuance", + "trailingOperatingCashFlow", + "trailingOperatingGainsLosses", + "trailingOtherCashAdjustmentInsideChangeinCash", + "trailingOtherCashAdjustmentOutsideChangeinCash", + "trailingOtherCashPaymentsfromOperatingActivities", + "trailingOtherCashReceiptsfromOperatingActivities", + "trailingOtherNonCashItems", + "trailingPaymentsonBehalfofEmployees", + "trailingPaymentstoSuppliersforGoodsandServices", + "trailingPensionAndEmployeeBenefitExpense", + "trailingPreferredStockDividendPaid", + "trailingPreferredStockIssuance", + "trailingPreferredStockPayments", + "trailingProceedsFromStockOptionExercised", + "trailingProvisionandWriteOffofAssets", + "trailingPurchaseOfBusiness", + "trailingPurchaseOfIntangibles", + "trailingPurchaseOfInvestment", + "trailingPurchaseOfInvestmentProperties", + "trailingPurchaseOfPPE", + "trailingReceiptsfromCustomers", + "trailingReceiptsfromGovernmentGrants", + "trailingRepaymentOfDebt", + "trailingRepurchaseOfCapitalStock", + "trailingSaleOfBusiness", + "trailingSaleOfIntangibles", + "trailingSaleOfInvestment", + "trailingSaleOfInvestmentProperties", + "trailingSaleOfPPE", + "trailingShortTermDebtIssuance", + "trailingShortTermDebtPayments", + "trailingStockBasedCompensation", + "trailingTaxesRefundPaid", + "trailingTaxesRefundPaidDirect", + "trailingUnrealizedGainLossOnInvestmentSecurities", + ], + "monthly": [ + "monthlyAdjustedGeographySegmentData", + "monthlyAmortizationCashFlow", + "monthlyAmortizationOfIntangibles", + "monthlyAmortizationOfSecurities", + "monthlyAssetImpairmentCharge", + "monthlyBeginningCashPosition", + "monthlyCapitalExpenditure", + "monthlyCapitalExpenditureReported", + "monthlyCashDividendsPaid", + "monthlyCashFlowFromContinuingFinancingActivities", + "monthlyCashFlowFromContinuingInvestingActivities", + "monthlyCashFlowFromContinuingOperatingActivities", + "monthlyCashFlowFromDiscontinuedOperation", + "monthlyCashFlowsfromusedinOperatingActivitiesDirect", + "monthlyCashFromDiscontinuedFinancingActivities", + "monthlyCashFromDiscontinuedInvestingActivities", + "monthlyCashFromDiscontinuedOperatingActivities", + "monthlyChangeInAccountPayable", + "monthlyChangeInAccruedExpense", + "monthlyChangeInCashSupplementalAsReported", + "monthlyChangeInDividendPayable", + "monthlyChangeInIncomeTaxPayable", + "monthlyChangeInInterestPayable", + "monthlyChangeInInventory", + "monthlyChangeInOtherCurrentAssets", + "monthlyChangeInOtherCurrentLiabilities", + "monthlyChangeInOtherWorkingCapital", + "monthlyChangeInPayable", + "monthlyChangeInPayablesAndAccruedExpense", + "monthlyChangeInPrepaidAssets", + "monthlyChangeInReceivables", + "monthlyChangeInTaxPayable", + "monthlyChangeInWorkingCapital", + "monthlyChangesInAccountReceivables", + "monthlyChangesInCash", + "monthlyClassesofCashPayments", + "monthlyClassesofCashReceiptsfromOperatingActivities", + "monthlyCommonStockDividendPaid", + "monthlyCommonStockIssuance", + "monthlyCommonStockPayments", + "monthlyDeferredIncomeTax", + "monthlyDeferredTax", + "monthlyDepletion", + "monthlyDepreciation", + "monthlyDepreciationAmortizationDepletion", + "monthlyDepreciationAndAmortization", + "monthlyDividendPaidCFO", + "monthlyDividendReceivedCFO", + "monthlyDividendsPaidDirect", + "monthlyDividendsReceivedCFI", + "monthlyDividendsReceivedDirect", + "monthlyDomesticSales", + "monthlyEarningsLossesFromEquityInvestments", + "monthlyEffectOfExchangeRateChanges", + "monthlyEndCashPosition", + "monthlyExcessTaxBenefitFromStockBasedCompensation", + "monthlyFinancingCashFlow", + "monthlyForeignSales", + "monthlyFreeCashFlow", + "monthlyGainLossOnInvestmentSecurities", + "monthlyGainLossOnSaleOfBusiness", + "monthlyGainLossOnSaleOfPPE", + "monthlyIncomeTaxPaidSupplementalData", + "monthlyInterestPaidCFF", + "monthlyInterestPaidCFO", + "monthlyInterestPaidDirect", + "monthlyInterestPaidSupplementalData", + "monthlyInterestReceivedCFI", + "monthlyInterestReceivedCFO", + "monthlyInterestReceivedDirect", + "monthlyInvestingCashFlow", + "monthlyIssuanceOfCapitalStock", + "monthlyIssuanceOfDebt", + "monthlyLongTermDebtIssuance", + "monthlyLongTermDebtPayments", + "monthlyNetBusinessPurchaseAndSale", + "monthlyNetCommonStockIssuance", + "monthlyNetForeignCurrencyExchangeGainLoss", + "monthlyNetIncome", + "monthlyNetIncomeFromContinuingOperations", + "monthlyNetIntangiblesPurchaseAndSale", + "monthlyNetInvestmentPropertiesPurchaseAndSale", + "monthlyNetInvestmentPurchaseAndSale", + "monthlyNetIssuancePaymentsOfDebt", + "monthlyNetLongTermDebtIssuance", + "monthlyNetOtherFinancingCharges", + "monthlyNetOtherInvestingChanges", + "monthlyNetPPEPurchaseAndSale", + "monthlyNetPreferredStockIssuance", + "monthlyNetShortTermDebtIssuance", + "monthlyOperatingCashFlow", + "monthlyOperatingGainsLosses", + "monthlyOtherCashAdjustmentInsideChangeinCash", + "monthlyOtherCashAdjustmentOutsideChangeinCash", + "monthlyOtherCashPaymentsfromOperatingActivities", + "monthlyOtherCashReceiptsfromOperatingActivities", + "monthlyOtherNonCashItems", + "monthlyPaymentsonBehalfofEmployees", + "monthlyPaymentstoSuppliersforGoodsandServices", + "monthlyPensionAndEmployeeBenefitExpense", + "monthlyPreferredStockDividendPaid", + "monthlyPreferredStockIssuance", + "monthlyPreferredStockPayments", + "monthlyProceedsFromStockOptionExercised", + "monthlyProvisionandWriteOffofAssets", + "monthlyPurchaseOfBusiness", + "monthlyPurchaseOfIntangibles", + "monthlyPurchaseOfInvestment", + "monthlyPurchaseOfInvestmentProperties", + "monthlyPurchaseOfPPE", + "monthlyReceiptsfromCustomers", + "monthlyReceiptsfromGovernmentGrants", + "monthlyRepaymentOfDebt", + "monthlyRepurchaseOfCapitalStock", + "monthlySaleOfBusiness", + "monthlySaleOfIntangibles", + "monthlySaleOfInvestment", + "monthlySaleOfInvestmentProperties", + "monthlySaleOfPPE", + "monthlyShortTermDebtIssuance", + "monthlyShortTermDebtPayments", + "monthlyStockBasedCompensation", + "monthlyTaxesRefundPaid", + "monthlyTaxesRefundPaidDirect", + "monthlyUnrealizedGainLossOnInvestmentSecurities", + "trailingAdjustedGeographySegmentData", + "trailingAmortizationCashFlow", + "trailingAmortizationOfIntangibles", + "trailingAmortizationOfSecurities", + "trailingAssetImpairmentCharge", + "trailingBeginningCashPosition", + "trailingCapitalExpenditure", + "trailingCapitalExpenditureReported", + "trailingCashDividendsPaid", + "trailingCashFlowFromContinuingFinancingActivities", + "trailingCashFlowFromContinuingInvestingActivities", + "trailingCashFlowFromContinuingOperatingActivities", + "trailingCashFlowFromDiscontinuedOperation", + "trailingCashFlowsfromusedinOperatingActivitiesDirect", + "trailingCashFromDiscontinuedFinancingActivities", + "trailingCashFromDiscontinuedInvestingActivities", + "trailingCashFromDiscontinuedOperatingActivities", + "trailingChangeInAccountPayable", + "trailingChangeInAccruedExpense", + "trailingChangeInCashSupplementalAsReported", + "trailingChangeInDividendPayable", + "trailingChangeInIncomeTaxPayable", + "trailingChangeInInterestPayable", + "trailingChangeInInventory", + "trailingChangeInOtherCurrentAssets", + "trailingChangeInOtherCurrentLiabilities", + "trailingChangeInOtherWorkingCapital", + "trailingChangeInPayable", + "trailingChangeInPayablesAndAccruedExpense", + "trailingChangeInPrepaidAssets", + "trailingChangeInReceivables", + "trailingChangeInTaxPayable", + "trailingChangeInWorkingCapital", + "trailingChangesInAccountReceivables", + "trailingChangesInCash", + "trailingClassesofCashPayments", + "trailingClassesofCashReceiptsfromOperatingActivities", + "trailingCommonStockDividendPaid", + "trailingCommonStockIssuance", + "trailingCommonStockPayments", + "trailingDeferredIncomeTax", + "trailingDeferredTax", + "trailingDepletion", + "trailingDepreciation", + "trailingDepreciationAmortizationDepletion", + "trailingDepreciationAndAmortization", + "trailingDividendPaidCFO", + "trailingDividendReceivedCFO", + "trailingDividendsPaidDirect", + "trailingDividendsReceivedCFI", + "trailingDividendsReceivedDirect", + "trailingDomesticSales", + "trailingEarningsLossesFromEquityInvestments", + "trailingEffectOfExchangeRateChanges", + "trailingEndCashPosition", + "trailingExcessTaxBenefitFromStockBasedCompensation", + "trailingFinancingCashFlow", + "trailingForeignSales", + "trailingFreeCashFlow", + "trailingGainLossOnInvestmentSecurities", + "trailingGainLossOnSaleOfBusiness", + "trailingGainLossOnSaleOfPPE", + "trailingIncomeTaxPaidSupplementalData", + "trailingInterestPaidCFF", + "trailingInterestPaidCFO", + "trailingInterestPaidDirect", + "trailingInterestPaidSupplementalData", + "trailingInterestReceivedCFI", + "trailingInterestReceivedCFO", + "trailingInterestReceivedDirect", + "trailingInvestingCashFlow", + "trailingIssuanceOfCapitalStock", + "trailingIssuanceOfDebt", + "trailingLongTermDebtIssuance", + "trailingLongTermDebtPayments", + "trailingNetBusinessPurchaseAndSale", + "trailingNetCommonStockIssuance", + "trailingNetForeignCurrencyExchangeGainLoss", + "trailingNetIncome", + "trailingNetIncomeFromContinuingOperations", + "trailingNetIntangiblesPurchaseAndSale", + "trailingNetInvestmentPropertiesPurchaseAndSale", + "trailingNetInvestmentPurchaseAndSale", + "trailingNetIssuancePaymentsOfDebt", + "trailingNetLongTermDebtIssuance", + "trailingNetOtherFinancingCharges", + "trailingNetOtherInvestingChanges", + "trailingNetPPEPurchaseAndSale", + "trailingNetPreferredStockIssuance", + "trailingNetShortTermDebtIssuance", + "trailingOperatingCashFlow", + "trailingOperatingGainsLosses", + "trailingOtherCashAdjustmentInsideChangeinCash", + "trailingOtherCashAdjustmentOutsideChangeinCash", + "trailingOtherCashPaymentsfromOperatingActivities", + "trailingOtherCashReceiptsfromOperatingActivities", + "trailingOtherNonCashItems", + "trailingPaymentsonBehalfofEmployees", + "trailingPaymentstoSuppliersforGoodsandServices", + "trailingPensionAndEmployeeBenefitExpense", + "trailingPreferredStockDividendPaid", + "trailingPreferredStockIssuance", + "trailingPreferredStockPayments", + "trailingProceedsFromStockOptionExercised", + "trailingProvisionandWriteOffofAssets", + "trailingPurchaseOfBusiness", + "trailingPurchaseOfIntangibles", + "trailingPurchaseOfInvestment", + "trailingPurchaseOfInvestmentProperties", + "trailingPurchaseOfPPE", + "trailingReceiptsfromCustomers", + "trailingReceiptsfromGovernmentGrants", + "trailingRepaymentOfDebt", + "trailingRepurchaseOfCapitalStock", + "trailingSaleOfBusiness", + "trailingSaleOfIntangibles", + "trailingSaleOfInvestment", + "trailingSaleOfInvestmentProperties", + "trailingSaleOfPPE", + "trailingShortTermDebtIssuance", + "trailingShortTermDebtPayments", + "trailingStockBasedCompensation", + "trailingTaxesRefundPaid", + "trailingTaxesRefundPaidDirect", + "trailingUnrealizedGainLossOnInvestmentSecurities", + ] + }, + "valuation": { + "quarterly": [ + "quarterlyForwardPeRatio", + "quarterlyPsRatio", + "quarterlyPbRatio", + "quarterlyEnterprisesValueEBITDARatio", + "quarterlyEnterprisesValueRevenueRatio", + "quarterlyPeRatio", + "quarterlyMarketCap", + "quarterlyEnterpriseValue", + "quarterlyPegRatio", + "trailingForwardPeRatio", + "trailingPsRatio", + "trailingPbRatio", + "trailingEnterprisesValueEBITDARatio", + "trailingEnterprisesValueRevenueRatio", + "trailingPeRatio", + "trailingMarketCap", + "trailingEnterpriseValue", + "trailingPegRatio", + ], + } +} + +FUNDAMENTALS_TIME_MAP = { + "annual": {"prefix": "annual", "period_type": "12M"}, + "quarterly": {"prefix": "quarterly", "period_type": "3M"}, + "monthly": {"prefix": "monthly", "period_type": "1M"}, +} + +MODULES_MAP = { + "assetProfile": { + "convert_dates": ["governanceEpochDate", "compensationAsOfEpochDate"] + }, + "balanceSheetHistory": { + "filter": "balanceSheetStatements", + "convert_dates": ["endDate"], + }, + "balanceSheetHistoryQuarterly": { + "filter": "balanceSheetStatements", + "convert_dates": ["endDate"], + }, + "calendarEvents": { + "convert_dates": ["earningsDate", "exDividendDate", "dividendDate"] + }, + "cashflowStatementHistory": { + "filter": "cashflowStatements", + "convert_dates": ["endDate"], + }, + "cashflowStatementHistoryQuarterly": { + "filter": "cashflowStatements", + "convert_dates": ["endDate"], + }, + "defaultKeyStatistics": { + "convert_dates": [ + "sharesShortPreviousMonthDate", + "dateShortInterest", + "lastFiscalYearEnd", + "nextFiscalYearEnd", + "fundInceptionDate", + "lastSplitDate", + "mostRecentQuarter", + ] + }, + "earnings": {"convert_dates": ["earningsDate"]}, + "earningsHistory": {"filter": "history", "convert_dates": ["quarter"]}, + "earningsTrend": {"convert_dates": []}, + "esgScores": {"convert_dates": []}, + "financialData": {"convert_dates": []}, + "fundOwnership": {"filter": "ownershipList", "convert_dates": ["reportDate"]}, + "fundPerformance": {"convert_dates": ["asOfDate"]}, + "fundProfile": {"convert_dates": []}, + "indexTrend": {"convert_dates": []}, + "incomeStatementHistory": { + "filter": "incomeStatementHistory", + "convert_dates": ["endDate"], + }, + "incomeStatementHistoryQuarterly": { + "filter": "incomeStatementHistory", + "convert_dates": ["endDate"], + }, + "industryTrend": {"convert_dates": []}, + "insiderHolders": { + "filter": "holders", + "convert_dates": ["latestTransDate", "positionDirectDate"], + }, + "insiderTransactions": { + "filter": "transactions", + "convert_dates": ["startDate"], + }, + "institutionOwnership": { + "filter": "ownershipList", + "convert_dates": ["reportDate"], + }, + "majorHoldersBreakdown": {"convert_dates": []}, + "pageViews": {"convert_dates": []}, + "price": {"convert_dates": [ + "postMarketTime", "preMarketTime", "regularMarketTime" + ]}, + "quoteType": {"convert_dates": ["firstTradeDateEpochUtc"]}, + "recommendationTrend": {"filter": "trend", "convert_dates": []}, + "secFilings": {"filter": "filings", "convert_dates": ["epochDate"]}, + "netSharePurchaseActivity": {"convert_dates": []}, + "sectorTrend": {"convert_dates": []}, + "summaryDetail": { + "convert_dates": ["exDividendDate", "expireDate", "startDate"] + }, + "summaryProfile": {"convert_dates": []}, + "topHoldings": {"convert_dates": []}, + "upgradeDowngradeHistory": { + "filter": "history", + "convert_dates": ["epochGradeDate"], + }, +} + +REQUEST_MAP = { + "quoteSummary": { + "path": "https://query2.finance.yahoo.com/v10/finance/quoteSummary/{symbol}", + "response_field": "quoteSummary", + "request": { + "formatted": {"required": False, "default": False}, + "modules": { + "required": True, + "default": None, + "options": list(MODULES_MAP.keys()), + }, + }, + }, + "fundamentals": { + "path": "https://query2.finance.yahoo.com/ws/fundamentals-timeseries/v1/finance/timeseries/{symbol}", + "response_field": "timeseries", + "request": { + "period1": {"required": True, "default": 493590046}, + "period2": {"required": True, "default": int(time.time())}, + "type": { + "required": True, + "default": None, + "options": FUNDAMENTALS_MAP, + }, + "merge": {"required": False, "default": False}, + "padTimeSeries": {"required": False, "default": False}, + }, + }, +} diff --git a/yahoofinancials/utils.py b/yahoofinancials/utils.py new file mode 100644 index 0000000..0655b15 --- /dev/null +++ b/yahoofinancials/utils.py @@ -0,0 +1,19 @@ + +def remove_prefix(s, prefix): + return s[len(prefix):] if s.startswith(prefix) else s + + +def get_request_config(tech_type, req_map): + if tech_type == '': + r_map = req_map['fundamentals'] + else: + r_map = req_map['quoteSummary'] + return r_map + + +def get_request_category(tech_type, fin_types, statement_type): + if tech_type == '': + r_cat = fin_types.get(statement_type, [])[0] + else: + r_cat = tech_type + return r_cat diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 24e143a..5e67341 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.12 +Version: 1.13 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 01/27/2023 +Version Released: 02/14/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders @@ -45,7 +45,7 @@ from yahoofinancials.calcs import num_shares_outstanding, eps from yahoofinancials.etl import YahooFinanceETL -__version__ = "1.12" +__version__ = "1.13" __author__ = "Connor Sanders" @@ -72,24 +72,25 @@ class YahooFinancials(YahooFinanceETL): """ # Private method that handles financial statement extraction - def _run_financial_stmt(self, statement_type, report_num, reformat): + def _run_financial_stmt(self, statement_type, report_num, frequency, reformat): + hist_obj = {"interval": frequency} report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] if reformat: - raw_data = self.get_stock_data(statement_type, report_name=report_name) + raw_data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj) data = self.get_reformatted_stmt_data(raw_data, statement_type) else: - data = self.get_stock_data(statement_type, report_name=report_name) + data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj) return data # Public Method for the user to get financial statement data def get_financial_stmts(self, frequency, statement_type, reformat=True): report_num = self.get_report_type(frequency) if isinstance(statement_type, str): - data = self._run_financial_stmt(statement_type, report_num, reformat) + data = self._run_financial_stmt(statement_type, report_num, frequency, reformat) else: data = {} for stmt_type in statement_type: - re_data = self._run_financial_stmt(stmt_type, report_num, reformat) + re_data = self._run_financial_stmt(stmt_type, report_num, frequency, reformat) data.update(re_data) return data @@ -111,9 +112,10 @@ def get_key_statistics_data(self, reformat=True): def get_stock_profile_data(self, reformat=True): if reformat: return self.get_clean_data( - self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile'), 'earnings') + self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile'), + 'earnings') else: - return self.get_stock_data(statement_type='profile', tech_type='', report_name='assetProfile') + return self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile') # Public Method for the user to get stock earnings data def get_stock_earnings_data(self, reformat=True): @@ -195,7 +197,7 @@ def _financial_statement_data(self, stmt_type, stmt_code, field_name, freq): date_key = re_data[self.ticker][0].keys()[0] except (IndexError, AttributeError, TypeError): date_key = list(re_data[self.ticker][0])[0] - data = re_data[self.ticker][0][date_key][field_name] + data = re_data[self.ticker][0][date_key].get(field_name) else: data = {} for tick in self.ticker: @@ -317,7 +319,8 @@ def get_interest_expense(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'interestExpense', 'annual') def get_operating_income(self): - return self._financial_statement_data('income', 'incomeStatementHistory', 'operatingIncome', 'annual') + return self._financial_statement_data('income', 'incomeStatementHistory', 'netIncomeContinuousOperations', + 'annual') def get_total_operating_expense(self): return self._financial_statement_data('income', 'incomeStatementHistory', 'totalOperatingExpenses', 'annual') From db563f9b10e11047d1590231708dee5da68c19e9 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Mon, 13 Feb 2023 23:00:48 -0600 Subject: [PATCH 101/108] updated CHANGES --- CHANGES | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index a8ae929..191dd1f 100644 --- a/CHANGES +++ b/CHANGES @@ -42,4 +42,5 @@ 1.9 01/14/2023 -- Fixed new data encryption issue & hardened. 1.10 01/25/2023 -- Fixed new decryption issue. 1.11 01/26/2023 -- Added a dynamic fix for the decryption issue. -1.12 01/27/2023 -- Fixed get profile function for #127 and added additional unit test +1.12 01/27/2023 -- Fixed get profile function for #127 and added additional unit test. +1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping. \ No newline at end of file From 79f72c4568bcd0999e9a825c29c3c2b6f841a4b0 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Tue, 14 Feb 2023 00:05:22 -0600 Subject: [PATCH 102/108] Added additional unit tests and esg scores as requested in #48. --- CHANGES | 4 +++- README.rst | 15 ++++++--------- test/test_yahoofinancials.py | 35 ++++++++++++++++++++++++++++++++++- yahoofinancials/etl.py | 28 +++++++++++++++------------- yahoofinancials/maps.py | 14 ++++++++++++++ yahoofinancials/utils.py | 1 - yahoofinancials/yf.py | 18 +++++++++--------- 7 files changed, 81 insertions(+), 34 deletions(-) diff --git a/CHANGES b/CHANGES index 191dd1f..7158915 100644 --- a/CHANGES +++ b/CHANGES @@ -43,4 +43,6 @@ 1.10 01/25/2023 -- Fixed new decryption issue. 1.11 01/26/2023 -- Added a dynamic fix for the decryption issue. 1.12 01/27/2023 -- Fixed get profile function for #127 and added additional unit test. -1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping. \ No newline at end of file +1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping. +1.13 02/14/2023 -- Added method to retrieve ESG data as requested in #48. +1.13 02/14/2023 -- Added additional unit tests. diff --git a/README.rst b/README.rst index 61b3ff9..36d3584 100644 --- a/README.rst +++ b/README.rst @@ -35,9 +35,8 @@ A powerful financial data module used for pulling both fundamental and technical balance_sheet_data_qt = yahoo_financials.get_financial_stmts('quarterly', 'balance') print(balance_sheet_data_qt) -- New methods in Version 1.9: - - get_stock_profile_data() - - get_financial_data() +- New methods in Version 1.13: + - get_esg_score_data() Installation @@ -98,8 +97,7 @@ Featured Methods - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 2. get_stock_price_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -3. get_stock_earnings_data(reformat=True) +3. get_stock_earnings_data() - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 4. get_summary_data(reformat=True) @@ -119,12 +117,11 @@ Featured Methods - price_type can also be set to 'average' to calculate the shares outstanding with the daily average price. -Methods Added in v1.5 -^^^^^^^^^^^^^^^^^^^^^^^ -- get_daily_dividend_data(start_date, end_date) - Additional Module Methods ^^^^^^^^^^^^^^^^^^^^^^^^^ +- get_daily_dividend_data(start_date, end_date) +- get_stock_profile_data() +- get_financial_data() - get_interest_expense() - get_operating_income() - get_total_operating_expense() diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index b97b5a6..41865f5 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -95,19 +95,52 @@ def test_yf_dividend_price(self): # Extra Module Methods Test def test_yf_module_methods(self): - # Stocks + # Stock Current Price if isinstance(self.test_yf_stock_single.get_current_price(), float): self.assertEqual(True, True) else: self.assertEqual(False, True) + # Stock Net Income if isinstance(self.test_yf_stock_single.get_net_income(), float): self.assertEqual(True, True) else: self.assertEqual(False, True) + # Stock Profile Data if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services": self.assertEqual(True, True) else: self.assertEqual(False, True) + + # Stock Financial Data + if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Stock Summary Data + if self.test_yf_stock_single.get_summary_data().get("C").get("currency") == "USD": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Stock Price Data + if self.test_yf_stock_single.get_stock_price_data().get("C").get("exchangeName") == "NYSE": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Stock Key Statistics + if isinstance(self.test_yf_stock_single.get_key_statistics_data().get("C").get("forwardPE"), float): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + + # Stock ESG SCORES + if self.test_yf_stock_single.get_esg_score_data().get("C").get("peerGroup") == "Banks": + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + # Treasuries if isinstance(self.test_yf_treasuries_single.get_current_price(), float): self.assertEqual(True, True) diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 7dbd3fc..f72b861 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -9,10 +9,9 @@ import pytz import requests as requests -from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP +from yahoofinancials.maps import COUNTRY_MAP, REQUEST_MAP, USER_AGENTS from yahoofinancials.utils import remove_prefix, get_request_config, get_request_category - # track the last get timestamp to add a minimum delay between gets - be nice! _lastget = 0 @@ -27,20 +26,30 @@ class ManagedException(Exception): # Class used to get data from urls class UrlOpener: - user_agent_headers = { - 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36' + + request_headers = { + "accept": "*/*", + "accept-encoding": "gzip, deflate, br", + "accept-language": "en-US,en;q=0.9", + "origin": "https://finance.yahoo.com", + "referer": "https://finance.yahoo.com", + "sec-fetch-dest": "empty", + "sec-fetch-mode": "cors", + "sec-fetch-site": "same-site", } + user_agent = random.choice(USER_AGENTS) + request_headers["User-Agent"] = user_agent def __init__(self, session=None): self._session = session or requests - def open(self, url, user_agent_headers=None, params=None, proxy=None, timeout=30): + def open(self, url, request_headers=None, params=None, proxy=None, timeout=30): response = self._session.get( url=url, params=params, proxies=proxy, timeout=timeout, - headers=user_agent_headers or self.user_agent_headers + headers=request_headers or self.request_headers ) return response @@ -323,13 +332,6 @@ def _encode_ticker(ticker_str): encoded_ticker = ticker_str.replace('=', '%3D') return encoded_ticker - # Private method to get time interval code - def _build_historical_url(self, ticker, hist_oj): - url = self._BASE_YAHOO_URL + self._encode_ticker(ticker) + '/history?period1=' + str(hist_oj['start']) + \ - '&period2=' + str(hist_oj['end']) + '&interval=' + hist_oj['interval'] + '&filter=history&frequency=' + \ - hist_oj['interval'] - return url - # Private Method to clean the dates of the newly returns historical stock data into readable format def _clean_historical_data(self, hist_data, last_attempt=False): data = {} diff --git a/yahoofinancials/maps.py b/yahoofinancials/maps.py index 530a03e..9110dbd 100644 --- a/yahoofinancials/maps.py +++ b/yahoofinancials/maps.py @@ -2376,3 +2376,17 @@ }, }, } + +USER_AGENTS = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 " + "Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 ' + 'Safari/537.36' + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.129 " + "Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 " + "Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36", +] diff --git a/yahoofinancials/utils.py b/yahoofinancials/utils.py index 0655b15..622d442 100644 --- a/yahoofinancials/utils.py +++ b/yahoofinancials/utils.py @@ -1,4 +1,3 @@ - def remove_prefix(s, prefix): return s[len(prefix):] if s.startswith(prefix) else s diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 5e67341..8bf1ee5 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -19,8 +19,7 @@ - statement_type can be 'income', 'balance', 'cash'. - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 2) get_stock_price_data(reformat=True) - - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. -3) get_stock_earnings_data(reformat=True) +3) get_stock_earnings_data() - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. 4) get_summary_data(reformat=True) - reformat optional value defaulted to true. Enter False for unprocessed raw data from Yahoo Finance. @@ -112,17 +111,14 @@ def get_key_statistics_data(self, reformat=True): def get_stock_profile_data(self, reformat=True): if reformat: return self.get_clean_data( - self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile'), + self.get_stock_data(statement_type='profile', tech_type='assetProfile', report_name='assetProfile'), 'earnings') else: - return self.get_stock_data(statement_type='profile', tech_type='summaryProfile', report_name='assetProfile') + return self.get_stock_data(statement_type='profile', tech_type='assetProfile', report_name='assetProfile') # Public Method for the user to get stock earnings data - def get_stock_earnings_data(self, reformat=True): - if reformat: - return self.get_clean_data(self.get_stock_tech_data('earnings'), 'earnings') - else: - return self.get_stock_tech_data('earnings') + def get_stock_earnings_data(self): + return self.get_stock_tech_data('earnings') # Public Method for the user to return financial data def get_financial_data(self, reformat=True): @@ -149,6 +145,10 @@ def get_stock_summary_url(self): def get_stock_quote_type_data(self): return self.get_stock_tech_data('quoteType') + # Public Method for the user to get stock quote data + def get_esg_score_data(self): + return self.get_stock_tech_data('esgScores') + # Public Method for user to get historical price data with def get_historical_price_data(self, start_date, end_date, time_interval): interval_code = self.get_time_code(time_interval) From 15e2d41a072891cf2efc3804301e2350f053c09b Mon Sep 17 00:00:00 2001 From: connorsanders Date: Tue, 14 Feb 2023 00:27:52 -0600 Subject: [PATCH 103/108] Updated README.rst --- README.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.rst b/README.rst index 36d3584..10c44a0 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,12 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://static.pepy.tech/badge/yahoofinancials :target: https://pepy.tech/project/yahoofinancials +.. image:: https://static.pepy.tech/badge/yahoofinancials/month + :target: https://pepy.tech/project/yahoofinancials + +.. image:: https://static.pepy.tech/badge/yahoofinancials/week + :target: https://pepy.tech/project/yahoofinancials + Current Version: v1.13 Version Released: 02/14/2023 From bef934e1aa686fb32e5cad2aa9b254ca7a601e11 Mon Sep 17 00:00:00 2001 From: connorsanders Date: Tue, 21 Feb 2023 20:26:32 -0600 Subject: [PATCH 104/108] fixed get_ten_day_avg_daily_volume() and removed get_three_month_avg_daily_volume() to address #137. Three month data no longer seems available from Yahoo Finance --- CHANGES | 3 +++ README.rst | 5 ++--- setup.py | 4 ++-- test/test_yahoofinancials.py | 11 +++++++++-- yahoofinancials/yf.py | 11 ++++------- 5 files changed, 20 insertions(+), 14 deletions(-) diff --git a/CHANGES b/CHANGES index 7158915..2dc8c26 100644 --- a/CHANGES +++ b/CHANGES @@ -46,3 +46,6 @@ 1.13 02/14/2023 -- Implemented fixes for #132 and #128 by refactoring package to use Yahoo API instead of scraping. 1.13 02/14/2023 -- Added method to retrieve ESG data as requested in #48. 1.13 02/14/2023 -- Added additional unit tests. +1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137. +1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data. +1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume \ No newline at end of file diff --git a/README.rst b/README.rst index 10c44a0..4b36631 100644 --- a/README.rst +++ b/README.rst @@ -16,9 +16,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://static.pepy.tech/badge/yahoofinancials/week :target: https://pepy.tech/project/yahoofinancials -Current Version: v1.13 +Current Version: v1.14 -Version Released: 02/14/2023 +Version Released: 02/21/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -145,7 +145,6 @@ Additional Module Methods - get_prev_close_price() - get_open_price() - get_ten_day_avg_daily_volume() -- get_three_month_avg_daily_volume() - get_stock_exchange() - get_market_cap() - get_daily_low() diff --git a/setup.py b/setup.py index 8a81d46..d2309dd 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.13', + version='1.14', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.13.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.14.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 41865f5..4567714 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,5 +1,5 @@ -# YahooFinancials Unit Tests v1.13 -# Version Released: 02/14/2023 +# YahooFinancials Unit Tests v1.14 +# Version Released: 02/21/2023 # Author: Connor Sanders # Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 # Copyright (c) 2023 Connor Sanders @@ -95,6 +95,13 @@ def test_yf_dividend_price(self): # Extra Module Methods Test def test_yf_module_methods(self): + + # 10 Day Average Daily Volume + if isinstance(self.test_yf_stock_single.get_ten_day_avg_daily_volume(), int): + self.assertEqual(True, True) + else: + self.assertEqual(False, True) + # Stock Current Price if isinstance(self.test_yf_stock_single.get_current_price(), float): self.assertEqual(True, True) diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 8bf1ee5..41e09b0 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,12 +1,12 @@ """ ============================== The Yahoo Financials Module -Version: 1.13 +Version: 1.14 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 02/14/2023 +Version Released: 02/21/2023 Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders @@ -44,7 +44,7 @@ from yahoofinancials.calcs import num_shares_outstanding, eps from yahoofinancials.etl import YahooFinanceETL -__version__ = "1.13" +__version__ = "1.14" __author__ = "Connor Sanders" @@ -241,10 +241,7 @@ def get_open_price(self): return self._stock_price_data('regularMarketOpen') def get_ten_day_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume10Day') - - def get_three_month_avg_daily_volume(self): - return self._stock_price_data('averageDailyVolume3Month') + return self._stock_summary_data('averageDailyVolume10Day') def get_stock_exchange(self): return self._stock_price_data('exchangeName') From c7e9b1feb6e7b54690d29034562ae76366809c0e Mon Sep 17 00:00:00 2001 From: connorsanders Date: Tue, 21 Feb 2023 20:29:17 -0600 Subject: [PATCH 105/108] updated README --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4b36631..500c9de 100644 --- a/README.rst +++ b/README.rst @@ -48,7 +48,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- - yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11. -- This package depends on beautifulsoup4, pytz, requests, and cryptography to work. +- This package depends on pytz & requests to work. 1. Installation using pip: From 63143a53b5ed321cc639c271a8c661f4d25a5d0e Mon Sep 17 00:00:00 2001 From: Victor Korelsky Date: Sun, 30 Apr 2023 15:16:25 +0200 Subject: [PATCH 106/108] do not retry requests in case of response with 4xx status code. Slight refactoring --- CHANGES | 3 +- setup.py | 2 +- test/__init__.py | 0 yahoofinancials/etl.py | 80 ++++++++++++++++++++++++++---------------- 4 files changed, 53 insertions(+), 32 deletions(-) create mode 100644 test/__init__.py diff --git a/CHANGES b/CHANGES index 2dc8c26..298a59e 100644 --- a/CHANGES +++ b/CHANGES @@ -48,4 +48,5 @@ 1.13 02/14/2023 -- Added additional unit tests. 1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137. 1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data. -1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume \ No newline at end of file +1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume +1.15 04/30/2023 -- Don't retry requests if response code is a client error (4xx). \ No newline at end of file diff --git a/setup.py b/setup.py index d2309dd..579bfa0 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='yahoofinancials', - version='1.14', + version='1.15', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index f72b861..3380366 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -181,21 +181,28 @@ def _construct_url(self, symbol, config, params, freq, request_type): # Private method to execute a web scrape request and decrypt the return def _request_handler(self, url, res_field=""): urlopener = UrlOpener() - # Try to open the URL up to 10 times sleeping random time if something goes wrong + # Try to open the URL up to 10 times sleeping random time if the server responds with a 5xx code max_retry = 10 + for i in range(0, max_retry): response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout) - if response.status_code != 200: - time.sleep(random.randrange(10, 20)) - response.close() - else: - res_content = response.text + + if response.status_code == 200: + res_content = response.text response.close() self._cache[url] = loads(res_content).get(res_field) break + + if 400 <= response.status_code < 500: + raise ManagedException("Server replied with client error code, HTTP " + str(response.status_code) + + " code while opening the url: " + str(url) + " . Not retrying.") + if response.status_code >= 500: + time.sleep(random.randrange(10, 20)) + response.close() + if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with HTTP " + str(response.status_code) + + raise ManagedException("Server replied with server error code, HTTP " + str(response.status_code) + " code while opening the url: " + str(url)) @staticmethod @@ -464,6 +471,16 @@ def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): return self._recursive_api_request(hist_obj, up_ticker, clean, i) elif clean: return self._clean_historical_data(re_data, True) + + # Private method, acting as wrapper to call _create_dict_ent() and log exceptions as warnings + def _safe_create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): + try: + return self._create_dict_ent(up_ticker, statement_type, tech_type, report_name, hist_obj) + except ManagedException as e: + logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", + str(up_ticker), str(statement_type), str(e)) + + return {up_ticker: None} # Private Method to take scrapped data and build a data dictionary with, used by get_stock_data() def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): @@ -539,30 +556,33 @@ def get_time_code(self, time_interval): # Public Method to get stock data def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): data = {} - if isinstance(self.ticker, str): - dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) + tickers = [self.ticker] if isinstance(self.ticker, str) else self.ticker + + if self.concurrent: + data = self._get_stock_data_concurrently(tickers, statement_type, tech_type, report_name, hist_obj) else: - if self.concurrent: - with Pool(self._get_worker_count()) as pool: - dict_ents = pool.map(partial(self._create_dict_ent, - statement_type=statement_type, - tech_type=tech_type, - report_name=report_name, - hist_obj=hist_obj), self.ticker) - for dict_ent in dict_ents: - data.update(dict_ent) - pool.close() - pool.join() - else: - for tick in self.ticker: - try: - dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - data.update(dict_ent) - except ManagedException: - logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", - str(tick), statement_type, str(ManagedException)) - continue + for tick in tickers: + dict_ent = self._safe_create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + if dict_ent[tick]: + data.update(dict_ent) + + return data + + def _get_stock_data_concurrently(self, tickers, statement_type='income', tech_type='', report_name='', hist_obj={}): + data = {} + + with Pool(self._get_worker_count()) as pool: + dict_ents = pool.map(partial(self._safe_create_dict_ent, + statement_type=statement_type, + tech_type=tech_type, + report_name=report_name, + hist_obj=hist_obj), tickers) + for dict_ent in dict_ents: + if dict_ent: + data.update(dict_ent) + + pool.close() + pool.join() return data # Public Method to get technical stock data From 2482bd0830451c2fff472436a0a2b6b2a4c0ff93 Mon Sep 17 00:00:00 2001 From: Connor Sanders <32915591+JECSand@users.noreply.github.com> Date: Tue, 20 Jun 2023 23:34:31 -0500 Subject: [PATCH 107/108] Revert "Do not retry requests in case of response with 4xx status code " --- CHANGES | 3 +- setup.py | 2 +- test/__init__.py | 0 yahoofinancials/etl.py | 80 ++++++++++++++++-------------------------- 4 files changed, 32 insertions(+), 53 deletions(-) delete mode 100644 test/__init__.py diff --git a/CHANGES b/CHANGES index 298a59e..2dc8c26 100644 --- a/CHANGES +++ b/CHANGES @@ -48,5 +48,4 @@ 1.13 02/14/2023 -- Added additional unit tests. 1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137. 1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data. -1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume -1.15 04/30/2023 -- Don't retry requests if response code is a client error (4xx). \ No newline at end of file +1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume \ No newline at end of file diff --git a/setup.py b/setup.py index 579bfa0..d2309dd 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='yahoofinancials', - version='1.15', + version='1.14', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', diff --git a/test/__init__.py b/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index 3380366..f72b861 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -181,28 +181,21 @@ def _construct_url(self, symbol, config, params, freq, request_type): # Private method to execute a web scrape request and decrypt the return def _request_handler(self, url, res_field=""): urlopener = UrlOpener() - # Try to open the URL up to 10 times sleeping random time if the server responds with a 5xx code + # Try to open the URL up to 10 times sleeping random time if something goes wrong max_retry = 10 - for i in range(0, max_retry): response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout) - - if response.status_code == 200: - res_content = response.text + if response.status_code != 200: + time.sleep(random.randrange(10, 20)) + response.close() + else: + res_content = response.text response.close() self._cache[url] = loads(res_content).get(res_field) break - - if 400 <= response.status_code < 500: - raise ManagedException("Server replied with client error code, HTTP " + str(response.status_code) + - " code while opening the url: " + str(url) + " . Not retrying.") - if response.status_code >= 500: - time.sleep(random.randrange(10, 20)) - response.close() - if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with server error code, HTTP " + str(response.status_code) + + raise ManagedException("Server replied with HTTP " + str(response.status_code) + " code while opening the url: " + str(url)) @staticmethod @@ -471,16 +464,6 @@ def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): return self._recursive_api_request(hist_obj, up_ticker, clean, i) elif clean: return self._clean_historical_data(re_data, True) - - # Private method, acting as wrapper to call _create_dict_ent() and log exceptions as warnings - def _safe_create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): - try: - return self._create_dict_ent(up_ticker, statement_type, tech_type, report_name, hist_obj) - except ManagedException as e: - logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", - str(up_ticker), str(statement_type), str(e)) - - return {up_ticker: None} # Private Method to take scrapped data and build a data dictionary with, used by get_stock_data() def _create_dict_ent(self, up_ticker, statement_type, tech_type, report_name, hist_obj): @@ -556,33 +539,30 @@ def get_time_code(self, time_interval): # Public Method to get stock data def get_stock_data(self, statement_type='income', tech_type='', report_name='', hist_obj={}): data = {} - tickers = [self.ticker] if isinstance(self.ticker, str) else self.ticker - - if self.concurrent: - data = self._get_stock_data_concurrently(tickers, statement_type, tech_type, report_name, hist_obj) + if isinstance(self.ticker, str): + dict_ent = self._create_dict_ent(self.ticker, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) else: - for tick in tickers: - dict_ent = self._safe_create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) - if dict_ent[tick]: - data.update(dict_ent) - - return data - - def _get_stock_data_concurrently(self, tickers, statement_type='income', tech_type='', report_name='', hist_obj={}): - data = {} - - with Pool(self._get_worker_count()) as pool: - dict_ents = pool.map(partial(self._safe_create_dict_ent, - statement_type=statement_type, - tech_type=tech_type, - report_name=report_name, - hist_obj=hist_obj), tickers) - for dict_ent in dict_ents: - if dict_ent: - data.update(dict_ent) - - pool.close() - pool.join() + if self.concurrent: + with Pool(self._get_worker_count()) as pool: + dict_ents = pool.map(partial(self._create_dict_ent, + statement_type=statement_type, + tech_type=tech_type, + report_name=report_name, + hist_obj=hist_obj), self.ticker) + for dict_ent in dict_ents: + data.update(dict_ent) + pool.close() + pool.join() + else: + for tick in self.ticker: + try: + dict_ent = self._create_dict_ent(tick, statement_type, tech_type, report_name, hist_obj) + data.update(dict_ent) + except ManagedException: + logging.warning("yahoofinancials ticker: %s error getting %s - %s\n\tContinuing extraction...", + str(tick), statement_type, str(ManagedException)) + continue return data # Public Method to get technical stock data From bf30c47cdd0a1eb56db8e8708a35ee8fb012fcaa Mon Sep 17 00:00:00 2001 From: Connor Sanders Date: Wed, 21 Jun 2023 02:24:03 -0500 Subject: [PATCH 108/108] enhancements to address 400 error related performance issues, #148 --- .github/workflows/test.yml | 2 +- CHANGES | 6 ++- README.rst | 6 +-- setup.py | 5 +-- test/__init__.py | 0 test/test_yahoofinancials.py | 81 ++++++++++++++++++------------------ yahoofinancials/etl.py | 40 +++++++++++++----- yahoofinancials/yf.py | 10 ++--- 8 files changed, 85 insertions(+), 65 deletions(-) create mode 100644 test/__init__.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff76a5e..6834a7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/CHANGES b/CHANGES index 2dc8c26..2d15466 100644 --- a/CHANGES +++ b/CHANGES @@ -48,4 +48,8 @@ 1.13 02/14/2023 -- Added additional unit tests. 1.14 02/21/2023 -- Fixed get_ten_day_avg_daily_volume as reported in #137. 1.14 02/21/2023 -- Removed get_three_month_avg_daily_volume due to value now missing in Yahoo data. -1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume \ No newline at end of file +1.14 02/21/2023 -- Added unit test for get_ten_day_avg_daily_volume +1.15 06/21/2023 -- Dropped Python 3.6 support +1.15 06/21/2023 -- Enhanced cache to cover all api requests +1.15 06/21/2023 -- Enhanced api url to better toggle between query1 and query2 subdomains +1.15 06/21/2023 -- Minimized sleeps in between 400 errors to less than 10 seconds diff --git a/README.rst b/README.rst index 500c9de..4a1f7d0 100644 --- a/README.rst +++ b/README.rst @@ -16,9 +16,9 @@ A python module that returns stock, cryptocurrency, forex, mutual fund, commodit .. image:: https://static.pepy.tech/badge/yahoofinancials/week :target: https://pepy.tech/project/yahoofinancials -Current Version: v1.14 +Current Version: v1.15 -Version Released: 02/21/2023 +Version Released: 06/21/2023 Report any bugs by opening an issue here: https://github.com/JECSand/yahoofinancials/issues @@ -47,7 +47,7 @@ A powerful financial data module used for pulling both fundamental and technical Installation ------------- -- yahoofinancials runs on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11. +- yahoofinancials runs on Python 3.7, 3.8, 3.9, 3.10, and 3.11. - This package depends on pytz & requests to work. 1. Installation using pip: diff --git a/setup.py b/setup.py index d2309dd..aa339f8 100644 --- a/setup.py +++ b/setup.py @@ -10,11 +10,11 @@ setup( name='yahoofinancials', - version='1.14', + version='1.15', description='A powerful financial data module used for pulling both fundamental and technical data from Yahoo Finance', long_description=long_description, url='https://github.com/JECSand/yahoofinancials', - download_url='https://github.com/JECSand/yahoofinancials/archive/1.14.tar.gz', + download_url='https://github.com/JECSand/yahoofinancials/archive/1.15.tar.gz', author='Connor Sanders', author_email='connor@exceleri.com', license='MIT', @@ -32,7 +32,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: OS Independent', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_yahoofinancials.py b/test/test_yahoofinancials.py index 4567714..99952ba 100644 --- a/test/test_yahoofinancials.py +++ b/test/test_yahoofinancials.py @@ -1,13 +1,12 @@ -# YahooFinancials Unit Tests v1.14 -# Version Released: 02/21/2023 +# YahooFinancials Unit Tests v1.15 +# Version Released: 06/21/2023 # Author: Connor Sanders -# Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 -# Copyright (c) 2023 Connor Sanders +# Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11 +# Copyright (c) 2023 Connor Sanders # MIT License -from unittest import main as test_main, TestCase - -from yahoofinancials import YahooFinancials +from yahoofinancials import YahooFinancials as yf +from unittest import main as t_main, TestCase # Test Configuration Variables stocks = ['AAPL', 'MSFT', 'C', 'IL&FSTRANS.NS'] @@ -40,12 +39,12 @@ def check_fundamental(test_data, test_type): class TestModule(TestCase): def setUp(self): - self.test_yf_stock_single = YahooFinancials('C') - self.test_yf_stock_multi = YahooFinancials(stocks) - self.test_yf_treasuries_single = YahooFinancials('^IRX') - self.test_yf_treasuries_multi = YahooFinancials(us_treasuries) - self.test_yf_currencies = YahooFinancials(currencies) - self.test_yf_concurrent = YahooFinancials(stocks, concurrent=True) + self.test_yf_stock_single = yf('C') + self.test_yf_stock_multi = yf(stocks) + self.test_yf_treasuries_single = yf('^IRX') + self.test_yf_treasuries_multi = yf(us_treasuries) + self.test_yf_currencies = yf(currencies) + self.test_yf_concurrent = yf(stocks, concurrent=True) # Fundamentals Test def test_yf_fundamentals(self): @@ -78,20 +77,34 @@ def test_yf_fundamentals(self): # Historical Price Test def test_yf_historical_price(self): single_stock_prices = self.test_yf_stock_single.get_historical_price_data('2015-01-15', '2017-10-15', 'weekly') - expect_dict = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', - 'low': 46.599998474121094, 'date': 1421038800, - 'close': 47.61000061035156, 'open': 48.959999084472656} + expected = {'high': 49.099998474121094, 'volume': 125737200, 'formatted_date': '2015-01-12', + 'low': 46.599998474121094, 'date': 1421038800, + 'close': 47.61000061035156, 'open': 48.959999084472656} # ignore adjclose as it will change with every dividend paid in the future del single_stock_prices['C']['prices'][0]['adjclose'] - self.assertDictEqual(single_stock_prices['C']['prices'][0], expect_dict) + self.assertDictEqual(single_stock_prices['C']['prices'][0], expected) # Historical Stock Daily Dividend Test def test_yf_dividend_price(self): single_stock_dividend = self.test_yf_stock_single.get_daily_dividend_data('1986-09-15', '1987-09-15') - expect_dict = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332}, - {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}] - } - self.assertDictEqual(single_stock_dividend, expect_dict) + expected = {"C": [{'date': 544714200, 'formatted_date': '1987-04-06', 'amount': 0.332}, + {'date': 552576600, 'formatted_date': '1987-07-06', 'amount': 0.332}]} + self.assertDictEqual(single_stock_dividend, expected) + + # Test concurrent functionality of module + def test_yf_concurrency(self): + # Multi stock test + multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance') + multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income') + multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', + ['income', 'cash', 'balance']) + # Multi stock check + result = check_fundamental(multi_balance_sheet_data_qt, 'bal') + self.assertEqual(result, True) + result = check_fundamental(multi_income_statement_data_qt, 'inc') + self.assertEqual(result, True) + result = check_fundamental(multi_all_statement_data_qt, 'all') + self.assertEqual(result, True) # Extra Module Methods Test def test_yf_module_methods(self): @@ -112,14 +125,15 @@ def test_yf_module_methods(self): self.assertEqual(True, True) else: self.assertEqual(False, True) - # Stock Profile Data - if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services": + + # Stock Financial Data + if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD": self.assertEqual(True, True) else: self.assertEqual(False, True) - # Stock Financial Data - if self.test_yf_stock_single.get_financial_data().get("C").get("financialCurrency") == "USD": + # Stock Profile Data + if self.test_yf_stock_single.get_stock_profile_data().get("C").get("sector") == "Financial Services": self.assertEqual(True, True) else: self.assertEqual(False, True) @@ -154,21 +168,6 @@ def test_yf_module_methods(self): else: self.assertEqual(False, True) - # Test concurrent functionality of module - def test_yf_concurrency(self): - # Multi stock test - multi_balance_sheet_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'balance') - multi_income_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', 'income') - multi_all_statement_data_qt = self.test_yf_concurrent.get_financial_stmts('quarterly', - ['income', 'cash', 'balance']) - # Multi stock check - result = check_fundamental(multi_balance_sheet_data_qt, 'bal') - self.assertEqual(result, True) - result = check_fundamental(multi_income_statement_data_qt, 'inc') - self.assertEqual(result, True) - result = check_fundamental(multi_all_statement_data_qt, 'all') - self.assertEqual(result, True) - if __name__ == "__main__": - test_main() + t_main() diff --git a/yahoofinancials/etl.py b/yahoofinancials/etl.py index f72b861..a21f536 100644 --- a/yahoofinancials/etl.py +++ b/yahoofinancials/etl.py @@ -182,12 +182,19 @@ def _construct_url(self, symbol, config, params, freq, request_type): def _request_handler(self, url, res_field=""): urlopener = UrlOpener() # Try to open the URL up to 10 times sleeping random time if something goes wrong + cur_url = url max_retry = 10 for i in range(0, max_retry): - response = urlopener.open(url, proxy=self._get_proxy(), timeout=self.timeout) + response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout) if response.status_code != 200: - time.sleep(random.randrange(10, 20)) + time.sleep(random.randrange(1, 5)) response.close() + time.sleep(random.randrange(1, 5)) + if response.status_code == 404 and i % 2 == 0: + if 'query2.' in cur_url: + cur_url = cur_url.replace("query2.", "query1.") + elif 'query1.' in cur_url: + cur_url = cur_url.replace("query1.", "query2.") else: res_content = response.text response.close() @@ -195,8 +202,8 @@ def _request_handler(self, url, res_field=""): break if i == max_retry - 1: # Raise a custom exception if we can't get the web page within max_retry attempts - raise ManagedException("Server replied with HTTP " + str(response.status_code) + - " code while opening the url: " + str(url)) + raise ManagedException("Server replied with server error code, HTTP " + str(response.status_code) + + " code while opening the url: " + str(cur_url)) @staticmethod def _format_raw_fundamental_data(raw_data): @@ -373,7 +380,7 @@ def _clean_historical_data(self, hist_data, last_attempt=False): # Private Static Method to build API url for GET Request def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): if events is None: - events = [" div", "split", "earn"] + events = ["div", "split", "earn"] event_str = '' for idx, s in enumerate(events, start=1): if idx < len(events): @@ -390,18 +397,31 @@ def _build_api_url(self, hist_obj, up_ticker, v="2", events=None): # Private Method to get financial data via API Call def _get_api_data(self, api_url, tries=0): + if tries == 0 and self._cache.get(api_url): + return self._cache[api_url] + cur_url = api_url + if tries > 0 and tries % 2 == 0: + if 'query2.' in cur_url: + cur_url = cur_url.replace("query2.", "query1.") + elif 'query1.' in cur_url: + cur_url = cur_url.replace("query1.", "query2.") urlopener = UrlOpener() - response = urlopener.open(api_url, proxy=self._get_proxy(), timeout=self.timeout) + response = urlopener.open(cur_url, proxy=self._get_proxy(), timeout=self.timeout) if response.status_code == 200: res_content = response.text response.close() - return loads(res_content) + data = loads(res_content) + self._cache[api_url] = data + return data else: if tries < 5: - time.sleep(random.randrange(10, 20)) + time.sleep(random.randrange(1, 5)) + response.close() + time.sleep(random.randrange(1, 5)) tries += 1 return self._get_api_data(api_url, tries) else: + response.close() return None # Private Method to clean API data @@ -448,8 +468,6 @@ def _clean_api_data(self, api_url): # Private Method to Handle Recursive API Request def _recursive_api_request(self, hist_obj, up_ticker, clean=True, i=0): v = "2" - if i == 3: # After 3 tries querying against 'query2.finance.yahoo.com' try 'query1.finance.yahoo.com' instead - v = "1" if clean: re_data = self._clean_api_data(self._build_api_url(hist_obj, up_ticker, v)) cleaned_re_data = self._clean_historical_data(re_data) @@ -573,7 +591,7 @@ def get_stock_tech_data(self, tech_type): return self.get_stock_data(tech_type=tech_type) # Public Method to get reformatted statement data - def get_reformatted_stmt_data(self, raw_data, statement_type): + def get_reformatted_stmt_data(self, raw_data): sub_dict, data_dict = {}, {} data_type = raw_data['dataType'] if isinstance(self.ticker, str): diff --git a/yahoofinancials/yf.py b/yahoofinancials/yf.py index 41e09b0..e8d550c 100644 --- a/yahoofinancials/yf.py +++ b/yahoofinancials/yf.py @@ -1,13 +1,13 @@ """ ============================== The Yahoo Financials Module -Version: 1.14 +Version: 1.15 ============================== Author: Connor Sanders Email: sandersconnor1@gmail.com -Version Released: 02/21/2023 -Tested on Python 3.6, 3.7, 3.8, 3.9, 3.10, and 3.11 +Version Released: 06/21/2023 +Tested on Python 3.7, 3.8, 3.9, 3.10, and 3.11 Copyright (c) 2023 Connor Sanders MIT License @@ -44,7 +44,7 @@ from yahoofinancials.calcs import num_shares_outstanding, eps from yahoofinancials.etl import YahooFinanceETL -__version__ = "1.14" +__version__ = "1.15" __author__ = "Connor Sanders" @@ -76,7 +76,7 @@ def _run_financial_stmt(self, statement_type, report_num, frequency, reformat): report_name = self.YAHOO_FINANCIAL_TYPES[statement_type][report_num] if reformat: raw_data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj) - data = self.get_reformatted_stmt_data(raw_data, statement_type) + data = self.get_reformatted_stmt_data(raw_data) else: data = self.get_stock_data(statement_type, report_name=report_name, hist_obj=hist_obj) return data