diff --git a/mgm-hurry b/mgm-hurry index 55b152d3b..5034370ca 100755 --- a/mgm-hurry +++ b/mgm-hurry @@ -50,7 +50,7 @@ class MGMHurry: Attributes: monigomani_config The monigomani_config object freqtrade_cli The FreqtradeCli object - logger The logging function out of MoniGoManiLogger + logger The logging function out of MoniGoManiLogger """ monigomani_config: MoniGoManiConfig freqtrade_cli: FreqtradeCli @@ -131,6 +131,14 @@ class MGMHurry: {'name': 'Yes, Dry-Run please', 'value': 1}, {'name': 'Yes, for real!', 'value': 2} ] + }, { + 'type': 'confirm', + 'name': 'export_trades_backtest', + 'message': '💨 Do you want to export trades from backtest now?' + }, { + 'type': 'confirm', + 'name': 'export_trades_hyperopt', + 'message': '💨 Do you want to export trades from hyperopt now?' }]) if answers == {}: @@ -211,6 +219,51 @@ class MGMHurry: if answers.get('do_backtest') is True: self.backtest() + if answers.get('export_trades_backtest') is True: + last_result = prompt(questions=[{ + 'type': 'list', + 'name': 'last_result', + 'message': 'Do you want to export trades from the latest backtest results ?', + 'choices': [{'name': 'Yes', 'value': 1}, {'name': 'No', 'value': 0}]} + ]) + + input_file = None + if last_result.get('last_result') == 0: + input_file = prompt(questions=[{ + 'type': 'input', + 'name': 'input_file', + 'message': 'Please specify the filename (.json) you want to use ?' + }]) + input_file = input_file.get('input_file') + + self.export_trades_backtest(input_file_name=input_file) + + if answers.get('export_trades_hyperopt') is True: + last_result = prompt(questions=[{ + 'type': 'list', + 'name': 'last_result', + 'message': 'Do you want to export trades from the latest hyperopt results ?', + 'choices': [{'name': 'Yes', 'value': 1}, {'name': 'No', 'value': 0}]} + ]) + + input_file = None + if last_result.get('last_result') == 0: + input_file = prompt(questions=[{ + 'type': 'input', + 'name': 'input_file', + 'message': 'Please specify the filename (.fthypt) you want to use ?' + }]) + input_file = input_file.get('input_file') + + epoch_choice = prompt(questions=[{ + 'type': 'input', + 'name': 'ho_epoch', + 'message': 'Choose the epoch which you want to export trades from (0 = export all epochs)', + 'filter': lambda val: int(val) + }]) + + self.export_trades_hyperopt(input_file_name=input_file, epoch=epoch_choice.get('ho_epoch')) + if answers.get('start_trading') > 0: if answers.get('start_trading') == 2: self.start_trader(False) @@ -810,7 +863,7 @@ class MGMHurry: fthypt_name = None if fthypt is not None: - fthypt_name = self.freqtrade_cli.parse_fthypt_name(fthypt_name=fthypt) + fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=fthypt) if epoch == 0: self.logger.error(Color.red('🤷 Please pass the epoch number through. ' @@ -1053,7 +1106,7 @@ class MGMHurry: fthypt_name = None fthypt_file_path = None if fthypt is not None: - fthypt_name = self.freqtrade_cli.parse_fthypt_name(fthypt_name=fthypt) + fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=fthypt) fthypt_file_path = f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}' if output_file_name is None: @@ -1073,6 +1126,83 @@ class MGMHurry: message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Results ⬇️', results_paths=[output_file_path]) + def export_trades_backtest(self, strategy: str = 'MoniGoManiHyperStrategy', + output_file_name: str = None, input_file_name: str = None) -> None: + """ + Export the detailed trades from backtest result to an easy to interpret/sort/filter '.csv' SpreadSheet. + + :param strategy: Name of the used Strategy, defaults to 'MoniGoManiHyperStrategy' + :param output_file_name: (str, Optional) Custom filename for the CsvTrades '.csv' file + that will be stored in the format: 'CsvTrades-.csv'. + Defaults to 'StrategyName + Current DateTime'. + :param input_file_name: (str, Optional) '.fthypt' file to export results from.. If 'true' is passed, + an interactive prompt will launch to pick an '.fthypt' file of choice. Defaults to latest + """ + + self.logger.info(Color.title(f'👉 Exporting Backtest Trades to SpreadSheet (.csv) format.')) + + fthypt_name = None + fthypt_file_path = None + if input_file_name is not None: + fthypt_name = self.freqtrade_cli.parse_backtest_filename(bt_filename=input_file_name) + fthypt_file_path = f'{self.basedir}/user_data/backtest_results/{fthypt_name}' + + if output_file_name is None: + output_file_name = f'MoniGoManiHyperStrategy-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + csv_results_file_name = f'CsvTrades-{output_file_name}' + output_file_path = f'{self.basedir}/user_data/csv_results/{csv_results_file_name}.csv' + + command = (f'python3 {self.basedir}/user_data/mgm_tools/ExportCsvBacktestTrades.py -o {output_file_path}') + + if fthypt_name is not None: + command += f' -i {fthypt_file_path}' + + self.monigomani_cli.run_command(command=command) + + MoniGoManiLogger(self.basedir).post_message(username=self.monigomani_config.config['username'], + message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Trades ⬇️', + results_paths=[output_file_path]) + + def export_trades_hyperopt(self, strategy: str = 'MoniGoManiHyperStrategy', + output_file_name: str = None, input_file_name: str = None, epoch: int = None) -> None: + """ + Export the detailed trades from hyperopt results to an easy to interpret/sort/filter '.csv' SpreadSheet. + + :param strategy: Name of the used Strategy, defaults to 'MoniGoManiHyperStrategy' + :param output_file_name: (str, Optional) Custom filename for the CsvTrades '.csv' file + that will be stored in the format: 'CsvTrades-.csv'. + Defaults to 'StrategyName + Current DateTime'. + :param input_file_name: (str, Optional) '.fthypt' file to export results from.. If 'true' is passed, + an interactive prompt will launch to pick an '.fthypt' file of choice. Defaults to latest + """ + + self.logger.info(Color.title(f'👉 Exporting HyperOpt Trades to SpreadSheet (.csv) format.')) + + fthypt_name = None + fthypt_file_path = None + if input_file_name is not None: + fthypt_name = self.freqtrade_cli.parse_hyperopt_filename(fthypt_name=input_file_name) + fthypt_file_path = f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}' + + if output_file_name is None: + output_file_name = f'MoniGoManiHyperStrategy-{datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}' + csv_results_file_name = f'CsvTrades-{output_file_name}' + output_file_path = f'{self.basedir}/user_data/csv_results/{csv_results_file_name}.csv' + + command = (f'python3 {self.basedir}/user_data/mgm_tools/ExportCsvHyperoptTrades.py -o {output_file_path}') + + if fthypt_name is not None: + command += f' -i {fthypt_file_path}' + + if epoch is not None and epoch > 0: + command += f' -n {epoch}' + + self.monigomani_cli.run_command(command=command) + + MoniGoManiLogger(self.basedir).post_message(username=self.monigomani_config.config['username'], + message=f'🚀 Fresh **{strategy}** SpreadSheet (.csv) Trades ⬇️', + results_paths=[output_file_path]) + def export_results(self): """ Export the results that are selected. Creates a '.zip' archive of the various files created by HyperOpt runs after the user selects which HyperOpt run they would like to export. diff --git a/user_data/mgm_tools/ExportCsvBacktestTrades.py b/user_data/mgm_tools/ExportCsvBacktestTrades.py new file mode 100644 index 000000000..47c1fc433 --- /dev/null +++ b/user_data/mgm_tools/ExportCsvBacktestTrades.py @@ -0,0 +1,78 @@ +import getopt +import os +import sys +from pathlib import Path + +import pandas as pd +import rapidjson +from pandas import json_normalize + + +def ExportCsvBacktestTrades(input_file, output_file): + basedir = os.getcwd() + + # Fetch the latest '.json' filename from last_result + if input_file is None or input_file == '': + last_result_file = Path(f'{basedir}/user_data/backtest_results/.last_result.json') + with last_result_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + last_rf = json_normalize(data, max_level=1) + + results_file = Path(f'{basedir}/user_data/backtest_results/{last_rf["latest_backtest"].loc[0]}') + else: + results_file = Path(input_file) + + run_id = results_file.name.split('.')[0] + + # Open and normalize '.json' to dataframe + with results_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + backtest = json_normalize(data, max_level=0) + + # Define result dataframe columns + list_of_columns = ['pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration', + 'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open'] + + backtest_result = json_normalize(backtest["strategy"], max_level=1) + trades = pd.DataFrame.from_dict(backtest_result.iloc[0, 0]) + + if len(trades) > 0: + trades = trades.loc[:, list_of_columns] + trades['stake_amount'] = trades['stake_amount'].apply(lambda x: round(x, 3)) + trades['amount'] = trades['amount'].apply(lambda x: round(x, 3)) + trades['trade_duration'] = trades['trade_duration'].apply(lambda x: round(x / 3600, 2)) + trades['open_rate'] = trades['open_rate'].apply(lambda x: round(x, 3)) + trades['close_rate'] = trades['close_rate'].apply(lambda x: round(x, 3)) + trades['profit_ratio'] = trades['profit_ratio'].apply(lambda x: round(x * 100, 2)) + trades['profit_abs'] = trades['profit_abs'].apply(lambda x: round(x, 3)) + trades.insert(0, 'run_id', run_id) + + # Export result as '.csv' file for readable result + if output_file is None or output_file == '': + output_file = f'{basedir}/user_data/csv_results/{run_id}_trades.csv' + + trades.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8') + + +def main(argv): + input_file = '' + output_file = '' + try: + opts, args = getopt.getopt(argv, 'h:i:o:', ['cfile=', 'ifile=', 'ofile=']) + except getopt.GetoptError: + print('ExportCsvBacktestTrades.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('ExportCsvBacktestTrades.py -i -o ') + sys.exit() + elif opt in ('-i', '--ifile', '--input_file'): + input_file = arg + elif opt in ('-o', '--ofile', '--output_file'): + output_file = arg + + ExportCsvBacktestTrades(input_file, output_file) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/user_data/mgm_tools/ExportCsvHyperoptTrades.py b/user_data/mgm_tools/ExportCsvHyperoptTrades.py new file mode 100644 index 000000000..f9acc3eef --- /dev/null +++ b/user_data/mgm_tools/ExportCsvHyperoptTrades.py @@ -0,0 +1,96 @@ +import getopt +import os +import sys +from pathlib import Path + +import rapidjson +from pandas import DataFrame, json_normalize + + +def ExportCsvHyperoptTrades(config_file, input_file, output_file, epoch_n): + # Load the 'mgm-config' file as an object and parse it as a dictionary + basedir = os.getcwd() + + # Fetch the latest '.fthypt' filename from last_result + if input_file is None or input_file == '': + last_result_file = Path(f'{basedir}/user_data/hyperopt_results/.last_result.json') + with last_result_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + last_rf = json_normalize(data, max_level=1) + + results_file = Path(f'{basedir}/user_data/hyperopt_results/{last_rf["latest_hyperopt"].loc[0]}') + else: + results_file = Path(input_file) + + run_id = results_file.name.split('.')[0] + + # Open '.fthypt' file and normalize '.json' to dataframe + with results_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + hyperopt_results = json_normalize(data, max_level=2) + + # Filter results + if epoch_n != 0: + # Filter chosen epoch only + hyperopt_results = hyperopt_results.loc[hyperopt_results['current_epoch'] == epoch_n] + else: + # Filter out epochs without profit + hyperopt_results = hyperopt_results.loc[hyperopt_results['total_profit'] > 0] + + # Define result dataframe columns + list_of_columns = ['epoch', 'pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration', + 'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open'] + results_df = DataFrame(columns=list_of_columns) + + # Populate results df with selected values + rearrange format + for idx, row in hyperopt_results.iterrows(): + trades = json_normalize(row['results_metrics.trades'], max_level=1) + trades["epoch"] = row["current_epoch"] + results_df = results_df.append(trades) + + if len(results_df) > 0: + results_df = results_df.loc[:, list_of_columns] + results_df['stake_amount'] = results_df['stake_amount'].apply(lambda x: round(x, 3)) + results_df['amount'] = results_df['amount'].apply(lambda x: round(x, 3)) + results_df['trade_duration'] = results_df['trade_duration'].apply(lambda x: round(x / 3600, 2)) + results_df['open_rate'] = results_df['open_rate'].apply(lambda x: round(x, 3)) + results_df['close_rate'] = results_df['close_rate'].apply(lambda x: round(x, 3)) + results_df['profit_ratio'] = results_df['profit_ratio'].apply(lambda x: round(x * 100, 2)) + results_df['profit_abs'] = results_df['profit_abs'].apply(lambda x: round(x, 3)) + results_df.insert(0, 'run_id', run_id) + + # Export result as '.csv' file for readable result + if output_file is None or output_file == '': + output_file = f'{basedir}/user_data/csv_results/{run_id}_trades.csv' + + results_df.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8') + + +def main(argv): + input_file = '' + output_file = '' + config_file = f'{os.getcwd()}/user_data/mgm-config.json' + epoch = 0 + try: + opts, args = getopt.getopt(argv, 'hc:i:o:n:', ['cfile=', 'ifile=', 'ofile=']) + except getopt.GetoptError: + print('ExportCsvHyperoptTrades.py -c -i -o -n ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('ExportCsvHyperoptTrades.py -c -i -o -n ') + sys.exit() + elif opt in ('-c', '--cfile', '--config_file'): + config_file = arg + elif opt in ('-i', '--ifile', '--input_file'): + input_file = arg + elif opt in ('-o', '--ofile', '--output_file'): + output_file = arg + elif opt in ('-n'): + epoch = int(arg) + + ExportCsvHyperoptTrades(config_file, input_file, output_file, epoch) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py b/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py index 2b71742a3..838678f32 100644 --- a/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py +++ b/user_data/mgm_tools/mgm_hurry/FreqtradeCli.py @@ -356,9 +356,9 @@ def choose_fthypt_file(self) -> str: answers = prompt(questions=questions) return answers.get('fthypt_file') - def parse_fthypt_name(self, fthypt_name: str) -> str: + def parse_hyperopt_filename(self, fthypt_name: str) -> str: """ - Helper method to parse the '.fthypt' filename provided/asked by the user + Helper method to parse the hyperopt '.fthypt' filename provided/asked by the user :param fthypt_name: '.fthypt' filename provided by the user :return: fthypt_name usable for the code @@ -370,9 +370,26 @@ def parse_fthypt_name(self, fthypt_name: str) -> str: elif os.path.isfile(f'{self.basedir}/user_data/hyperopt_results/{fthypt_name}'): return fthypt_name else: - self.cli_logger.warning(Color.yellow('🤷 Provided fthypt file not exist, please select fthypt file:')) + self.cli_logger.warning(Color.yellow('🤷 Provided fthypt file not exist, please select a new file:')) return self.choose_fthypt_file() + def parse_backtest_filename(self, bt_filename: str) -> str: + """ + Helper method to parse the backtest '.json' filename provided/asked by the user + + :param bt_filename: '.json' filename provided by the user + :return: bt_filename usable for the code + """ + if bt_filename is True or bt_filename.lower() == 'true': + return self.choose_backtest_results_file() + elif os.path.isfile(f'{self.basedir}/user_data/backtest_results/{bt_filename}.json'): + return f'{bt_filename}.json' + elif os.path.isfile(f'{self.basedir}/user_data/backtest_results/{bt_filename}'): + return bt_filename + else: + self.cli_logger.warning(Color.yellow('🤷 Provided backtest file not exist, please select a new file:')) + return self.choose_backtest_results_file() + def choose_backtest_results_file(self, choose_results: bool = True) -> str: """ Interactive prompt to choose a 'backtest-result-.json' file.