diff --git a/user_data/mgm_tools/ExportCsvBacktest.py b/user_data/mgm_tools/ExportCsvBacktest.py new file mode 100644 index 000000000..089ae771d --- /dev/null +++ b/user_data/mgm_tools/ExportCsvBacktest.py @@ -0,0 +1,81 @@ +import getopt +import json +import os +import sys +from pathlib import Path + +import rapidjson +import pandas as pd +from pandas import DataFrame, json_normalize, read_json + + +def ExportCsvBacktest(input_file, output_file): + basedir = os.getcwd() + + # Fetch the latest '.json' filename from last_result + if input_file is None or input_file == '': + last_result_file = Path(f'{basedir}/user_data/backtest_results/.last_result.json') + with last_result_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + last_rf = json_normalize(data, max_level=1) + + results_file = Path(f'{basedir}/user_data/backtest_results/{last_rf["latest_backtest"].loc[0]}') + else: + results_file = Path(input_file) + + run_id = results_file.name.split('.')[0] + + # Open and normalize '.json' to dataframe + with results_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + backtest = json_normalize(data, max_level=0) + + # Define result dataframe columns + results_df = DataFrame(columns=[ + 'epoch', 'pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration', + 'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open' + ]) + + backtest_result = json_normalize(backtest["strategy"], max_level=1) + trades = pd.DataFrame.from_dict(backtest_result.iloc[0,0]) + + if len(trades) > 0 : + trades = trades.drop(['fee_open','fee_close','initial_stop_loss_abs','initial_stop_loss_ratio','stop_loss_abs', + 'stop_loss_ratio','min_rate','max_rate','buy_tag','open_timestamp','close_timestamp'], axis=1) + trades['stake_amount'] = trades['stake_amount'].apply(lambda x: round(x, 3)) + trades['amount'] = trades['amount'].apply(lambda x: round(x, 3)) + trades['trade_duration'] = trades['trade_duration'].apply(lambda x: round(x / 3600, 2)) + trades['open_rate'] = trades['open_rate'].apply(lambda x: round(x, 3)) + trades['close_rate'] = trades['close_rate'].apply(lambda x: round(x, 3)) + trades['profit_ratio'] = trades['profit_ratio'].apply(lambda x: round(x * 100, 2)) + trades['profit_abs'] = trades['profit_abs'].apply(lambda x: round(x, 3)) + trades.insert(0, 'run_id', run_id) + + # Export result as '.csv' file for readable result + if output_file is None or output_file == '': + output_file = f'{basedir}/user_data/backtest_results/{run_id}_trades.csv' + + trades.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8') + +def main(argv): + input_file = '' + output_file = '' + try: + opts, args = getopt.getopt(argv, 'h:i:o:', ['cfile=', 'ifile=', 'ofile=']) + except getopt.GetoptError: + print('ExportCsvBacktest.py -i -o ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('ExportCsvBacktest.py -i -o ') + sys.exit() + elif opt in ('-i', '--ifile', '--input_file'): + input_file = arg + elif opt in ('-o', '--ofile', '--output_file'): + output_file = arg + + ExportCsvBacktest(input_file, output_file) + + +if __name__ == '__main__': + main(sys.argv[1:]) diff --git a/user_data/mgm_tools/ExportCsvTrades.py b/user_data/mgm_tools/ExportCsvTrades.py new file mode 100644 index 000000000..be7b2a726 --- /dev/null +++ b/user_data/mgm_tools/ExportCsvTrades.py @@ -0,0 +1,106 @@ +import getopt +import json +import os +import sys +from pathlib import Path + +import rapidjson +from pandas import DataFrame, json_normalize + + +def ExportCsvTrades(config_file, input_file, output_file, epoch_n): + # Load the 'mgm-config' file as an object and parse it as a dictionary + + basedir = os.getcwd() + + mgm_config = {} + if os.path.isfile(config_file) is True: + file_object = open(config_file, ) + json_data = json.load(file_object) + mgm_config = json_data['monigomani_settings'] + + # Fetch the latest '.fthypt' filename from last_result + if input_file is None or input_file == '': + last_result_file = Path(f'{basedir}/user_data/hyperopt_results/.last_result.json') + with last_result_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + last_rf = json_normalize(data, max_level=1) + + results_file = Path(f'{basedir}/user_data/hyperopt_results/{last_rf["latest_hyperopt"].loc[0]}') + else: + results_file = Path(input_file) + + run_id = results_file.name.split('.')[0] + + # Open '.fthypt' file and normalize '.json' to dataframe + with results_file.open('r') as f: + data = [rapidjson.loads(line) for line in f] + epochs = json_normalize(data, max_level=2) + + #Filter results + if epoch_n != 0 : + #Filter choozen epoch only + epochs = epochs.loc[epochs['current_epoch'] == epoch_n] + else : + # Filter out epochs without profit + epochs = epochs.loc[epochs['total_profit'] > 0] + + # Define result dataframe columns + results_df = DataFrame(columns=[ + 'epoch', 'pair', 'stake_amount', 'amount', 'open_date', 'close_date', 'trade_duration', + 'open_rate', 'close_rate', 'profit_ratio', 'profit_abs', 'sell_reason', 'is_open' + ]) + + # Populate results df with selected values + rearrange format + trades = DataFrame() + for idx, row in epochs.iterrows(): + trades = json_normalize(row['results_metrics.trades'], max_level=1) + trades["epoch"] = row["current_epoch"] + results_df = results_df.append(trades) + + if len(results_df) > 0 : + results_df = results_df.drop(['fee_open','fee_close','initial_stop_loss_abs','initial_stop_loss_ratio','stop_loss_abs', + 'stop_loss_ratio','min_rate','max_rate','buy_tag','open_timestamp','close_timestamp'], axis=1) + results_df['stake_amount'] = results_df['stake_amount'].apply(lambda x: round(x, 3)) + results_df['amount'] = results_df['amount'].apply(lambda x: round(x, 3)) + results_df['trade_duration'] = results_df['trade_duration'].apply(lambda x: round(x / 3600, 2)) + results_df['open_rate'] = results_df['open_rate'].apply(lambda x: round(x, 3)) + results_df['close_rate'] = results_df['close_rate'].apply(lambda x: round(x, 3)) + results_df['profit_ratio'] = results_df['profit_ratio'].apply(lambda x: round(x * 100, 2)) + results_df['profit_abs'] = results_df['profit_abs'].apply(lambda x: round(x, 3)) + results_df.insert(0, 'run_id', run_id) + + # Export result as '.csv' file for readable result + if output_file is None or output_file == '': + output_file = f'{basedir}/user_data/hyperopt_results/{run_id}_trades.csv' + + results_df.to_csv(output_file, index=False, header=True, mode='w', encoding='UTF-8') + +def main(argv): + input_file = '' + output_file = '' + config_file = f'{os.getcwd()}/user_data/mgm-config.json' + epoch = 0 + try: + opts, args = getopt.getopt(argv, 'hc:i:o:n:', ['cfile=', 'ifile=', 'ofile=']) + except getopt.GetoptError: + print('ExportCsvTrades.py -c -i -o -n ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print('ExportCsvTrades.py -c -i -o -n ') + sys.exit() + elif opt in ('-c', '--cfile', '--config_file'): + config_file = arg + elif opt in ('-i', '--ifile', '--input_file'): + input_file = arg + elif opt in ('-o', '--ofile', '--output_file'): + output_file = arg + elif opt in ('-n'): + epoch = int(arg) + + ExportCsvTrades(config_file, input_file, output_file, epoch) + + +if __name__ == '__main__': + main(sys.argv[1:])