Change data paths, minor fixes

kushalbabel · kushalbabel · commit 227b8ef42b21 · 2020-12-17T17:13:19.000Z
diff --git a/data-scripts/exchanges.py b/data-scripts/exchanges.py
@@ -51,9 +51,7 @@
 
 bancor_relayers = open('data/bancor_relayers').read().strip().splitlines()
 kyber_relayers = open('data/kyber_relayers').read().strip().splitlines()
-uniswap_relayers = open('data/uniswap_relayers').read().strip().splitlines()
-uniswapv2_relayers = open('data/uniswapv2_relayers').read().strip().splitlines()
-sushiswap_relayers = open('data/sushiswap_relayers').read().strip().splitlines()
+uniswap_relayers = open('latest-data/uniswap_relayers').read().strip().splitlines()
 
 
 dex_list = dex_list + bancor_relayers + kyber_relayers + uniswap_relayers
@@ -217,7 +215,6 @@ def get_trade_data_from_log_item(topics, data, address):
             exchange = "Kyber"
             parser = parse_kyber
     elif address in uniswap_relayers:
-        print("PARSING", topics)
         if topics[0] == '0x7f4091b46c33e918a0f3aa42307641d17bb67029427a5369e54b353984238705':
             exchange = "Uniswap"
             parser = parse_uniswap_ethpurchase
diff --git a/data-scripts/get_top_uniswapv2_pairs.py b/data-scripts/get_top_uniswapv2_pairs.py
@@ -8,7 +8,7 @@
 pairs_to_tokens = {}
 
 
-df = pd.read_csv('data/all_logs_uniswapv2_factory.csv')
+df = pd.read_csv('latest-data/all_logs_uniswapv2_factory.csv')
 for _, row in df.iterrows():
     topics = row['topics'][1:-1]
     data = row['data'][2:]
@@ -29,8 +29,8 @@
         pairs_to_tokens[(address, pair_address)] = (token0_addr, token1_addr)
                         
 
-uniswapv2_top_tokens = 'data/uniswapv2_top_tokens.csv'
-uniswapv2_pairs = 'data/uniswapv2_pairs.csv'
+uniswapv2_top_tokens = 'latest-data/uniswapv2_top_tokens.csv'
+uniswapv2_pairs = 'latest-data/uniswapv2_pairs.csv'
 
 with open(uniswapv2_top_tokens, 'w') as csvfile:
     spamwriter = csv.writer(csvfile, delimiter=',',
diff --git a/data-scripts/get_uniswapv2_pairs.py b/data-scripts/get_uniswapv2_pairs.py
@@ -6,7 +6,7 @@
 import csv, os
 from google.cloud import bigquery
 
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "etharbskey.json"
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "bq.json"
 client = bigquery.Client()
 
 
@@ -27,7 +27,7 @@
     job_config=job_config)  # API request - starts the query
 
 
-with open('data/all_logs_uniswapv2_factory.csv', 'w') as csvfile:
+with open('latest-data/all_logs_uniswapv2_factory.csv', 'w') as csvfile:
     spamwriter = csv.writer(csvfile, delimiter=',',
                             quotechar='"', quoting=csv.QUOTE_MINIMAL)
 
diff --git a/data-scripts/get_uniswapv2_relayers.py b/data-scripts/get_uniswapv2_relayers.py
@@ -9,7 +9,7 @@
 uniswapv2_pairs = set()
 
 
-df = pd.read_csv('data/all_logs_uniswapv2_factory.csv')
+df = pd.read_csv('latest-data/all_logs_uniswapv2_factory.csv')
 for _, row in df.iterrows():
     topics = row['topics'][1:-1]
     data = row['data'][2:]
@@ -35,12 +35,12 @@
             print(pair_address)
                         
 
-sushiswap_file = 'data/sushiswap_relayers'
+sushiswap_file = 'latest-data/sushiswap_relayers'
 f_sushiswap = open(sushiswap_file, 'w')
 for pair_address in sushiswap_pairs:
     f_sushiswap.write('0x'+ pair_address + '\n')
 
-uniswapv2_file = 'data/uniswapv2_relayers'
+uniswapv2_file = 'latest-data/uniswapv2_relayers'
 f_uniswapv2 = open(uniswapv2_file, 'w')
 for pair_address in uniswapv2_pairs:
     f_uniswapv2.write('0x' + pair_address + '\n')
diff --git a/data-scripts/get_uswapv2_logs.py b/data-scripts/get_uswapv2_logs.py
@@ -1,11 +1,15 @@
 import csv, os
 from google.cloud import bigquery
-from exchanges import uniswapv2_relayers, sushiswap_relayers
 
-os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "etharbskey.json"
+
+os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "bq.json"
 client = bigquery.Client()
 
 
+uniswapv2_relayers = open('latest-data/uniswapv2_relayers').read().strip().splitlines()
+sushiswap_relayers = open('latest-data/sushiswap_relayers').read().strip().splitlines()
+
+
 query = """SELECT log_index,transaction_hash,logs.transaction_index,address,data,topics,logs.block_timestamp,logs.block_number,gas,gas_price,receipt_gas_used FROM 
   `bigquery-public-data.crypto_ethereum.logs` AS logs
   JOIN `bigquery-public-data.crypto_ethereum.transactions` AS transactions ON logs.transaction_hash = transactions.hash
@@ -22,7 +26,7 @@
         # Location must match that of the dataset(s) referenced in the query.
         location='US',
         job_config=job_config)  # API request - starts the query
-    with open('data/all_logs_%s.csv' % (exchange_relayers[0]), 'w') as csvfile:
+    with open('latest-data/all_logs_%s.csv' % (exchange_relayers[0]), 'w') as csvfile:
         spamwriter = csv.writer(csvfile, delimiter=',',
                                 quotechar='"', quoting=csv.QUOTE_MINIMAL)
 
diff --git a/data-scripts/uniswapv2_reserves.py b/data-scripts/uniswapv2_reserves.py
@@ -26,12 +26,12 @@
 
 exchange_name = args.exchange
 
-uniswapv2_logs = 'data/all_logs_uniswapv2.csv'
-sushiswap_logs = 'data/all_logs_sushiswap.csv'
+uniswapv2_logs = 'latest-data/all_logs_uniswapv2.csv'
+sushiswap_logs = 'latest-data/all_logs_sushiswap.csv'
 
 exchange_logs = {'uniswapv2' : uniswapv2_logs, 'sushiswap' : sushiswap_logs}
 
-uniswapv2_pairs = pd.read_csv('data/uniswapv2_pairs.csv').set_index('pair')
+uniswapv2_pairs = pd.read_csv('latest-data/uniswapv2_pairs.csv').set_index('pair')
 
 logsdict = csv.DictReader(open(exchange_logs[exchange_name]), delimiter=',',
                             quotechar='"', quoting=csv.QUOTE_MINIMAL)
@@ -78,7 +78,7 @@
             logger.info("Parsed %d" %(parsed))
 
 
-filepath = '%s-reserves.csv' % (exchange_name)
+filepath = 'latest-data/%s-reserves.csv' % (exchange_name)
 
 logger.info("Writing to %s" % (filepath))
 
diff --git a/data-scripts/uniswapv2_trades.py b/data-scripts/uniswapv2_trades.py
@@ -1,4 +1,4 @@
-import csv, csv_hack, os
+import csv, os
 import pandas as pd
 import logging
 
@@ -11,13 +11,13 @@
 
 exchange_name = 'uniswapv2'
 
-uniswapv2_logs = 'data/all_logs_uniswapv2.csv'
-sushiswap_logs = 'data/all_logs_sushiswap.csv'
+uniswapv2_logs = 'latest-data/all_logs_uniswapv2.csv'
+sushiswap_logs = 'latest-data/all_logs_sushiswap.csv'
 
 exchange_logs = {'uniswapv2' : uniswapv2_logs, 'sushiswap' : sushiswap_logs}
-outputdir = exchange_name + '-processed'
+outputdir = 'latest-data/' + exchange_name + '-processed'
 
-uniswapv2_pairs = pd.read_csv('data/uniswapv2_pairs.csv').set_index('pair')
+uniswapv2_pairs = pd.read_csv('latest-data/uniswapv2_pairs.csv').set_index('pair')
 
 logsdict = csv.DictReader(open(exchange_logs[exchange_name]), delimiter=',',
                             quotechar='"', quoting=csv.QUOTE_MINIMAL)
diff --git a/find_mev_krun_maker.py b/find_mev_krun_maker.py
@@ -9,22 +9,33 @@
 def all_orderings(all_transactions):
     ret = list(itertools.permutations(all_transactions))
     random.shuffle(ret)
+    print("Num all reorderings ", len(ret))
+    ret = [x for x in ret if valid_ordering(x)]
+    print("Num valid reorderings", len(ret))
     return ret
 
-def reordering_mev(program, program_file, outfile, acc, tokens, balances, pre_price, post_price, pair_address, maker_prologue):
+def valid_ordering(transaction_ordering):
+    for transaction in transaction_ordering:
+        if 'locks' in transaction:
+            return True
+        elif 'draws' in transaction:
+            return False
+    return True
+
+def reordering_mev(program, program_file, outfile, acc, tokens, balances, pre_price, post_price, pair_address, maker_prologue, maker_epilogue):
 
     program = program.strip()
 
     addresses = set()
     transactions = program.split('\n')
     all_transactions = [transaction for transaction in transactions if not transaction.strip().startswith('//')]
-    print(all_transactions)
+    #print(all_transactions)
     for i in range(0, len(all_transactions)):
         chunks = all_transactions[i].split()
-        print(chunks)
+        #print(chunks)
         addresses.add(chunks[0])
 
-    print(addresses)
+    #print(addresses)
 
     lower_balance_bounds = {}
     upper_balance_bounds = {}
@@ -47,7 +58,7 @@ def reordering_mev(program, program_file, outfile, acc, tokens, balances, pre_pr
         output = ""
         Path(os.path.dirname(program_file)).mkdir(parents=True, exist_ok=True)
         #print("Writing program to", program_file)
-        open(program_file, "w").write(PROLOGUE + '\n'.join(transaction_ordering))
+        open(program_file, "w").write(PROLOGUE + '\n'.join(transaction_ordering) + maker_epilogue)
         sys.stdout.flush()
         pipe = Popen("krun " + program_file, shell=True, stdout=PIPE, stderr=PIPE)
         output = pipe.stdout.read() + pipe.stderr.read()
diff --git a/find_mev_uniswapv1.py b/find_mev_uniswapv1.py
@@ -51,15 +51,15 @@ def reordering_mev(program, program_file, outfile, exchange_acc, tokens, balance
 
     Path(os.path.dirname(program_file)).mkdir(parents=True, exist_ok=True)
     fout = open(program_file, 'w')
-    #path_to_mev = {}
+    path_to_mev = {}
     
-    #path_num = 0
+    path_num = 0
     for transaction_ordering in all_orderings(all_transactions):
         u = UniswapV1({tokens[0] : balances[0], tokens[1] : balances[1]}, exchange_acc)
         for transaction in transaction_ordering:
             u.process(transaction)
         token_balances = u.config()
-        #mev = 0
+        mev = 0
         for acc in token_balances:
             if acc == exchange_acc:
                 continue
@@ -72,21 +72,27 @@ def reordering_mev(program, program_file, outfile, exchange_acc, tokens, balance
             if total_balance > upper_bounds[acc][token0]:
                 upper_bounds[acc][token0] = total_balance
                 upper_bound_paths[acc] = ('\n'.join(transaction_ordering), token_balances)
-            #extortion = upper_bounds[acc][token0] - lower_bounds[acc][token0]
+            extortion = upper_bounds[acc][token0] - lower_bounds[acc][token0]
             #mev += extortion
-        #path_num += 1
-        #path_to_mev[path_num] = mev
+            mev = max(mev,extortion)
+        path_num += 1
+        path_to_mev[path_num] = mev
 
-    #sorted_items = sorted(path_to_mev.items())
+    sorted_items = sorted(path_to_mev.items())
     #print("Writing hill climbing data to {} ...".format(program_file))
-    #fout.write('pathnum,mev\n')
-    #fout.write('\n'.join(["{},{}".format(path_num, mev) for path_num, mev in sorted_items]))
+    fout.write('pathnum,mev\n')
+    fout.write('\n'.join(["{},{}".format(path_num, mev) for path_num, mev in sorted_items]))
         
         
-    mev = 0    
+    mev = 0
+    argmax_acc = 0
     for acc in lower_bounds:
         extortion = upper_bounds[acc][token0] - lower_bounds[acc][token0]
-        mev += extortion
+        #mev += extortion
+        if extortion > mev :
+            mev = extortion
+            argmax_acc = acc
+    print(argmax_acc)
     '''        
         print(acc, extortion)
         
diff --git a/find_mev_uniswapv2.py b/find_mev_uniswapv2.py
@@ -73,7 +73,8 @@ def reordering_mev(program, program_file, outfile, exchange_acc, tokens, balance
                 upper_bounds[acc][token0] = total_balance
                 upper_bound_paths[acc] = ('\n'.join(transaction_ordering), token_balances)
             extortion = upper_bounds[acc][token0] - lower_bounds[acc][token0]
-            mev += extortion
+            #mev += extortion
+            mev = max(mev, extortion)
         path_num += 1
         path_to_mev[path_num] = mev
 
@@ -86,7 +87,9 @@ def reordering_mev(program, program_file, outfile, exchange_acc, tokens, balance
     mev = 0    
     for acc in lower_bounds:
         extortion = upper_bounds[acc][token0] - lower_bounds[acc][token0]
-        mev += extortion
+        # mev += extortion
+        mev = max(mev,extortion)
+    '''        
         print(acc, extortion)
         
     for acc in lower_bounds:
@@ -96,7 +99,6 @@ def reordering_mev(program, program_file, outfile, exchange_acc, tokens, balance
         print(default_to_regular(upper_bound_paths[acc][0]))
         print('')
         print(default_to_regular(lower_bound_paths[acc][0]))
-    '''        
     '''
 
     print(exchange_acc, pair_address, token0, token1, block, len(all_transactions), mev, sep=',')
diff --git a/maker.k b/maker.k
@@ -157,8 +157,9 @@ rule
      <k> exec(Address:ETHAddress bites vault Vault:ETHAddress) =>
          Vault in SAI gets (0 -Int TotalDebt) ~>
          Vault in 0 gets 0 -Int ((130 *Int TotalDebt) /Int (Px *Int 100)) ~> // 30% penalty
-         Address in SAI gets TotalDebt ~>
-         Address in 0 gets (0 -Int ((90 *Int TotalDebt) /Int (Px *Int 100)) ) //10% discount
+         Address in SAI gets (0 -Int TotalDebt) ~>
+         Address in 0 gets (TotalDebt /Int Px ) //10% discount
+         //Address in 0 gets (0 -Int ((90 *Int TotalDebt) /Int (Px *Int 100)) ) //10% discount
      ... </k>  
      <P> ... (0 , SAI) |-> Px ... </P>
      <S> ... (Vault in 0) |-> TotalCollateral (Vault in SAI) |-> TotalDebt  ... </S>
diff --git a/mcd_maker.k b/mcd_maker.k
@@ -140,8 +140,9 @@ rule
      <k> exec(Address:ETHAddress bites vault Vault:ETHAddress) =>
      Vault in DAI gets (0 -Int TotalDebt) ~>
      Vault in CAddress gets 0 -Int TotalCollateral ~>
-     Address in DAI gets TotalDebt *Int Rate /Int 1000000000000000000000000000 ~>
-     Address in CAddress gets (0 -Int ((90 *Int TotalDebt *Int Rate) /Int (Px *Int 100 *Int 1000000000000000000000000000)) ) //10% discount, abstraction for auction
+     Address in DAI gets (0 -Int TotalDebt *Int Rate /Int 1000000000000000000000000000) ~>
+     Address in CAddress gets TotalCollateral // win the reverse auction
+//     Address in CAddress gets (0 -Int ((90 *Int TotalDebt *Int Rate) /Int (Px *Int 100 *Int 1000000000000000000000000000)) ) //10% discount, abstraction for auction
      ...
      </k>
      <R> ... CAddress |-> Rate ... </R>
diff --git a/run_all_experiments.sh b/run_all_experiments.sh
@@ -1,19 +1,18 @@
 #!/bin/bash
-exchange_name=uniswapv2
+exchange_name=sushiswap
 pairs_data=data-scripts/data/uniswapv2_pairs.csv
 echo exchange,pair,token0,token1,block,numtransactions,mev
 for file in `ls -S data-scripts/$exchange_name-processed/0x* | head -n 10`
 do
     temp=${file%.csv}
     address=${temp##*/}
-    #for block in `sort -rt, -k2 -n data-scripts/active-region/$exchange_name/txcount_$address.csv | grep ,[0-9]$ | head -n 5 | cut -f1 -d,`
-    for block in `sort -rt, -k2 -n data-scripts/active-region/$exchange_name/txcount_$address.csv | head -n 10 | cut -f1 -d,`
+    for block in `sort -rt, -k2 -n data-scripts/active-region/$exchange_name/txcount_$address.csv | grep ,[0-9]$ | head -n 30 | cut -f1 -d,`
+    #for block in `sort -rt, -k2 -n data-scripts/active-region/$exchange_name/txcount_$address.csv | head -n 30 | cut -f1 -d,`
     do
         cmd="time python3 run_uniswapv2_experiments.py -b $block -a $address -e $exchange_name &"
-        #echo $cmd
-        #eval $cmd
-        mkdir -p /tmp/plot_approx_mev/$exchange_name/
-        cp experiments/$block-$address/bound.k /tmp/plot_approx_mev/$exchange_name/convergence-$block-$address.csv
+        eval $cmd
+        mkdir -p /tmp/plot_mev/$exchange_name/
+        cp experiments/$block-$address/bound.k /tmp/plot_mev/$exchange_name/convergence-$block-$address.csv
     done
     wait
 done
diff --git a/run_maker_experiments.py b/run_maker_experiments.py
@@ -52,8 +52,8 @@
     required=True
 )
 
-#sai_token_address = 89d24a6b4ccb1b6faa2625fe562bdd9a23260359
-#sai_token = 786821374916005576892310737142965798721793950553
+sai_token_address = '89d24a6b4ccb1b6faa2625fe562bdd9a23260359'
+sai_token = '786821374916005576892310737142965798721793950553'
 
 
 args = parser.parse_args()    
@@ -93,7 +93,7 @@
 balances = (0,0)
 
 pre_reserve = reserves[(reserves.Address == args.address) & (reserves.Block <  int(args.start_block))]
-tokens = (str(pre_reserve.iloc[0]['Token0']), str(pre_reserve.iloc[0]['Token1']))
+tokens = (str(pre_reserve.iloc[0]['Token0']).replace(sai_token, 'SAI'), str(pre_reserve.iloc[0]['Token1']).replace(sai_token, 'SAI'))
 if len(pre_reserve) < 1:
     pre_price = (0,0) # TODO : subtle issue wrt MEV here
 else:
@@ -112,8 +112,6 @@
 outfile = 'output/'+ identifier +'.out'
 
 
-# move cdp opening transactions to prologue
-logger.info(transactions)
 
 
 transactions = transactions.split('\n')
@@ -122,9 +120,15 @@
 
 transactions = '\n'.join([transaction for transaction in transactions if 'opens' not in transaction])
 
-transactions = transactions + '0 bites vault {} ;'.format(args.cdp)
+maker_epilogue = '\n0 bites vault {} ;'.format(args.cdp)
+
+# replace address w/ semantics keyword
+transactions = transactions.replace(sai_token, 'SAI')
+
+logger.info(maker_prologue)
 
 logger.info(transactions)
 
+logger.info(maker_epilogue)
     
-reordering_mev(transactions, spec_file, outfile, acc, tokens, balances, pre_price, post_price, args.address, maker_prologue)
+reordering_mev(transactions, spec_file, outfile, acc, tokens, balances, pre_price, post_price, args.address, maker_prologue, maker_epilogue)
diff --git a/run_uniswapv1_experiments.sh b/run_uniswapv1_experiments.sh
diff --git a/run_uniswapv2_experiments.py b/run_uniswapv2_experiments.py
