Computing inter-group performance.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4cdcc526",
   "metadata": {},
   "source": [
    "# The code in this jupyter notebook \n",
    "* reads the models and the datasets\n",
    "* calculates the accuracy of each model on its own dataset (the personal dataset of the peer within the shard), then on the entire shard's dataset\n",
    "* implements the ensemble method and calculates its accuracy when the model pool is composed of the entire pool of models (models from all trained shards)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "327a5160",
   "metadata": {},
   "source": [
    "### Reading datasets and models from the trained shards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35fb0141",
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from collections import defaultdict\n",
    "from transformers import T5ForConditionalGeneration, T5Tokenizer, AdamW\n",
    "import pickle\n",
    "import torch\n",
    "def defaultdict_to_dict(d):\n",
    "    \"\"\" Recursively convert defaultdict to dict. \"\"\"\n",
    "    if isinstance(d, defaultdict):\n",
    "        d = {key: defaultdict_to_dict(value) for key, value in d.items()}\n",
    "    return d\n",
    "\n",
    "\n",
    "\n",
    "# Reading data and models\n",
    "groups = [str(i) for i in range(0,3)] # the index number of the shards trained (should be found in the aggregate_models folder)\n",
    "peers = [str(i) for i in range(8090, 8100)] # the port under which the models were trained in a decentralized manner\n",
    "epoch_cutoff = 6000 # the epoch checkpoint for identifying which models to use\n",
    "# epoch_cutoff is generally the largest number of epochs trained by any peer (peers may train at different speeds when using 'main.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "786849ca",
   "metadata": {
    "is_executing": true
   },
   "outputs": [],
   "source": [
    "def find_first_files_with_str(directory, str_contain, epoch_cutoff):\n",
    "    return os.path.join(str_contain + '_' + str(epoch_cutoff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0865bbe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = T5Tokenizer.from_pretrained('t5-small')\n",
    "\n",
    "\n",
    "#reading entire dataset for all groups: train_df_group1\n",
    "\n",
    "\n",
    "# reading individual peer datasets & group datasets: train_df_group1, train_df_group1_peer1\n",
    "for group in groups:\n",
    "    # creating train_df's\n",
    "    exec(f'train_df_group{group} = pd.DataFrame()')\n",
    "    for peer in peers:\n",
    "        datasets_folder = os.path.join('aggregated_results',f'group{group}','datasets')\n",
    "        exec_str = f\"train_df_group{group}_peer{int(peer) - 8089} = pd.read_csv(os.path.join(datasets_folder,'{peer}_df.csv'))\"\n",
    "        exec(exec_str)\n",
    "        exec(f'train_df_group{group} = pd.concat([train_df_group{group}, train_df_group{group}_peer{int(peer) - 8089}])')\n",
    "       \n",
    "    exec(f'train_df_group{group} = train_df_group{group}.drop_duplicates()')\n",
    "    \n",
    "    # creating test_df's: test_df_group1, test_df_group1_peer1\n",
    "    \n",
    "    datasets_folder = os.path.join('aggregated_results',f'group{group}','datasets')\n",
    "    exec (f\"test_df_group{group} = pd.read_csv(os.path.join(datasets_folder,'test_df.csv')) \")\n",
    "    exec (f\"test_df_group{group} = test_df_group{group}[test_df_group{group}['doc_id'].isin(train_df_group{group}['doc_id'].unique())]\")\n",
    "    for peer in peers:\n",
    "        exec (f\"test_df_group{group}_peer{int(peer) - 8089} = test_df_group{group}[test_df_group{group}['doc_id'].isin(train_df_group{group}_peer{int(peer) - 8089}['doc_id'].unique())]\")\n",
    "    \n",
    "    \n",
    "    \n",
    "\n",
    "#reading models: model_group1_peer1\n",
    "for group in groups:\n",
    "    for peer in peers:\n",
    "        model_folder = os.path.join('aggregated_results',f'group{group}', 'models')\n",
    "        model_file = find_first_files_with_str(model_folder, peer, epoch_cutoff) # 10 is the largest number of saved models that all peers have finished training\n",
    "        print (group, peer, model_file)\n",
    "        exec_str = f\"model_group{group}_peer{str(int(peer)-8089)} = T5ForConditionalGeneration.from_pretrained(os.path.join(model_folder, model_file))\"\n",
    "        \n",
    "        \n",
    "        exec(exec_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8a65048",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_group1_peer1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "736cafa8",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_group0[train_df_group0['doc_id'].isin(train_df_group1['doc_id'].unique())]['doc_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2c9dbad",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df_group0['doc_id'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d55c8c6",
   "metadata": {},
   "source": [
    "### Implementing testing algorithm to check accuracy for top1 of individual models on their local and shard-wide dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fde6e06d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_model_and_evaluate(group, peer, mode = 'global'):\n",
    "    global_scope = globals()\n",
    "    acc_train = -1\n",
    "    print ('group ', group, 'peer ', peer)\n",
    "    exec (f'model = model_group{group}_peer{peer}', global_scope)\n",
    "    if mode == 'global':\n",
    "        exec (f'train_df = train_df_group{group}.copy()', global_scope)\n",
    "        exec (f'test_df = test_df_group{group}.copy()', global_scope)\n",
    "    elif mode == 'local':\n",
    "        exec (f'train_df = train_df_group{group}_peer{peer}.copy()', global_scope)\n",
    "        exec (f'test_df = test_df_group{group}_peer{peer}.copy()', global_scope)\n",
    "        \n",
    "    df_tot = train_df.copy()\n",
    "    df_tst = test_df.copy()\n",
    "    print (df_tot.shape, df_tst.shape)\n",
    "    \n",
    "    df_tot['generated_doc_id'] = df_tot['query'].apply(lambda x: generate_text(x, model))\n",
    "    df_tst['generated_doc_id'] = df_tst['query'].apply(lambda x: generate_text(x, model))\n",
    "    acc_train = df_tot[df_tot['doc_id'] == df_tot['generated_doc_id']].shape[0]/df_tot.shape[0]\n",
    "    acc_test = df_tst[df_tst['doc_id'] == df_tst['generated_doc_id']].shape[0]/df_tst.shape[0]\n",
    "    \n",
    "    \n",
    "    print (f'{mode} training set accuracy: ', acc_train)\n",
    "    print (f'{mode} test set accuracy: ', acc_test)\n",
    "    \n",
    "    \n",
    "    df_tot['generated_doc_id_log'] = df_tot['query'].apply(lambda x: generate_text_through_logits(x, model, df_tot))\n",
    "    df_tst['generated_doc_id_log'] = df_tst['query'].apply(lambda x: generate_text_through_logits(x, model, df_tst))\n",
    "\n",
    "    \n",
    "    acc_train_log = df_tot[df_tot['doc_id'] == df_tot['generated_doc_id_log']].shape[0]/df_tot.shape[0]\n",
    "    acc_test_log = df_tst[df_tst['doc_id'] == df_tst['generated_doc_id_log']].shape[0]/df_tst.shape[0]\n",
    "    \n",
    "    print (f'{mode} training set accuracy log: ', acc_train_log)\n",
    "    print (f'{mode} test set accuracy log: ', acc_test_log)\n",
    "    return acc_train, acc_test, df_tot, df_tst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9bc08ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to generate text\n",
    "def generate_text(query, model):\n",
    "    input_ids = tokenizer.encode(query, return_tensors='pt')\n",
    "    output = model.generate(input_ids, max_length = 20)\n",
    "    return tokenizer.decode(output[0], skip_special_tokens=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe58e0d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_text_through_logits(query,model, df_tst):\n",
    "    doc_id = df_tst[df_tst['query'] == query]['doc_id'].iloc[0]\n",
    "#     print (query, doc_id)\n",
    "    inputs = tokenizer(query, padding=False, return_tensors=\"pt\", truncation=True).input_ids\n",
    "    labels = tokenizer(doc_id, padding=True, return_tensors=\"pt\", truncation=True).input_ids\n",
    "\n",
    "    # Forward pass\n",
    "    outputs = model(input_ids=inputs, labels = labels)\n",
    "    loss = outputs.loss\n",
    "\n",
    "    # Extract logits and convert to token IDs\n",
    "    logits = outputs.logits\n",
    "    predicted_token_ids = tokenizer.decode(torch.argmax(logits, dim=-1)[0], skip_special_tokens=True)\n",
    "#     print (predicted_token_ids)\n",
    "    return predicted_token_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "328a3ec7",
   "metadata": {},
   "outputs": [],
   "source": [
    "global_accuracies = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(float))))\n",
    "for group in groups:\n",
    "    for peer in peers:\n",
    "        \n",
    "        p = int(peer) - 8089\n",
    "        exec(f\"acc_train_local, acc_test_local, df_tot_l, df_tst_l = read_model_and_evaluate({group}, {p}, 'local')\")\n",
    "        global_accuracies[group][peer]['train']['local'] = acc_train_local\n",
    "        global_accuracies[group][peer]['test']['local'] = acc_test_local\n",
    "        \n",
    "        exec(f\"acc_train_global, acc_test_global, df_tot_g, df_tst_g = read_model_and_evaluate({group}, {p}, 'global')\")\n",
    "        global_accuracies[group][peer]['train']['global'] = acc_train_global\n",
    "        global_accuracies[group][peer]['test']['global'] = acc_test_global"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a643fa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'my_defaultdict' is your existing defaultdict\n",
    "# Convert it to a regular dictionary\n",
    "regular_dict = defaultdict_to_dict(global_accuracies)\n",
    "\n",
    "# Serialize and save to a file\n",
    "with open('localnglobal_accuracies_allgroups_allpeers.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict, file)\n",
    "\n",
    "# To load and optionally convert back to defaultdict\n",
    "# (You'll need to redefine your defaultdict structure as before)\n",
    "with open('localnglobal_accuracies_allgroups_allpeers.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d03e7e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# To load and optionally convert back to defaultdict\n",
    "# (You'll need to redefine your defaultdict structure as before)\n",
    "with open('localnglobal_accuracies_allgroups_allpeers.pkl', 'rb') as file:\n",
    "    global_accuracies = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56730e20",
   "metadata": {},
   "source": [
    "### Calculating top5 accuracy for each peer under above conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0a5c94c4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import threading\n",
    "\n",
    "# Global dictionary to store models for each group and peer\n",
    "global_objects = {}\n",
    "global_accuracies = defaultdict(lambda: defaultdict(lambda: defaultdict(lambda: defaultdict(float))))\n",
    "class ModelEvaluator:\n",
    "    def __init__(self, group, peer):\n",
    "        self.group = group\n",
    "        self.peer = peer\n",
    "        self.counter = 0\n",
    "\n",
    "    def read_model_and_evaluate(self, mode='global'):\n",
    "        global global_accuracies\n",
    "        acc_train = -1.0\n",
    "        acc_test = -1.0\n",
    "        print('group', self.group, 'peer', self.peer, 'mode', mode)\n",
    "\n",
    "        model = globals()[f'model_group{self.group}_peer{self.peer}']\n",
    "        \n",
    "        if mode == 'global':\n",
    "            df_tot = globals()[f'train_df_group{group}'].copy()\n",
    "            self.df_tst = globals()[f'test_df_group{group}'].copy()\n",
    "        elif mode == 'local':\n",
    "            df_tot = globals()[f'train_df_group{group}_peer{peer}'].copy()\n",
    "            self.df_tst = globals()[f'test_df_group{group}_peer{peer}'].copy()\n",
    "        \n",
    "        print(df_tot.shape, self.df_tst.shape)\n",
    "\n",
    "        self.df_tst['generated_doc_id'] = self.df_tst['query'].apply(lambda x: self.generate_text_beams(x, model))\n",
    "        acc_test = self.df_tst.apply(lambda row: row['doc_id'] in row['generated_doc_id'], axis=1).sum() / self.df_tst.shape[0]\n",
    "        \n",
    "        \n",
    "        global global_objects\n",
    "        global_objects[(self.group, self.peer)] = self.df_tst\n",
    "        \n",
    "        print(f'{mode} training set accuracy: ', acc_train)\n",
    "        print(f'{mode} test set accuracy: ', acc_test)\n",
    "        return acc_train, acc_test\n",
    "\n",
    "    def generate_text_beams(self, query, model):\n",
    "        self.counter += 1\n",
    "        if self.counter % 1000 == 0:\n",
    "            print(f\"Processed {self.counter} queries\")\n",
    "        input_ids = tokenizer.encode(query, return_tensors='pt')\n",
    "        output = model.generate(input_ids, do_sample=False, max_length=20,\n",
    "                                num_beams=5, num_return_sequences=5)\n",
    "        return [tokenizer.decode(i, skip_special_tokens=True) for i in output]\n",
    "\n",
    "    def thread_function(self):\n",
    "        global global_accuracies_20samples\n",
    "        acc_train_global, acc_test_global = self.read_model_and_evaluate('global')\n",
    "        global_accuracies[self.group][self.peer]['train']['global'] = acc_train_global\n",
    "        global_accuracies[self.group][self.peer]['test']['global'] = acc_test_global\n",
    "        print(f'finished global work for group {self.group} and peer {self.peer}, acc test global :{acc_test_global}')\n",
    "\n",
    "def evaluate_in_thread(group, peer):\n",
    "    try:\n",
    "        evaluator = ModelEvaluator(group, peer)\n",
    "        evaluator.thread_function()\n",
    "    except Exception as e:\n",
    "        print(f\"Error in thread for group {group} and peer {peer}: {e}\")\n",
    "\n",
    "    \n",
    "# Start threads directly in the main script body\n",
    "threads = []\n",
    "for group in groups:\n",
    "    for peer in peers:\n",
    "        p = int(peer) - 8089\n",
    "        thread = threading.Thread(target=evaluate_in_thread, args=(group, p,))\n",
    "        thread.start()\n",
    "        threads.append(thread)\n",
    "\n",
    "    # Wait for all threads to complete\n",
    "    for thread in threads:\n",
    "        thread.join()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "941f5575",
   "metadata": {},
   "outputs": [],
   "source": [
    "global_accuracies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b620405c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'my_defaultdict' is your existing defaultdict\n",
    "# Convert it to a regular dictionary\n",
    "regular_dict = defaultdict_to_dict(global_accuracies)\n",
    "\n",
    "# Serialize and save to a file\n",
    "with open('global_accuracies_5beams.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2bcdeb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('global_accuracies_5beams.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "loaded_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a98d88a2",
   "metadata": {},
   "source": [
    "### Implementing and saving results for ensemble method under inter-group conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13f25364",
   "metadata": {},
   "outputs": [],
   "source": [
    "three_groups_list = defaultdict(list)\n",
    "for group in groups:\n",
    "    print (f'group {group}')\n",
    "    for i, peer in enumerate(peers):\n",
    "        print (f'peer {int(peer) - 8089}')\n",
    "        exec(f'three_groups_list[\"group{group}\"].append(model_group{group}_peer{int(peer)-8089})')\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5eaaf6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "from collections import defaultdict\n",
    "import threading\n",
    "from torch.nn.functional import softmax\n",
    "\n",
    "class ModelManager:\n",
    "    def __init__(self, model_list, test_df, tokenizer):\n",
    "        self.model_list = model_list\n",
    "        self.test_df = test_df.copy()\n",
    "        print ('test set size:', self.test_df.shape[0])\n",
    "    \n",
    "        self.tokenizer = tokenizer\n",
    "        self.counter = 0\n",
    "\n",
    "    def generate_text_beams(self, query):\n",
    "        self.counter += 1\n",
    "        if self.counter % 500 == 0:\n",
    "            print(f\"Processed {self.counter} queries\")\n",
    "        results = defaultdict(float)\n",
    "        \n",
    "        sampled_models = random.sample(self.model_list['group1'], 3)\n",
    "        sampled_models.extend(random.sample(self.model_list['group2'], 3))\n",
    "#         sampled_models.extend(random.sample(self.model_list['group3'], 3))\n",
    "        for model in sampled_models:\n",
    "            # Ensure query is properly encoded\n",
    "            input_ids = self.tokenizer.encode(query, return_tensors='pt')\n",
    "            output = model.generate(input_ids, do_sample=False, return_dict_in_generate=True, output_scores=True,\n",
    "                                    num_beams=5, num_return_sequences=5, max_length = 20)\n",
    "\n",
    "            beam_scores = output.sequences_scores\n",
    "#             print (beam_scores)\n",
    "            probabilities = softmax(beam_scores, dim=0).tolist()\n",
    "#             print (probabilities)\n",
    "            model_res = [self.tokenizer.decode(output_id, skip_special_tokens=True) for output_id in output.sequences]\n",
    "            \n",
    "\n",
    "            for res, prob in zip(model_res, probabilities):\n",
    "                results[res] += prob\n",
    "\n",
    "        return self.top_5_generated_texts(results)\n",
    "\n",
    "    def top_5_generated_texts(self, input_dict):\n",
    "        sorted_responses = sorted(input_dict.items(), key=lambda x: x[1], reverse=True)\n",
    "        return [response[0] for response in sorted_responses[:5]]\n",
    "\n",
    "    def evaluate_accuracy(self):\n",
    "#         self.train_df['generated_doc_id'] = self.train_df['query'].apply(self.generate_text_beams)\n",
    "        self.test_df['generated_doc_id'] = self.test_df['query'].apply(self.generate_text_beams)\n",
    "\n",
    "#         acc_train = self.calculate_accuracy(self.train_df)\n",
    "        acc_test = self.calculate_accuracy(self.test_df)\n",
    "\n",
    "        return acc_test\n",
    "\n",
    "    def calculate_accuracy(self, df):\n",
    "        return df.apply(lambda row: row['doc_id'] in row['generated_doc_id'], axis=1).sum() / df.shape[0]\n",
    "\n",
    "# Assuming you have predefined dictionaries/lists for models and datasets, such as:\n",
    "# two_groups_list = [...]\n",
    "# three_groups_list = [...]\n",
    "# train_df_group1 = ...\n",
    "# test_df_group1 = ...\n",
    "# ... and so on for other groups\n",
    "# And a tokenizer instance\n",
    "\n",
    "def run_evaluation(group_nbr, tokenizer):\n",
    "    model_list = three_groups_list\n",
    "\n",
    "    test_df = globals()[f'test_df_group{group_nbr}']\n",
    "    \n",
    "    manager = ModelManager(model_list, test_df, tokenizer)\n",
    "    \n",
    "    key = group_nbr\n",
    "    model_managers[key] = manager\n",
    "    \n",
    "    \n",
    "    acc_test = manager.evaluate_accuracy()\n",
    "    global_accuracies[key] = {'acc_test': acc_test}\n",
    "\n",
    "    print(f\"Group: {group_nbr}, Test Acc: {acc_test}\")\n",
    "\n",
    "    \n",
    "# Global collection to store ModelManager instances\n",
    "model_managers = {}\n",
    "global_accuracies = {}\n",
    "    \n",
    "# Threading\n",
    "threads = []\n",
    "\n",
    "for group_nbr in groups:\n",
    "        thread = threading.Thread(target=run_evaluation, args=(group_nbr, tokenizer))\n",
    "        thread.start()\n",
    "        threads.append(thread)\n",
    "\n",
    "for thread in threads:\n",
    "    thread.join()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9be52d6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for group in model_managers:\n",
    "    df_temp = model_managers[group].test_df.copy()\n",
    "    df_temp['generated_doc_id_1beam'] = df_temp['generated_doc_id'].apply(lambda x: [x[0]])\n",
    "    print (group, df_temp.apply(lambda row: row['doc_id'] in row['generated_doc_id_1beam'], \n",
    "                                axis=1).sum() / df_temp.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70580685",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming 'my_defaultdict' is your existing defaultdict\n",
    "# Convert it to a regular dictionary\n",
    "regular_dict = defaultdict_to_dict(global_accuracies)\n",
    "\n",
    "# Serialize and save to a file\n",
    "with open('inter_group_accs_5beams.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e36e2d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('inter_group_accs_5beams.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "loaded_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f670d78d",
   "metadata": {},
   "outputs": [],
   "source": [
    "regular_dict2 = {}\n",
    "for group in model_managers:\n",
    "    df_temp = model_managers[group].test_df.copy()\n",
    "    df_temp['generated_doc_id_1beam'] = df_temp['generated_doc_id'].apply(lambda x: [x[0]])\n",
    "    regular_dict2[group] = df_temp.apply(lambda row: row['doc_id'] in row['generated_doc_id_1beam'], \n",
    "                                axis=1).sum() / df_temp.shape[0]\n",
    "    print (group, df_temp.apply(lambda row: row['doc_id'] in row['generated_doc_id_1beam'], \n",
    "                                axis=1).sum() / df_temp.shape[0])\n",
    "    \n",
    "    \n",
    "# display(df_temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "300a88ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('inter_group_accs_1beams.pkl', 'wb') as file:\n",
    "    pickle.dump(regular_dict2, file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3f561b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('inter_group_accs_1beams.pkl', 'rb') as file:\n",
    "    loaded_dict = pickle.load(file)\n",
    "    # Optionally convert back to defaultdict\n",
    "    # my_defaultdict = convert_to_defaultdict(loaded_dict)\n",
    "    \n",
    "loaded_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e5ba140",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}