0_trace_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "import glob\n",
    "from collections import defaultdict\n",
    "import random\n",
    "import copy\n",
    "import json\n",
    "import os\n",
    "from collections import Counter\n",
    "import pickle\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Read dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# How many lines to read\n",
    "# lines = 224667\n",
    "lines = 500000\n",
    "dataset = 'russian'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.1. Import file path\n",
    "\n",
    "Base path: Datasets folder\n",
    "- Russian: \"russian_rtid.txt\" (twid ts uid rtid)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_path = \"./Datasets/\"\n",
    "filename = dataset +\".txt\"\n",
    "path_to_file = os.path.join(base_path, filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.2. Add headers "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Read file with headers and print its first 5 rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "if dataset == 'covid':\n",
    "    header_list = [\"twid\", \"uid\", \"ts\", \"rtid\"]\n",
    "\n",
    "    # reading data\n",
    "    initial_file = pd.read_csv(path_to_file, sep=\"\\t\", names=header_list)\n",
    "\n",
    "    # in case of covid.txt\n",
    "    initial_file[\"ts\"] = initial_file[\"ts\"].astype('datetime64[ns]') \n",
    "    initial_file[\"ts\"] = initial_file.ts.dt.to_pydatetime()\n",
    "\n",
    "    # sorting by time\n",
    "    initial_file.sort_values(\"ts\", inplace = True)\n",
    "\n",
    "    # dropping ALL duplicate values \n",
    "    initial_file.drop_duplicates(subset =\"twid\", \n",
    "                         keep = 'first', inplace = True)\n",
    "elif dataset == 'russian':\n",
    "    header_list = [\"twid\", \"ts\", \"uid\", \"rtid\"]\n",
    "\n",
    "    # reading data\n",
    "    initial_file = pd.read_csv('./Datasets/russian_rtid.txt', sep=\"\\s+\", names=header_list)\n",
    "\n",
    "    # sorting by time\n",
    "    initial_file.sort_values(\"ts\", inplace = True)\n",
    "\n",
    "    # dropping ALL duplicate values \n",
    "    initial_file.drop_duplicates(subset =\"twid\", \n",
    "                         keep = 'first', inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_file = initial_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twid</th>\n",
       "      <th>ts</th>\n",
       "      <th>uid</th>\n",
       "      <th>rtid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>955515379299778561</td>\n",
       "      <td>1516647594</td>\n",
       "      <td>201677046</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>955515380449054721</td>\n",
       "      <td>1516647594</td>\n",
       "      <td>917409759225139200</td>\n",
       "      <td>955496328808853504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>955515420177510400</td>\n",
       "      <td>1516647603</td>\n",
       "      <td>4363056442</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>955515458458914816</td>\n",
       "      <td>1516647613</td>\n",
       "      <td>1590944965</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>955515460996366337</td>\n",
       "      <td>1516647613</td>\n",
       "      <td>980583553</td>\n",
       "      <td>955512752373673984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945361</th>\n",
       "      <td>976042785601216513</td>\n",
       "      <td>1521541709</td>\n",
       "      <td>850236221456437248</td>\n",
       "      <td>976030607116591104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945359</th>\n",
       "      <td>976042785731162112</td>\n",
       "      <td>1521541709</td>\n",
       "      <td>2974210367</td>\n",
       "      <td>976016190647697408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945362</th>\n",
       "      <td>976042787450867712</td>\n",
       "      <td>1521541709</td>\n",
       "      <td>1110976454</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945363</th>\n",
       "      <td>976042788956594177</td>\n",
       "      <td>1521541710</td>\n",
       "      <td>3306364731</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945364</th>\n",
       "      <td>976042792408571904</td>\n",
       "      <td>1521541710</td>\n",
       "      <td>242733415</td>\n",
       "      <td>976030994523402240</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1943830 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       twid          ts                 uid  \\\n",
       "0        955515379299778561  1516647594           201677046   \n",
       "1        955515380449054721  1516647594  917409759225139200   \n",
       "2        955515420177510400  1516647603          4363056442   \n",
       "3        955515458458914816  1516647613          1590944965   \n",
       "4        955515460996366337  1516647613           980583553   \n",
       "...                     ...         ...                 ...   \n",
       "1945361  976042785601216513  1521541709  850236221456437248   \n",
       "1945359  976042785731162112  1521541709          2974210367   \n",
       "1945362  976042787450867712  1521541709          1110976454   \n",
       "1945363  976042788956594177  1521541710          3306364731   \n",
       "1945364  976042792408571904  1521541710           242733415   \n",
       "\n",
       "                       rtid  \n",
       "0                        -1  \n",
       "1        955496328808853504  \n",
       "2                        -1  \n",
       "3                        -1  \n",
       "4        955512752373673984  \n",
       "...                     ...  \n",
       "1945361  976030607116591104  \n",
       "1945359  976016190647697408  \n",
       "1945362                  -1  \n",
       "1945363                  -1  \n",
       "1945364  976030994523402240  \n",
       "\n",
       "[1943830 rows x 4 columns]"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1943830"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.3 Filter data:\n",
    "- We want an original twid to be retweeted from at least one user \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "data1 = input_file[((input_file['twid'].isin(input_file['rtid']))) & ((input_file['rtid']==-1))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We want all rtid (!=1) to exist in the twid column for completeness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = input_file[((input_file['rtid'].isin(input_file['twid']))) & ((input_file['rtid']!=-1)) & ((input_file['rtid']!=974014568073695232))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data1, data2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "827268"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twid</th>\n",
       "      <th>ts</th>\n",
       "      <th>uid</th>\n",
       "      <th>rtid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>206</th>\n",
       "      <td>956216939172499456</td>\n",
       "      <td>1516814859</td>\n",
       "      <td>28859692</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>346</th>\n",
       "      <td>956218141184577536</td>\n",
       "      <td>1516815145</td>\n",
       "      <td>4086524967</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>401</th>\n",
       "      <td>956218574967902208</td>\n",
       "      <td>1516815249</td>\n",
       "      <td>264912416</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>420</th>\n",
       "      <td>956218666508521472</td>\n",
       "      <td>1516815270</td>\n",
       "      <td>2613120188</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>956219412377407488</td>\n",
       "      <td>1516815448</td>\n",
       "      <td>386829998</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945353</th>\n",
       "      <td>976042746426413057</td>\n",
       "      <td>1521541699</td>\n",
       "      <td>782976485133000704</td>\n",
       "      <td>975614873118871552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945355</th>\n",
       "      <td>976042758371790848</td>\n",
       "      <td>1521541702</td>\n",
       "      <td>1863574340</td>\n",
       "      <td>976042274625900544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945357</th>\n",
       "      <td>976042761286799360</td>\n",
       "      <td>1521541703</td>\n",
       "      <td>731574742344466432</td>\n",
       "      <td>976041960426352640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945361</th>\n",
       "      <td>976042785601216513</td>\n",
       "      <td>1521541709</td>\n",
       "      <td>850236221456437248</td>\n",
       "      <td>976030607116591104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1945364</th>\n",
       "      <td>976042792408571904</td>\n",
       "      <td>1521541710</td>\n",
       "      <td>242733415</td>\n",
       "      <td>976030994523402240</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>827268 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       twid          ts                 uid  \\\n",
       "206      956216939172499456  1516814859            28859692   \n",
       "346      956218141184577536  1516815145          4086524967   \n",
       "401      956218574967902208  1516815249           264912416   \n",
       "420      956218666508521472  1516815270          2613120188   \n",
       "479      956219412377407488  1516815448           386829998   \n",
       "...                     ...         ...                 ...   \n",
       "1945353  976042746426413057  1521541699  782976485133000704   \n",
       "1945355  976042758371790848  1521541702          1863574340   \n",
       "1945357  976042761286799360  1521541703  731574742344466432   \n",
       "1945361  976042785601216513  1521541709  850236221456437248   \n",
       "1945364  976042792408571904  1521541710           242733415   \n",
       "\n",
       "                       rtid  \n",
       "206                      -1  \n",
       "346                      -1  \n",
       "401                      -1  \n",
       "420                      -1  \n",
       "479                      -1  \n",
       "...                     ...  \n",
       "1945353  975614873118871552  \n",
       "1945355  976042274625900544  \n",
       "1945357  976041960426352640  \n",
       "1945361  976030607116591104  \n",
       "1945364  976030994523402240  \n",
       "\n",
       "[827268 rows x 4 columns]"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.4 Count number of original tweets in the whole trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "# original tweets are the tweets with rtid = -1\n",
    "o_ids = list(set(list(data['twid'].loc[data['rtid'] == -1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of original tweets in the trace: 60426\n"
     ]
    }
   ],
   "source": [
    "print('Number of original tweets in the trace:', len(o_ids))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.5 Create set of users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "U = data.uid.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of unique users in the trace: 93892\n"
     ]
    }
   ],
   "source": [
    "print('Number of unique users in the trace:', len(U))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(U, open(\"./Extracted/\"+dataset+\"/U\" + str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>twid</th>\n",
       "      <th>ts</th>\n",
       "      <th>uid</th>\n",
       "      <th>rtid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>706</th>\n",
       "      <td>956221240225746944</td>\n",
       "      <td>1516815884</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162688</th>\n",
       "      <td>958170756088782848</td>\n",
       "      <td>1517280685</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>427677</th>\n",
       "      <td>961264449742168064</td>\n",
       "      <td>1518018279</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>443396</th>\n",
       "      <td>961514646195851264</td>\n",
       "      <td>1518077930</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556169</th>\n",
       "      <td>963078985608519680</td>\n",
       "      <td>1518450898</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>588905</th>\n",
       "      <td>963495796778946560</td>\n",
       "      <td>1518550273</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>684767</th>\n",
       "      <td>964772264565788672</td>\n",
       "      <td>1518854607</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797364</th>\n",
       "      <td>966705872335462400</td>\n",
       "      <td>1519315615</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797378</th>\n",
       "      <td>966706049796407296</td>\n",
       "      <td>1519315657</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797419</th>\n",
       "      <td>966706389870678016</td>\n",
       "      <td>1519315739</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>870185</th>\n",
       "      <td>967991239730745344</td>\n",
       "      <td>1519622071</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>985613</th>\n",
       "      <td>969132366793990144</td>\n",
       "      <td>1519894137</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1198327</th>\n",
       "      <td>970324762923855872</td>\n",
       "      <td>1520178426</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1236284</th>\n",
       "      <td>970649675194863616</td>\n",
       "      <td>1520255891</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1506614</th>\n",
       "      <td>973122401922842624</td>\n",
       "      <td>1520845435</td>\n",
       "      <td>358147382</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3584</th>\n",
       "      <td>956244584350257154</td>\n",
       "      <td>1516821450</td>\n",
       "      <td>358147382</td>\n",
       "      <td>956221240225746944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172099</th>\n",
       "      <td>958281027511902208</td>\n",
       "      <td>1517306976</td>\n",
       "      <td>358147382</td>\n",
       "      <td>958170756088782848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>432466</th>\n",
       "      <td>961311506276212743</td>\n",
       "      <td>1518029498</td>\n",
       "      <td>358147382</td>\n",
       "      <td>961264449742168064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>457527</th>\n",
       "      <td>961629570737823751</td>\n",
       "      <td>1518105331</td>\n",
       "      <td>358147382</td>\n",
       "      <td>961514646195851264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>568263</th>\n",
       "      <td>963271761260367872</td>\n",
       "      <td>1518496859</td>\n",
       "      <td>358147382</td>\n",
       "      <td>963078985608519680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593568</th>\n",
       "      <td>963617087842324481</td>\n",
       "      <td>1518579192</td>\n",
       "      <td>358147382</td>\n",
       "      <td>963495796778946560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>693311</th>\n",
       "      <td>964893289517342722</td>\n",
       "      <td>1518883462</td>\n",
       "      <td>358147382</td>\n",
       "      <td>964772264565788672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>717174</th>\n",
       "      <td>965464216588181505</td>\n",
       "      <td>1519019581</td>\n",
       "      <td>358147382</td>\n",
       "      <td>964903486700900352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>756009</th>\n",
       "      <td>965998625527554048</td>\n",
       "      <td>1519146994</td>\n",
       "      <td>358147382</td>\n",
       "      <td>965951423375728640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797410</th>\n",
       "      <td>966706331599110146</td>\n",
       "      <td>1519315725</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966681030236213248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797515</th>\n",
       "      <td>966707166831890433</td>\n",
       "      <td>1519315924</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966591532487860224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>797521</th>\n",
       "      <td>966707244522967040</td>\n",
       "      <td>1519315942</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966591532487860224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799722</th>\n",
       "      <td>966727469775671297</td>\n",
       "      <td>1519320764</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966706389870678016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799732</th>\n",
       "      <td>966727514902220802</td>\n",
       "      <td>1519320775</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966706049796407296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799736</th>\n",
       "      <td>966727533914984448</td>\n",
       "      <td>1519320780</td>\n",
       "      <td>358147382</td>\n",
       "      <td>966705872335462400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>871468</th>\n",
       "      <td>968016298860077057</td>\n",
       "      <td>1519628045</td>\n",
       "      <td>358147382</td>\n",
       "      <td>967991239730745344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>916828</th>\n",
       "      <td>968577772544241670</td>\n",
       "      <td>1519761911</td>\n",
       "      <td>358147382</td>\n",
       "      <td>968557387308306432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1039351</th>\n",
       "      <td>969244894396059648</td>\n",
       "      <td>1519920965</td>\n",
       "      <td>358147382</td>\n",
       "      <td>969132366793990144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1202884</th>\n",
       "      <td>970346981175554049</td>\n",
       "      <td>1520183723</td>\n",
       "      <td>358147382</td>\n",
       "      <td>970324762923855872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1235741</th>\n",
       "      <td>970647087980466176</td>\n",
       "      <td>1520255274</td>\n",
       "      <td>358147382</td>\n",
       "      <td>970632626682892288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1235800</th>\n",
       "      <td>970647367555969024</td>\n",
       "      <td>1520255341</td>\n",
       "      <td>358147382</td>\n",
       "      <td>970633270546305024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1245409</th>\n",
       "      <td>970701222838767617</td>\n",
       "      <td>1520268181</td>\n",
       "      <td>358147382</td>\n",
       "      <td>970649675194863616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1512467</th>\n",
       "      <td>973165435007983616</td>\n",
       "      <td>1520855695</td>\n",
       "      <td>358147382</td>\n",
       "      <td>973122401922842624</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       twid          ts        uid                rtid\n",
       "706      956221240225746944  1516815884  358147382                  -1\n",
       "162688   958170756088782848  1517280685  358147382                  -1\n",
       "427677   961264449742168064  1518018279  358147382                  -1\n",
       "443396   961514646195851264  1518077930  358147382                  -1\n",
       "556169   963078985608519680  1518450898  358147382                  -1\n",
       "588905   963495796778946560  1518550273  358147382                  -1\n",
       "684767   964772264565788672  1518854607  358147382                  -1\n",
       "797364   966705872335462400  1519315615  358147382                  -1\n",
       "797378   966706049796407296  1519315657  358147382                  -1\n",
       "797419   966706389870678016  1519315739  358147382                  -1\n",
       "870185   967991239730745344  1519622071  358147382                  -1\n",
       "985613   969132366793990144  1519894137  358147382                  -1\n",
       "1198327  970324762923855872  1520178426  358147382                  -1\n",
       "1236284  970649675194863616  1520255891  358147382                  -1\n",
       "1506614  973122401922842624  1520845435  358147382                  -1\n",
       "3584     956244584350257154  1516821450  358147382  956221240225746944\n",
       "172099   958281027511902208  1517306976  358147382  958170756088782848\n",
       "432466   961311506276212743  1518029498  358147382  961264449742168064\n",
       "457527   961629570737823751  1518105331  358147382  961514646195851264\n",
       "568263   963271761260367872  1518496859  358147382  963078985608519680\n",
       "593568   963617087842324481  1518579192  358147382  963495796778946560\n",
       "693311   964893289517342722  1518883462  358147382  964772264565788672\n",
       "717174   965464216588181505  1519019581  358147382  964903486700900352\n",
       "756009   965998625527554048  1519146994  358147382  965951423375728640\n",
       "797410   966706331599110146  1519315725  358147382  966681030236213248\n",
       "797515   966707166831890433  1519315924  358147382  966591532487860224\n",
       "797521   966707244522967040  1519315942  358147382  966591532487860224\n",
       "799722   966727469775671297  1519320764  358147382  966706389870678016\n",
       "799732   966727514902220802  1519320775  358147382  966706049796407296\n",
       "799736   966727533914984448  1519320780  358147382  966705872335462400\n",
       "871468   968016298860077057  1519628045  358147382  967991239730745344\n",
       "916828   968577772544241670  1519761911  358147382  968557387308306432\n",
       "1039351  969244894396059648  1519920965  358147382  969132366793990144\n",
       "1202884  970346981175554049  1520183723  358147382  970324762923855872\n",
       "1235741  970647087980466176  1520255274  358147382  970632626682892288\n",
       "1235800  970647367555969024  1520255341  358147382  970633270546305024\n",
       "1245409  970701222838767617  1520268181  358147382  970649675194863616\n",
       "1512467  973165435007983616  1520855695  358147382  973122401922842624"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.loc[data['uid']==358147382]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Create episodes\n",
    "\n",
    "- For each original tweet t that we detect in T, we construct the set of its retweets, which we call episode and denote by $E_{t}$.\n",
    "- The whole set of episodes is denoted by $E$ and includes o_ids episodes in total, which is the number of original tweets\n",
    "- We count $X_{ij}$ out of the o_ids total episodes where $(i,j)$ appears as an ordered pair."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Create tweets dictionaries\n",
    "\n",
    "- We name it E and it includes the u_ids of the users that retweeted each t \n",
    "- The source $s_{t}$ will be included in dictionary $s_{t}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# group dataframe by rtid id\n",
    "gb = data.groupby(['rtid'])\n",
    "E = gb['uid'].unique()\n",
    "# result = result.reset_index(inplace=False) #lastly, reset the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd = defaultdict(list)\n",
    "E = E.to_dict(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(E[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60426"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(E)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example, for tweet t = 976039941472956416 the retweeters are: \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(E[976039941472956416])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Create $s_t$ dictionary with the original tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "s_t = data.loc[data['rtid']==-1][['twid', 'uid']]\n",
    "s_t = s_t.set_index('twid').T.to_dict('list')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example, for tweet t = 976039941472956416 the original tweeter is:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# s_t[976039941472956416]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The number of original tweets / episodes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "60426"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(s_t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save S dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(s_t, open(\"./Extracted/\"+dataset+\"/S\"+ str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Insert the user who originally tweeted each tweet in the first position of each episode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ep in E:\n",
    "    E[ep] = np.insert(E[ep], 0, s_t[ep], axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for ep in E:\n",
    "#     a_set = set(E[ep])\n",
    "#     contains_duplicates = len(E[ep]) != len(a_set)\n",
    "#     if contains_duplicates == True: print(ep, E[ep])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete the duplicates from each tweet and remove tweets with no retweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "for ep in list(E):\n",
    "    E[ep] = list(dict.fromkeys(E[ep]))   \n",
    "    if len(E[ep])==1:\n",
    "        E.pop(ep)\n",
    "# removes duplicates from list and maintains order, and removes tweets with no retweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(E, open(\"./Extracted/\"+dataset+\"/E\"+ str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Count $M_{ij}$ out of the o_ids total episodes where $(i,j)$ appears as an ordered pair.\n",
    "\n",
    "Create dictionary d_f that has for key a user and for values the users that appear after him and then create a $M$ dictionary with the total times for each user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = defaultdict()\n",
    "for s in E:\n",
    "    rw_list = E[s]\n",
    "    for u in rw_list:\n",
    "        u_after = rw_list[rw_list.index(u)+1:]\n",
    "        if u in d:\n",
    "            if len(u_after)!=0:\n",
    "                d[u].append(u_after)\n",
    "        else:\n",
    "            if len(u_after)!=0:\n",
    "                d[u] = [u_after]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_f = defaultdict()\n",
    "for i in d: \n",
    "    d_f[i] = [item for sublist in d[i] for item in sublist]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = defaultdict()\n",
    "for i in d_f:\n",
    "    M[i] = Counter(d_f[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l1 = list(data['rtid'].loc[data['uid']==358147382])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# l2 = list(data['rtid'].loc[data['uid']==842298422505926656])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data.loc[data['twid']==964903486700900352]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in M:\n",
    "#     for j in M[i]:\n",
    "#         if i==358147382 or j==358147382: print(i,j,M[i][j])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Save M in files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(M, open(\"./Extracted/\"+dataset+\"/M\"+ str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Create D for Saito"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a $D$ dictionary, which contains the users that retweeted each tweet, in each timestamp:\n",
    "- key1: episode $s$\n",
    "- key2: timestamps $ts $of episode $s$\n",
    "- values: list of users that retweeted $s$ at $ts$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "D = dict()\n",
    "for s in E:\n",
    "    D[s] = dict()\n",
    "    D[s][0] = [list(data['uid'].loc[data['twid']==s])[0]]\n",
    "\n",
    "for s in E:\n",
    "    visited_users = []\n",
    "    retweets = data[['uid','ts']].loc[data['rtid']==s]\n",
    "    for index, row in retweets.iterrows():\n",
    "        if row['uid']!=D[s][0][0] and row['uid'] not in visited_users: # remove cases where user retweets again  and # remove cases where user retweets himself\n",
    "            visited_users.append(row['uid'])\n",
    "            if row['ts'] in D[s]: \n",
    "                D[s][row['ts']].append(row['uid'])\n",
    "            else:\n",
    "                D[s][row['ts']] = []\n",
    "                D[s][row['ts']].append(row['uid'])\n",
    "                \n",
    "# removes tweets with no retweets\n",
    "# for s in list(D):\n",
    "#     if len(D[s])==1:\n",
    "#         D.pop(s)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(D, open(\"./Extracted/\"+dataset+\"/D\"+ str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 Count $M_{ij}$ out of the o_ids total episodes where $(i,j)$ appears as an ordered pair according to D. (difference that users at the same timestamp do not count as an ordered pair) \n",
    "\n",
    "Create dictionary d_f that has for key a user and for values the users that appear after him and then create a $M$ dictionary with the total times for each user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = defaultdict()\n",
    "for s in D:\n",
    "    for users_list in list(D[s].values()):\n",
    "        index_now = list(D[s].values()).index(users_list)\n",
    "        for u in users_list:\n",
    "            u_after = list(D[s].values())[index_now+1:]\n",
    "            if u in d:\n",
    "                if len(u_after)!=0:\n",
    "                    d[u].append(u_after)\n",
    "            else:\n",
    "                if len(u_after)!=0:\n",
    "                    d[u] = [u_after]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_f = defaultdict()\n",
    "for i in d: \n",
    "    d_f[i] = [user for sublist in d[i] for item in sublist for user in item]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "M = defaultdict()\n",
    "for i in d_f:\n",
    "    M[i] = Counter(d_f[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D[956234331961389056]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(M, open(\"./Extracted/\"+dataset+\"/M_d\" + str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Create E_newman for Newman"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_id_to_retweet = defaultdict(list)\n",
    "total_retweets = 0 \n",
    "for index, tweet in input_file.iterrows():\n",
    "    user_id = tweet['uid']\n",
    "    tweet_id = tweet['twid']\n",
    "    retweet_id = tweet['rtid']\n",
    "    if retweet_id!=-1:\n",
    "        try:\n",
    "            retweeted_by_id = int(input_file['uid'].loc[input_file['twid']==retweet_id])\n",
    "            total_retweets += 1\n",
    "            user_id_to_retweet[user_id].append(retweeted_by_id)\n",
    "        except:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "Eij_newman = defaultdict(dict)\n",
    "\n",
    "for user in user_id_to_retweet:\n",
    "        for retweet in user_id_to_retweet[user]:\n",
    "            if not retweet in Eij_newman[user]:\n",
    "                Eij_newman[user][retweet] = 1\n",
    "            else:\n",
    "                Eij_newman[user][retweet] += 1\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "try: \n",
    "    pickle.dump(Eij_newman, open(\"./Extracted/\"+dataset+\"/E_newman\"+ str(lines) + \".p\", \"wb\"))\n",
    "except: \n",
    "    print(\"Unable to write to file\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}