PageRankTemp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "import json\n",
    "import warnings\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import networkx as nx\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# Local dependencies\n",
    "from modules.dataset.tweets import Tweets\n",
    "from modules.dataset.entities import Entities\n",
    "\n",
    "# Constants\n",
    "alpha = 0.85\n",
    "max_iter = 100\n",
    "data_path = \"data/communities/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_adjacency_matrix(data):\n",
    "    \"\"\"\n",
    "    Input:\n",
    "        - data: pandas.DataFrame with columns names = ['index_id', 'index_tag']\n",
    "    Output:\n",
    "        - numpy.matrix A - Adjacency matrix\n",
    "    \"\"\"\n",
    "    # Create networkx graph object\n",
    "    graph = nx.from_pandas_edgelist(data, source='index_id', target='index_tag')\n",
    "    # Check if the graph is connected\n",
    "    cc = nx.number_connected_components(graph)\n",
    "    if cc > 1:\n",
    "        print('The bipartite graph is not connected! The GC will be used')\n",
    "        # Extract giant component\n",
    "        giant = max(nx.connected_component_subgraphs(graph), key=len)\n",
    "        # Select the GC\n",
    "        graph = graph.subgraph(giant)\n",
    "    # Extract adjacency matrix\n",
    "    A = nx.to_numpy_matrix(graph)\n",
    "    #print(A.shape)\n",
    "    return A"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_google_matrix(A, e2i, cluster, alpha):\n",
    "    \"\"\"\n",
    "    Input:\n",
    "        - A       : numpy.matrix of dimension [n_nodes, n_nodes]\n",
    "        - e2i     : dictionary that associates each node name to its number in the graph\n",
    "        - cluster : list of strings (node names)\n",
    "        - alpha   : float between 0 and 1 -- Dumping factor (which is 1 - teleport probability)\n",
    "    Output:\n",
    "        - numpy.matrix - Google matrix G = alpha A + (1-alpha) C\n",
    "    \"\"\"\n",
    "    # Normalize A (stochastic on columns)\n",
    "    A /= A.sum(axis=0)\n",
    "    # Mask of indices in the cluster\n",
    "    mask = [ e2i[e] for e in cluster ]\n",
    "    # Compute google matrix\n",
    "    G = alpha*A\n",
    "    G[mask, :] += (1-alpha)/len(cluster)\n",
    "\n",
    "    return G"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def power_iteration(G, max_iter: int, tolerance=1e-6):\n",
    "    \"\"\"\n",
    "    Input:\n",
    "        - G         : squared numpy.matrix -- Google matrix\n",
    "        - max_iter  : int -- maximum number of iterations\n",
    "        - tolerance : float -- maximum accepted error\n",
    "    Output:\n",
    "        - approximate eigenvector of G (unique if G is a Google matrix)\n",
    "    \"\"\"\n",
    "    # Choose a random vector to decrease the chance that our vector is orthogonal to the eigenvector\n",
    "    b_k = np.random.rand(G.shape[1])\n",
    "\n",
    "    for _ in range(max_iter):\n",
    "        # Calculate the matrix-by-vector product Ab\n",
    "        b_k1 = G @ np.reshape(b_k, (-1,1)) \n",
    "\n",
    "        # Calculate the norm\n",
    "        b_k1_norm = np.linalg.norm(b_k1)\n",
    "\n",
    "        # Re-normalize the vector\n",
    "        b_k = b_k1 / b_k1_norm\n",
    "\n",
    "        # If the precision increment is uniformly lower than the tolerance, break\n",
    "        if np.allclose(b_k, b_k1, atol=tolerance):\n",
    "            #print(b_k1_norm)\n",
    "            break\n",
    "\n",
    "    return b_k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load communities data\n",
    "communities = pd.read_csv(data_path+\"hashtags_community_selected.csv\", header=0)\n",
    "\n",
    "# Load tweet_id - hashtag map\n",
    "hashtags = Entities()\n",
    "hashtags.from_json(\"data/db/hashtags.json\")\n",
    "hashtags.df = hashtags.df[['tweet_id','entity_text']].rename(columns={'entity_text': 'hashtag'})\n",
    "hashtags.df.hashtag = hashtags.df.hashtag.apply(lambda x: x.lower())\n",
    "# Drop rows with search hashtags\n",
    "seed_list = [\"#climatechange\", \"#climate\", \"#sdgs\", \"#sustainability\", \"#environment\", \"#globalwarming\"]\n",
    "hashtags.df = hashtags.df[~hashtags.df.hashtag.isin(seed_list)]\n",
    "\n",
    "# Load tweet data\n",
    "tweets = Tweets()\n",
    "tweets.from_json(\"data/db/tweets.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/laura/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "/home/laura/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network 2017\n",
      "\n",
      "The bipartite graph is not connected! The GC will be used\n",
      "det(A):  0.0\n",
      "\n",
      "Network 2018\n",
      "\n",
      "The bipartite graph is not connected! The GC will be used\n",
      "det(A):  0.0\n",
      "\n",
      "Network 2019\n",
      "\n",
      "The bipartite graph is not connected! The GC will be used\n",
      "det(A):  0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "years = set(communities.year.values)\n",
    "\n",
    "for year in years:    \n",
    "    # Select ids of the year\n",
    "    curr_ids = list(tweets.df.tweet_id[tweets.df.tweet_date.dt.year == year].values)\n",
    "    # Select communities of the year\n",
    "    curr_communities = communities[communities.year == year]\n",
    "    # Select hashtags of interest\n",
    "    data = hashtags.df[hashtags.df.tweet_id.isin(curr_ids)]\n",
    "    \n",
    "    # Map entities in index\n",
    "    nodes = list(data.hashtag.unique())\n",
    "    nodes.extend(data.tweet_id.unique())\n",
    "    e2i = dict(zip(nodes, range(len(nodes))))\n",
    "    # Map index in entities\n",
    "    i2e = dict(zip(range(len(nodes)), nodes))\n",
    "\n",
    "    # Add indices to data\n",
    "    data['index_id'] = data.tweet_id.apply(lambda x: e2i[x])\n",
    "    data['index_tag'] = data.hashtag.apply(lambda x: e2i[x])\n",
    "    # Keep only indices\n",
    "    data = data[['index_id', 'index_tag']]\n",
    "    \n",
    "    # Init metrics container for year\n",
    "    clusters = curr_communities.community.unique()\n",
    "    community_similarity = pd.DataFrame(columns=clusters, dtype=float)\n",
    "    \n",
    "    print(\"Network {:d}\".format(year), end='\\n\\n')\n",
    "    \n",
    "    # Compute adjacency matrix\n",
    "    A = get_adjacency_matrix(data)    \n",
    "    print(\"det(A): \", np.linalg.det(A))\n",
    "    # Loop through communities\n",
    "    for cluster in clusters:\n",
    "        # Compute Google matrix\n",
    "        G = get_google_matrix(A, e2i, curr_communities.hashtag[curr_communities.community == cluster], alpha)\n",
    "        # Compute eigenvector\n",
    "        v = power_iteration(G, 100)\n",
    "        # Add eigenvector to metrics container\n",
    "        community_similarity[cluster] = np.array(v).squeeze()\n",
    "    print()    \n",
    "    # Remove hashtag from nodes\n",
    "    community_similarity.index = pd.Series(community_similarity.index).apply(lambda x: i2e[x])\n",
    "    mask_tweets = community_similarity.index.isin(curr_ids)\n",
    "    community_similarity = community_similarity.loc[mask_tweets]\n",
    "\n",
    "    # Save results\n",
    "    community_similarity.to_csv(data_path+\"tweet_communities{}.csv\".format(year))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>9</th>\n",
       "      <th>12</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>0.000369</td>\n",
       "      <td>0.002738</td>\n",
       "      <td>0.000617</td>\n",
       "      <td>0.000357</td>\n",
       "      <td>0.002133</td>\n",
       "      <td>0.000373</td>\n",
       "      <td>0.001195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>0.000218</td>\n",
       "      <td>0.001795</td>\n",
       "      <td>0.000585</td>\n",
       "      <td>0.000659</td>\n",
       "      <td>0.005833</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>0.000254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574476297314311</td>\n",
       "      <td>0.000511</td>\n",
       "      <td>0.000661</td>\n",
       "      <td>0.000512</td>\n",
       "      <td>0.000524</td>\n",
       "      <td>0.000587</td>\n",
       "      <td>0.000395</td>\n",
       "      <td>0.000638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574488225918976</td>\n",
       "      <td>0.002658</td>\n",
       "      <td>0.001162</td>\n",
       "      <td>0.000342</td>\n",
       "      <td>0.000286</td>\n",
       "      <td>0.001092</td>\n",
       "      <td>0.000258</td>\n",
       "      <td>0.001009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574491430375428</td>\n",
       "      <td>0.001358</td>\n",
       "      <td>0.001863</td>\n",
       "      <td>0.001931</td>\n",
       "      <td>0.002736</td>\n",
       "      <td>0.002146</td>\n",
       "      <td>0.001339</td>\n",
       "      <td>0.001802</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574513450459138</td>\n",
       "      <td>0.000390</td>\n",
       "      <td>0.000532</td>\n",
       "      <td>0.000558</td>\n",
       "      <td>0.000773</td>\n",
       "      <td>0.000607</td>\n",
       "      <td>0.000385</td>\n",
       "      <td>0.000508</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574525622329344</td>\n",
       "      <td>0.008266</td>\n",
       "      <td>0.000425</td>\n",
       "      <td>0.000297</td>\n",
       "      <td>0.001051</td>\n",
       "      <td>0.000963</td>\n",
       "      <td>0.000553</td>\n",
       "      <td>0.000346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574533465681920</td>\n",
       "      <td>0.003521</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>0.000148</td>\n",
       "      <td>0.000444</td>\n",
       "      <td>0.000408</td>\n",
       "      <td>0.000242</td>\n",
       "      <td>0.000144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574542147780609</td>\n",
       "      <td>0.001452</td>\n",
       "      <td>0.002004</td>\n",
       "      <td>0.001598</td>\n",
       "      <td>0.001908</td>\n",
       "      <td>0.002061</td>\n",
       "      <td>0.003321</td>\n",
       "      <td>0.001498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574542261141506</td>\n",
       "      <td>0.000311</td>\n",
       "      <td>0.000428</td>\n",
       "      <td>0.000344</td>\n",
       "      <td>0.000404</td>\n",
       "      <td>0.000437</td>\n",
       "      <td>0.000708</td>\n",
       "      <td>0.000317</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0         1         2         3         9  \\\n",
       "1101574442575167489  0.000369  0.002738  0.000617  0.000357  0.002133   \n",
       "1101574446341607424  0.000218  0.001795  0.000585  0.000659  0.005833   \n",
       "1101574476297314311  0.000511  0.000661  0.000512  0.000524  0.000587   \n",
       "1101574488225918976  0.002658  0.001162  0.000342  0.000286  0.001092   \n",
       "1101574491430375428  0.001358  0.001863  0.001931  0.002736  0.002146   \n",
       "1101574513450459138  0.000390  0.000532  0.000558  0.000773  0.000607   \n",
       "1101574525622329344  0.008266  0.000425  0.000297  0.001051  0.000963   \n",
       "1101574533465681920  0.003521  0.000188  0.000148  0.000444  0.000408   \n",
       "1101574542147780609  0.001452  0.002004  0.001598  0.001908  0.002061   \n",
       "1101574542261141506  0.000311  0.000428  0.000344  0.000404  0.000437   \n",
       "\n",
       "                           12        15  \n",
       "1101574442575167489  0.000373  0.001195  \n",
       "1101574446341607424  0.000246  0.000254  \n",
       "1101574476297314311  0.000395  0.000638  \n",
       "1101574488225918976  0.000258  0.001009  \n",
       "1101574491430375428  0.001339  0.001802  \n",
       "1101574513450459138  0.000385  0.000508  \n",
       "1101574525622329344  0.000553  0.000346  \n",
       "1101574533465681920  0.000242  0.000144  \n",
       "1101574542147780609  0.003321  0.001498  \n",
       "1101574542261141506  0.000708  0.000317  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "year = 2019\n",
    "\n",
    "community_similarity = pd.read_csv(data_path+\"tweet_communities{}.csv\".format(year), index_col=0,\n",
    "                                   dtype={'index': np.unicode_})\n",
    "community_similarity.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/laura/anaconda3/lib/python3.7/site-packages/numpy/core/fromnumeric.py:61: FutureWarning: \n",
      "The current behaviour of 'Series.argmax' is deprecated, use 'idxmax'\n",
      "instead.\n",
      "The behavior of 'argmax' will be corrected to return the positional\n",
      "maximum in the future. For now, use 'series.values.argmax' or\n",
      "'np.argmax(np.array(values))' to get the position of the maximum\n",
      "row.\n",
      "  return bound(*args, **kwds)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOd0lEQVR4nO3cf4hlZ33H8ffHbPzRKm5MJiHsbjtSl6JIs4YhLgTEJlI2ibhRDCiiW7vtthAholBX+4cV+sdKqRGhBLbdkLVYNfiDLElqXWJS6R+JTjTGpKvNNmzNdEN2bH6oBG2j3/5xn6WT3buZOzN39s48vl9wOed5znPv+R7u3M+ceeacm6pCktSXF026AEnS+BnuktQhw12SOmS4S1KHDHdJ6tCGSRcAcMEFF9T09PSky5CkdeX+++//cVVNDdu2JsJ9enqa2dnZSZchSetKkv880zanZSSpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUNr4g7V9Wp67x0T2e+xfddMZL+S1g/P3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6tBI4Z7kWJLvJ3kgyWzre1WSw0keacvzWn+SfCbJ0SQPJrl0NQ9AknS6pZy5/35VbauqmdbeC9xVVVuBu1ob4Cpga3vsAW4aV7GSpNGsZFpmJ3CwrR8Erl3Q/9kauBfYmOTiFexHkrREo4Z7AV9Pcn+SPa3voqp6HKAtL2z9m4DHFjx3rvU9T5I9SWaTzM7Pzy+veknSUKN+t8zlVXU8yYXA4SQ/eIGxGdJXp3VU7Qf2A8zMzJy2XZK0fCOduVfV8bY8AXwVuAx44uR0S1ueaMPngC0Lnr4ZOD6ugiVJi1s03JP8ZpJXnFwH/gB4CDgE7GrDdgG3tfVDwPvaVTPbgWdOTt9Iks6OUaZlLgK+muTk+H+sqq8l+TZwa5LdwI+A69r4O4GrgaPAs8D7x161JOkFLRruVfUocMmQ/v8GrhzSX8D1Y6lOkrQs3qEqSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdGvVbIdes6b13TLoESVpz1n24S72a1InLsX3XTGS/Gi+nZSSpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOjRzuSc5J8t0kt7f2q5Pcl+SRJF9M8uLW/5LWPtq2T69O6ZKkM1nKmfsNwJEF7U8CN1bVVuApYHfr3w08VVWvAW5s4yRJZ9FI4Z5kM3AN8PetHeAK4EttyEHg2ra+s7Vp269s4yVJZ8moZ+6fBv4c+FVrnw88XVXPtfYcsKmtbwIeA2jbn2njJUlnyaLhnuStwImqun9h95ChNcK2ha+7J8lsktn5+fmRipUkjWaUM/fLgbclOQZ8gcF0zKeBjUk2tDGbgeNtfQ7YAtC2vxJ48tQXrar9VTVTVTNTU1MrOghJ0vMtGu5V9dGq2lxV08C7gG9U1XuAu4F3tmG7gNva+qHWpm3/RlWdduYuSVo9K7nO/SPAh5IcZTCnfqD1HwDOb/0fAvaurERJ0lJtWHzI/6uqe4B72vqjwGVDxvwcuG4MtUmSlsk7VCWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdWtJNTNKvm+m9d0y6BGlZPHOXpA4Z7pLUIadlJK0Zk5oGO7bvmonsdzV55i5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOLhnuSlyb5VpLvJXk4ySda/6uT3JfkkSRfTPLi1v+S1j7atk+v7iFIkk41ypn7L4ArquoSYBuwI8l24JPAjVW1FXgK2N3G7waeqqrXADe2cZKks2jRcK+Bn7Xmue1RwBXAl1r/QeDatr6ztWnbr0ySsVUsSVrUSHPuSc5J8gBwAjgM/AfwdFU914bMAZva+ibgMYC2/Rng/HEWLUl6YSOFe1X9sqq2AZuBy4DXDhvWlsPO0uvUjiR7kswmmZ2fnx+1XknSCJZ0tUxVPQ3cA2wHNibZ0DZtBo639TlgC0Db/krgySGvtb+qZqpqZmpqannVS5KGGuVqmakkG9v6y4C3AEeAu4F3tmG7gNva+qHWpm3/RlWdduYuSVo9GxYfwsXAwSTnMPhlcGtV3Z7k34AvJPkr4LvAgTb+APAPSY4yOGN/1yrULUl6AYuGe1U9CLxhSP+jDObfT+3/OXDdWKqTJC2Ld6hKUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktShUW5ikqSuTe+9Y2L7PrbvmlV5Xc/cJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShxYN9yRbktyd5EiSh5Pc0PpfleRwkkfa8rzWnySfSXI0yYNJLl3tg5AkPd+GEcY8B3y4qr6T5BXA/UkOA38I3FVV+5LsBfYCHwGuAra2xxuBm9pSHZjee8dE9nts3zUT2a+0Xi165l5Vj1fVd9r6T4EjwCZgJ3CwDTsIXNvWdwKfrYF7gY1JLh575ZKkM1rSnHuSaeANwH3ARVX1OAx+AQAXtmGbgMcWPG2u9UmSzpKRwz3Jy4EvAx+sqp+80NAhfTXk9fYkmU0yOz8/P2oZkqQRjBTuSc5lEOyfq6qvtO4nTk63tOWJ1j8HbFnw9M3A8VNfs6r2V9VMVc1MTU0tt35J0hCjXC0T4ABwpKo+tWDTIWBXW98F3Lag/33tqpntwDMnp28kSWfHKFfLXA68F/h+kgda38eAfcCtSXYDPwKua9vuBK4GjgLPAu8fa8WSpEUtGu5V9a8Mn0cHuHLI+AKuX2FdkqQV8A5VSeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6tGi4J7k5yYkkDy3oe1WSw0keacvzWn+SfCbJ0SQPJrl0NYuXJA03ypn7LcCOU/r2AndV1VbgrtYGuArY2h57gJvGU6YkaSkWDfeq+ibw5CndO4GDbf0gcO2C/s/WwL3AxiQXj6tYSdJoljvnflFVPQ7Qlhe2/k3AYwvGzbW+0yTZk2Q2yez8/Pwyy5AkDTPuf6hmSF8NG1hV+6tqpqpmpqamxlyGJP16W264P3FyuqUtT7T+OWDLgnGbgePLL0+StBzLDfdDwK62vgu4bUH/+9pVM9uBZ05O30iSzp4Niw1I8nngzcAFSeaAjwP7gFuT7AZ+BFzXht8JXA0cBZ4F3r8KNUuSFrFouFfVu8+w6cohYwu4fqVFSZJWxjtUJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUocMd0nqkOEuSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KHDHdJ6pDhLkkdMtwlqUOGuyR1yHCXpA4Z7pLUIcNdkjpkuEtShwx3SeqQ4S5JHTLcJalDhrskdchwl6QOGe6S1CHDXZI6ZLhLUodWJdyT7EjywyRHk+xdjX1Iks5s7OGe5Bzgb4GrgNcB707yunHvR5J0Zqtx5n4ZcLSqHq2q/wG+AOxchf1Iks4gVTXeF0zeCeyoqj9u7fcCb6yqD5wybg+wpzV/F/jhMnd5AfDjZT53rfFY1p5ejgM8lrVqJcfy21U1NWzDhuXXc0YZ0nfab5Cq2g/sX/HOktmqmlnp66wFHsva08txgMeyVq3WsazGtMwcsGVBezNwfBX2I0k6g9UI928DW5O8OsmLgXcBh1ZhP5KkMxj7tExVPZfkA8A/A+cAN1fVw+PezwIrntpZQzyWtaeX4wCPZa1alWMZ+z9UJUmT5x2qktQhw12SOrRuwz3JzUlOJHlo0rWsVJIbkjyU5OEkH5x0PSvRy1dPJHlpkm8l+V57Xz4x6ZqWYtjnI8lfJ/lBkgeTfDXJxknWuFRJtiS5O8mR9p7cMOmaluIM78lfJvmvJA+0x9Xj2t+6DXfgFmDHpItYqSSvB/6EwZ29lwBvTbJ1slUtT2dfPfEL4IqqugTYBuxIsn3CNS3FLZz++TgMvL6qfg/4d+CjZ7uoFXoO+HBVvRbYDly/zn6+bmF4Zt1YVdva485x7WzdhntVfRN4ctJ1jMFrgXur6tmqeg74F+DtE65pubr56oka+Flrntse6+bqg2Gfj6r6evsZA7iXwT0o60ZVPV5V32nrPwWOAJsmW9XoznZmrdtw78hDwJuSnJ/kN4Cref5NYOvJJuCxBe051tGH71RJzknyAHACOFxV9026pjH6I+CfJl3EciWZBt4A9PCefKBNld2c5LxxvajhPmFVdQT4JIM/mb8GfI/Bn5/r0UhfPbFeVNUvq2obgzPcy9oU2rqX5C8Y/Ix9btK1LEeSlwNfBj5YVT+ZdD0rdBPwOwym/h4H/mZcL2y4rwFVdaCqLq2qNzH4s+2RSde0TF1+9URVPQ3cQx//49kFvBV4T63Dm1ySnMsg2D9XVV+ZdD0rVVVPtJOIXwF/x2BqcywM9zUgyYVt+VvAO4DPT7aiZevmqyeSTJ28miTJy4C3AD+YbFUrk2QH8BHgbVX17KTrWaokAQ4AR6rqU5OuZxySXLyg+XYG07RjsRrfCnlWJPk88GbggiRzwMer6sBkq1q2Lyc5H/hf4PqqemrSBS3HBL56YjVdDBxsVwC9CLi1qm6fcE0jG/b5YHB1zEuAw4Oc5N6q+rOJFbl0lwPvBb7f/hcC8LFxXmGyms7wnrw5yTYG05fHgD8d2/7W4V9mkqRFOC0jSR0y3CWpQ4a7JHXIcJekDhnuktQhw12SOmS4S1KH/g8faqH2p5mePAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Distribution of tweets in communities\n",
    "comm_list = community_similarity.apply(lambda x: np.argmax(x), axis=1)\n",
    "plt.hist(comm_list.values)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcMAAAHFCAYAAACO35LDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcdklEQVR4nO3df8ydZ33f8ffHxlEKBFhnSqPYNGmVskasEGSZtpFooNAaghIxqimpykbXYk0ilK7tulBVaZtpmlZN/SVlVC6k9AeQobRsLnUTWEvGOvHDTgk/nJDWyuj8YLqQpCSELhg//u6P57g7fWI/5/jyOc/tc533C93K+XH7Ot+jB/v7fK77vq87VYUkSctsy9AFSJI0NJuhJGnp2QwlSUvPZihJWno2Q0nS0rMZSpKWns1QkrRQktyW5KEknz3D+0ny60mOJPl0kpdMGtNmKElaNO8C9mzw/quBy0fbXuDtkwa0GUqSFkpVfQR4dINdrgN+p9Z8DHhOkos3GtNmKEnqzSXA0bHnK6PXzuhpcy1HktStH3j5M+qRR1dnPu49n/7aYeDJsZf2VdW+sxgip3ltw7VHbYaSpCaPPLrKJ+56/szH3XrxXz5ZVbvOYYgVYOfY8x3AsY3+gNOkkqQmBZycw/9mYD/wz0ZnlX4X8FhVfXGjP2AylCQtlCTvBa4GtidZAX4e2AZQVb8BHABeAxwB/hb4kUlj2gwlSY2K1ZpJkju7T626YcL7Bbz5bMZ0mlSStPRMhpKkJmvHDPu4QbzNUJLUbEYnvAzOaVJJ0tIzGUqSmhTFavUxTWoylCQtPZOhJKmZJ9BIkpZaAaudNEOnSSVJS89kKElq1ss0qclQkrT0TIaSpCYF3VxaYTOUJDXrY/0Zp0klSTIZSpLaFOWlFZIk9cJkKElqU7DaRzA0GUqSZDKUJDVZu7lvH2yGkqRGYZUMXcRMOE0qSVp6JkNJUpMCTnoCjSRJfTAZSpKa9XLM0GYoSWqydnPfPpqh06SSpKVnMpQkNTtZJkNJkrpgMpQkNenpmKHNUJLUpAirnUww9vEtJEk6ByZDSVIzT6CRJKkTJkNJUhNPoJlg+zdurUt3bpvH0Jvugb/aPnQJM5PH/3boEmYmF1wwdAkzU8e/PnQJM5Wt/Uw41Wofd+t7sr7K8XpyDl0rrFYfP++5NMNLd27jE3ftnMfQm+7qH3vT0CXMzIUf/OTQJczMluf38f8vgJNHjw1dwkxtueiZQ5cwMye/8sTQJczEx772x0OXcN5zmlSS1GTtTvd9JMM+voUkSefAZChJatbLCTQmQ0nS0jMZSpKaVHk2qSRJnHSaVJKkPpgMJUlN1lag6SNT9fEtJEk6ByZDSVIjT6CRJC05V6CRJKkjJkNJUrNVb+4rSVIfTIaSpCZFurm0wmYoSWp2spOzSfv4FpKkpZFkT5IHkhxJctNp3v+WJH+S5NNJ7k6yY9KYNkNJUpNTK9DMettIkq3ArcCrgSuAG5JcsW63/wj8TlV9J3AL8O8nfReboSRpkewGjlTVg1V1HLgduG7dPlcAfzJ6/OHTvP8UNkNJUpMirNbstwkuAY6OPV8ZvTbuU8DrR49fB1yU5B9uNOhUzXDS/KwkSTO0PcmhsW3v2Hun65a17vlPA9+b5JPA9wJfAE5s9IETzyYdm599FWsd+GCS/VV136Q/K0nq25yWY3u4qnad4b0VYOfY8x3AsfEdquoY8E8AkjwTeH1VPbbRB05zacXfzc+OBj41P2szlKQlVsUQC3UfBC5Pchlrie964IfGd0iyHXi0qk4CbwNumzToNN9imvlZkuw9FWm/9MjqFMNKknR2quoEcCNwF3A/8L6qOpzkliTXjna7GnggyV8AzwP+3aRxp0mG08zPUlX7gH0Au1504VPelyT1Jpw8bYuYr6o6ABxY99rNY4/vAO44mzGnSYYT52clSVpk0yTDifOzkqTlUwxyzHAuJjbDqjqR5NT87Fbgtqo6PPfKJEnnvaVaqPt087OSJPXCu1ZIkpoU4aQ395UkqQ8mQ0lSs6U6ZihJ0nqFN/eVJKkbJkNJUqOwOsAKNPNgMpQkLT2ToSSpiccMJUnqiMlQktSsl2OGNkNJUpOqOE0qSVIvTIaSpGa93MKpj28hSdI5MBlKkpoUcNITaCRJyy1Ok0qS1AuToSSpydoKNH1Mk5oMJUlLby7J8IG/2s7VP/ameQy96e5+x28OXcLM7LnuDUOXMDOPXfr0oUuYmRPf/c1DlzBTW1Zr6BJmZnVbH6ln9Q//+/zG7iRTOU0qSWpSxGlSSZJ6YTKUJDU72Umm6uNbSJJ0DkyGkqQmVbDqMUNJkvpgMpQkNevlbFKboSSpydqlFX1MMPbxLSRJOgcmQ0lSs9VObuFkMpQkLT2ToSSpSU93rbAZSpIaeQKNJEndMBlKkpqd9AQaSZL6YDKUJDXpaW1Sm6EkqZkn0EiS1AmToSSpydrapH1Mk5oMJUlLz2QoSWq2NJdWJLktyUNJPrsZBUmStNmmmSZ9F7BnznVIkhbMqbVJZ70NYeI0aVV9JMml8y9FkrRovLRinSR7kxxKcujrx786q2ElSZq7mZ1AU1X7gH0AFz1nR81qXEnSeWrAac1Z6yPfSpJ0DmyGkqQmxdqlFbPeJkmyJ8kDSY4kuek07z8/yYeTfDLJp5O8ZtKY01xa8V7go8ALkqwk+dGJlUqSlsJmn02aZCtwK/Bq4ArghiRXrNvt54D3VdWVwPXAf5r0PaY5m/SGSftIkrRJdgNHqupBgCS3A9cB943tU8CzRo+fDRybNKgr0EiSmpy6znCTXQIcHXu+Arx03T6/AHwwyVuAZwCvnDSoxwwlSeeb7acu1Rtte8feO133XX8Fww3Au6pqB/Aa4HeTbNjvTIaSpGZzSoYPV9WuM7y3Auwce76Dp06D/iijldOq6qNJLgS2Aw+d6QNNhpKkJqdu4bTJy7EdBC5PclmSC1g7QWb/un3+N/B9AEm+A7gQ+NJGg9oMJUkLo6pOADcCdwH3s3bW6OEktyS5drTbTwFvSvIp4L3AG6tqw8VgnCaVJDUb4hZOVXUAOLDutZvHHt8HXHU2Y5oMJUlLz2QoSWpTg1xaMRcmQ0nS0jMZSpKaDHTR/VzYDCVJzXpphk6TSpKWnslQktTk1EX3PTAZSpKWnslQktSsOkmGNkNJUrMhVqCZB6dJJUlLz2QoSWpSrkAjSVI/5pIM8/jfcuEHPzmPoTfdnuveMHQJM3Pnf/3doUuYmWt2XzN0CTNTTzwxdAkztfrY40OXMDsb3/VnYWytr85tbE+gkSQtOa8zlCSpGyZDSVKzXqZJTYaSpKVnMpQkNenpFk4mQ0nS0jMZSpLaVDdXn9gMJUntXJtUkqROmAwlSU0KL62QJKkbJkNJUqN+lmOzGUqSmvVyNqnTpJKkpWcylCQ18wQaSZI6YTKUJDWp6icZ2gwlSc16OZvUaVJJ0tIzGUqSmnlphSRJnTAZSpKa9XICzcRkmGRnkg8nuT/J4SRv3YzCJEnntyJUzX4bwjTJ8ATwU1X150kuAu5J8qGqum/OtUmStCkmNsOq+iLwxdHjryS5H7gEsBlK0pLr5PyZsztmmORS4Erg46d5by+wF+BCnj6D0iRJ2hxTN8MkzwR+H/iJqnp8/ftVtQ/YB/CsLd/Yyy8LkqQz6WgFmqkurUiyjbVG+O6q+oP5liRJ0uaamAyTBHgncH9V/fL8S5IkLYxO5gGnmSa9CngD8Jkk945e+9mqOjC/siRJi6CXadJpzib9M6CPbytJ0mm4Ao0kqZlrk0qS1AmToSSpSbFExwwlSTqtAjpphk6TSpKWnslQktTME2gkSeqEzVCS1K7msE2QZE+SB5IcSXLTad7/lST3jra/SPLlSWM6TSpJarT5N+NNshW4FXgVsAIcTLJ//B67VfWvxvZ/C2t3W9qQyVCStEh2A0eq6sGqOg7cDly3wf43AO+dNKjNUJLUbvOnSS8Bjo49Xxm99hRJvgW4DPjTSYM6TSpJOt9sT3Jo7Pm+0T1z4fRrZZ+phV4P3FFVq5M+0GYoSWozv5v7PlxVu87w3gqwc+z5DuDYGfa9HnjzNB/oNKkkaZEcBC5PclmSC1hrePvX75TkBcA/AD46zaA2Q0lSu00+ZlhVJ4AbgbuA+4H3VdXhJLckuXZs1xuA26umWxbAaVJJ0jnY/LVJRzeXP7DutZvXPf+FsxnTZChJWnomQ0lSO9cmlSSpD3NJhrngArY8f+fkHRfAY5c+fegSZuaa3dcMXcLM/NEn/mjoEmbmNS973dAlzNSW5188dAkzs+WRx4cuYSbyf7bNb/BOkqHTpJKkNt7cV5KkfpgMJUnNvLmvJEmdMBlKktp1kgxthpKkdp5AI0lSH0yGkqRm6WSa1GQoSVp6JkNJUpspbrm0KEyGkqSlZzKUJDVKN2eT2gwlSe2cJpUkqQ8mQ0lSO5OhJEl9MBlKktp1kgxthpKkNt7cV5KkfpgMJUnNXJtUkqROmAwlSe2WJRkmuTDJJ5J8KsnhJL+4GYVJkrRZpkmGXwNeUVVPJNkG/FmSP66qj825NkmSNsXEZlhVBTwxerpttHUSjCVJ52KpTqBJsjXJvcBDwIeq6uOn2WdvkkNJDh1f/b+zrlOSpLmZqhlW1WpVvRjYAexO8sLT7LOvqnZV1a4Ltn7DrOuUJJ2PKrPfBnBWl1ZU1ZeBu4E9c6lGkqQBTHM26XOTPGf0+BuAVwKfm3dhkqTzXM1pG8A0Z5NeDPx2kq2sNc/3VdUH5luWJGkhdHICzTRnk34auHITapEkaRCuQCNJarZUl1ZIktQzk6EkqV0nydBmKElq10kzdJpUkrT0TIaSpCYpT6CRJKkbJkNJUruB1hKdNZuhJKmd06SSJPXBZChJauYJNJIkdcJkKElqZzKUJKkPJkNJUpuOLrq3GUqS2nXSDJ0mlSQtPZuhJKldzWGbIMmeJA8kOZLkpjPs80+T3JfkcJL3TBrTaVJJ0sJIshW4FXgVsAIcTLK/qu4b2+dy4G3AVVX1N0m+adK4c2mGdfzrnDx6bB5Db7oT3/3NQ5cwM/XEE0OXMDOvednrhi5hZg585P1DlzBTPf1s2NZJXsj81g8d4ASa3cCRqnoQIMntwHXAfWP7vAm4tar+BqCqHpo0qNOkkqRFcglwdOz5yui1cd8OfHuS/5nkY0n2TBq0k197JEkd2Z7k0NjzfVW1b/T4dDF3fT59GnA5cDWwA/gfSV5YVV8+0wfaDCVJ7eYzTfpwVe06w3srwM6x5zuA9cflVoCPVdXXgf+V5AHWmuPBM32g06SSpEVyELg8yWVJLgCuB/av2+e/AC8HSLKdtWnTBzca1GQoSWozwAo0VXUiyY3AXcBW4LaqOpzkFuBQVe0fvff9Se4DVoF/XVWPbDSuzVCS1G6AFWiq6gBwYN1rN489LuAnR9tUnCaVJC09k6EkqZ1rk0qS1AeToSSpSejnFk4mQ0nS0jMZSpLadZIMbYaSpDYd3eneaVJJ0tIzGUqS2pkMJUnqg8lQktSuk2RoM5QkNfMEGkmSOmEylCS1MxlKktQHk6EkqU3RTTKcuhkm2QocAr5QVa+dX0mSpEWxjCfQvBW4f16FSJI0lKmaYZIdwDXAO+ZbjiRpodQctgFMmwx/FfgZ4OSZdkiyN8mhJIe+Xk/OpDhJkjbDxGaY5LXAQ1V1z0b7VdW+qtpVVbu25cKZFShJOn+lZr8NYZpkeBVwbZLPA7cDr0jye3OtSpKkTTSxGVbV26pqR1VdClwP/GlV/fDcK5Mknf86OWbodYaSpDbLeJ0hQFXdDdw9l0okSRqIyVCS1CSjrQeuTSpJWnomQ0lSu2U8ZihJ0rhlXJtUkqQumQwlSe1MhpIk9cFkKElq10kytBlKktoMuLD2rDlNKklaeiZDSVI7k6EkSX0wGUqSmnnMUJKkTpgMJUntOkmGNkNJUjOnSSVJ6oTJUJLUpuhmmtRkKElaeiZDSVK7TpLhXJphtm5hy0XPnMfQm27Laic/aWD1sceHLmFmtjz/4qFLmJnXvOx1Q5cwUwc+8v6hS5iZa1762qFLmJH5/DsWPIFGkqRuOE0qSWpnMpQkqQ8mQ0lSs1Qf0dBmKElq43WGkiT1w2YoSWqWmv028TOTPUkeSHIkyU2nef+NSb6U5N7R9mOTxnSaVJK0MJJsBW4FXgWsAAeT7K+q+9bt+p+r6sZpxzUZSpLa1Ry2je0GjlTVg1V1HLgduO5cv4bNUJLUbIBp0kuAo2PPV0avrff6JJ9OckeSnZMGtRlKks4325McGtv2jr2X0+y/voX+IXBpVX0n8N+A3570gR4zlCS1m8+lFQ9X1a4zvLcCjCe9HcCxv1dS1SNjT38T+A+TPtBkKElaJAeBy5NcluQC4Hpg//gOScZX8r8WuH/SoCZDSVKbKS+FmOlHVp1IciNwF7AVuK2qDie5BThUVfuBH09yLXACeBR446RxbYaSpIVSVQeAA+teu3ns8duAt53NmDZDSVK7TpZjsxlKkpp4c19JkjpiMpQktevkFk4mQ0nS0jMZSpKa9XLMcKpmmOTzwFeAVeDEBisDSJKWRUc39z2bZPjyqnp4bpVIkjQQp0klSc1ycugKZmPaE2gK+GCSe9atHv53kuw9tcL48ZNPzq5CSZLmbNpkeFVVHUvyTcCHknyuqj4yvkNV7QP2ATx723M7mUWWJG2ok3/tp0qGVXVs9N+HgPezdqdhSdKSG+DmvnMxsRkmeUaSi049Br4f+Oy8C5MkabNMM036POD9SU7t/56qunOuVUmSzn9FNyvQTGyGVfUg8KJNqEWSpEF4aYUkqVkvK9C4NqkkaemZDCVJ7TpJhjZDSVITb+4rSVJHTIaSpDZV3VxaYTKUJC09k6EkqVkvxwxthpKkdp00Q6dJJUlLz2QoSWrWyzSpyVCStPRMhpKkNgWc7CMa2gwlSe366IVOk0qSZDKUJDXzBBpJkjphMpQktXNtUkmS+mAylCQ16+WY4VyaYa2e5ORXnpjH0JtudVuGLmF2OpnOANjyyONDlzA72/r6nfSal7526BJm5o8+/oGhS5iJ3T/w2HwGLry0QpKkXvT1K6kkadMESCczTiZDSdLSMxlKktqdHLqA2bAZSpKaOU0qSVInTIaSpDZeWiFJUj9MhpKkRtXNYh42Q0lSs16WY3OaVJK09EyGkqR2nUyTmgwlSUvPZChJalOQTlagMRlKkpaeyVCS1M5jhpKkpVdz2CZIsifJA0mOJLlpg/1+MEkl2TVpTJuhJGlhJNkK3Aq8GrgCuCHJFafZ7yLgx4GPTzOuzVCS1CxVM98m2A0cqaoHq+o4cDtw3Wn2+7fALwFPTvM9bIaSpEVyCXB07PnK6LW/k+RKYGdVfWDaQT2BRpLUbj4n0GxPcmjs+b6q2jd6nNNVcepBki3ArwBvPJsPnKoZJnkO8A7ghaMP/RdV9dGz+SBJUmeKed3p/uGqOtNJLyvAzrHnO4BjY88vYq1X3Z0E4JuB/UmurarxBvv3TJsMfw24s6p+MMkFwNOn/HOSJM3SQeDyJJcBXwCuB37o1JtV9Riw/dTzJHcDP71RI4QpmmGSZwEvYxQ5Rwcsj591+ZKkroSpTniZqao6keRG4C5gK3BbVR1OcgtwqKr2t4w7TTL8VuBLwG8leRFwD/DWqvrq+E5J9gJ7AS40OEqS5qSqDgAH1r128xn2vXqaMac5m/RpwEuAt1fVlcBXgadc5FhV+6pqV1Xt2pYLp/lsSdKiq5r9NoBpmuEKsFJVpy5cvIO15ihJWnbL0gyr6q+Bo0leMHrp+4D75lqVJEmbaNqzSd8CvHt0JumDwI/MryRJ0kKY36UVm26qZlhV9wITFzqVJGkRuQKNJKnZZl9aMS+uTSpJWnomQ0lSu06Soc1QktRouEshZs1pUknS0jMZSpLaFCZDSZJ6YTKUJLVbpovuJUk6Ha8zlCSpEyZDSVI7k6EkSX0wGUqS2hRwso9kaDOUJDVyBRpJkrphMpQktTMZSpLUB5OhJKmdyVCSpD6YDCVJbby0YmNfqUcf/tCT7/6reYw9Zjvw8Jw/A9717rl/BJv1XTbH5nyXlbl/AvhzOV9t2nfZevHcP2Kzvsu3zGfYgupjpe65NMOqeu48xh2X5FBV7Zr352wGv8v5ye9yfvK7aB6cJpUktfMEGkmS+rDIyXDf0AXMkN/l/OR3OT/5Xc4XHZ1Ak+ok4kqSNtezL3hefc/zrp/5uHeu/Po9m30s1WlSSdLSW8hmmGRPkgeSHEly09D1tEpyW5KHknx26FrOVZKdST6c5P4kh5O8deiaWiW5MMknknxq9F1+ceiazkWSrUk+meQDQ9dyrpJ8Pslnktyb5NDQ9ZyLJM9JckeSz43+3nz30DU1qZr9NoCFa4ZJtgK3Aq8GrgBuSHLFsFU1exewZ+giZuQE8FNV9R3AdwFvXuCfy9eAV1TVi4AXA3uSfNfANZ2LtwL3D13EDL28ql7cwSUJvwbcWVX/CHgRff2MFs7CNUNgN3Ckqh6squPA7cB1A9fUpKo+Ajw6dB2zUFVfrKo/Hz3+Cmt/sS8Ztqo2teaJ0dNto20hD64n2QFcA7xj6Fr0/yV5FvAy4J0AVXW8qr48bFUt5pAKTYZTuwQ4OvZ8hQX9R7dXSS4FrgQ+Pmwl7UZTi/cCDwEfqqpF/S6/CvwM0McyIWu/lHwwyT1J9g5dzDn4VuBLwG+NprDfkeQZQxd11go4eXL22wAWsRnmNK8t5G/tPUryTOD3gZ+oqseHrqdVVa1W1YuBHcDuJC8cuqazleS1wENVdc/QtczQVVX1EtYOk7w5ycuGLqjR04CXAG+vqiuBrwILe/5DDxaxGa4AO8ee7wCODVSLxiTZxlojfHdV/cHQ9czCaOrqbhbz2O5VwLVJPs/a4YRXJPm9YUs6N1V1bPTfh4D3s3bYZBGtACtjMw53sNYcF4/TpIM5CFye5LIkFwDXA/sHrmnpJQlrxz/ur6pfHrqec5HkuUmeM3r8DcArgc8NW9XZq6q3VdWOqrqUtb8nf1pVPzxwWc2SPCPJRaceA98PLOSZ2FX118DRJC8YvfR9wH0DlrT0Fm4Fmqo6keRG4C5gK3BbVR0euKwmSd4LXA1sT7IC/HxVvXPYqppdBbwB+MzoWBvAz1bVgQFranUx8NujM5e3AO+rqoW/LKEDzwPev/Z7F08D3lNVdw5b0jl5C/Du0S/1DwI/MnA9bTpZuMUVaCRJTZ697Zvqe77x9TMf986HfmPTV6BZuGQoSTpfVDdrk9oMJUltCqqTm/su4gk0kiTNlMlQktSuk2lSk6EkaemZDCVJ7Tq5IsFmKElqUzXYWqKz5jSpJGnpmQwlSe06mSY1GUqSlp7JUJLUrDxmKElabsPc6T7JniQPJDmS5Cn3gUzyL5N8Jsm9Sf4syRWTxrQZSpIWxuhuMreydoPnK4AbTtPs3lNV/3h0g+5fAibeVs5pUklSm2KIFWh2A0eq6kGAJLcD1zF2P8iqenxs/2ewVumGbIaSpEVyCXB07PkK8NL1OyV5M/CTwAXAKyYN6jSpJKldnZz9tnbD80Nj296xT8zpqnjKC1W3VtW3Af8G+LlJX8NkKEk63zy8wc19V4CdY893AMc2GOt24O2TPtBmKElqUkBt/jHDg8DlSS4DvgBcD/zQ+A5JLq+qvxw9vQb4SyawGUqS2lSdmtbcxI+sE0luBO4CtgK3VdXhJLcAh6pqP3BjklcCXwf+Bvjnk8a1GUqSFkpVHQAOrHvt5rHHbz3bMW2GkqRmA0yTzoVnk0qSlp7JUJLUbpOPGc5LqpPbb0iSNleSO4Htcxj64araM4dxz8hmKElaeh4zlCQtPZuhJGnp2QwlSUvPZihJWno2Q0nS0vt/PcopCjLaW1MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x576 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Correlation matrix\n",
    "fig, axs = plt.subplots(1, 1, figsize=(8,8))\n",
    "axs.imshow(community_similarity.corr())\n",
    "plt.colorbar(axs.imshow(community_similarity.corr()))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4641238670694864"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check numbero of duplicates in row\n",
    "df = community_similarity.copy()\n",
    "(community_similarity.shape[0] - df.drop_duplicates().shape[0] )/community_similarity.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Other useful data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save networks only with the hashtags in a community (selected)\n",
    "\n",
    "years = set(communities.year.values)\n",
    "for year in years:\n",
    "    # Select ids of the year\n",
    "    curr_ids = list(tweets.df.tweet_id[tweets.df.tweet_date.dt.year == year].values)\n",
    "    # Select communities of the year\n",
    "    curr_communities = communities[communities.year == year]\n",
    "    # Select hashtags of interest\n",
    "    curr_hashtags = hashtags.df[(hashtags.df.tweet_id.isin(curr_ids)) & (hashtags.df.hashtag.isin(curr_communities.hashtag.unique())) ]\n",
    "    # Initialize net class\n",
    "    curr_edges = pd.merge(curr_hashtags, curr_hashtags, on='tweet_id')\n",
    "    curr_edges = curr_edges[['hashtag_x', 'hashtag_y']]\n",
    "    # Discard self loops\n",
    "    self_loops = curr_edges.apply(lambda x: x.hashtag_x == x.hashtag_y, axis = 1)\n",
    "    curr_edges = curr_edges[~self_loops]\n",
    "    # Define net\n",
    "    curr_net= nx.from_pandas_edgelist(curr_edges, source='hashtag_x', target='hashtag_y')\n",
    "    # Save net\n",
    "    nx.write_gexf(curr_net, 'data/communities/hash_net_communities_{}.gexf'.format(year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rename columns with the communities labels + normalize on columns\n",
    "\n",
    "years = set(communities.year.values)\n",
    "# Import name-id map\n",
    "labels = pd.read_csv(\"data/communities/name_index_community_map.csv\", dtype=str)\n",
    "\n",
    "for year in years:\n",
    "    # Import files\n",
    "    community_similarity = pd.read_csv(data_path+\"tweet_communities{}.csv\".format(year), index_col=0,\n",
    "                                   dtype={'index': np.unicode_})\n",
    "\n",
    "    # Define dict {cluster_id : cluster_label}\n",
    "    id2name = labels.loc[labels.year==str(year), [\"label\", \"index\"]].set_index(\"index\", drop=True)\n",
    "    id2name = id2name.to_dict()[\"label\"]\n",
    "    \n",
    "    # Rename columns using community names\n",
    "    community_similarity = community_similarity.rename(columns=id2name)\n",
    "    # Normalize columns (L1-norm)\n",
    "    norms = community_similarity.apply(lambda x: sum(x), axis = 0)\n",
    "    community_similarity = community_similarity.apply(lambda col: col/norms[col.name], axis = 0)\n",
    "    # Save results\n",
    "    community_similarity.to_csv(data_path+\"tweet_communities{}.csv\".format(year))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sustainable_development</th>\n",
       "      <th>greta</th>\n",
       "      <th>endangered_habitats</th>\n",
       "      <th>innovation</th>\n",
       "      <th>renewable_energy</th>\n",
       "      <th>mixed</th>\n",
       "      <th>developement_goals</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.000670</td>\n",
       "      <td>0.000191</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.000507</td>\n",
       "      <td>0.000094</td>\n",
       "      <td>0.000326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>0.000439</td>\n",
       "      <td>0.000182</td>\n",
       "      <td>0.000166</td>\n",
       "      <td>0.001387</td>\n",
       "      <td>0.000062</td>\n",
       "      <td>0.000069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574476297314311</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>0.000132</td>\n",
       "      <td>0.000140</td>\n",
       "      <td>0.000100</td>\n",
       "      <td>0.000174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574488225918976</td>\n",
       "      <td>0.000787</td>\n",
       "      <td>0.000284</td>\n",
       "      <td>0.000106</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.000260</td>\n",
       "      <td>0.000065</td>\n",
       "      <td>0.000276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1101574491430375428</td>\n",
       "      <td>0.000402</td>\n",
       "      <td>0.000456</td>\n",
       "      <td>0.000599</td>\n",
       "      <td>0.000691</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>0.000337</td>\n",
       "      <td>0.000492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     sustainable_development     greta  endangered_habitats  \\\n",
       "1101574442575167489                 0.000109  0.000670             0.000191   \n",
       "1101574446341607424                 0.000065  0.000439             0.000182   \n",
       "1101574476297314311                 0.000152  0.000162             0.000159   \n",
       "1101574488225918976                 0.000787  0.000284             0.000106   \n",
       "1101574491430375428                 0.000402  0.000456             0.000599   \n",
       "\n",
       "                     innovation  renewable_energy     mixed  \\\n",
       "1101574442575167489    0.000090          0.000507  0.000094   \n",
       "1101574446341607424    0.000166          0.001387  0.000062   \n",
       "1101574476297314311    0.000132          0.000140  0.000100   \n",
       "1101574488225918976    0.000072          0.000260  0.000065   \n",
       "1101574491430375428    0.000691          0.000510  0.000337   \n",
       "\n",
       "                     developement_goals  \n",
       "1101574442575167489            0.000326  \n",
       "1101574446341607424            0.000069  \n",
       "1101574476297314311            0.000174  \n",
       "1101574488225918976            0.000276  \n",
       "1101574491430375428            0.000492  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show last computed result\n",
    "community_similarity.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sustainable_development    1.0\n",
       "greta                      1.0\n",
       "endangered_habitats        1.0\n",
       "innovation                 1.0\n",
       "renewable_energy           1.0\n",
       "mixed                      1.0\n",
       "developement_goals         1.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "community_similarity.apply(lambda x: sum(x), axis = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}