CommunityDetection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.dataset.entities import Entities\n",
    "from modules.dataset.tweets import Tweets\n",
    "from modules.network import HashNet\n",
    "\n",
    "import networkx as nx\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import community"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "TWEETS_DB_PATH = 'data/db/tweets.json'  # Path to tweets dataset\n",
    "HASHTAGS_DB_PATH = 'data/db/hashtags.json' # Path to hashtags dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define networks container, indexed by periods\n",
    "networks = {\n",
    "    2017: None,\n",
    "    2018: None,\n",
    "    2019: None\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>836950901495631872</td>\n",
       "      <td>2017-03-01 14:46:59</td>\n",
       "      <td>TT SINGAPORE 22:46\\n1.Hong Kong\\n2.#JointAddre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>836950882528989184</td>\n",
       "      <td>2017-03-01 14:46:54</td>\n",
       "      <td>Letting #snapchat prepare me for the day's uns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>836950869639835649</td>\n",
       "      <td>2017-03-01 14:46:51</td>\n",
       "      <td>\"The bill would require the state to get all o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>836950847380668416</td>\n",
       "      <td>2017-03-01 14:46:46</td>\n",
       "      <td>Style-Lead don't Follow #recycledfashion https...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>836950839101116421</td>\n",
       "      <td>2017-03-01 14:46:44</td>\n",
       "      <td>‘Shell knew’: oil giant's 1991 film warned of ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id          tweet_date  \\\n",
       "0  836950901495631872 2017-03-01 14:46:59   \n",
       "1  836950882528989184 2017-03-01 14:46:54   \n",
       "2  836950869639835649 2017-03-01 14:46:51   \n",
       "3  836950847380668416 2017-03-01 14:46:46   \n",
       "4  836950839101116421 2017-03-01 14:46:44   \n",
       "\n",
       "                                          tweet_text  \n",
       "0  TT SINGAPORE 22:46\\n1.Hong Kong\\n2.#JointAddre...  \n",
       "1  Letting #snapchat prepare me for the day's uns...  \n",
       "2  \"The bill would require the state to get all o...  \n",
       "3  Style-Lead don't Follow #recycledfashion https...  \n",
       "4  ‘Shell knew’: oil giant's 1991 film warned of ...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import tweets dataset\n",
    "tweets = Tweets()\n",
    "tweets.from_json(TWEETS_DB_PATH)\n",
    "tweets.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>entity_index</th>\n",
       "      <th>entity_text</th>\n",
       "      <th>entity_tag</th>\n",
       "      <th>entity_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>0</td>\n",
       "      <td>#Humans</td>\n",
       "      <td>#</td>\n",
       "      <td>0.6243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>7</td>\n",
       "      <td>#climatechange</td>\n",
       "      <td>^</td>\n",
       "      <td>0.4925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>27</td>\n",
       "      <td>#ClimateChange</td>\n",
       "      <td>#</td>\n",
       "      <td>0.6968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>36</td>\n",
       "      <td>#ActOnClimate</td>\n",
       "      <td>#</td>\n",
       "      <td>0.7929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>37</td>\n",
       "      <td>#climate</td>\n",
       "      <td>#</td>\n",
       "      <td>0.9559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  entity_index     entity_text entity_tag  entity_conf\n",
       "0  1101574442575167489             0         #Humans          #       0.6243\n",
       "1  1101574442575167489             7  #climatechange          ^       0.4925\n",
       "2  1101574446341607424            27  #ClimateChange          #       0.6968\n",
       "3  1101574446341607424            36   #ActOnClimate          #       0.7929\n",
       "4  1101574446341607424            37        #climate          #       0.9559"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hashtags = Entities()\n",
    "hashtags.from_json(HASHTAGS_DB_PATH)\n",
    "hashtags.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the list of hashtags used as search seeds\n",
    "seed_list = [\"#climatechange\", \"#climate\", \"#sdgs\", \"#sustainability\", \"#environment\", \"#globalwarming\"]\n",
    "\n",
    "for period in networks.keys():\n",
    "    # Subset tweets for current period\n",
    "    curr_tweets = Tweets()\n",
    "    curr_tweets.df = tweets.df.loc[tweets.df.tweet_date.dt.year == period].copy()\n",
    "    # Get ids of tweets for current period\n",
    "    curr_tweets_id = curr_tweets.df.tweet_id.unique()\n",
    "    # Subset hashtags and words associated to current tweets\n",
    "    curr_hashtags = Entities()\n",
    "    curr_hashtags.df = hashtags.df.loc[hashtags.df.tweet_id.isin(curr_tweets_id)].copy()\n",
    "    # Lower hashtags\n",
    "    curr_hashtags.df[\"entity_text\"] = curr_hashtags.df[\"entity_text\"].str.lower()\n",
    "    # Remove seeds from hashtag dataset\n",
    "    curr_hashtags.df = curr_hashtags.df.loc[~curr_hashtags.df.entity_text.isin(seed_list)]\n",
    "    #curr_words.df = curr_words.df.loc[~curr_words.df.entity_text.isin(seed_list)]\n",
    "    # Generate a dictionary containing words and hashtags networks for each period\n",
    "    networks[period] = HashNet.from_entities(curr_hashtags)\n",
    "    \n",
    "    del curr_hashtags, curr_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2017 hashtags network consists in 218 connected components\n",
      "The largest cc corresponds to 77% of total\n",
      "2018 hashtags network consists in 206 connected components\n",
      "The largest cc corresponds to 86% of total\n",
      "2019 hashtags network consists in 185 connected components\n",
      "The largest cc corresponds to 88% of total\n"
     ]
    }
   ],
   "source": [
    "for period in networks.keys():\n",
    "    net_type = \"hashtags\"\n",
    "    # Get the list of connected components sorted by size\n",
    "    cc = networks[period].get_connected_components()\n",
    "    # If there is only one cc\n",
    "    if len(cc) == 1:\n",
    "        print(\"{} {} network is connected\".format(period, net_type))\n",
    "    else:\n",
    "        # Compute the ratio between the size of the largest cc and the sum of all the cc sizes\n",
    "        gc_ratio = int(100*cc[0][\"size\"]/sum([cc[i][\"size\"] for i in range(len(cc))]))\n",
    "        # Print results\n",
    "        print(\"{} {} network consists in {} connected components\".format(period, net_type, len(cc)))\n",
    "        print(\"The largest cc corresponds to {}% of total\".format(gc_ratio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep only the largest connected components\n",
    "for period in networks.keys():\n",
    "    # Get the largest connected component\n",
    "    lcc = networks[period].get_connected_components()[0][\"component\"]\n",
    "    # Project the network on the lcc\n",
    "    networks[period] = networks[period].project_component(lcc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Community detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getCommunitiesLouvain(network, resolution=1.0, threshold = 100):\n",
    "    \"\"\"\n",
    "    Filter communities with less than threshold hashtags\n",
    "    return: \n",
    "        comm: map community_id -> list of hashtags\n",
    "        partitions: map hashtag -> community_id\n",
    "    \"\"\" \n",
    "    # compute best partitions (fixed random state for reproducibility)\n",
    "    partition = community.best_partition(graph=network, weight='weight', resolution=resolution, random_state=100)\n",
    "    size = float(len(set(partition.values())))\n",
    "    print('There are {} communities'.format(size))\n",
    "    \n",
    "    # partition is a dictionary in the form {'hashtag':community_id, ...}\n",
    "    # we want ot transform it in the form {'community_id':[hashtag1, hashtag2, ...]}\n",
    "    communities = {}\n",
    "    for p in partition:\n",
    "        if partition[p] in communities:\n",
    "            communities[partition[p]].append(p)\n",
    "        else:\n",
    "            communities[partition[p]] = [p]\n",
    "\n",
    "    # delete small communities (size(community)<threshold)\n",
    "    communities = {k:v for k, v in communities.items() if len(v)>threshold}\n",
    "    print('-> {} communities remaining after filtering'.format(len(communities)))\n",
    "    return communities, partition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Year: 2017 ---\n",
      "There are 26.0 communities\n",
      "-> 7 communities remaining after filtering\n",
      "\n",
      "--- Year: 2018 ---\n",
      "There are 51.0 communities\n",
      "-> 7 communities remaining after filtering\n",
      "\n",
      "--- Year: 2019 ---\n",
      "There are 45.0 communities\n",
      "-> 7 communities remaining after filtering\n",
      "\n"
     ]
    }
   ],
   "source": [
    "communities = {\n",
    "    2017: None,\n",
    "    2018: None,\n",
    "    2019: None\n",
    "}\n",
    "\n",
    "parameters = {\n",
    "    2017: {'resolution':1.1, 'threshold':100},\n",
    "    2018: {'resolution':0.6, 'threshold':135},\n",
    "    2019: {'resolution':0.6, 'threshold':160},\n",
    "}\n",
    "\n",
    "for period in networks.keys():\n",
    "    print(\"--- Year: {} ---\".format(period))\n",
    "    comm, partitions = getCommunitiesLouvain(\n",
    "        networks[period].net,\n",
    "        resolution=parameters[period]['resolution'],\n",
    "        threshold=parameters[period]['threshold']\n",
    "    )\n",
    "    communities[period] = {\n",
    "        'communities': comm,\n",
    "        'partitions': partitions\n",
    "    }\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sort_dict(d, descending=True):\n",
    "    \"\"\"\n",
    "    Sort a dictionary based on items\n",
    "    \"\"\"\n",
    "    return {k: v for k, v in sorted(d.items(), key=lambda item: item[1], reverse=True)}\n",
    "\n",
    "def printCommunitiElemnts(\n",
    "    network, communities, num_communities=5,\n",
    "    community_id=None, metric_name='centrality',\n",
    "    showTopK=5, print_metric=True\n",
    "):\n",
    "    \"\"\"\n",
    "    Print most important nodes based on certain metric\n",
    "    \"\"\"\n",
    "    if metric_name=='centrality':\n",
    "        centrality = nx.degree_centrality(network)\n",
    "        metric = centrality\n",
    "        \n",
    "    if metric_name=='degree':\n",
    "        degree = nx.degree(network, weight='weight')\n",
    "        metric = degree\n",
    "        \n",
    "    # find most relevant (based on metric) terms for each community\n",
    "    if community_id == None:\n",
    "        counter = 1\n",
    "        for k,v in communities.items():\n",
    "            print('---Community: {}, Size: {}---'.format(k, len(v)))\n",
    "            tag_dict = {tag: metric[tag] for tag in v}\n",
    "            tag_dict = sort_dict(tag_dict)\n",
    "            tag_list = list(tag_dict.keys())\n",
    "            for tag in tag_list[:showTopK]:\n",
    "                if print_metric:\n",
    "                    print(tag, ',{}: {:.4f}'.format(metric_name, metric[tag]))\n",
    "                else:\n",
    "                    print(tag)\n",
    "            print()\n",
    "            ## Show only num_communities\n",
    "            if counter == num_communities:\n",
    "                break\n",
    "            counter += 1\n",
    "    else:\n",
    "        k = community_id\n",
    "        if k not in communities:\n",
    "            print(\"Community {} has been discarded (too small)\".format(k))\n",
    "            return\n",
    "        v = communities[community_id]\n",
    "        print('---Community: {}---'.format(k))\n",
    "        tag_dict = {tag: metric[tag] for tag in v}\n",
    "        tag_dict = sort_dict(tag_dict)\n",
    "        tag_list = list(tag_dict.keys())\n",
    "        for tag in tag_list[:showTopK]:\n",
    "            if print_metric:\n",
    "                print(tag, ',{}: {:.4f}'.format(metric_name, metric[tag]))\n",
    "            else:\n",
    "                print(tag)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Community: 0, Size: 177---\n",
      "#un ,centrality: 0.0331\n",
      "#sustainabledevelopment ,centrality: 0.0249\n",
      "#india ,centrality: 0.0198\n",
      "#peace ,centrality: 0.0169\n",
      "#sdg ,centrality: 0.0135\n",
      "#nigeria ,centrality: 0.0109\n",
      "#unitednations ,centrality: 0.0094\n",
      "#china ,centrality: 0.0092\n",
      "#growth ,centrality: 0.0090\n",
      "#fridayfeeling ,centrality: 0.0087\n",
      "#ngo ,centrality: 0.0073\n",
      "#campaign ,centrality: 0.0070\n",
      "#people ,centrality: 0.0058\n",
      "#sustainabledevelopmentgoals ,centrality: 0.0053\n",
      "#literacy ,centrality: 0.0053\n",
      "#youth ,centrality: 0.0053\n",
      "#human ,centrality: 0.0053\n",
      "#goals ,centrality: 0.0051\n",
      "#partnership ,centrality: 0.0051\n",
      "#socialgood ,centrality: 0.0051\n",
      "#mdgs ,centrality: 0.0048\n",
      "#pt2019 ,centrality: 0.0048\n",
      "#lagos ,centrality: 0.0046\n",
      "#2030 ,centrality: 0.0044\n",
      "#mains2019 ,centrality: 0.0044\n",
      "\n",
      "---Community: 1, Size: 261---\n",
      "#climateaction ,centrality: 0.0777\n",
      "#climatestrike ,centrality: 0.0556\n",
      "#fridaysforfuture ,centrality: 0.0404\n",
      "#climatecrisis ,centrality: 0.0324\n",
      "#extinctionrebellion ,centrality: 0.0252\n",
      "#climateactionnow ,centrality: 0.0167\n",
      "#climateemergency ,centrality: 0.0138\n",
      "#schoolstrike4climate ,centrality: 0.0138\n",
      "#gretathunberg ,centrality: 0.0116\n",
      "#iwd2019 ,centrality: 0.0092\n",
      "#fridayforfuture ,centrality: 0.0082\n",
      "#internationalwomensday ,centrality: 0.0077\n",
      "#climatebreakdown ,centrality: 0.0075\n",
      "#klimaschutz ,centrality: 0.0063\n",
      "#climatemarch ,centrality: 0.0060\n",
      "#nowplaying ,centrality: 0.0060\n",
      "#schoolstrike ,centrality: 0.0056\n",
      "#climatechangestrike ,centrality: 0.0056\n",
      "#radio ,centrality: 0.0056\n",
      "#rebelforlife ,centrality: 0.0053\n",
      "#balanceforbetter ,centrality: 0.0046\n",
      "#toronto ,centrality: 0.0046\n",
      "#tellthetruth ,centrality: 0.0041\n",
      "#strike4climate ,centrality: 0.0041\n",
      "#climatecatastrophe ,centrality: 0.0041\n",
      "\n",
      "---Community: 2, Size: 184---\n",
      "#nature ,centrality: 0.0803\n",
      "#pollution ,centrality: 0.0532\n",
      "#conservation ,centrality: 0.0344\n",
      "#usa ,centrality: 0.0235\n",
      "#savetheplanet ,centrality: 0.0220\n",
      "#trees ,centrality: 0.0167\n",
      "#wildlife ,centrality: 0.0148\n",
      "#future ,centrality: 0.0145\n",
      "#extinction ,centrality: 0.0131\n",
      "#animal ,centrality: 0.0116\n",
      "#america ,centrality: 0.0114\n",
      "#climatejustice ,centrality: 0.0111\n",
      "#paris ,centrality: 0.0111\n",
      "#fish ,centrality: 0.0102\n",
      "#forests ,centrality: 0.0097\n",
      "#1o5c ,centrality: 0.0087\n",
      "#amazon ,centrality: 0.0087\n",
      "#xr ,centrality: 0.0080\n",
      "#fff ,centrality: 0.0075\n",
      "#living ,centrality: 0.0073\n",
      "#iran ,centrality: 0.0073\n",
      "#pakistan ,centrality: 0.0070\n",
      "#insect ,centrality: 0.0068\n",
      "#cop25 ,centrality: 0.0065\n",
      "#knowledge ,centrality: 0.0065\n",
      "\n",
      "---Community: 3, Size: 193---\n",
      "#innovation ,centrality: 0.0462\n",
      "#ai ,centrality: 0.0319\n",
      "#iot ,centrality: 0.0254\n",
      "#news ,centrality: 0.0189\n",
      "#startups ,centrality: 0.0152\n",
      "#policy ,centrality: 0.0143\n",
      "#engineering ,centrality: 0.0138\n",
      "#automation ,centrality: 0.0133\n",
      "#futureofwork ,centrality: 0.0123\n",
      "#tree ,centrality: 0.0116\n",
      "#design ,centrality: 0.0109\n",
      "#leadership ,centrality: 0.0109\n",
      "#construction ,centrality: 0.0106\n",
      "#energyefficiency ,centrality: 0.0104\n",
      "#hrtech ,centrality: 0.0104\n",
      "#marketing ,centrality: 0.0104\n",
      "#digital ,centrality: 0.0102\n",
      "#robotics ,centrality: 0.0102\n",
      "#sundaythoughts ,centrality: 0.0097\n",
      "#smartcity ,centrality: 0.0094\n",
      "#bigdata ,centrality: 0.0092\n",
      "#greentech ,centrality: 0.0085\n",
      "#stem ,centrality: 0.0085\n",
      "#economics ,centrality: 0.0085\n",
      "#digitaltransformation ,centrality: 0.0082\n",
      "\n",
      "---Community: 9, Size: 205---\n",
      "#energy ,centrality: 0.0748\n",
      "#greennewdeal ,centrality: 0.0455\n",
      "#renewables ,centrality: 0.0436\n",
      "#solar ,centrality: 0.0411\n",
      "#renewableenergy ,centrality: 0.0409\n",
      "#actonclimate ,centrality: 0.0399\n",
      "#cdnpoli ,centrality: 0.0368\n",
      "#tech ,centrality: 0.0264\n",
      "#greenenergy ,centrality: 0.0191\n",
      "#power ,centrality: 0.0177\n",
      "#ev ,centrality: 0.0174\n",
      "#france ,centrality: 0.0165\n",
      "#wind ,centrality: 0.0165\n",
      "#electricity ,centrality: 0.0150\n",
      "#solarenergy ,centrality: 0.0116\n",
      "#solarpower ,centrality: 0.0116\n",
      "#renewable ,centrality: 0.0114\n",
      "#pv ,centrality: 0.0109\n",
      "#panelsnotpipelines ,centrality: 0.0106\n",
      "#gosolar ,centrality: 0.0090\n",
      "#alternativeenergy ,centrality: 0.0090\n",
      "#sun ,centrality: 0.0087\n",
      "#go100re ,centrality: 0.0085\n",
      "#solarpanels ,centrality: 0.0085\n",
      "#cleantech ,centrality: 0.0070\n",
      "\n",
      "---Community: 12, Size: 173---\n",
      "#auspol ,centrality: 0.0348\n",
      "#co2 ,centrality: 0.0206\n",
      "#biodiversity ,centrality: 0.0165\n",
      "#coal ,centrality: 0.0155\n",
      "#ausvotes2019 ,centrality: 0.0131\n",
      "#fossilfuels ,centrality: 0.0126\n",
      "#stopadani ,centrality: 0.0082\n",
      "#climateelection ,centrality: 0.0068\n",
      "#hambacherforst ,centrality: 0.0063\n",
      "#onpoli ,centrality: 0.0063\n",
      "#endcoal ,centrality: 0.0060\n",
      "#wtf ,centrality: 0.0058\n",
      "#hydro ,centrality: 0.0056\n",
      "#adani ,centrality: 0.0053\n",
      "#polarbears ,centrality: 0.0053\n",
      "#credlin ,centrality: 0.0044\n",
      "#mine ,centrality: 0.0041\n",
      "#ausvotes ,centrality: 0.0039\n",
      "#qldpol ,centrality: 0.0036\n",
      "#climatism ,centrality: 0.0031\n",
      "#antarctica ,centrality: 0.0029\n",
      "#alarmism ,centrality: 0.0027\n",
      "#ausvotes19 ,centrality: 0.0027\n",
      "#extremeweather ,centrality: 0.0027\n",
      "#warringahvotes ,centrality: 0.0027\n",
      "\n",
      "---Community: 15, Size: 179---\n",
      "#recycling ,centrality: 0.0264\n",
      "#recycle ,centrality: 0.0249\n",
      "#globalgoals ,centrality: 0.0242\n",
      "#australia ,centrality: 0.0177\n",
      "#reuse ,centrality: 0.0114\n",
      "#socent ,centrality: 0.0104\n",
      "#teachsdgs ,centrality: 0.0104\n",
      "#2030agenda ,centrality: 0.0092\n",
      "#agenda2030 ,centrality: 0.0082\n",
      "#waste ,centrality: 0.0080\n",
      "#foodwaste ,centrality: 0.0075\n",
      "#sdg13 ,centrality: 0.0065\n",
      "#melbourne ,centrality: 0.0063\n",
      "#followback ,centrality: 0.0060\n",
      "#sydney ,centrality: 0.0060\n",
      "#brisbane ,centrality: 0.0058\n",
      "#uhc ,centrality: 0.0056\n",
      "#followbacks ,centrality: 0.0053\n",
      "#canberra ,centrality: 0.0051\n",
      "#sdg17 ,centrality: 0.0048\n",
      "#upcycle ,centrality: 0.0048\n",
      "#hlpf ,centrality: 0.0041\n",
      "#adelaide ,centrality: 0.0039\n",
      "#climateambition ,centrality: 0.0039\n",
      "#climattitude ,centrality: 0.0039\n",
      "\n"
     ]
    }
   ],
   "source": [
    "year = 2019\n",
    "\n",
    "printCommunitiElemnts(\n",
    "    networks[year].net, communities[year]['communities'],\n",
    "    num_communities=10, metric_name='centrality',\n",
    "    print_metric=True, showTopK=25, community_id=None\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Community: 2, Size: 241---\n",
      "#energy\n",
      "#solar\n",
      "#renewables\n",
      "#climateaction\n",
      "#renewableenergy\n",
      "#cdnpoli\n",
      "#tech\n",
      "#bcpoli\n",
      "#waterislife\n",
      "#actonclimate\n",
      "\n",
      "---Community: 3, Size: 232---\n",
      "#nature\n",
      "#wildlife\n",
      "#animals\n",
      "#oceans\n",
      "#photography\n",
      "#spring\n",
      "#forest\n",
      "#travel\n",
      "#tuesdaythoughts\n",
      "#art\n",
      "\n",
      "---Community: 4, Size: 205---\n",
      "#agriculture\n",
      "#food\n",
      "#organic\n",
      "#biodiversity\n",
      "#foodsecurity\n",
      "#zerohunger\n",
      "#2030agenda\n",
      "#gardening\n",
      "#fossilfuel\n",
      "#nutrition\n",
      "\n",
      "---Community: 5, Size: 258---\n",
      "#globalgoals\n",
      "#africa\n",
      "#csr\n",
      "#youth\n",
      "#sdg7\n",
      "#climatejustice\n",
      "#esg\n",
      "#cop24\n",
      "#sdg5\n",
      "#sdg\n",
      "\n",
      "---Community: 11, Size: 139---\n",
      "#plastic\n",
      "#deforestation\n",
      "#reading\n",
      "#oneplanet\n",
      "#unitednations\n",
      "#extinction\n",
      "#mniwiconi\n",
      "#palmoil\n",
      "#lifeonland\n",
      "#indonesia\n",
      "\n",
      "---Community: 12, Size: 201---\n",
      "#innovation\n",
      "#science\n",
      "#education\n",
      "#technology\n",
      "#ecology\n",
      "#development\n",
      "#economy\n",
      "#poverty\n",
      "#leadership\n",
      "#security\n",
      "\n",
      "---Community: 17, Size: 150---\n",
      "#health\n",
      "#govegan\n",
      "#mondaymotivation\n",
      "#conference\n",
      "#animalcruelty\n",
      "#healthbenefits\n",
      "#healthforall\n",
      "#meatymarch\n",
      "#uhc\n",
      "#meatlessmonday\n",
      "\n"
     ]
    }
   ],
   "source": [
    "year = 2018\n",
    "\n",
    "printCommunitiElemnts(\n",
    "    networks[year].net, communities[year]['communities'],\n",
    "    num_communities=10, metric_name='centrality',\n",
    "    print_metric=False, showTopK=10, community_id=None\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---Community: 0, Size: 147---\n",
      "#globalgoals\n",
      "#education\n",
      "#iwd2017\n",
      "#community\n",
      "#agenda2030\n",
      "#logic\n",
      "#womensday\n",
      "#saveourplanet\n",
      "#equality\n",
      "#beboldforchange\n",
      "#nepal\n",
      "#youngleaders\n",
      "#internationalwomensday\n",
      "#2030agenda\n",
      "#gfi4sd\n",
      "\n",
      "---Community: 3, Size: 215---\n",
      "#energy\n",
      "#cdnpoli\n",
      "#trump\n",
      "#renewables\n",
      "#auspol\n",
      "#solar\n",
      "#renewableenergy\n",
      "#coal\n",
      "#waterislife\n",
      "#actonclimate\n",
      "#nokxl\n",
      "#future\n",
      "#wind\n",
      "#divest\n",
      "#bcpoli\n",
      "\n",
      "---Community: 4, Size: 130---\n",
      "#resist\n",
      "#humanrights\n",
      "#theresistance\n",
      "#maga\n",
      "#women\n",
      "#usa\n",
      "#republicans\n",
      "#progressive\n",
      "#donaldtrump\n",
      "#resistance\n",
      "#koch\n",
      "#keystonexl\n",
      "#grabyourwallet\n",
      "#womensmarch\n",
      "#sdg5\n",
      "\n",
      "---Community: 6, Size: 126---\n",
      "#food\n",
      "#agriculture\n",
      "#green\n",
      "#tree\n",
      "#outdoors\n",
      "#global\n",
      "#agtech\n",
      "#landscape\n",
      "#nutrition\n",
      "#agribusiness\n",
      "#globaldev\n",
      "#ag\n",
      "#countryside\n",
      "#cropland\n",
      "#farmland\n",
      "\n",
      "---Community: 10, Size: 134---\n",
      "#science\n",
      "#epa\n",
      "#gogreen\n",
      "#animals\n",
      "#arctic\n",
      "#earthday\n",
      "#oceans\n",
      "#environmental\n",
      "#uniteblue\n",
      "#sciencemarch\n",
      "#vegan\n",
      "#notmypresident\n",
      "#obama\n",
      "#tntweeters\n",
      "#savetheplanet\n",
      "\n",
      "---Community: 14, Size: 147---\n",
      "#sustainable\n",
      "#water\n",
      "#earth\n",
      "#eco\n",
      "#love\n",
      "#peace\n",
      "#biodegradable\n",
      "#fashion\n",
      "#design\n",
      "#solarenergy\n",
      "#style\n",
      "#spring\n",
      "#handmade\n",
      "#america\n",
      "#organic\n",
      "\n",
      "---Community: 15, Size: 126---\n",
      "#innovation\n",
      "#csr\n",
      "#business\n",
      "#resilience\n",
      "#security\n",
      "#impinv\n",
      "#ethics\n",
      "#esg\n",
      "#corpgov\n",
      "#home\n",
      "#cleantech\n",
      "#mining\n",
      "#ceo\n",
      "#greentech\n",
      "#wastemanagement\n",
      "\n"
     ]
    }
   ],
   "source": [
    "year = 2017\n",
    "\n",
    "printCommunitiElemnts(\n",
    "    networks[year].net, communities[year]['communities'],\n",
    "    num_communities=10, metric_name='centrality',\n",
    "    print_metric=False, showTopK=15, community_id=None\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save map hashtag-community\n",
    "\n",
    "SAVE_SELECTED = True\n",
    "out_path = \"data/communities/hashtags_community_{}.csv\".format(\"full\" if not SAVE_SELECTED else \"selected\")\n",
    "\n",
    "hashtags_map = pd.DataFrame([])\n",
    "\n",
    "for year in communities.keys():\n",
    "    hashtags_ = []\n",
    "    communities_ = []\n",
    "\n",
    "    if SAVE_SELECTED:\n",
    "        selected_comm = list(communities[year]['communities'].keys())\n",
    "        \n",
    "    for h, c in communities[year]['partitions'].items():\n",
    "        if SAVE_SELECTED and (c not in selected_comm):\n",
    "            continue\n",
    "        hashtags_.append(h)\n",
    "        communities_.append(c)\n",
    "        \n",
    "    hashtags_map = hashtags_map.append(pd.DataFrame({\n",
    "        'hashtag':hashtags_,\n",
    "        'community':communities_,\n",
    "        'year':year\n",
    "    }))\n",
    "    \n",
    "hashtags_map.to_csv(out_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}