extract_entities.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dependencies\n",
    "from modules.dataset.entities import Entities\n",
    "from modules.dataset.tweets import Tweets\n",
    "from langdetect import detect\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import json\n",
    "import re\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paths\n",
    "TWEETS_RAW_PATH = 'data/tweets.jsonl'\n",
    "TWEETS_DB_PATH = 'data/db/tweets.json'\n",
    "HASHTAGS_DB_PATH = 'data/db/hashtags.json'\n",
    "WORDS_DB_PATH = 'data/db/words.json'\n",
    "SUBS_HASHTAGS_PATH = 'data/hashtag_subs.json'\n",
    "SUBS_CONTRACT_PATH = 'data/contract_forms.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# States wether to overvrite dataset with a new one\n",
    "OVERWRITE = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set network years\n",
    "YEARS = [2017, 2018, 2019]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset creation\n",
    "\n",
    "Once words and hashtags are extracted, they are stored in Pandas DataFrames with the following attributes:\n",
    "\n",
    "1. <code> tweet_id </code> : Reference to the tweet ID\n",
    "2. <code> entity_index </code> : Position of the word in the sentence (tweet) - useful to delete self loops \n",
    "3. <code> entity_text </code> : Text\n",
    "4. <code> entity_tag </code> : Part Of Speech tag \n",
    "\t\t\t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>836950901495631872</td>\n",
       "      <td>2017-03-01 14:46:59</td>\n",
       "      <td>TT SINGAPORE 22:46\\n1.Hong Kong\\n2.#JointAddre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>836950882528989184</td>\n",
       "      <td>2017-03-01 14:46:54</td>\n",
       "      <td>Letting #snapchat prepare me for the day's uns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>836950869639835649</td>\n",
       "      <td>2017-03-01 14:46:51</td>\n",
       "      <td>\"The bill would require the state to get all o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>836950847380668416</td>\n",
       "      <td>2017-03-01 14:46:46</td>\n",
       "      <td>Style-Lead don't Follow #recycledfashion https...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>836950839101116421</td>\n",
       "      <td>2017-03-01 14:46:44</td>\n",
       "      <td>‘Shell knew’: oil giant's 1991 film warned of ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             tweet_id          tweet_date  \\\n",
       "0  836950901495631872 2017-03-01 14:46:59   \n",
       "1  836950882528989184 2017-03-01 14:46:54   \n",
       "2  836950869639835649 2017-03-01 14:46:51   \n",
       "3  836950847380668416 2017-03-01 14:46:46   \n",
       "4  836950839101116421 2017-03-01 14:46:44   \n",
       "\n",
       "                                          tweet_text  \n",
       "0  TT SINGAPORE 22:46\\n1.Hong Kong\\n2.#JointAddre...  \n",
       "1  Letting #snapchat prepare me for the day's uns...  \n",
       "2  \"The bill would require the state to get all o...  \n",
       "3  Style-Lead don't Follow #recycledfashion https...  \n",
       "4  ‘Shell knew’: oil giant's 1991 film warned of ...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load Tweets dataset\n",
    "\n",
    "# Initialize Tweets class\n",
    "tweets = Tweets()\n",
    "# Case dataset must be completely re-loaded and overwritten\n",
    "if OVERWRITE:\n",
    "     # Load raw dataset\n",
    "    tweets.from_json_list(TWEETS_RAW_PATH)\n",
    "    # Define which tweets are in english\n",
    "    is_english = tweets.df.tweet_text.apply(lambda x: detect(x) == 'en')\n",
    "    # Filter out dataset language\n",
    "    tweets.df = tweets.df[is_english]\n",
    "    # Store dataset to disk\n",
    "    tweets.to_json(TWEETS_DB_PATH)\n",
    "    \n",
    "# Load tweets from disk\n",
    "tweets = Tweets()\n",
    "tweets.from_json(TWEETS_DB_PATH)\n",
    "tweets.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extract entities (i.e. hashtags and words)\n",
    "\n",
    "# Load substitutions dictionary\n",
    "subs = {}\n",
    "# Load hahstag substitutions\n",
    "with open(SUBS_HASHTAGS_PATH, 'r') as file:\n",
    "    subs = {**subs, **json.load(file)}\n",
    "# Load contact forms substitutions\n",
    "with open(SUBS_CONTRACT_PATH, 'r') as file:\n",
    "    subs = {**subs, **json.load(file)}\n",
    "\n",
    "# Case dataset must be overwritten\n",
    "if OVERWRITE:\n",
    "    # Retrieve hastags and words entities tables\n",
    "    hashtags, words = tweets.get_entities(subs=subs)\n",
    "    # Store entities tables to disk\n",
    "    hashtags.to_json(HASHTAGS_DB_PATH)\n",
    "    words.to_json(WORDS_DB_PATH)\n",
    "    \n",
    "# Load entities tables from disk\n",
    "hashtags, words = Entities(), Entities()\n",
    "hashtags.from_json(HASHTAGS_DB_PATH)\n",
    "words.from_json(WORDS_DB_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>entity_index</th>\n",
       "      <th>entity_text</th>\n",
       "      <th>entity_tag</th>\n",
       "      <th>entity_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>0</td>\n",
       "      <td>#Humans</td>\n",
       "      <td>#</td>\n",
       "      <td>0.6243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>7</td>\n",
       "      <td>#climatechange</td>\n",
       "      <td>^</td>\n",
       "      <td>0.4925</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>27</td>\n",
       "      <td>#ClimateChange</td>\n",
       "      <td>#</td>\n",
       "      <td>0.6968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>36</td>\n",
       "      <td>#ActOnClimate</td>\n",
       "      <td>#</td>\n",
       "      <td>0.7929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101574446341607424</td>\n",
       "      <td>37</td>\n",
       "      <td>#climate</td>\n",
       "      <td>#</td>\n",
       "      <td>0.9559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  entity_index     entity_text entity_tag  entity_conf\n",
       "0  1101574442575167489             0         #Humans          #       0.6243\n",
       "1  1101574442575167489             7  #climatechange          ^       0.4925\n",
       "2  1101574446341607424            27  #ClimateChange          #       0.6968\n",
       "3  1101574446341607424            36   #ActOnClimate          #       0.7929\n",
       "4  1101574446341607424            37        #climate          #       0.9559"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show hashtags dataset header\n",
    "hashtags.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>entity_index</th>\n",
       "      <th>entity_text</th>\n",
       "      <th>entity_tag</th>\n",
       "      <th>entity_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>1</td>\n",
       "      <td>frog</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>3</td>\n",
       "      <td>hot</td>\n",
       "      <td>A</td>\n",
       "      <td>0.9848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>4</td>\n",
       "      <td>water</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>6</td>\n",
       "      <td>climate</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>7</td>\n",
       "      <td>change</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  entity_index entity_text entity_tag  entity_conf\n",
       "0  1101574442575167489             1        frog          N       0.9797\n",
       "1  1101574442575167489             3         hot          A       0.9848\n",
       "2  1101574442575167489             4       water          N       0.9996\n",
       "3  1101574442575167489             6     climate          N       0.9742\n",
       "4  1101574442575167489             7      change          N       0.9579"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show words dataset header\n",
    "words.df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Words filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recognize pronouns and clean from symbols\n",
    "words.clean_entities()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13553"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Automatically and manually found pronouns\n",
    "words.df[words.df.entity_tag == 'O'].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13521"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Manually found pronouns\n",
    "words.df[(words.df.entity_conf == 1.0) & (words.df.entity_tag == 'O')].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>entity_index</th>\n",
       "      <th>entity_text</th>\n",
       "      <th>entity_tag</th>\n",
       "      <th>entity_conf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>1</td>\n",
       "      <td>frog</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>3</td>\n",
       "      <td>hot</td>\n",
       "      <td>A</td>\n",
       "      <td>0.9848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>4</td>\n",
       "      <td>water</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>6</td>\n",
       "      <td>climate</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1101574442575167489</td>\n",
       "      <td>7</td>\n",
       "      <td>change</td>\n",
       "      <td>N</td>\n",
       "      <td>0.9579</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id  entity_index entity_text entity_tag  entity_conf\n",
       "0  1101574442575167489             1        frog          N       0.9797\n",
       "1  1101574442575167489             3         hot          A       0.9848\n",
       "2  1101574442575167489             4       water          N       0.9996\n",
       "3  1101574442575167489             6     climate          N       0.9742\n",
       "4  1101574442575167489             7      change          N       0.9579"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter on POS tag\n",
    "words.df = words.df[words.df.entity_tag.isin(['N', 'V', 'R', 'A', 'O', 'S'])]\n",
    "words.df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4QAAAFBCAYAAAAv9GEVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3debhsVX3n//dHcMCBgIKEMOSaiCbor5sQGkGNIhimgIgDoImAjWIE0qEDiTgFAzilRZFfhBYjDWoUECcUEBG52iogKMgY5AZRLiKjiopR0W//sfbRos5wz3DPqbpV79fz7Keq1l577+/ed51z63vW2munqpAkSZIkjZ+HDDoASZIkSdJgmBBKkiRJ0pgyIZQkSZKkMWVCKEmSJEljyoRQkiRJksaUCaEkSZIkjSkTQklahSQvTPKFJD9M8vMk30ryziS/t0jHe0aSbyT5zyTVld2S5B2r2O6pSSrJDosR11JKctjEuXefd+jO7amz3P7xSd6UZNkcjllJDuv5vDzJ2XOJe4Z975PkwCnKV9sxZhnHo5KckeSe7nwnxbQajrGs2/fE8uMkVyTZZ4q6T0lyZpI7u/b+rSTHJHnUFHWfmeTCJHcl+WmSm5KclmTTGWKZczuQpHGz9qADkKRhluR44HDg/wDvAu4DtgT+GngCsPciHPa9wJ3ALsDPu7K9gXsW4Vhrim8A2wP/Mcv6jweOBpYDt8xym+2Bb881sFnaB9gAOK2v/BDgl4t0zKm8GtgT2B+4jdlfz/k4EvgKsC7wcuDMJPdX1WcAkjwHOBe4Cvgb4PvANsDrgN2SPKeqftLVfSbt3/KTwEHAz2g/hy8Ffh9YOU0M82kHkjRWTAglaRpJ9gT+Djioqk7tWfXFJKcAOy/Sof8IOKWqvjhRUFVXLtKx1ghVdR9w6WLsO8k6VfWzqlqU/c+kqq5f4kP+EXBjVX1soTuauG4zVLlx4pom+TywNS0h/UySRwL/Bnwd2LGqJpLiLya5sCs/jvbHGLrtbgBeXFUTPccXAu9OkoWeiySNM4eMStL0/ifwjb5kEICq+lVVnT/xOckGSU7vhuLd3w0F3KZ3m4lhn0n+Z5KVSX7QDd9br1u/QzdMci3aF91Kclrvtn37OyTJrd3wuU8DG/fHmeQhSY5KsqJnuOsBfXWWJzk7yUu7evclOb9/KF6SdZL8c5LvdPv6dpK39tV5RZLruvXfSfIPq7rISR6e5F+6Ibn3JnkX8NC+OpOGjCY5KMn1SX6W5O4kX+yGIC4DrumqXTwxdLFvP7skOSfJT4B/6dY9aMhoz3EO7q7/z5Kcm2STmeLqvabd+9OAFwLP7hlG+ab+ej3b7pjksm4I5R1JTkry6CmOuUOSjyb5SZKbkxyyiut8C6137U96r8kcjznpus1GVf2a1hO4rCt6Ma29vr4nGZyoezXwIeAVXeIIsB5wZ08y2Ft/UlkX8zKmbweP6trcjd3P67eTvCfJun37WL/7Gf1pku8leU33M3xLT531kvxrt/4/k3w3yftme20kadBMCCVpCkkeCjwd+OwsN/kkbYjnkcC+tN+vFyd5Yl+9fYCdgIOB1wB7AG/p1k0MiwQ4vnt/7DTx7QW8B/gM8ALaF99JiSvw/wNvAE4B/gL4BHBqkj366j0NOAw4oott626bieMF+BStp+Y9wO60oXgb9NT5e+Dk7lrs0b0/dqokq8/bgFd05/qXtCGAR8y0QZJnAf8b+CCwG/Dfga8CvwPc3u0H4FDaddy+bxfvB74JPK97P53tacMZ/46WTP2X7vzm4ljgYuDKnlj+daqKSZ5Ca3N305LIo2nDIqe6z/B93TnsTRsS+Z4k284Qx97AecC/98Qx12PO9rpNZRltWCjAs4AfVNWXpqn7SeBRtHYI7WfjOUnemOQPZnm8mdrBI2l/eHk9rf28EdgR+GjfPk4D/hz4W9rPxc60n+9e7wSeSfsD0i60Ia9TJqmSNJSqysXFxcWlbwF+l/al7lWzqLtrV/fZPWWPAu4C3ttTdgvtnq21e8pOAL7ft78CDusruwV4R8/nrwHn99V5X7ftDt3nJwK/Bg7oq/cB4PKez8uBHwHr95Qd3u1rne7zLt3n501zDdYFfgIc3Vd+DC0JWGua7R5Hux/sNT1lD6ElLdVTtkN3/Kd2n48Evj7Dv8lTe6/FFPt51xTbPOi6d9fll8DmPWXP6OrtOlVcfdue3fP5bGD5FMfsr3cGcFPv9aL9EaGA7fuOeUxPnYd27e1tq2irpwFX9JXN5ZiTrtsUx1g20VZot6Y8FviH3utLS0CvnGEfW3X19+1pX1/oygr4Hu0PAk9aRSxTtoMp6q3d82+7ed+2L+6ptw4tcb6lp+xa4G9WdV1cXFxchnWxh1CSZjabv/RvSxvO1nvP309pvXfP7Kt7cVU90PP5euDxXY/krCRZm9Zz8qm+VR/v+7wTLSH8RJK1JxbgImCrJGv11L28qn7QFxfAxPDIHYF7q+qcacLanpYEf7TvWF8ANgKmmwny/wMe0Xsu1YYX9p9bv6toQx/fleRZSR62ivr9zp1lvW9U1Xd7YvsKbcKfmXriFmJb4BNV9aueso8BDzC5LX2uJ65f0pK6aWfcXE3HnO11g/Zv+EvaZEjH0XrSTp5HfFS7h3QnWq/9W2h/WHkF8I0kW8+07XSSvCzJld3w118CX+5WPal7nRjy/emeOH4GfL5vV1cBf582hPtJSNIaxoRQkqZ2D22Gz81nUXdjWpLQ7w5a70ivH/Z9/gUQ4OFziG0D2nC3/mP2f56o9yPaF96J5TRaj0jvPYdTxQUtWYPWk3f7KmICuK7vWBd35ZtNs93vThP7VNfzN6rq87SZK59F62W7u7sHbNLjCqZxxyzrTRXHnUxxv+ZqsjF9sXWJ2j3Mri09grmbyzFne92gDaH8b7SJbB5dVUf0JJ230YYGT+f3e+pNxFRVdUlVvb6q/oyWsP2aNtxzTpLsTespv4R2P+N2/HbG4Ilr+LvAj6vqP/s2v6vv82G0Ia7/CNyY9jiM/eYakyQNirOMStIUquqXSb5CGyr5hlVUv502vX2/jYB7V3dstCFrv5rimP2f76X18jyD9sW534xJV597mDkJmjjPPZg6abhxmu0m7il7PA++VlNdzwepqtOB05NsSLuP8l3Aj4GjVrUts7/Ha6o4Hs9vk+OJZKG/h3J92r/TXE1qS11P7uNYnLY012PO5d64FVV1xTTrvgT89yTPrKovT7H+ecBPabONTqmqrkqbkXTLOcQ04cXAZVX1m4l4kjy7r873gcckeURfUrhhXxw/BP4H8D+S/Bfa8Nh/S3J1Lf0sspI0Z/YQStL0TgC2Sd+snPCb2Tt37T5eRhv2+aye9Y+kTeIy1ZfdBemGnF4J7NW36gV9n79A6yH8naq6YorlF8zeRcBjp5iMZsIltHsBf2+aY/14mu2uoSVVvzmXJA+Z4tymVVV3VdV7gf/Lb5OD/h7O+do6yW96iZM8g5Y8fa0rmnj+3R/31NmM1ivWa7a9d5cBe/cN530B7Q+4q70tDfCYH6Ulom/uhhb/Rjdj68uA93VDNEkyKTHvJjr6Q2butZyuHazDb5/xOeEv+z5PJLPP6znmOrRJZqZUbYbUv6d9v+pvA5I0lOwhlKRpVNWnk7wTeH+XCHyKNnHKH9EeTH8L8NmquiDJV2kP3j6K1pt2JO1L5/9apPDeAnw8ycm0mUOfTZvcpjf+G5P8b+CMJP9M+4L7COAptMk4XjGH410IXAB8OMkxtFkfNwaeVVWvqqofpj1K4d1Jfp/WA/QQ2v1Yz6mqvafaaVXdk/ZMx39K8gBtyOkrgUdPVX9Ckn+iDWdcTuuJ+5PuGkz0Dn6XlqAekORHwC9n6K2ayV3AuUmOpl27t9PuK/xsF//KJFfQZlO9vzvn1zG5Z+3fgb2SPJ+WRH6vqr43xfGOoyX7n+z+bTftjnlBVV0yj/hnY8mPWVX3J/lL2j2Jy5OcSEvs/pR2/b7Jg4eC/mv3h4KP0e4fXJ82ZPi/0nr7pjNdO7iQNivr62kJ8e60exR7Y7w27XEuJyd5DK3H8O+A++npcU/yZdrP4LW0HtRX0no3v4YkrQFMCCVpBlV1RJfsHQZ8mJbk3QKcA/Q+F/D5tEdFnEBLHL5Ge+D2ikWK6xNJ/oaWAB1AS4wOoiVtvQ4FvkX7knoMcB9twpg5PTKgqqq77+pY2gykG9JmevxwT51/TvI92r1jR9B6/r4FnLmK3f8DbZbMf6R90f4QbQKS42fY5vLuOPsBjwG+A7wJeHcXy38meSXtEQpf7PY/nweYf5U2icgJtHNeTnv8QK+X0B4j8SFasvcPXWy9TqIlrafSkpl/6uJ9kKq6LsludAk/7d/rI90+F8Ugjtkd9+LuMRn/SHuUybq0f8eTgLd3EzNNOAk4sKu7Me3+yeuAXarqc0xjhnbwXuAPaI+TeAQtQXwpcGnfLg6kTYRzIu2PQe8BbqbdGznhkq7eMtpQ7iuB3apqJZK0BkiVj8qRJElalW5467W0+w8nDSWXpDWRPYSSJElTSPJi4Pdo97quS+tp3wLYf5BxSdLqZEIoSZI0tZ/S7lV8Im2CpmuAPavK+wMljQyHjEqSJEnSmPKxE5IkSZI0pkwIJUmSJGlMrbH3EG6wwQa1bNmyQYehYXfjje31yU8ebBySJEnSavb1r3/97qracCH7WGMTwmXLlnHFFfN5xrDGymtf217f+tbBxiFJkiStZkm+s9B9rLEJoTQrJoKSJEnStLyHUJIkSZLGlAmhRtsLX9gWSZIkSZM4ZFSj7Z57Bh2BJEmSNLTsIZQkSZKkMWVCKEmSJEljyoRQkiRJksaU9xBqtO2006AjkCRJkoaWCaFG2xvfOOgIJEmSpKHlkFFJkiRJGlMmhBptu+3WFkmSJEmTOGRUo+1nPxt0BJIkSdK8LTvq3EXdvz2EkiRJkjSmTAglSZIkaUyZEEqSJEnSmPIeQo22PfYYdASSJEnS0DIh1Gg78shBRyBJkiQNLYeMSpIkSdKYMiHUaNthh7ZIkiRJmsSEUJIkSZLGlAmhJEmSJI0pE0JJkiRJGlMmhJIkSZI0pnzshEbbPvsMOgJJkiRpaJkQarQdcsigI5AkSZKGlkNGNdruv78tkiRJkiaxh1Cjbffd2+vy5QMNQ5IkSRpG9hBKkiRJ0pgyIZQkSZKkMbXKhDDJZkkuTnJ9kuuS/G1X/qYktyW5qlt279nmtUlWJLkxyS495bt2ZSuSHNVT/oQkl3XlZyZ52Oo+UUmSJEnSg82mh/AB4Iiq2hLYDjg0yZbdundV1Vbdch5At24/4CnArsBJSdZKshbwHmA3YEvgJT37eXu3rycCPwAOWk3nJ0mSJEmaxionlamq24Hbu/c/TnIDsMkMm+wFnFFVPwe+nWQFsG23bkVV3QyQ5Axgr25/OwIv7eqcDrwJOHnupyP1OfDAQUcgSZIkDa053UOYZBnwJ8BlXdFhSa5OcmqS9buyTYBbezZb2ZVNV/444IdV9UBfubRwBx5oUihJkiRNY9YJYZJHAx8DDq+q+2g9eH8IbEXrQTx+USJ8cAwHJ7kiyRV33XXXYh9Oo+Duu9siSZIkaZJZJYRJHkpLBv+tqj4OUFV3VNWvqurXwPv47bDQ24DNejbftCubrvweYL0ka/eVT1JVp1TVNlW1zYYbbjib0DXuXvSitkiSJEmaZDazjAZ4P3BDVb2zp3zjnmp7A9d2788B9kvy8CRPALYAvgZcDmzRzSj6MNrEM+dUVQEXAxPf2g8APrWw05IkSZIkrcoqJ5UBngG8DLgmyVVd2etos4RuBRRwC/AqgKq6LslZwPW0GUoPrapfASQ5DLgAWAs4taqu6/b3GuCMJMcBV9ISUEmSJEnSIprNLKNfBjLFqvNm2ObNwJunKD9vqu26mUe37S+XJEmSJC2eOc0yKkmSJEkaHbMZMiqtuV796kFHIEmSJA0tE0KNtn33HXQEkiRJ0tByyKhG2623tkWSJEnSJPYQarS97GXtdfnygYYhSZIkDSN7CCVJkiRpTJkQSpIkSdKYMiGUJEmSpDFlQihJkiRJY8pJZTTajjhi0BFIkiRJQ8uEUKNtzz0HHYEkSZI0tBwyqtF2441tkSRJkjSJPYQaba96VXv1OYSSJEnSJPYQSpIkSdKYMiGUJEmSpDFlQihJkiRJY8qEUJIkSZLGlJPKaLS94Q2DjkCSJEkaWiaEGm3Pfe6gI5AkSZKGlkNGNdquuqotkiRJkiaxh1Cj7fDD26vPIZQkSZImsYdQkiRJksaUCaEkSZIkjSkTQkmSJEkaUyaEkiRJkjSmnFRGo+0tbxl0BJIkSdLQMiHUaHv60wcdgSRJkjS0HDKq0fbVr7ZFkiRJ0iT2EGq0ve517dXnEEqSJEmT2EMoSZIkSWPKhFCSJEmSxpQJoSRJkiSNKRNCSZIkSRpTTiqj0XbCCYOOQJIkSRpaJoQabVttNegIJEmSpKHlkFGNts9/vi2SJEmSJrGHUKPtuOPa63OfO9g4JEmSpCFkD6EkSZIkjSkTQkmSJEkaU6tMCJNsluTiJNcnuS7J33blj01yYZKbutf1u/IkOTHJiiRXJ9m6Z18HdPVvSnJAT/mfJrmm2+bEJFmMk5UkSZIk/dZseggfAI6oqi2B7YBDk2wJHAVcVFVbABd1nwF2A7boloOBk6ElkMDRwNOAbYGjJ5LIrs4re7bbdeGnJkmSJEmaySonlamq24Hbu/c/TnIDsAmwF7BDV+10YDnwmq78A1VVwKVJ1kuycVf3wqq6FyDJhcCuSZYD61bVpV35B4DnA+evnlPUWHvvewcdgSRJkjS05jTLaJJlwJ8AlwEbdckiwPeBjbr3mwC39my2siubqXzlFOVTHf9gWq8jm2+++VxC17h68pMHHYEkSZI0tGY9qUySRwMfAw6vqvt613W9gbWaY5ukqk6pqm2qapsNN9xwsQ+nUfDpT7dFkiRJ0iSzSgiTPJSWDP5bVX28K76jGwpK93pnV34bsFnP5pt2ZTOVbzpFubRwxx/fFkmSJEmTzGaW0QDvB26oqnf2rDoHmJgp9ADgUz3l+3ezjW4H/KgbWnoBsHOS9bvJZHYGLujW3Zdku+5Y+/fsS5IkSZK0SGZzD+EzgJcB1yS5qit7HfA24KwkBwHfAfbp1p0H7A6sAO4HXg5QVfcmORa4vKt3zMQEM8AhwGnAOrTJZJxQRpIkSZIW2WxmGf0yMN1zAXeaon4Bh06zr1OBU6covwJ46qpikSRJkiStPrOeVEaSJEmSNFrm9NgJaY3zwQ8OOgJJkiRpaJkQarRtttmq60iSJEljyiGjGm1nntkWSZIkSZPYQ6jRdvLJ7XXffQcbhyRJkjSE7CGUJEmSpDFlQihJkiRJY8qEUJIkSZLGlAmhJEmSJI0pJ5XRaDv77EFHIEmSJA0tE0KNtg02GHQEkiRJ0tByyKhG22mntUWSJEnSJCaEGm0mhJIkSdK0TAglSZIkaUyZEEqSJEnSmDIhlCRJkqQxZUIoSZIkSWPKx05otJ133qAjkCRJkoaWCaFG2yMfOegIJEmSpKHlkFGNtpNOaoskSZKkSUwINdrOOqstkiRJkiYxIZQkSZKkMWVCKEmSJEljyoRQkiRJksaUCaEkSZIkjSkfO6HRtnz5oCOQJEmShpY9hJIkSZI0pkwINdre8Y62SJIkSZrEhFCj7TOfaYskSZKkSUwIJUmSJGlMmRBKkiRJ0pgyIZQkSZKkMeVjJzTa1lln0BFIkiRJQ8uEUKPt/PMHHYEkSZI0tBwyKkmSJEljyoRQo+3YY9siSZIkaRITQo22iy5qiyRJkqRJTAglSZIkaUytMiFMcmqSO5Nc21P2piS3JbmqW3bvWffaJCuS3Jhkl57yXbuyFUmO6il/QpLLuvIzkzxsdZ6gJEmSJGlqs+khPA3YdYryd1XVVt1yHkCSLYH9gKd025yUZK0kawHvAXYDtgRe0tUFeHu3rycCPwAOWsgJSZIkSZJmZ5UJYVV9Cbh3lvvbCzijqn5eVd8GVgDbdsuKqrq5qn4BnAHslSTAjsDZ3fanA8+f4zlI03vc49oiSZIkaZKFPIfwsCT7A1cAR1TVD4BNgEt76qzsygBu7St/GvA44IdV9cAU9aWF+9jHBh2BJEmSNLTmO6nMycAfAlsBtwPHr7aIZpDk4CRXJLnirrvuWopDSpIkSdLImldCWFV3VNWvqurXwPtoQ0IBbgM266m6aVc2Xfk9wHpJ1u4rn+64p1TVNlW1zYYbbjif0DVuXvvatkiSJEmaZF4JYZKNez7uDUzMQHoOsF+Shyd5ArAF8DXgcmCLbkbRh9Emnjmnqgq4GHhRt/0BwKfmE5M0pUsuaYskSZKkSVZ5D2GSjwA7ABskWQkcDeyQZCuggFuAVwFU1XVJzgKuBx4ADq2qX3X7OQy4AFgLOLWqrusO8RrgjCTHAVcC719tZydJkiRJmtYqE8KqeskUxdMmbVX1ZuDNU5SfB5w3RfnN/HbIqSRJkiRpicx3UhlJkiRJ0hpuIY+dkIbfppsOOgJJkiRpaJkQarR96EODjkCSJEkaWg4ZlSRJkqQxZUKo0Xb44W2RJEmSNIlDRjXarrpq0BFIkiRJQ8seQkmSJEkaUyaEkiRJkjSmTAglSZIkaUx5D6FG25OeNOgIJEmSpKFlQqjRdsopg45AkiRJGloOGZUkSZKkMWVCqNF28MFtkSRJkjSJQ0Y12r71rUFHIEmSJA0tewglSZIkaUyZEEqSJEnSmDIhlCRJkqQx5T2EGm1bbTXoCCRJkqShZUKo0XbCCYOOQJIkSRpaDhmVJEmSpDFlQqjR9ld/1RZJkiRJkzhkVKNt5cpBRyBJkiQNLXsIJUmSJGlMmRBKkiRJ0pgyIZQkSZKkMeU9hBpt228/6AgkSZKkoWVCqNH21rcOOgJJkiRpaDlkVJIkSZLGlAmhRtsLX9gWSZIkSZM4ZFSj7Z57Bh2BJEmSNLTsIZQkSZKkMWUPoSRJkiQN0LKjzh3Yse0hlCRJkqQxZQ+hRttOOw06AkmSJGlomRBqtL3xjYOOQJIkSRpaDhmVJEmSpDFlQqjRtttubZEkSZI0iUNGNdp+9rNBRyBJkiQNrVX2ECY5NcmdSa7tKXtskguT3NS9rt+VJ8mJSVYkuTrJ1j3bHNDVvynJAT3lf5rkmm6bE5NkdZ+kJEmSJGmy2QwZPQ3Yta/sKOCiqtoCuKj7DLAbsEW3HAycDC2BBI4GngZsCxw9kUR2dV7Zs13/sSRJkiRJi2CVCWFVfQm4t694L+D07v3pwPN7yj9QzaXAekk2BnYBLqyqe6vqB8CFwK7dunWr6tKqKuADPfuSJEmSJC2i+d5DuFFV3d69/z6wUfd+E+DWnnoru7KZyldOUS6tHnvsMegIJEmSpKG14EllqqqS1OoIZlWSHEwbisrmm2++FIfUmu7IIwcdgSRJkjS05vvYiTu64Z50r3d25bcBm/XU27Qrm6l80ynKp1RVp1TVNlW1zYYbbjjP0CVJkiRJMP+E8BxgYqbQA4BP9ZTv3802uh3wo25o6QXAzknW7yaT2Rm4oFt3X5LtutlF9+/Zl7RwO+zQFkmSJEmTrHLIaJKPADsAGyRZSZst9G3AWUkOAr4D7NNVPw/YHVgB3A+8HKCq7k1yLHB5V++YqpqYqOYQ2kym6wDnd4skSZIkaZGtMiGsqpdMs2qnKeoWcOg0+zkVOHWK8iuAp64qDkmSJEnS6jXfIaOSJEmSpDWcCaEkSZIkjakFP3ZCGmr77LPqOpIkSdKYMiHUaDvkkEFHIEmSJA0th4xqtN1/f1skSZIkTWIPoUbb7ru31+XLBxqGJEmSNIzsIZQkSZKkMWVCKEmSJEljyoRQkiRJksaUCaEkSZIkjSknldFoO/DAQUcgSZIkDS0TQo02E0JJkiRpWg4Z1Wi7++62SJIkSZrEHkKNthe9qL36HEJJkiRpEnsIJUmSJGlMmRBKkiRJ0phyyKgkSZIkLbJlR5076BCmZA+hJEmSJI0pewg12l796kFHIEmSJA0tE0KNtn33HXQEkiRJ0tByyKhG2623tkWSJEnSJPYQarS97GXt1ecQSpIkSZPYQyhJkiRJY8qEUJIkSZLGlAmhJEmSJI0pE0JJkiRJGlNOKqPRdsQRg45AkiRJGlomhBpte+456AgkSZKkoeWQUY22G29siyRJkqRJ7CHUaHvVq9qrzyGUJEmSJrGHUJIkSZLGlAmhJEmSJI0pE0JJkiRJGlPeQyhJkiRJq8Gyo84ddAhzZkKo0faGNww6AkmSJGlomRBqtD33uYOOQJIkSRpa3kOo0XbVVW2RJEmSNIk9hBpthx/eXn0OoSRJkjSJPYSSJEmSNKYWlBAmuSXJNUmuSnJFV/bYJBcmual7Xb8rT5ITk6xIcnWSrXv2c0BX/6YkByzslCRJkiRJs7E6egifU1VbVdU23eejgIuqagvgou4zwG7AFt1yMHAytAQSOBp4GrAtcPREEilJkiRJWjyLMWR0L+D07v3pwPN7yj9QzaXAekk2BnYBLqyqe6vqB8CFwK6LEJckSZIkqcdCJ5Up4HNJCnhvVZ0CbFRVt3frvw9s1L3fBLi1Z9uVXdl05ZMkOZjWu8jmm2++wNA1Ft7ylkFHIEmSJA2thSaEz6yq25I8Hrgwyb/3rqyq6pLF1aJLOE8B2GabbVbbfjXCnv70QUcgSZIkDa0FJYRVdVv3emeST9DuAbwjycZVdXs3JPTOrvptwGY9m2/ald0G7NBXvnwhcUm/8dWvtlcTQ0mSJK0Gy446d9AhrFbzvocwyaOSPGbiPbAzcC1wDjAxU+gBwKe69+cA+3ezjW4H/KgbWnoBsHOS9bvJZHbuyqSFe93r2iJJkiRpkoX0EG4EfCLJxH4+XFWfTXI5cFaSg4DvAPt09c8DdgdWAPcDLweoqnuTHAtc3tU7pqruXUBckiRJkqRZmHdCWFU3A/91ivJ7gJ2mKC/g0Gn2dSpw6nxjkSRJkiTN3WI8dkKSJEmStAZY6CyjkiRJkjRSRm3imJmYEGq0nXDCoCOQJEmShpYJoUbbVlsNOgJJkiRpaHkPoUbb5z/fFkmSJEmT2IlqA+cAAAjESURBVEOo0Xbcce31uc8dbBySJEnSELKHUJIkSZLGlD2EkiRJksbOOM0kOhN7CCVJkiRpTJkQSpIkSdKYcsioRtt73zvoCCRJkqShZUKo0fbkJw86AkmSJA2I9wmumkNGNdo+/em2SJIkSZrEHkKNtuOPb6977jnYOCRJkrQo7AVcGHsIJUmSJGlMmRBKkiRJ0pgyIZQkSZKkMWVCKEmSJEljykllNNo++MFBRyBJkqQFcuKYxWNCqNG22WaDjkCSJEkaWiaEGm1nntle9913sHFIkiRpRvYCDoYJoUbbySe3VxNCSZKkgTPpGz5OKiNJkiRJY8oeQkmSJElzMlNP3y1v+4sljEQLZUIoSZIkabVxWOiaxSGjkiRJkjSm7CHUaDv77EFHIEmStFo4TFOLwYRQo22DDQYdgSRJ0hrJoZ/jwYRQo+2009rrgQcOMgpJkjRgw9S7thixmLxpvryHUKPttNN+mxRKkiRJehB7CCVJkrTkhqnHbpjY06elZkIoSZIkrWYmdlpTmBBKkiRpJCxGEmZip1FnQihJkqShstTDSU36NM5MCDXazjtv0BFI0lgZhfvCRuEcFsOwXBeTN2n1MiHUaHvkIwcdgSRJM7I3TNIgmRBqtJ10Uns95JDBxiFJGgnzTd7mm4StKcnbmhKnpMlMCDXazjqrvZoQSiNvWIazDZtRuC5LnYTNl0mRpDXR0CSESXYF3g2sBfxrVb1twCFJkjStxUhSRj25WVP2KUnjZCgSwiRrAe8B/hxYCVye5Jyqun6wkUmai6X+gjxM5vuldCnvD7rlbX+xxn95nu/1WlOSqTUlTknS6BiKhBDYFlhRVTcDJDkD2AswIdSCXXrzPeznl56BW1O+IC+1UT+/1c3rJUnS6vWQQQfQ2QS4tefzyq5MkiRJkrRIhqWHcFaSHAwc3H38eZJrBxmP1hgb8PY97h50EFpjbACMZHvJ2wcdwcgZ2baiRWF70WzZVjQXT17oDoYlIbwN2Kzn86Zd2YNU1SnAKQBJrqiqbZYmPK3JbCuaC9uLZsu2ormwvWi2bCuaiyRXLHQfwzJk9HJgiyRPSPIwYD/gnAHHJEmSJEkjbSh6CKvqgSSHARfQHjtxalVdN+CwJEmSJGmkDUVCCFBV5wHnzWGTUxYrFo0c24rmwvai2bKtaC5sL5ot24rmYsHtJVW1OgKRJEmSJK1hhuUeQkmSJEnSEhu6hDDJrkluTLIiyVFTrN88ycVJrkxydZLdu/KHJjk9yTVJbkjy2qWPXkttFu3l95Nc1LWV5Uk27Vl3QJKbuuWApY1cS22+bSXJVkkuSXJdt27fpY9eS20hv1u69esmWZnkX5Yuag3CAv8f2jzJ57rvLdcnWbaUsWvpLbC9/HP3f9ENSU5MkqWNXkspyalJ7pzuMXtpTuza0tVJtu5ZN7fvuFU1NAttQpn/AP4AeBjwTWDLvjqnAK/u3m8J3NK9fylwRvf+kcAtwLJBn5PLwNvLR4EDuvc7Ah/s3j8WuLl7Xb97v/6gz8llKNvKk4Atuve/B9wOrDfoc3IZzvbSs/7dwIeBfxn0+bgMb1sBlgN/3r1/NPDIQZ+Ty3C2F+DpwFe6fawFXALsMOhzclnU9vIsYGvg2mnW7w6cDwTYDrisK5/zd9xh6yHcFlhRVTdX1S+AM4C9+uoUsG73/neA7/WUPyrJ2sA6wC+A+xY/ZA3QbNrLlsAXuvcX96zfBbiwqu6tqh8AFwK7LkHMGox5t5Wq+lZV3dS9/x5wJ7DhkkStQVnI7xaS/CmwEfC5JYhVgzXvtpJkS2DtqroQoKp+UlX3L03YGpCF/G4p4BG0RPLhwEOBOxY9Yg1MVX0JuHeGKnsBH6jmUmC9JBszj++4w5YQbgLc2vN5ZVfW603AXyVZSZuV9G+68rOBn9L+ev9d4B1VNdNF1JpvNu3lm8ALuvd7A49J8rhZbqvRsZC28htJtqX9Z/wfixSnhsO820uShwDHA0cuepQaBgv53fIk4IdJPt7dBvO/kqy16BFrkObdXqrqElqCeHu3XFBVNyxyvBpu07WnOX/HHbaEcDZeApxWVZvSuko/2P0HvC3wK9qQricARyT5g8GFqSFxJPDsJFcCzwZuo7UTqd+MbaX7q9sHgZdX1a8HE6KGyHTt5RDgvKpaOcjgNFSmaytrA3/Wrf9vtGGEBw4oRg2PKdtLkicCfwxsSvtyv2OSPxtcmBolQ/Mcws5twGY9nzftynodRNftWVWXJHkEsAHtHsLPVtUvgTuTfAXYhjZuVqNple2lG+L3AoAkjwZeWFU/THIbsEPftssXM1gN1LzbSvd5XeBc4PXdsAyNtoX8btke+LMkh9DuCXtYkp9U1aTJIzQSFtJWVgJXVdXN3bpP0u4Dev9SBK6BWEh7eSVwaVX9pFt3PrA98H+XInANpena05y/4w5bD+HlwBZJnpDkYcB+wDl9db4L7ASQ5I9p46nv6sp37MofRful+u9LFLcGY5XtJckGXQ8ywGuBU7v3FwA7J1k/yfrAzl2ZRtO820pX/xO0cfpnL2HMGpx5t5eq+suq2ryqltH+0v8Bk8GRtpD/hy6n3fMzcU/yjsD1SxCzBmch7eW7tJ7DtZM8lNZ76JDR8XYOsH832+h2wI+q6nbm8R13qBLCqnoAOIwW9A3AWVV1XZJjkjyvq3YE8Mok3wQ+AhxYbUqd9wCPTnId7Qfu/1TV1Ut/Floqs2wvOwA3JvkWbZKHN3fb3gscS2srlwPHeM/p6FpIWwH2oc30dWCSq7plq6U9Ay2lBbYXjZEF/j/0K9ofDS5Kcg1tpsD3LfEpaAkt8HfL2bT716+h3Wf4zar69FLGr6WV5CO02WSfnPYYo4OS/HWSv+6qnEcbCbmC9rvjEJjfd9x005NKkiRJksbMUPUQSpIkSZKWjgmhJEmSJI0pE0JJkiRJGlMmhJIkSZI0pkwIJUmSJGlMmRBKkiRJ0pgyIZQkSZKkMWVCKEmSJElj6v8BVZ6GL9BUyHkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1080x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 0 (0.00%) words under 0.90 confidence interval\n",
      "There will be 127725 (100.00%) words remaining inside 0.90 confidence interval\n"
     ]
    }
   ],
   "source": [
    "# Set threshold on confidence\n",
    "conf = 0.9\n",
    "\n",
    "# Show confidence distribution\n",
    "fig, ax = plt.subplots(figsize=(15, 5))\n",
    "_ = ax.set_title('Confidence distribution for POS tags',fontsize=15)\n",
    "_ = ax.hist(words.df.entity_conf, bins=100)\n",
    "_ = ax.set_xlim(left=0.88, right=1.0)\n",
    "_ = ax.axvline(x=conf, c='r', ls='--')\n",
    "# _ = plt.savefig('images/preprocessing/tag_conf.png')\n",
    "_ = plt.show()\n",
    "\n",
    "# info\n",
    "print('There are {:d} ({:.02f}%) words under {:.2f} confidence interval'.format(\n",
    "    sum(words.df.entity_conf < conf), sum(words.df.entity_conf < conf) / words.df.shape[0] * 100, conf\n",
    "))\n",
    "# info\n",
    "print('There will be {:d} ({:.02f}%) words remaining inside {:.2f} confidence interval'.format(\n",
    "    sum(words.df.entity_conf >= conf), sum(words.df.entity_conf >= conf) / words.df.shape[0] * 100, conf\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Filter on confidence\n",
    "words.df = words.df[words.df.entity_conf >= conf]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13553"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Number of pronouns after cleaning\n",
    "words.df[words.df.entity_tag == 'O'].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127725"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Total number of words extracted \n",
    "words.df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save words data\n",
    "if OVERWRITE:\n",
    "    words.to_json(WORDS_DB_PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tweets filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make a copy of tweets dataset\n",
    "filtered = Tweets()\n",
    "filtered.df = tweets.df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>836950901495631872</td>\n",
       "      <td>2017-03-01 14:46:59</td>\n",
       "      <td>TT SINGAPORE 22:46 1.Hong Kong 2.#JointAddress...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>836950882528989184</td>\n",
       "      <td>2017-03-01 14:46:54</td>\n",
       "      <td>Letting #snapchat prepare me for the day's uns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>836950869639835649</td>\n",
       "      <td>2017-03-01 14:46:51</td>\n",
       "      <td>\"The bill would require the state to get all o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>836950847380668416</td>\n",
       "      <td>2017-03-01 14:46:46</td>\n",
       "      <td>Style-Lead don't Follow #recycledfashion https...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>836950839101116421</td>\n",
       "      <td>2017-03-01 14:46:44</td>\n",
       "      <td>‘Shell knew’: oil giant's 1991 film warned of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>836950831958224896</td>\n",
       "      <td>2017-03-01 14:46:42</td>\n",
       "      <td>Mythbusting #climate fact: Current Arctic sea ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>836950753520427008</td>\n",
       "      <td>2017-03-01 14:46:24</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>836950737502494720</td>\n",
       "      <td>2017-03-01 14:46:20</td>\n",
       "      <td>https://t.co/7sDZTyB9hD States need environmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>836950728967081984</td>\n",
       "      <td>2017-03-01 14:46:18</td>\n",
       "      <td>With environmental &amp;amp; social impact less sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>836950727599788035</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>so what does solar radiation management look l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>836950724122710016</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>17 Goals - 17 #SDGAdvocates highlighting the #...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>836950706489798656</td>\n",
       "      <td>2017-03-01 14:46:12</td>\n",
       "      <td>I explain how @TuckerCarlson induced cognitive...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>836950679159721984</td>\n",
       "      <td>2017-03-01 14:46:06</td>\n",
       "      <td>We can't just focus on the metric of economic ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>836950658703974400</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>#Throwback #environment #sketch. Back when we ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>836950658079154176</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>It's time to get back to the game and invest i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>836950652186087424</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>Don't be exciting about what he said last nigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>836950650051313665</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>836950648197382144</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>#ClimateChange Still Kills More People Worldwi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>836950645051703296</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>#Shell issued a stark warning of the catastrop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>836950634582654976</td>\n",
       "      <td>2017-03-01 14:45:55</td>\n",
       "      <td>Making some noise from #대한민국 for #ZeroDiscrimi...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id          tweet_date  \\\n",
       "0   836950901495631872 2017-03-01 14:46:59   \n",
       "1   836950882528989184 2017-03-01 14:46:54   \n",
       "2   836950869639835649 2017-03-01 14:46:51   \n",
       "3   836950847380668416 2017-03-01 14:46:46   \n",
       "4   836950839101116421 2017-03-01 14:46:44   \n",
       "5   836950831958224896 2017-03-01 14:46:42   \n",
       "6   836950753520427008 2017-03-01 14:46:24   \n",
       "7   836950737502494720 2017-03-01 14:46:20   \n",
       "8   836950728967081984 2017-03-01 14:46:18   \n",
       "9   836950727599788035 2017-03-01 14:46:17   \n",
       "10  836950724122710016 2017-03-01 14:46:17   \n",
       "11  836950706489798656 2017-03-01 14:46:12   \n",
       "12  836950679159721984 2017-03-01 14:46:06   \n",
       "13  836950658703974400 2017-03-01 14:46:01   \n",
       "14  836950658079154176 2017-03-01 14:46:01   \n",
       "15  836950652186087424 2017-03-01 14:45:59   \n",
       "16  836950650051313665 2017-03-01 14:45:59   \n",
       "17  836950648197382144 2017-03-01 14:45:58   \n",
       "18  836950645051703296 2017-03-01 14:45:58   \n",
       "19  836950634582654976 2017-03-01 14:45:55   \n",
       "\n",
       "                                           tweet_text  \n",
       "0   TT SINGAPORE 22:46 1.Hong Kong 2.#JointAddress...  \n",
       "1   Letting #snapchat prepare me for the day's uns...  \n",
       "2   \"The bill would require the state to get all o...  \n",
       "3   Style-Lead don't Follow #recycledfashion https...  \n",
       "4   ‘Shell knew’: oil giant's 1991 film warned of ...  \n",
       "5   Mythbusting #climate fact: Current Arctic sea ...  \n",
       "6   A revolution is required in the way we think a...  \n",
       "7   https://t.co/7sDZTyB9hD States need environmen...  \n",
       "8   With environmental &amp; social impact less sc...  \n",
       "9   so what does solar radiation management look l...  \n",
       "10  17 Goals - 17 #SDGAdvocates highlighting the #...  \n",
       "11  I explain how @TuckerCarlson induced cognitive...  \n",
       "12  We can't just focus on the metric of economic ...  \n",
       "13  #Throwback #environment #sketch. Back when we ...  \n",
       "14  It's time to get back to the game and invest i...  \n",
       "15  Don't be exciting about what he said last nigh...  \n",
       "16  A revolution is required in the way we think a...  \n",
       "17  #ClimateChange Still Kills More People Worldwi...  \n",
       "18  #Shell issued a stark warning of the catastrop...  \n",
       "19  Making some noise from #대한민국 for #ZeroDiscrimi...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter out \\r and \\n characters from tweets\n",
    "\n",
    "# Get tweet text\n",
    "tweet_text = filtered.df.tweet_text\n",
    "# Filter out \\t, \\r and \\n characters from tweets\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[\\t\\n\\r]+', ' ', txt))\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[ ]+', ' ', txt))\n",
    "# Save dataset to disk\n",
    "filtered.df.tweet_text = tweet_text\n",
    "filtered.to_tsv('data/db/tweets_filtered_1.csv')\n",
    "filtered.df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>836950901495631872</td>\n",
       "      <td>2017-03-01 14:46:59</td>\n",
       "      <td>TT SINGAPORE 22:46 1.Hong Kong 2.#JointAddress...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>836950882528989184</td>\n",
       "      <td>2017-03-01 14:46:54</td>\n",
       "      <td>Letting #snapchat prepare me for the day's uns...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>836950869639835649</td>\n",
       "      <td>2017-03-01 14:46:51</td>\n",
       "      <td>\"The bill would require the state to get all o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>836950847380668416</td>\n",
       "      <td>2017-03-01 14:46:46</td>\n",
       "      <td>Style-Lead don't Follow #recycledfashion https...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>836950839101116421</td>\n",
       "      <td>2017-03-01 14:46:44</td>\n",
       "      <td>Shell knew : oil giant's 1991 film warned of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>836950831958224896</td>\n",
       "      <td>2017-03-01 14:46:42</td>\n",
       "      <td>Mythbusting #climate fact: Current Arctic sea ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>836950753520427008</td>\n",
       "      <td>2017-03-01 14:46:24</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>836950737502494720</td>\n",
       "      <td>2017-03-01 14:46:20</td>\n",
       "      <td>https://t.co/7sDZTyB9hD States need environmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>836950728967081984</td>\n",
       "      <td>2017-03-01 14:46:18</td>\n",
       "      <td>With environmental &amp;amp; social impact less sc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>836950727599788035</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>so what does solar radiation management look l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>836950724122710016</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>17 Goals - 17 #SDGAdvocates highlighting the #...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>836950706489798656</td>\n",
       "      <td>2017-03-01 14:46:12</td>\n",
       "      <td>I explain how @TuckerCarlson induced cognitive...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>836950679159721984</td>\n",
       "      <td>2017-03-01 14:46:06</td>\n",
       "      <td>We can't just focus on the metric of economic ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>836950658703974400</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>#Throwback #environment #sketch. Back when we ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>836950658079154176</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>It's time to get back to the game and invest i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>836950652186087424</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>Don't be exciting about what he said last nigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>836950650051313665</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>836950648197382144</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>#ClimateChange Still Kills More People Worldwi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>836950645051703296</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>#Shell issued a stark warning of the catastrop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>836950634582654976</td>\n",
       "      <td>2017-03-01 14:45:55</td>\n",
       "      <td>Making some noise from # for #ZeroDiscriminati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id          tweet_date  \\\n",
       "0   836950901495631872 2017-03-01 14:46:59   \n",
       "1   836950882528989184 2017-03-01 14:46:54   \n",
       "2   836950869639835649 2017-03-01 14:46:51   \n",
       "3   836950847380668416 2017-03-01 14:46:46   \n",
       "4   836950839101116421 2017-03-01 14:46:44   \n",
       "5   836950831958224896 2017-03-01 14:46:42   \n",
       "6   836950753520427008 2017-03-01 14:46:24   \n",
       "7   836950737502494720 2017-03-01 14:46:20   \n",
       "8   836950728967081984 2017-03-01 14:46:18   \n",
       "9   836950727599788035 2017-03-01 14:46:17   \n",
       "10  836950724122710016 2017-03-01 14:46:17   \n",
       "11  836950706489798656 2017-03-01 14:46:12   \n",
       "12  836950679159721984 2017-03-01 14:46:06   \n",
       "13  836950658703974400 2017-03-01 14:46:01   \n",
       "14  836950658079154176 2017-03-01 14:46:01   \n",
       "15  836950652186087424 2017-03-01 14:45:59   \n",
       "16  836950650051313665 2017-03-01 14:45:59   \n",
       "17  836950648197382144 2017-03-01 14:45:58   \n",
       "18  836950645051703296 2017-03-01 14:45:58   \n",
       "19  836950634582654976 2017-03-01 14:45:55   \n",
       "\n",
       "                                           tweet_text  \n",
       "0   TT SINGAPORE 22:46 1.Hong Kong 2.#JointAddress...  \n",
       "1   Letting #snapchat prepare me for the day's uns...  \n",
       "2   \"The bill would require the state to get all o...  \n",
       "3   Style-Lead don't Follow #recycledfashion https...  \n",
       "4    Shell knew : oil giant's 1991 film warned of ...  \n",
       "5   Mythbusting #climate fact: Current Arctic sea ...  \n",
       "6   A revolution is required in the way we think a...  \n",
       "7   https://t.co/7sDZTyB9hD States need environmen...  \n",
       "8   With environmental &amp; social impact less sc...  \n",
       "9   so what does solar radiation management look l...  \n",
       "10  17 Goals - 17 #SDGAdvocates highlighting the #...  \n",
       "11  I explain how @TuckerCarlson induced cognitive...  \n",
       "12  We can't just focus on the metric of economic ...  \n",
       "13  #Throwback #environment #sketch. Back when we ...  \n",
       "14  It's time to get back to the game and invest i...  \n",
       "15  Don't be exciting about what he said last nigh...  \n",
       "16  A revolution is required in the way we think a...  \n",
       "17  #ClimateChange Still Kills More People Worldwi...  \n",
       "18  #Shell issued a stark warning of the catastrop...  \n",
       "19  Making some noise from # for #ZeroDiscriminati...  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter out non ascii symbols (e.g. emoticons)\n",
    "\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[^\\x00-\\x7F]+',' ', txt))\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[ ]+', ' ', txt))\n",
    "\n",
    "filtered.df.tweet_text = tweet_text\n",
    "filtered.to_tsv('data/db/tweets_filtered_2.csv')\n",
    "filtered.df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>tweet_date</th>\n",
       "      <th>tweet_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>836950901495631872</td>\n",
       "      <td>2017-03-01 14:46:59</td>\n",
       "      <td>TT SINGAPORE 22:46 1.Hong Kong 2. JointAddress...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>836950882528989184</td>\n",
       "      <td>2017-03-01 14:46:54</td>\n",
       "      <td>Letting snapchat prepare me for the day's unse...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>836950869639835649</td>\n",
       "      <td>2017-03-01 14:46:51</td>\n",
       "      <td>\"The bill would require the state to get all o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>836950847380668416</td>\n",
       "      <td>2017-03-01 14:46:46</td>\n",
       "      <td>Style-Lead don't Follow recycledfashion https:...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>836950839101116421</td>\n",
       "      <td>2017-03-01 14:46:44</td>\n",
       "      <td>Shell knew : oil giant's 1991 film warned of ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>836950831958224896</td>\n",
       "      <td>2017-03-01 14:46:42</td>\n",
       "      <td>Mythbusting climate fact: Current Arctic sea i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>836950753520427008</td>\n",
       "      <td>2017-03-01 14:46:24</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>836950737502494720</td>\n",
       "      <td>2017-03-01 14:46:20</td>\n",
       "      <td>https: t.co 7sDZTyB9hD States need environment...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>836950728967081984</td>\n",
       "      <td>2017-03-01 14:46:18</td>\n",
       "      <td>With environmental amp; social impact less scr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>836950727599788035</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>so what does solar radiation management look l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>836950724122710016</td>\n",
       "      <td>2017-03-01 14:46:17</td>\n",
       "      <td>17 Goals - 17 SDGAdvocates highlighting the SD...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>836950706489798656</td>\n",
       "      <td>2017-03-01 14:46:12</td>\n",
       "      <td>I explain how TuckerCarlson induced cognitive ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>836950679159721984</td>\n",
       "      <td>2017-03-01 14:46:06</td>\n",
       "      <td>We can't just focus on the metric of economic ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>836950658703974400</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>Throwback environment sketch. Back when we we...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>836950658079154176</td>\n",
       "      <td>2017-03-01 14:46:01</td>\n",
       "      <td>It's time to get back to the game and invest i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>836950652186087424</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>Don't be exciting about what he said last nigh...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>836950650051313665</td>\n",
       "      <td>2017-03-01 14:45:59</td>\n",
       "      <td>A revolution is required in the way we think a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>836950648197382144</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>ClimateChange Still Kills More People Worldwi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>836950645051703296</td>\n",
       "      <td>2017-03-01 14:45:58</td>\n",
       "      <td>Shell issued a stark warning of the catastrop...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>836950634582654976</td>\n",
       "      <td>2017-03-01 14:45:55</td>\n",
       "      <td>Making some noise from for ZeroDiscrimination ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tweet_id          tweet_date  \\\n",
       "0   836950901495631872 2017-03-01 14:46:59   \n",
       "1   836950882528989184 2017-03-01 14:46:54   \n",
       "2   836950869639835649 2017-03-01 14:46:51   \n",
       "3   836950847380668416 2017-03-01 14:46:46   \n",
       "4   836950839101116421 2017-03-01 14:46:44   \n",
       "5   836950831958224896 2017-03-01 14:46:42   \n",
       "6   836950753520427008 2017-03-01 14:46:24   \n",
       "7   836950737502494720 2017-03-01 14:46:20   \n",
       "8   836950728967081984 2017-03-01 14:46:18   \n",
       "9   836950727599788035 2017-03-01 14:46:17   \n",
       "10  836950724122710016 2017-03-01 14:46:17   \n",
       "11  836950706489798656 2017-03-01 14:46:12   \n",
       "12  836950679159721984 2017-03-01 14:46:06   \n",
       "13  836950658703974400 2017-03-01 14:46:01   \n",
       "14  836950658079154176 2017-03-01 14:46:01   \n",
       "15  836950652186087424 2017-03-01 14:45:59   \n",
       "16  836950650051313665 2017-03-01 14:45:59   \n",
       "17  836950648197382144 2017-03-01 14:45:58   \n",
       "18  836950645051703296 2017-03-01 14:45:58   \n",
       "19  836950634582654976 2017-03-01 14:45:55   \n",
       "\n",
       "                                           tweet_text  \n",
       "0   TT SINGAPORE 22:46 1.Hong Kong 2. JointAddress...  \n",
       "1   Letting snapchat prepare me for the day's unse...  \n",
       "2   \"The bill would require the state to get all o...  \n",
       "3   Style-Lead don't Follow recycledfashion https:...  \n",
       "4    Shell knew : oil giant's 1991 film warned of ...  \n",
       "5   Mythbusting climate fact: Current Arctic sea i...  \n",
       "6   A revolution is required in the way we think a...  \n",
       "7   https: t.co 7sDZTyB9hD States need environment...  \n",
       "8   With environmental amp; social impact less scr...  \n",
       "9   so what does solar radiation management look l...  \n",
       "10  17 Goals - 17 SDGAdvocates highlighting the SD...  \n",
       "11  I explain how TuckerCarlson induced cognitive ...  \n",
       "12  We can't just focus on the metric of economic ...  \n",
       "13   Throwback environment sketch. Back when we we...  \n",
       "14  It's time to get back to the game and invest i...  \n",
       "15  Don't be exciting about what he said last nigh...  \n",
       "16  A revolution is required in the way we think a...  \n",
       "17   ClimateChange Still Kills More People Worldwi...  \n",
       "18   Shell issued a stark warning of the catastrop...  \n",
       "19  Making some noise from for ZeroDiscrimination ...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Filter out everything that is not alphanumeric\n",
    "\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[^a-zA-Z0-9\\-\\,\\.\\!\\?\\\"\\'\\;\\:\\_]+',' ', txt))\n",
    "tweet_text = tweet_text.apply(lambda txt: re.sub(r'[ ]+', ' ', txt))\n",
    "\n",
    "filtered.df.tweet_text = tweet_text\n",
    "filtered.to_tsv('data/db/tweets_filtered_3.csv')\n",
    "filtered.df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}