02_Download_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tweepy\n",
    "import json\n",
    "from pandas.io.json import json_normalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup connection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"credentials.json\"\n",
    "with open(filename) as file:\n",
    "    keys = json.loads(file.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "consumer_key = keys['Twitter']['consumer_key']\n",
    "consumer_secret = keys['Twitter']['consumer_secret']\n",
    "access_token = keys['Twitter']['access_token']\n",
    "access_secret = keys['Twitter']['access_secret']\n",
    "auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "auth.set_access_token(access_token, access_secret)\n",
    "api = tweepy.API(auth,\n",
    "                 wait_on_rate_limit=True,\n",
    "                 wait_on_rate_limit_notify=True,\n",
    "                 retry_count=1000,\n",
    "                 retry_delay=60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download replies tweets\n",
    "- [Sony live streaming](https://twitter.com/Sony/status/1214350830981148673)\n",
    "- [AMD Conference announcement](https://twitter.com/AMD/status/1212119569487351808)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_replies(user_name, tweet_id):\n",
    "    replies = tweepy.Cursor(api.search,\n",
    "                            q='to:{} -filter:retweets'.format(user_name),\n",
    "                            since_id=tweet_id,\n",
    "                            tweet_mode='extended',\n",
    "                            lang='en').items()\n",
    "    all_repl = []\n",
    "    while True:\n",
    "        try:\n",
    "            reply = replies.next()\n",
    "            all_repl.append(reply)\n",
    "            if not hasattr(reply, 'in_reply_to_status_id_str'):\n",
    "                continue\n",
    "            if reply.in_reply_to_status_id == tweet_id:\n",
    "                logging.info(\"reply of tweet:{}\".format(reply.full_text))\n",
    "\n",
    "        except tweepy.RateLimitError as e:\n",
    "            logging.error(\"Twitter api rate limit reached\".format(e))\n",
    "            time.sleep(60)\n",
    "            continue\n",
    "\n",
    "        except tweepy.TweepError as e:\n",
    "            logging.error(\"Tweepy error occured:{}\".format(e))\n",
    "            break\n",
    "\n",
    "        except StopIteration:\n",
    "            break\n",
    "\n",
    "        except Exception as e:\n",
    "            logging.error(\"Failed while fetching replies {}\".format(e))\n",
    "            break\n",
    "\n",
    "    all_repl_processed = [0]*len(all_repl)\n",
    "    for i in range(len(all_repl)):\n",
    "        all_repl_processed[i] = {}\n",
    "        all_repl_processed[i][\"username\"] = all_repl[i].user.screen_name\n",
    "        all_repl_processed[i][\"text\"] = all_repl[i].full_text\n",
    "        all_repl_processed[i][\"lang\"] = all_repl[i].lang\n",
    "    return all_repl_processed"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download replies to Sony tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Rate limit reached. Sleeping for: 520\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RandomWebSurf</td>\n",
       "      <td>@Sony @PlayStation will I be able to play 4K a...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jkkgold</td>\n",
       "      <td>@Sony @bts_bighit @BTS_twt you have the BEST c...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nasiradeen03</td>\n",
       "      <td>@Sony @bts_bighit @BTS_twt Anyone knew what wa...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DoItLikeMeezy</td>\n",
       "      <td>@Sony @PlayStation set themselves up for these...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bowyluv</td>\n",
       "      <td>@Sony @bts_bighit @BTS_twt bts legends \\nhttps...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        username                                               text lang\n",
       "0  RandomWebSurf  @Sony @PlayStation will I be able to play 4K a...   en\n",
       "1        jkkgold  @Sony @bts_bighit @BTS_twt you have the BEST c...   en\n",
       "2   nasiradeen03  @Sony @bts_bighit @BTS_twt Anyone knew what wa...   en\n",
       "3  DoItLikeMeezy  @Sony @PlayStation set themselves up for these...   en\n",
       "4        bowyluv  @Sony @bts_bighit @BTS_twt bts legends \\nhttps...   en"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_name = \"@Sony\"\n",
    "tweet_id = \"1214350830981148673\"\n",
    "sony_tweets = download_replies(user_name=user_name, tweet_id=tweet_id)\n",
    "sony_tweets = json_normalize(sony_tweets)\n",
    "sony_tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download replies to AMD tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jtrathgeber</td>\n",
       "      <td>@AMD Apparently, it wasn’t too impressive, bec...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>J_M_1987_</td>\n",
       "      <td>@AMD Why comments section is disabled ? And wh...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>techthelead</td>\n",
       "      <td>@AMD has announced several new processors, lap...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bernhardtravis</td>\n",
       "      <td>@AMD @Radeon @AMDRyzen i would absolutely love...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>laszlo_1337</td>\n",
       "      <td>@AMD @LisaSu @anandtech Wen ray tracing</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         username                                               text lang\n",
       "0     jtrathgeber  @AMD Apparently, it wasn’t too impressive, bec...   en\n",
       "1       J_M_1987_  @AMD Why comments section is disabled ? And wh...   en\n",
       "2     techthelead  @AMD has announced several new processors, lap...   en\n",
       "3  bernhardtravis  @AMD @Radeon @AMDRyzen i would absolutely love...   en\n",
       "4     laszlo_1337            @AMD @LisaSu @anandtech Wen ray tracing   en"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_name = \"@AMD\"\n",
    "tweet_id = \"1212119569487351808\"\n",
    "amd_tweets = download_replies(user_name=user_name, tweet_id=tweet_id)\n",
    "amd_tweets = json_normalize(amd_tweets)\n",
    "amd_tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download replies to Samsung tweet "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>username</th>\n",
       "      <th>text</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>unionjack81</td>\n",
       "      <td>@SamsungMobile any chance you could send your ...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SirEdwin_</td>\n",
       "      <td>@SamsungMobile @SamsungMobileKE  can we have '...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>andygray27</td>\n",
       "      <td>@SamsungMobile I'm not asking for warranty rep...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nj_ramz</td>\n",
       "      <td>@SamsungMobile Hi I tried my best in this draw...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gashead1974</td>\n",
       "      <td>@SamsungMobile is the s10 the softest screen e...</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      username                                               text lang\n",
       "0  unionjack81  @SamsungMobile any chance you could send your ...   en\n",
       "1    SirEdwin_  @SamsungMobile @SamsungMobileKE  can we have '...   en\n",
       "2   andygray27  @SamsungMobile I'm not asking for warranty rep...   en\n",
       "3      nj_ramz  @SamsungMobile Hi I tried my best in this draw...   en\n",
       "4  gashead1974  @SamsungMobile is the s10 the softest screen e...   en"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_name = \"@SamsungMobile\"\n",
    "tweet_id = \"1159826737770856448\"\n",
    "samsung_tweets = download_replies(user_name=user_name, tweet_id=tweet_id)\n",
    "samsung_tweets = json_normalize(samsung_tweets)\n",
    "samsung_tweets.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Store tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1779, 3)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sony_tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(283, 3)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "amd_tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(512, 3)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samsung_tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "sony_tweets.to_csv('dataset/sony_replies.csv', index=False)\n",
    "amd_tweets.to_csv('dataset/amd_replies.csv', index=False)\n",
    "samsung_tweets.to_csv('dataset/samsung_tweets.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download tweets related to CES 2020"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_tweets_from_query(api, query, number_of_tweets = 2000):\n",
    "\n",
    "    result = []\n",
    "    backoff_counter = 1\n",
    "    tweets = tweepy.Cursor(api.search,\n",
    "                           q=\"{} -filter:retweets\".format(query), #for reducing the number of calls\n",
    "                           lang='en', tweet_mode='extended'\n",
    "                           ).items(number_of_tweets)\n",
    "\n",
    "    while True:\n",
    "        try:\n",
    "            tweet = tweets.next()\n",
    "            filtered_tweet = {\n",
    "                \"text\": tweet.full_text,\n",
    "                \"lang\": tweet.lang,\n",
    "                \"username\": tweet.user.screen_name,\n",
    "            }\n",
    "            result.append(filtered_tweet)\n",
    "        except StopIteration:\n",
    "            break\n",
    "        except tweepy.TweepError:\n",
    "            print(\"Reached Tweet limits, waiting for {} seconds\".format(60 * backoff_counter))\n",
    "            sleep(60 * backoff_counter)\n",
    "            backoff_counter += 1\n",
    "            continue\n",
    "    return result\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Rate limit reached. Sleeping for: 745\n"
     ]
    }
   ],
   "source": [
    "ces2020_tweets = collect_tweets_from_query(api, \"#CES2020\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "ces2020_tweets = json_normalize(ces2020_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lang</th>\n",
       "      <th>text</th>\n",
       "      <th>username</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>en</td>\n",
       "      <td>@Razer debuted an incredibly compact all-in-on...</td>\n",
       "      <td>techthelead</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>en</td>\n",
       "      <td>In keynote address, @Delta Unveils New #OOH Pa...</td>\n",
       "      <td>YourOAAA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>en</td>\n",
       "      <td>👍 We are ready for Day 2 at #CES2020. Discover...</td>\n",
       "      <td>Sio_db</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>en</td>\n",
       "      <td>GO-&amp;gt; #CES2020 #France #USA !\\n#BusinessFran...</td>\n",
       "      <td>dillardmarg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>en</td>\n",
       "      <td>The industry's smallest and lightest 4K60P pro...</td>\n",
       "      <td>HoldanBlog</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  lang                                               text     username\n",
       "0   en  @Razer debuted an incredibly compact all-in-on...  techthelead\n",
       "1   en  In keynote address, @Delta Unveils New #OOH Pa...     YourOAAA\n",
       "2   en  👍 We are ready for Day 2 at #CES2020. Discover...       Sio_db\n",
       "3   en  GO-&gt; #CES2020 #France #USA !\\n#BusinessFran...  dillardmarg\n",
       "4   en  The industry's smallest and lightest 4K60P pro...   HoldanBlog"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ces2020_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ces2020_tweets.to_csv(\"dataset/ces2020_tweets_full_text.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}