Get_user_timelines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from twarc import Twarc\n",
    "import json as json\n",
    "\n",
    "import pandas as pd\n",
    "import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"twitter_credentials.txt\") as fp:\n",
    "    credentials = tuple(map(str.strip, fp.readlines()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "twarc_obj = Twarc(*credentials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>u_id</th>\n",
       "      <th>tid</th>\n",
       "      <th>u_statuses</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>492532196</td>\n",
       "      <td>511885958992904192</td>\n",
       "      <td>6510541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2169277098</td>\n",
       "      <td>511448287573213184</td>\n",
       "      <td>4014482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1338951085</td>\n",
       "      <td>513229038035673088</td>\n",
       "      <td>3377886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>279118291</td>\n",
       "      <td>623859753568051200</td>\n",
       "      <td>3307959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>59804598</td>\n",
       "      <td>513170741408169985</td>\n",
       "      <td>2762552</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         u_id                 tid  u_statuses\n",
       "0   492532196  511885958992904192     6510541\n",
       "1  2169277098  511448287573213184     4014482\n",
       "2  1338951085  513229038035673088     3377886\n",
       "3   279118291  623859753568051200     3307959\n",
       "4    59804598  513170741408169985     2762552"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/tid_uid.txt\", sep=\"\\t\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "User 1338951085: 100%|██████████| 3/3 [00:00<00:00,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collected 570 tweets.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "with open(\"./data/user_timelines.json\", \"w+\") as fp:\n",
    "    total = 0\n",
    "    pbar = tqdm.tqdm(df.values[:3])\n",
    "    for uid, tid, u_statuses in pbar:\n",
    "        found = 0\n",
    "        pbar.set_description(\"User {}\".format(uid))\n",
    "        for tweet_json in twarc_obj.timeline(user_id=\"{}\".format(uid)):\n",
    "            found += 1\n",
    "            if found > 190:\n",
    "                break\n",
    "            total += 1\n",
    "            print(json.dumps(tweet_json), file=fp)\n",
    "    pbar.close()\n",
    "print(\"Collected {} tweets.\".format(total))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}