Dataframe_Merge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from dateutil.parser import parse\n",
    "import json\n",
    "from pandas.io.json import json_normalize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('D:\\\\Springboard_Capstone2\\\\CapstoneDataset\\\\NewData\\\\US_category_id.json') as f:\n",
    "   data = json.load(f)\n",
    "\n",
    "df2=json_normalize(data['items'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>etag</th>\n",
       "      <th>id</th>\n",
       "      <th>kind</th>\n",
       "      <th>snippet.assignable</th>\n",
       "      <th>snippet.channelId</th>\n",
       "      <th>snippet.title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/Xy1mB4_yLrHy_BmKm...</td>\n",
       "      <td>1</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Film &amp; Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/UZ1oLIIz2dxIhO45Z...</td>\n",
       "      <td>2</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Autos &amp; Vehicles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/nqRIq97-xe5XRZTxb...</td>\n",
       "      <td>10</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/HwXKamM1Q20q9BN-o...</td>\n",
       "      <td>15</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Pets &amp; Animals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/9GQMSRjrZdHeb1OEM...</td>\n",
       "      <td>17</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                etag  id  \\\n",
       "0  \"m2yskBQFythfE4irbTIeOgYYfBU/Xy1mB4_yLrHy_BmKm...   1   \n",
       "1  \"m2yskBQFythfE4irbTIeOgYYfBU/UZ1oLIIz2dxIhO45Z...   2   \n",
       "2  \"m2yskBQFythfE4irbTIeOgYYfBU/nqRIq97-xe5XRZTxb...  10   \n",
       "3  \"m2yskBQFythfE4irbTIeOgYYfBU/HwXKamM1Q20q9BN-o...  15   \n",
       "4  \"m2yskBQFythfE4irbTIeOgYYfBU/9GQMSRjrZdHeb1OEM...  17   \n",
       "\n",
       "                    kind  snippet.assignable         snippet.channelId  \\\n",
       "0  youtube#videoCategory                True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "1  youtube#videoCategory                True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "2  youtube#videoCategory                True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "3  youtube#videoCategory                True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "4  youtube#videoCategory                True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "\n",
       "      snippet.title  \n",
       "0  Film & Animation  \n",
       "1  Autos & Vehicles  \n",
       "2             Music  \n",
       "3    Pets & Animals  \n",
       "4            Sports  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = pd.DataFrame(df2)\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>etag</th>\n",
       "      <th>category_id</th>\n",
       "      <th>kind</th>\n",
       "      <th>assignable</th>\n",
       "      <th>channelId</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/Xy1mB4_yLrHy_BmKm...</td>\n",
       "      <td>1</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Film &amp; Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/UZ1oLIIz2dxIhO45Z...</td>\n",
       "      <td>2</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Autos &amp; Vehicles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/nqRIq97-xe5XRZTxb...</td>\n",
       "      <td>10</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/HwXKamM1Q20q9BN-o...</td>\n",
       "      <td>15</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Pets &amp; Animals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"m2yskBQFythfE4irbTIeOgYYfBU/9GQMSRjrZdHeb1OEM...</td>\n",
       "      <td>17</td>\n",
       "      <td>youtube#videoCategory</td>\n",
       "      <td>True</td>\n",
       "      <td>UCBR8-60-B28hp2BmDPdntcQ</td>\n",
       "      <td>Sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                etag category_id  \\\n",
       "0  \"m2yskBQFythfE4irbTIeOgYYfBU/Xy1mB4_yLrHy_BmKm...           1   \n",
       "1  \"m2yskBQFythfE4irbTIeOgYYfBU/UZ1oLIIz2dxIhO45Z...           2   \n",
       "2  \"m2yskBQFythfE4irbTIeOgYYfBU/nqRIq97-xe5XRZTxb...          10   \n",
       "3  \"m2yskBQFythfE4irbTIeOgYYfBU/HwXKamM1Q20q9BN-o...          15   \n",
       "4  \"m2yskBQFythfE4irbTIeOgYYfBU/9GQMSRjrZdHeb1OEM...          17   \n",
       "\n",
       "                    kind  assignable                 channelId  \\\n",
       "0  youtube#videoCategory        True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "1  youtube#videoCategory        True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "2  youtube#videoCategory        True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "3  youtube#videoCategory        True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "4  youtube#videoCategory        True  UCBR8-60-B28hp2BmDPdntcQ   \n",
       "\n",
       "              title  \n",
       "0  Film & Animation  \n",
       "1  Autos & Vehicles  \n",
       "2             Music  \n",
       "3    Pets & Animals  \n",
       "4            Sports  "
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Renaming column names\n",
    "df2.columns = ['etag','category_id','kind','assignable','channelId','title']\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>category_id</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Film &amp; Animation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Autos &amp; Vehicles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10</td>\n",
       "      <td>Music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>Pets &amp; Animals</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>Sports</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  category_id             title\n",
       "0           1  Film & Animation\n",
       "1           2  Autos & Vehicles\n",
       "2          10             Music\n",
       "3          15    Pets & Animals\n",
       "4          17            Sports"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2 = df2[['category_id','title']]\n",
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2[['category_id']] = df2[['category_id']].apply(pd.to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>trending_date</th>\n",
       "      <th>title_description</th>\n",
       "      <th>channel_title</th>\n",
       "      <th>category_id</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>tags</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>thumbnail_link</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2kyS6SvSYSE</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>WE WANT TO TALK ABOUT OUR MARRIAGE</td>\n",
       "      <td>CaseyNeistat</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-13T17:13:01.000Z</td>\n",
       "      <td>SHANtell martin</td>\n",
       "      <td>748374</td>\n",
       "      <td>57527</td>\n",
       "      <td>2966</td>\n",
       "      <td>15954</td>\n",
       "      <td>https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>SHANTELL'S CHANNEL - https://www.youtube.com/s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1ZAPwfrtAFY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>The Trump Presidency: Last Week Tonight with J...</td>\n",
       "      <td>LastWeekTonight</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T07:30:00.000Z</td>\n",
       "      <td>last week tonight trump presidency|\"last week ...</td>\n",
       "      <td>2418783</td>\n",
       "      <td>97185</td>\n",
       "      <td>6146</td>\n",
       "      <td>12703</td>\n",
       "      <td>https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>One year after the presidential election, John...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5qpjK5DgCt4</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Racist Superman | Rudy Mancuso, King Bach &amp; Le...</td>\n",
       "      <td>Rudy Mancuso</td>\n",
       "      <td>23</td>\n",
       "      <td>2017-11-12T19:05:24.000Z</td>\n",
       "      <td>racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...</td>\n",
       "      <td>3191434</td>\n",
       "      <td>146033</td>\n",
       "      <td>5339</td>\n",
       "      <td>8181</td>\n",
       "      <td>https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>puqaWrEC7tY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Nickelback Lyrics: Real or Fake?</td>\n",
       "      <td>Good Mythical Morning</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T11:00:04.000Z</td>\n",
       "      <td>rhett and link|\"gmm\"|\"good mythical morning\"|\"...</td>\n",
       "      <td>343168</td>\n",
       "      <td>10172</td>\n",
       "      <td>666</td>\n",
       "      <td>2146</td>\n",
       "      <td>https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Today we find out if Link is a Nickelback amat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d380meD0W0M</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>I Dare You: GOING BALD!?</td>\n",
       "      <td>nigahiga</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-12T18:01:41.000Z</td>\n",
       "      <td>ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...</td>\n",
       "      <td>2095731</td>\n",
       "      <td>132235</td>\n",
       "      <td>1989</td>\n",
       "      <td>17518</td>\n",
       "      <td>https://i.ytimg.com/vi/d380meD0W0M/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>I know it's been a while since we did this sho...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id trending_date  \\\n",
       "0  2kyS6SvSYSE      17.14.11   \n",
       "1  1ZAPwfrtAFY      17.14.11   \n",
       "2  5qpjK5DgCt4      17.14.11   \n",
       "3  puqaWrEC7tY      17.14.11   \n",
       "4  d380meD0W0M      17.14.11   \n",
       "\n",
       "                                   title_description          channel_title  \\\n",
       "0                 WE WANT TO TALK ABOUT OUR MARRIAGE           CaseyNeistat   \n",
       "1  The Trump Presidency: Last Week Tonight with J...        LastWeekTonight   \n",
       "2  Racist Superman | Rudy Mancuso, King Bach & Le...           Rudy Mancuso   \n",
       "3                   Nickelback Lyrics: Real or Fake?  Good Mythical Morning   \n",
       "4                           I Dare You: GOING BALD!?               nigahiga   \n",
       "\n",
       "   category_id              publish_time  \\\n",
       "0           22  2017-11-13T17:13:01.000Z   \n",
       "1           24  2017-11-13T07:30:00.000Z   \n",
       "2           23  2017-11-12T19:05:24.000Z   \n",
       "3           24  2017-11-13T11:00:04.000Z   \n",
       "4           24  2017-11-12T18:01:41.000Z   \n",
       "\n",
       "                                                tags    views   likes  \\\n",
       "0                                    SHANtell martin   748374   57527   \n",
       "1  last week tonight trump presidency|\"last week ...  2418783   97185   \n",
       "2  racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...  3191434  146033   \n",
       "3  rhett and link|\"gmm\"|\"good mythical morning\"|\"...   343168   10172   \n",
       "4  ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...  2095731  132235   \n",
       "\n",
       "   dislikes  comment_count                                  thumbnail_link  \\\n",
       "0      2966          15954  https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg   \n",
       "1      6146          12703  https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg   \n",
       "2      5339           8181  https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg   \n",
       "3       666           2146  https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg   \n",
       "4      1989          17518  https://i.ytimg.com/vi/d380meD0W0M/default.jpg   \n",
       "\n",
       "   comments_disabled  ratings_disabled  video_error_or_removed  \\\n",
       "0              False             False                   False   \n",
       "1              False             False                   False   \n",
       "2              False             False                   False   \n",
       "3              False             False                   False   \n",
       "4              False             False                   False   \n",
       "\n",
       "                                         description  \n",
       "0  SHANTELL'S CHANNEL - https://www.youtube.com/s...  \n",
       "1  One year after the presidential election, John...  \n",
       "2  WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...  \n",
       "3  Today we find out if Link is a Nickelback amat...  \n",
       "4  I know it's been a while since we did this sho...  "
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Reading US YouTube Dataset\n",
    "df=pd.read_csv('D:\\\\Springboard_Capstone2\\\\CapstoneDataset\\\\USvideos.csv')\n",
    "df.columns = ['video_id', 'trending_date', 'title_description', 'channel_title', 'category_id',\n",
    "       'publish_time', 'tags', 'views', 'likes', 'dislikes', 'comment_count',\n",
    "       'thumbnail_link', 'comments_disabled', 'ratings_disabled',\n",
    "       'video_error_or_removed', 'description']\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[['category_id']] = df[['category_id']].apply(pd.to_numeric)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "category_id     int64\n",
      "title          object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "    print(df2.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "video_id                  object\n",
      "trending_date             object\n",
      "title_description         object\n",
      "channel_title             object\n",
      "category_id                int64\n",
      "publish_time              object\n",
      "tags                      object\n",
      "views                      int64\n",
      "likes                      int64\n",
      "dislikes                   int64\n",
      "comment_count              int64\n",
      "thumbnail_link            object\n",
      "comments_disabled           bool\n",
      "ratings_disabled            bool\n",
      "video_error_or_removed      bool\n",
      "description               object\n",
      "dtype: object\n"
     ]
    }
   ],
   "source": [
    "with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "    print(df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40949"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>trending_date</th>\n",
       "      <th>title_description</th>\n",
       "      <th>channel_title</th>\n",
       "      <th>category_id</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>tags</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>thumbnail_link</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>description</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2kyS6SvSYSE</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>WE WANT TO TALK ABOUT OUR MARRIAGE</td>\n",
       "      <td>CaseyNeistat</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-13T17:13:01.000Z</td>\n",
       "      <td>SHANtell martin</td>\n",
       "      <td>748374.0</td>\n",
       "      <td>57527.0</td>\n",
       "      <td>2966.0</td>\n",
       "      <td>15954.0</td>\n",
       "      <td>https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>SHANTELL'S CHANNEL - https://www.youtube.com/s...</td>\n",
       "      <td>People &amp; Blogs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0mlNzVSJrT0</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Me-O Cats Commercial</td>\n",
       "      <td>Nobrand</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-04-21T06:47:32.000Z</td>\n",
       "      <td>cute|\"cats\"|\"thai\"|\"eggs\"</td>\n",
       "      <td>98966.0</td>\n",
       "      <td>2486.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>532.0</td>\n",
       "      <td>https://i.ytimg.com/vi/0mlNzVSJrT0/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Kittens come out of the eggs in a Thai commerc...</td>\n",
       "      <td>People &amp; Blogs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>STI2fI7sKMo</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>AFFAIRS, EX BOYFRIENDS, $18MILLION NET WORTH -...</td>\n",
       "      <td>Shawn Johnson East</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-11T15:00:03.000Z</td>\n",
       "      <td>shawn johnson|\"andrew east\"|\"shawn east\"|\"shaw...</td>\n",
       "      <td>321053.0</td>\n",
       "      <td>4451.0</td>\n",
       "      <td>1772.0</td>\n",
       "      <td>895.0</td>\n",
       "      <td>https://i.ytimg.com/vi/STI2fI7sKMo/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Subscribe for weekly videos ▶ http://bit.ly/sj...</td>\n",
       "      <td>People &amp; Blogs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>KODzih-pYlU</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>BLIND(folded) CAKE DECORATING CONTEST (with Mo...</td>\n",
       "      <td>Grace Helbig</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-11T18:08:04.000Z</td>\n",
       "      <td>itsgrace|\"funny\"|\"comedy\"|\"vlog\"|\"grace\"|\"helb...</td>\n",
       "      <td>197062.0</td>\n",
       "      <td>7250.0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>456.0</td>\n",
       "      <td>https://i.ytimg.com/vi/KODzih-pYlU/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Molly is an god damn amazing human and she cha...</td>\n",
       "      <td>People &amp; Blogs</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8mhTWqWlQzU</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Wearing Online Dollar Store Makeup For A Week</td>\n",
       "      <td>Safiya Nygaard</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-11T01:19:33.000Z</td>\n",
       "      <td>wearing online dollar store makeup for a week|...</td>\n",
       "      <td>2744430.0</td>\n",
       "      <td>115426.0</td>\n",
       "      <td>1110.0</td>\n",
       "      <td>6541.0</td>\n",
       "      <td>https://i.ytimg.com/vi/8mhTWqWlQzU/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>I found this online dollar store called ShopMi...</td>\n",
       "      <td>People &amp; Blogs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id trending_date  \\\n",
       "0  2kyS6SvSYSE      17.14.11   \n",
       "1  0mlNzVSJrT0      17.14.11   \n",
       "2  STI2fI7sKMo      17.14.11   \n",
       "3  KODzih-pYlU      17.14.11   \n",
       "4  8mhTWqWlQzU      17.14.11   \n",
       "\n",
       "                                   title_description       channel_title  \\\n",
       "0                 WE WANT TO TALK ABOUT OUR MARRIAGE        CaseyNeistat   \n",
       "1                               Me-O Cats Commercial             Nobrand   \n",
       "2  AFFAIRS, EX BOYFRIENDS, $18MILLION NET WORTH -...  Shawn Johnson East   \n",
       "3  BLIND(folded) CAKE DECORATING CONTEST (with Mo...        Grace Helbig   \n",
       "4      Wearing Online Dollar Store Makeup For A Week      Safiya Nygaard   \n",
       "\n",
       "   category_id              publish_time  \\\n",
       "0           22  2017-11-13T17:13:01.000Z   \n",
       "1           22  2017-04-21T06:47:32.000Z   \n",
       "2           22  2017-11-11T15:00:03.000Z   \n",
       "3           22  2017-11-11T18:08:04.000Z   \n",
       "4           22  2017-11-11T01:19:33.000Z   \n",
       "\n",
       "                                                tags      views     likes  \\\n",
       "0                                    SHANtell martin   748374.0   57527.0   \n",
       "1                          cute|\"cats\"|\"thai\"|\"eggs\"    98966.0    2486.0   \n",
       "2  shawn johnson|\"andrew east\"|\"shawn east\"|\"shaw...   321053.0    4451.0   \n",
       "3  itsgrace|\"funny\"|\"comedy\"|\"vlog\"|\"grace\"|\"helb...   197062.0    7250.0   \n",
       "4  wearing online dollar store makeup for a week|...  2744430.0  115426.0   \n",
       "\n",
       "   dislikes  comment_count                                  thumbnail_link  \\\n",
       "0    2966.0        15954.0  https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg   \n",
       "1     184.0          532.0  https://i.ytimg.com/vi/0mlNzVSJrT0/default.jpg   \n",
       "2    1772.0          895.0  https://i.ytimg.com/vi/STI2fI7sKMo/default.jpg   \n",
       "3     217.0          456.0  https://i.ytimg.com/vi/KODzih-pYlU/default.jpg   \n",
       "4    1110.0         6541.0  https://i.ytimg.com/vi/8mhTWqWlQzU/default.jpg   \n",
       "\n",
       "  comments_disabled ratings_disabled video_error_or_removed  \\\n",
       "0             False            False                  False   \n",
       "1             False            False                  False   \n",
       "2             False            False                  False   \n",
       "3             False            False                  False   \n",
       "4             False            False                  False   \n",
       "\n",
       "                                         description           title  \n",
       "0  SHANTELL'S CHANNEL - https://www.youtube.com/s...  People & Blogs  \n",
       "1  Kittens come out of the eggs in a Thai commerc...  People & Blogs  \n",
       "2  Subscribe for weekly videos ▶ http://bit.ly/sj...  People & Blogs  \n",
       "3  Molly is an god damn amazing human and she cha...  People & Blogs  \n",
       "4  I found this online dollar store called ShopMi...  People & Blogs  "
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged = pd.merge(df,df2, on='category_id', how='outer')\n",
    "merged.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "merged = merged.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40379, 17)"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = 'D:\\\\Springboard_Capstone2\\\\CapstoneDataset\\\\NewData\\\\YouTubeDataSet.csv'\n",
    "merged.to_csv(file_name, encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>video_id</th>\n",
       "      <th>trending_date</th>\n",
       "      <th>title</th>\n",
       "      <th>channel_title</th>\n",
       "      <th>category_id</th>\n",
       "      <th>publish_time</th>\n",
       "      <th>tags</th>\n",
       "      <th>views</th>\n",
       "      <th>likes</th>\n",
       "      <th>dislikes</th>\n",
       "      <th>comment_count</th>\n",
       "      <th>thumbnail_link</th>\n",
       "      <th>comments_disabled</th>\n",
       "      <th>ratings_disabled</th>\n",
       "      <th>video_error_or_removed</th>\n",
       "      <th>description</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2kyS6SvSYSE</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>WE WANT TO TALK ABOUT OUR MARRIAGE</td>\n",
       "      <td>CaseyNeistat</td>\n",
       "      <td>22</td>\n",
       "      <td>2017-11-13T17:13:01.000Z</td>\n",
       "      <td>SHANtell martin</td>\n",
       "      <td>748374</td>\n",
       "      <td>57527</td>\n",
       "      <td>2966</td>\n",
       "      <td>15954</td>\n",
       "      <td>https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>SHANTELL'S CHANNEL - https://www.youtube.com/s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1ZAPwfrtAFY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>The Trump Presidency: Last Week Tonight with J...</td>\n",
       "      <td>LastWeekTonight</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T07:30:00.000Z</td>\n",
       "      <td>last week tonight trump presidency|\"last week ...</td>\n",
       "      <td>2418783</td>\n",
       "      <td>97185</td>\n",
       "      <td>6146</td>\n",
       "      <td>12703</td>\n",
       "      <td>https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>One year after the presidential election, John...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5qpjK5DgCt4</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Racist Superman | Rudy Mancuso, King Bach &amp; Le...</td>\n",
       "      <td>Rudy Mancuso</td>\n",
       "      <td>23</td>\n",
       "      <td>2017-11-12T19:05:24.000Z</td>\n",
       "      <td>racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...</td>\n",
       "      <td>3191434</td>\n",
       "      <td>146033</td>\n",
       "      <td>5339</td>\n",
       "      <td>8181</td>\n",
       "      <td>https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>puqaWrEC7tY</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>Nickelback Lyrics: Real or Fake?</td>\n",
       "      <td>Good Mythical Morning</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-13T11:00:04.000Z</td>\n",
       "      <td>rhett and link|\"gmm\"|\"good mythical morning\"|\"...</td>\n",
       "      <td>343168</td>\n",
       "      <td>10172</td>\n",
       "      <td>666</td>\n",
       "      <td>2146</td>\n",
       "      <td>https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>Today we find out if Link is a Nickelback amat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>d380meD0W0M</td>\n",
       "      <td>17.14.11</td>\n",
       "      <td>I Dare You: GOING BALD!?</td>\n",
       "      <td>nigahiga</td>\n",
       "      <td>24</td>\n",
       "      <td>2017-11-12T18:01:41.000Z</td>\n",
       "      <td>ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...</td>\n",
       "      <td>2095731</td>\n",
       "      <td>132235</td>\n",
       "      <td>1989</td>\n",
       "      <td>17518</td>\n",
       "      <td>https://i.ytimg.com/vi/d380meD0W0M/default.jpg</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>I know it's been a while since we did this sho...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      video_id trending_date  \\\n",
       "0  2kyS6SvSYSE      17.14.11   \n",
       "1  1ZAPwfrtAFY      17.14.11   \n",
       "2  5qpjK5DgCt4      17.14.11   \n",
       "3  puqaWrEC7tY      17.14.11   \n",
       "4  d380meD0W0M      17.14.11   \n",
       "\n",
       "                                               title          channel_title  \\\n",
       "0                 WE WANT TO TALK ABOUT OUR MARRIAGE           CaseyNeistat   \n",
       "1  The Trump Presidency: Last Week Tonight with J...        LastWeekTonight   \n",
       "2  Racist Superman | Rudy Mancuso, King Bach & Le...           Rudy Mancuso   \n",
       "3                   Nickelback Lyrics: Real or Fake?  Good Mythical Morning   \n",
       "4                           I Dare You: GOING BALD!?               nigahiga   \n",
       "\n",
       "   category_id              publish_time  \\\n",
       "0           22  2017-11-13T17:13:01.000Z   \n",
       "1           24  2017-11-13T07:30:00.000Z   \n",
       "2           23  2017-11-12T19:05:24.000Z   \n",
       "3           24  2017-11-13T11:00:04.000Z   \n",
       "4           24  2017-11-12T18:01:41.000Z   \n",
       "\n",
       "                                                tags    views   likes  \\\n",
       "0                                    SHANtell martin   748374   57527   \n",
       "1  last week tonight trump presidency|\"last week ...  2418783   97185   \n",
       "2  racist superman|\"rudy\"|\"mancuso\"|\"king\"|\"bach\"...  3191434  146033   \n",
       "3  rhett and link|\"gmm\"|\"good mythical morning\"|\"...   343168   10172   \n",
       "4  ryan|\"higa\"|\"higatv\"|\"nigahiga\"|\"i dare you\"|\"...  2095731  132235   \n",
       "\n",
       "   dislikes  comment_count                                  thumbnail_link  \\\n",
       "0      2966          15954  https://i.ytimg.com/vi/2kyS6SvSYSE/default.jpg   \n",
       "1      6146          12703  https://i.ytimg.com/vi/1ZAPwfrtAFY/default.jpg   \n",
       "2      5339           8181  https://i.ytimg.com/vi/5qpjK5DgCt4/default.jpg   \n",
       "3       666           2146  https://i.ytimg.com/vi/puqaWrEC7tY/default.jpg   \n",
       "4      1989          17518  https://i.ytimg.com/vi/d380meD0W0M/default.jpg   \n",
       "\n",
       "   comments_disabled  ratings_disabled  video_error_or_removed  \\\n",
       "0              False             False                   False   \n",
       "1              False             False                   False   \n",
       "2              False             False                   False   \n",
       "3              False             False                   False   \n",
       "4              False             False                   False   \n",
       "\n",
       "                                         description  \n",
       "0  SHANTELL'S CHANNEL - https://www.youtube.com/s...  \n",
       "1  One year after the presidential election, John...  \n",
       "2  WATCH MY PREVIOUS VIDEO ▶ \\n\\nSUBSCRIBE ► http...  \n",
       "3  Today we find out if Link is a Nickelback amat...  \n",
       "4  I know it's been a while since we did this sho...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_name = 'D:\\\\Springboard_Capstone2\\\\CapstoneDataset\\\\USvideos.csv'\n",
    "df=pd.read_csv(file_name, encoding='utf-8')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.tokenize import word_tokenize,sent_tokenize\n",
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import json\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib import cm\n",
    "from datetime import datetime\n",
    "import glob\n",
    "import seaborn as sns\n",
    "import re\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "import os\n",
    "# Any results you write to the current directory are saved as output.\n",
    "#2017-11-13T17:13:01.000Z\n",
    "#nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from stop_words import get_stop_words\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "\n",
    "top_N = 100\n",
    "#convert list of list into text\n",
    "#a=''.join(str(r) for v in df_usa['title'] for r in v)\n",
    "\n",
    "a = df['title'].str.lower().str.cat(sep=' ')\n",
    "\n",
    "# removes punctuation,numbers and returns list of words\n",
    "b = re.sub('[^A-Za-z]+', ' ', a)\n",
    "\n",
    "#remove all the stopwords from the text\n",
    "stop_words = list(get_stop_words('en'))         \n",
    "nltk_words = list(stopwords.words('english'))   \n",
    "stop_words.extend(nltk_words)\n",
    "\n",
    "word_tokens = word_tokenize(b)\n",
    "filtered_sentence = [w for w in word_tokens if not w in stop_words]\n",
    "filtered_sentence = []\n",
    "for w in word_tokens:\n",
    "    if w not in stop_words:\n",
    "        filtered_sentence.append(w)\n",
    "\n",
    "# Remove characters which have length less than 2  \n",
    "without_single_chr = [word for word in filtered_sentence if len(word) > 2]\n",
    "\n",
    "# Remove numbers\n",
    "cleaned_data_title = [word for word in without_single_chr if not word.isnumeric()]        \n",
    "\n",
    "# Calculate frequency distribution\n",
    "word_dist = nltk.FreqDist(cleaned_data_title)\n",
    "rslt = pd.DataFrame(word_dist.most_common(top_N),\n",
    "                    columns=['Word', 'Frequency'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Word</th>\n",
       "      <th>Frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>official</td>\n",
       "      <td>3991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>video</td>\n",
       "      <td>2864</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trailer</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>new</td>\n",
       "      <td>1008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>makeup</td>\n",
       "      <td>876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>audio</td>\n",
       "      <td>872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>music</td>\n",
       "      <td>838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>live</td>\n",
       "      <td>790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>first</td>\n",
       "      <td>751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>day</td>\n",
       "      <td>698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>challenge</td>\n",
       "      <td>670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>full</td>\n",
       "      <td>621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>game</td>\n",
       "      <td>604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>black</td>\n",
       "      <td>593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>star</td>\n",
       "      <td>572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>world</td>\n",
       "      <td>571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>make</td>\n",
       "      <td>569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>lyric</td>\n",
       "      <td>569</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>show</td>\n",
       "      <td>566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>time</td>\n",
       "      <td>562</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>love</td>\n",
       "      <td>535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>movie</td>\n",
       "      <td>509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>season</td>\n",
       "      <td>482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>best</td>\n",
       "      <td>468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>teaser</td>\n",
       "      <td>462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>christmas</td>\n",
       "      <td>443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>one</td>\n",
       "      <td>442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>like</td>\n",
       "      <td>436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>life</td>\n",
       "      <td>427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>test</td>\n",
       "      <td>422</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>everything</td>\n",
       "      <td>275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>battle</td>\n",
       "      <td>274</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>giant</td>\n",
       "      <td>273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>home</td>\n",
       "      <td>272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>every</td>\n",
       "      <td>271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>week</td>\n",
       "      <td>267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>beauty</td>\n",
       "      <td>266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>cardi</td>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>tutorial</td>\n",
       "      <td>263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>snl</td>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>wedding</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>people</td>\n",
       "      <td>259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>trailers</td>\n",
       "      <td>258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>episode</td>\n",
       "      <td>257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>box</td>\n",
       "      <td>251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>youtube</td>\n",
       "      <td>250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>voice</td>\n",
       "      <td>247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>kids</td>\n",
       "      <td>246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>take</td>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>questions</td>\n",
       "      <td>245</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>ball</td>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>war</td>\n",
       "      <td>239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>apple</td>\n",
       "      <td>238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>back</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>espn</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>nba</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>cast</td>\n",
       "      <td>227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>honest</td>\n",
       "      <td>226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>smith</td>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>jedi</td>\n",
       "      <td>224</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Word  Frequency\n",
       "0     official       3991\n",
       "1        video       2864\n",
       "2      trailer       2000\n",
       "3          new       1008\n",
       "4       makeup        876\n",
       "5        audio        872\n",
       "6        music        838\n",
       "7         live        790\n",
       "8        first        751\n",
       "9          day        698\n",
       "10   challenge        670\n",
       "11        full        621\n",
       "12        game        604\n",
       "13       black        593\n",
       "14        star        572\n",
       "15       world        571\n",
       "16        make        569\n",
       "17       lyric        569\n",
       "18        show        566\n",
       "19        time        562\n",
       "20        love        535\n",
       "21       movie        509\n",
       "22      season        482\n",
       "23        best        468\n",
       "24      teaser        462\n",
       "25   christmas        443\n",
       "26         one        442\n",
       "27        like        436\n",
       "28        life        427\n",
       "29        test        422\n",
       "..         ...        ...\n",
       "70  everything        275\n",
       "71      battle        274\n",
       "72       giant        273\n",
       "73        home        272\n",
       "74       every        271\n",
       "75        week        267\n",
       "76      beauty        266\n",
       "77       cardi        263\n",
       "78    tutorial        263\n",
       "79         snl        261\n",
       "80     wedding        260\n",
       "81      people        259\n",
       "82    trailers        258\n",
       "83     episode        257\n",
       "84         box        251\n",
       "85     youtube        250\n",
       "86       voice        247\n",
       "87        kids        246\n",
       "88        take        245\n",
       "89   questions        245\n",
       "90        ball        240\n",
       "91         war        239\n",
       "92       apple        238\n",
       "93        back        229\n",
       "94        espn        229\n",
       "95         nba        227\n",
       "96        cast        227\n",
       "97      honest        226\n",
       "98       smith        224\n",
       "99        jedi        224\n",
       "\n",
       "[100 rows x 2 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}