PAIGCP_Bag_of_Words.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PAIGCP_Bag_of_Words",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMaC0WHSnWsQoGaeTtdU7A3",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cxbxmxcx/PAIGCP/blob/master/PAIGCP_Bag_of_Words.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DbkMepZnYVHs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import urllib.request"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SLUJ5DYpY_yH",
        "colab_type": "text"
      },
      "source": [
        "Pull down Mody Dick"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dPyg06asYkY_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "url = \"https://www.gutenberg.org/files/2701/2701-0.txt\"\n",
        "file = urllib.request.urlopen(url)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XFw0-dPOZDCZ",
        "colab_type": "text"
      },
      "source": [
        "Load the text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ER0LYVxLYo2k",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "f0934ca1-dff6-4f83-893a-8b3daedc9760"
      },
      "source": [
        "text = [line.decode('utf-8') for line in file]\n",
        "text = ''.join(text)\n",
        "text[7600:8000]"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'ok whatsoever,\\r\\n  sacred or profane. Therefore you must not, in every case at least,\\r\\n  take the higgledy-piggledy whale statements, however authentic, in\\r\\n  these extracts, for veritable gospel cetology. Far from it. As\\r\\n  touching the ancient authors generally, as well as the poets here\\r\\n  appearing, these extracts are solely valuable or entertaining, as\\r\\n  affording a glancing bird’s eye view o'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yN2YwlWcY72h",
        "colab_type": "text"
      },
      "source": [
        "Tokenize\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WmXGjk_YY9Wc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "outputId": "b841031b-4127-438a-f396-9bed2d2bd687"
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk import word_tokenize\n",
        "tokens = word_tokenize(text)\n",
        "tokens[200:222]"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['.',\n",
              " 'A',\n",
              " 'Bosom',\n",
              " 'Friend',\n",
              " '.',\n",
              " 'CHAPTER',\n",
              " '11',\n",
              " '.',\n",
              " 'Nightgown',\n",
              " '.',\n",
              " 'CHAPTER',\n",
              " '12',\n",
              " '.',\n",
              " 'Biographical',\n",
              " '.',\n",
              " 'CHAPTER',\n",
              " '13',\n",
              " '.',\n",
              " 'Wheelbarrow',\n",
              " '.',\n",
              " 'CHAPTER',\n",
              " '14']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GHvzrkUsZUs8",
        "colab_type": "text"
      },
      "source": [
        "Then clean"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bb2pv0mLZLxV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 391
        },
        "outputId": "08e76702-62f0-460f-b175-aa43a9a7952f"
      },
      "source": [
        "import string\n",
        "tokens = [word for word in tokens if word.isalpha()]\n",
        "table = str.maketrans('', '', string.punctuation)\n",
        "tokens = [w.translate(table) for w in tokens]\n",
        "tokens = [word.lower() for word in tokens]\n",
        "tokens[200:222]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['specksnyder',\n",
              " 'chapter',\n",
              " 'the',\n",
              " 'chapter',\n",
              " 'the',\n",
              " 'chapter',\n",
              " 'the',\n",
              " 'chapter',\n",
              " 'sunset',\n",
              " 'chapter',\n",
              " 'dusk',\n",
              " 'chapter',\n",
              " 'first',\n",
              " 'chapter',\n",
              " 'midnight',\n",
              " 'forecastle',\n",
              " 'chapter',\n",
              " 'moby',\n",
              " 'dick',\n",
              " 'chapter',\n",
              " 'the',\n",
              " 'whiteness']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zA8tdJU0ZXZC",
        "colab_type": "text"
      },
      "source": [
        "Stop Words and Stemming"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_qQo2cT7ZZg5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 425
        },
        "outputId": "f7f2880e-7d28-4542-cc24-5a63166b9827"
      },
      "source": [
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "stop_words = set(stopwords.words('english'))\n",
        "tokens = [w for w in tokens if not w in stop_words]\n",
        "\n",
        "from nltk.stem.porter import PorterStemmer\n",
        "porter = PorterStemmer()\n",
        "tokens = [porter.stem(word) for word in tokens]\n",
        "tokens[200:222]"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['wood',\n",
              " 'stone',\n",
              " 'mountain',\n",
              " 'star',\n",
              " 'chapter',\n",
              " 'brit',\n",
              " 'chapter',\n",
              " 'squid',\n",
              " 'chapter',\n",
              " 'line',\n",
              " 'chapter',\n",
              " 'stubb',\n",
              " 'kill',\n",
              " 'whale',\n",
              " 'chapter',\n",
              " 'dart',\n",
              " 'chapter',\n",
              " 'crotch',\n",
              " 'chapter',\n",
              " 'stubb',\n",
              " 'supper',\n",
              " 'chapter']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2EHfQLMOZoOt",
        "colab_type": "text"
      },
      "source": [
        "Vocabulary - word counts"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Npw5JfpeZp-2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "e295aa34-995b-4615-ef3f-538eedbbe44d"
      },
      "source": [
        "from nltk.probability import FreqDist\n",
        "\n",
        "word_counts = FreqDist(tokens)\n",
        "\n",
        "top = 500\n",
        "vocabulary = word_counts.most_common(top)\n",
        "    \n",
        "vocabulary[:10]"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('whale', 1455),\n",
              " ('one', 920),\n",
              " ('like', 590),\n",
              " ('upon', 567),\n",
              " ('ship', 553),\n",
              " ('ye', 521),\n",
              " ('man', 496),\n",
              " ('ahab', 495),\n",
              " ('sea', 461),\n",
              " ('seem', 460)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mXj4YLGYbccP",
        "colab_type": "text"
      },
      "source": [
        "Count Vector - word vector"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i6txpap3bZmj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c001cdd5-972b-486f-d677-327132f2b1b4"
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "voc_size = len(vocabulary)\n",
        "doc_vector = np.zeros(voc_size)\n",
        "  \n",
        "word_vector = [(idx,word_counts[word[0]]) for idx, word in enumerate(vocabulary) if word[0] in word_counts.keys()] \n",
        "word_vector[10]"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(10, 443)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rrUAuUyhb0s8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 969
        },
        "outputId": "764493bb-1161-44d6-8a1e-7d817960a975"
      },
      "source": [
        "for idx, count in word_vector:\n",
        "  doc_vector[idx] = count\n",
        "\n",
        "doc_vector"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1455.,  920.,  590.,  567.,  553.,  521.,  496.,  495.,  461.,\n",
              "        460.,  443.,  434.,  429.,  424.,  364.,  342.,  338.,  337.,\n",
              "        331.,  322.,  317.,  315.,  312.,  312.,  311.,  307.,  307.,\n",
              "        298.,  292.,  284.,  280.,  278.,  277.,  277.,  268.,  268.,\n",
              "        266.,  256.,  255.,  251.,  249.,  247.,  243.,  241.,  240.,\n",
              "        238.,  236.,  231.,  230.,  228.,  224.,  222.,  217.,  217.,\n",
              "        215.,  211.,  211.,  205.,  204.,  204.,  203.,  201.,  201.,\n",
              "        196.,  196.,  193.,  192.,  191.,  190.,  189.,  184.,  182.,\n",
              "        182.,  180.,  179.,  176.,  176.,  175.,  171.,  171.,  168.,\n",
              "        168.,  168.,  167.,  167.,  164.,  161.,  159.,  159.,  159.,\n",
              "        153.,  153.,  153.,  152.,  148.,  143.,  142.,  140.,  139.,\n",
              "        138.,  137.,  134.,  132.,  132.,  130.,  129.,  129.,  128.,\n",
              "        128.,  127.,  126.,  126.,  125.,  125.,  125.,  124.,  123.,\n",
              "        122.,  122.,  122.,  121.,  121.,  121.,  120.,  119.,  119.,\n",
              "        119.,  119.,  118.,  118.,  115.,  115.,  114.,  113.,  113.,\n",
              "        113.,  112.,  112.,  110.,  110.,  107.,  107.,  106.,  106.,\n",
              "        105.,  105.,  105.,  105.,  104.,  104.,  103.,  103.,  102.,\n",
              "        102.,  100.,   99.,   98.,   98.,   97.,   97.,   97.,   96.,\n",
              "         96.,   94.,   94.,   94.,   93.,   93.,   93.,   93.,   92.,\n",
              "         91.,   91.,   91.,   91.,   90.,   90.,   90.,   90.,   90.,\n",
              "         89.,   89.,   88.,   88.,   88.,   87.,   87.,   87.,   87.,\n",
              "         87.,   87.,   86.,   86.,   86.,   84.,   84.,   84.,   83.,\n",
              "         83.,   83.,   82.,   82.,   82.,   81.,   81.,   80.,   80.,\n",
              "         80.,   80.,   80.,   80.,   80.,   80.,   80.,   79.,   79.,\n",
              "         78.,   78.,   78.,   78.,   78.,   78.,   77.,   77.,   76.,\n",
              "         76.,   76.,   76.,   76.,   76.,   76.,   76.,   75.,   75.,\n",
              "         75.,   75.,   75.,   75.,   75.,   75.,   75.,   74.,   74.,\n",
              "         74.,   74.,   73.,   72.,   72.,   72.,   72.,   71.,   71.,\n",
              "         71.,   70.,   70.,   69.,   69.,   69.,   68.,   68.,   68.,\n",
              "         68.,   68.,   68.,   68.,   67.,   67.,   67.,   67.,   67.,\n",
              "         66.,   66.,   65.,   65.,   65.,   64.,   64.,   64.,   64.,\n",
              "         64.,   64.,   63.,   63.,   63.,   63.,   62.,   62.,   62.,\n",
              "         61.,   61.,   61.,   60.,   60.,   60.,   60.,   60.,   60.,\n",
              "         60.,   60.,   59.,   59.,   59.,   59.,   59.,   59.,   57.,\n",
              "         57.,   57.,   57.,   57.,   56.,   56.,   56.,   56.,   56.,\n",
              "         56.,   56.,   56.,   56.,   56.,   56.,   56.,   56.,   55.,\n",
              "         55.,   55.,   55.,   55.,   55.,   54.,   54.,   54.,   54.,\n",
              "         54.,   54.,   54.,   54.,   54.,   54.,   53.,   53.,   53.,\n",
              "         53.,   53.,   53.,   53.,   53.,   52.,   52.,   52.,   52.,\n",
              "         52.,   52.,   52.,   51.,   51.,   51.,   51.,   51.,   51.,\n",
              "         51.,   51.,   51.,   51.,   50.,   50.,   50.,   50.,   50.,\n",
              "         50.,   50.,   50.,   50.,   50.,   49.,   49.,   49.,   49.,\n",
              "         49.,   49.,   48.,   48.,   48.,   48.,   48.,   48.,   48.,\n",
              "         48.,   47.,   47.,   47.,   47.,   47.,   47.,   47.,   47.,\n",
              "         47.,   47.,   47.,   47.,   46.,   46.,   46.,   46.,   46.,\n",
              "         46.,   46.,   45.,   45.,   45.,   45.,   45.,   45.,   45.,\n",
              "         45.,   45.,   45.,   45.,   45.,   45.,   45.,   44.,   44.,\n",
              "         44.,   44.,   44.,   44.,   44.,   44.,   44.,   44.,   44.,\n",
              "         43.,   43.,   43.,   43.,   43.,   43.,   43.,   43.,   43.,\n",
              "         43.,   43.,   43.,   43.,   43.,   43.,   42.,   42.,   42.,\n",
              "         42.,   42.,   42.,   42.,   42.,   42.,   42.,   42.,   42.,\n",
              "         42.,   42.,   42.,   42.,   42.,   42.,   41.,   41.,   41.,\n",
              "         41.,   41.,   41.,   41.,   41.,   41.,   41.,   41.,   41.,\n",
              "         41.,   41.,   41.,   41.,   41.,   40.,   40.,   40.,   40.,\n",
              "         40.,   40.,   40.,   40.,   40.,   40.,   40.,   40.,   40.,\n",
              "         40.,   39.,   39.,   39.,   39.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9NTcOHHQcBxI",
        "colab_type": "text"
      },
      "source": [
        "Bag of Words\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1zw-9VhdcAsM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "49fac49d-0f63-4129-a561-b72f8eb6fdae"
      },
      "source": [
        "from nltk import sent_tokenize\n",
        "\n",
        "docs = sent_tokenize(text)[703:706]\n",
        "docs"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['I began to twitch all over.',\n",
              " 'Besides, it was getting late, and my decent harpooneer ought to be home\\r\\nand going bedwards.',\n",
              " 'Suppose now, he should tumble in upon me at\\r\\nmidnight—how could I tell from what vile hole he had been coming?']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VXuVQoHjcv7M",
        "colab_type": "text"
      },
      "source": [
        "Import helpers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yZirOVoEc1AK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ddce57c6-1b04-4e54-8b6b-1ebcaaa23cc4"
      },
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "count_vectorizer=CountVectorizer(stop_words='english')\n",
        "\n",
        "word_count_vector=count_vectorizer.fit_transform(docs)\n",
        "word_count_vector.shape"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, 17)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAFg_lV8c_3H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "93ee33b0-dcf9-4c23-bb2e-c289b7e63296"
      },
      "source": [
        "word_count_vector.toarray()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],\n",
              "       [1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0],\n",
              "       [0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-6-tLvWEdEzX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "33b223ca-d27d-4bd7-98e3-49efbd2b785f"
      },
      "source": [
        "count_vectorizer.get_feature_names()"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['bedwards',\n",
              " 'began',\n",
              " 'coming',\n",
              " 'decent',\n",
              " 'getting',\n",
              " 'going',\n",
              " 'harpooneer',\n",
              " 'hole',\n",
              " 'home',\n",
              " 'late',\n",
              " 'midnight',\n",
              " 'ought',\n",
              " 'suppose',\n",
              " 'tell',\n",
              " 'tumble',\n",
              " 'twitch',\n",
              " 'vile']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    }
  ]
}