SentimentAnalysisMB.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [],
      "source": [
        "from gensim.models import KeyedVectors\n",
        "\n",
        "###Load vectors directly from the Google file \n",
        "embeddingWords \u003d KeyedVectors.load_word2vec_format(\u0027GoogleNews-vectors-negative300.bin\u0027, binary\u003dTrue)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "metadata": false,
          "name": "#%%\n"
        }
      },
      "outputs": [],
      "source": "###Import all classes needed\nimport re\nimport pandas as pd\nfrom nltk.corpus import stopwords\nfrom nltk.stem.snowball import EnglishStemmer\nfrom nltk.tokenize import word_tokenize\nfrom sklearn.naive_bayes import MultinomialNB\nimport numpy as np\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.metrics import confusion_matrix\nimport matplotlib.pyplot as plt\n\n###Import the training data from file\ndf_train \u003d pd.read_csv(\u0027TrainingTwitterFinal20K.csv\u0027)\ndf_train.dropna(axis\u003d0, inplace\u003dTrue)\n\n###Method to clean each sentence\ndef clean(text):\n    \n    if type(text) !\u003d str or text \u003d\u003d \u0027\u0027:\n        return \u0027\u0027\n     \n    text \u003d re.sub(\"\\\u0027s\", \"is\", text)\n    text \u003d re.sub(\" whats \", \"what is\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"\\\u0027ve\", \" have\", text)\n    text \u003d re.sub(\"can\\\u0027t\", \"cannot\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"don\\\u0027t\", \"do not\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"what\\\u0027s\", \"what is\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"aren\\\u0027t\", \"are not\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"it\\\u0027s\", \"it is\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"dont\", \"do not\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"how\\\u0027s\u0027\", \"how is\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"isn\\\u0027t\", \"is not\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"\\\u0027re\", \" are\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"\\\u0027m\", \" am\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"jrk\", \"jerk\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(\"shoulda\", \"should have\", text, flags\u003dre.IGNORECASE)\n    text \u003d re.sub(r\u0027\\W+\u0027, \u0027 URL \u0027, text)\n    \n    ###remove comma between numbers, i.e. 15,000 -\u003e 15000\n    text \u003d re.sub(\u0027(?\u003c\u003d[0-9])\\,(?\u003d[0-9])\u0027, \"\", text)\n    \n    ###remove stopwords\n    text_file \u003d open(\"list.txt\", \"r\")\n    lines \u003d text_file.readlines()\n    lines \u003d re.sub(\u0027\"\u0027, \u0027\u0027, lines[0])\n    lines \u003d re.sub(\u0027 \u0027, \u0027\u0027, lines)\n    lines \u003d re.sub(\u0027\\n\u0027, \u0027\u0027, lines)\n    \n    stopWords2 \u003d lines.split(\u0027,\u0027)\n    stopWords \u003d set(stopwords.words(\u0027english\u0027))\n    \n    ###stemmer\n    stemmer \u003d EnglishStemmer()\n\n    words \u003d word_tokenize(text)\n    wordsFiltered \u003d []\n      \n    lemmatizer \u003d WordNetLemmatizer() \n  \n    for w in words:\n        if w not in stopWords:\n            if w not in stopWords2:        \n                wordsFiltered.append(lemmatizer.lemmatize(w))\n        \n\n    ###Return a list of words\n    return sorted(list(set(wordsFiltered)))"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "metadata": false,
          "name": "#%%\n"
        }
      },
      "outputs": [],
      "source": [
        "###Apply clean to the train data\n",
        "df_train[\u0027comment\u0027] \u003d df_train[\u0027comment\u0027].apply(clean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "metadata": false,
          "name": "#%%\n"
        }
      },
      "outputs": [],
      "source": [
        "###sum the vectors given a certain size\n",
        "def sumvector(vec1, vec2 ,size):\n",
        "    \n",
        "    if vec1 is None:\n",
        "        vec1 \u003d [0] * size\n",
        "    \n",
        "    if vec2 is None:\n",
        "        vec2 \u003d [0] * size\n",
        "\n",
        "    vecsum \u003d []\n",
        "   \n",
        "    for i in range(len(vec1)):\n",
        "        vecsum.append(vec1[i] + vec2[i])\n",
        "\n",
        "    return vecsum"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "metadata": false,
          "name": "#%%\n"
        }
      },
      "outputs": [],
      "source": [
        "###create the vectors of each sentence with vectorizer\n",
        "allSentences \u003d []\n",
        "\n",
        "for i, sentence in enumerate(df_train[\u0027comment\u0027]):\n",
        "    string \u003d \u0027 \u0027.join(sentence)\n",
        "    allSentences.append(string)\n",
        "    \n",
        "vectorizer \u003d CountVectorizer()\n",
        "sentenceVectors \u003d vectorizer.fit_transform(allSentences)\n",
        "counter \u003d 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [],
      "source": [
        "###label the training set data\n",
        "labels \u003d df_train[\u0027positivity\u0027].values.tolist()\n",
        "\n",
        "tfidf \u003d TfidfTransformer()\n",
        "train \u003d tfidf.fit_transform(sentenceVectors)\n",
        "train.shape\n",
        "\n",
        "###create a Multinomial Naive Bayes Classifier, input array for X values and labels s\n",
        "clf \u003d MultinomialNB().fit(train, labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {
          "is_executing": true,
          "metadata": false,
          "name": "#%%\n"
        },
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "###Retrieving the test data\n",
        "df_test \u003d pd.read_csv(\u00272004RCnew.csv\u0027)\n",
        "\n",
        "###Apply clean\n",
        "df_test[\u0027comment\u0027] \u003d df_test[\u0027comment\u0027].apply(clean)\n",
        "\n",
        "###Compute labels for the test data\n",
        "labels \u003d df_test[\u0027positivity\u0027].values\n",
        " \n",
        "allSentences2 \u003d []\n",
        "\n",
        "for i, sentence in enumerate(df_test[\u0027comment\u0027]):\n",
        "    allSentences2.append(\u0027 \u0027.join(sentence))\n",
        "    \n",
        "sentenceVectors2 \u003d vectorizer.transform(allSentences2)\n",
        "\n",
        "###calculate accuracy of the classifier\n",
        "predicted \u003d clf.predict(sentenceVectors2)\n",
        "\n",
        "print(np.mean(predicted \u003d\u003d labels))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {},
        "scrolled": true
      },
      "outputs": [],
      "source": [
        "###Method to print a complete confusion matrix\n",
        "def plot_confusion_matrix(y_true, y_pred,\n",
        "                          normalize\u003dFalse,\n",
        "                          title\u003dNone,\n",
        "                          cmap\u003dplt.cm.Blues):\n",
        "    \"\"\"\n",
        "    This function prints and plots the confusion matrix.\n",
        "    Normalization can be applied by setting `normalize\u003dTrue`.\n",
        "    \"\"\"\n",
        "    if not title:\n",
        "        if normalize:\n",
        "            title \u003d \u0027Normalized confusion matrix\u0027\n",
        "        else:\n",
        "            title \u003d \u0027Confusion matrix, without normalization\u0027\n",
        "    ###Compute confusion matrix\n",
        "    cm \u003d confusion_matrix(y_true, y_pred)\n",
        "    \n",
        "    if normalize:\n",
        "        cm \u003d cm.astype(\u0027float\u0027) / cm.sum(axis\u003d1)[:, np.newaxis]\n",
        "        print(\"Normalized confusion matrix\")\n",
        "    else:\n",
        "        print(\u0027Confusion matrix, without normalization\u0027)\n",
        "\n",
        "    fig, ax \u003d plt.subplots()\n",
        "    im \u003d ax.imshow(cm, interpolation\u003d\u0027nearest\u0027, cmap\u003dcmap)\n",
        "    ax.figure.colorbar(im, ax\u003dax)\n",
        "    \n",
        "    classes \u003d (\u0027Negative\u0027, \u0027Positive\u0027)\n",
        "    \n",
        "    ###We want to show all ticks...\n",
        "    ax.set(xticks\u003dnp.arange(cm.shape[1]),\n",
        "           yticks\u003dnp.arange(cm.shape[0]),\n",
        "           # ... and label them with the respective list entries\n",
        "           xticklabels\u003dclasses, yticklabels\u003dclasses,\n",
        "           title\u003dtitle,\n",
        "           ylabel\u003d\u0027True label\u0027,\n",
        "           xlabel\u003d\u0027Predicted label\u0027)\n",
        "\n",
        "    ###Rotate the tick labels and set their alignment.\n",
        "    plt.setp(ax.get_xticklabels(), rotation\u003d45, ha\u003d\"right\",\n",
        "             rotation_mode\u003d\"anchor\")\n",
        "\n",
        "    ###Loop over data dimensions and create text annotations.\n",
        "    fmt \u003d \u0027.2f\u0027 if normalize else \u0027d\u0027\n",
        "    thresh \u003d cm.max() / 2.\n",
        "    for i in range(cm.shape[0]):\n",
        "        for j in range(cm.shape[1]):\n",
        "            ax.text(j, i, format(cm[i, j], fmt),\n",
        "                    ha\u003d\"center\", va\u003d\"center\",\n",
        "                    color\u003d\"white\" if cm[i, j] \u003e thresh else \"black\")\n",
        "    fig.tight_layout()\n",
        "    return ax"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "pycharm": {}
      },
      "outputs": [],
      "source": [
        "###Plot non-normalized confusion matrix\n",
        "plot_confusion_matrix(labels, predicted, title\u003d\u0027Confusion matrix, without normalization\u0027)\n",
        "\n",
        "plt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "stem_cell": {
      "cell_type": "raw",
      "metadata": {
        "pycharm": {
          "metadata": false
        }
      },
      "source": ""
    }
  },
  "nbformat": 4,
  "nbformat_minor": 1
}