Project_Code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import re\n",
    "import string\n",
    "import numpy as np\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "import pymorphy2\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation\n",
    "from keras.optimizers import SGD\n",
    "from keras.layers import Dense\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.ensemble import GradientBoostingClassifier as gb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpora = \"\"\n",
    "dump = r'data'\n",
    "for subdir in ['neg', 'pos']:\n",
    "    for file in os.listdir(dump + subdir):\n",
    "        with open(dump + subdir + '\\\\' + file, encoding='utf8') as f:\n",
    "            corpora += f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(corpora):\n",
    "    corpora = corpora.lower()\n",
    "    exclude = set(string.punctuation)\n",
    "    exclude = list(exclude) + [str(k) for k in range(10)] + ['«', '»'] + ['\\n', '\\r', '—']\n",
    "    corpora = ''.join(ch for ch in corpora if ch not in exclude)\n",
    "    words = word_tokenize(corpora)\n",
    "    return words\n",
    "\n",
    "words = preprocess(corpora)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_lemmas = []\n",
    "morph = pymorphy2.MorphAnalyzer()\n",
    "for word in list(set(words)):\n",
    "    p = morph.parse(word)[0]\n",
    "    unique_lemmas.append(p.normal_form)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.zeros((len(words), len(unique_lemmas)))\n",
    "\n",
    "for i in range(X.shape[0]):\n",
    "    p = morph.parse(words[i])[0]\n",
    "    X[i][unique_lemmas.index(p.normal_form)] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((64762, 1, 17201), (64762, 17201))"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = np.zeros((X.shape[0] - 4, 1, X.shape[1]))\n",
    "\n",
    "for i in range(2, X.shape[0] - 2):\n",
    "    for j in range(1):\n",
    "        if j < 2:\n",
    "            train_X[i - 2][j] = X[i - j - 1]\n",
    "        else:\n",
    "            train_X[i - 2][j] = X[i + j - 1]\n",
    "train_y =  X[2 : X.shape[0] - 2]\n",
    "train_X.shape, train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Neural Network\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(300, input_shape=(len(unique_lemmas),), activation=\"relu\"))\n",
    "model.add(Dense(len(unique_lemmas), input_shape=(300,)))\n",
    "sgd = SGD(lr=0.01)\n",
    "model.compile(loss=\"binary_crossentropy\", optimizer=sgd, metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "64762/64762 [==============================] - 441s 7ms/step - loss: 8.2115e-04 - acc: 0.9999\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1ba18d31400>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_X.reshape(X.shape[0] - 4, -1), train_y, epochs = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_weights = model.get_weights()\n",
    "weights = model_weights[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2vec(word):\n",
    "    try:\n",
    "        morph1 = pymorphy2.MorphAnalyzer()\n",
    "        p = morph.parse(word)[0]\n",
    "        ind = unique_lemmas.index(p.normal_form)\n",
    "        return weights[ind]\n",
    "    except:\n",
    "        return np.random.normal(0, 1, 300)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Testing on reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_neg = 'data\\\\neg' \n",
    "docs_neg = []\n",
    "os.chdir(path_neg)\n",
    "for fn in glob.glob(\"*.txt\"):\n",
    "    with open(fn, encoding=\"utf8\") as f: \n",
    "        docs_neg.append(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_pos = 'data\\\\pos' \n",
    "docs_pos = []\n",
    "os.chdir(path_pos)\n",
    "for fn in glob.glob(\"*.txt\"):\n",
    "    with open(fn, encoding=\"utf8\") as f:\n",
    "        docs_pos.append(f.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_neg = len(docs_neg)\n",
    "\n",
    "num_train_neg = int(0.9*num_neg)\n",
    "\n",
    "train_set_neg = []\n",
    "for i in range(0, num_train_neg):\n",
    "    train_set_neg.append(docs_neg[i])\n",
    "\n",
    "test_set_neg = []\n",
    "for i in range(num_train_neg, num_neg):\n",
    "    test_set_neg.append(docs_neg[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lenght of the negative train set 90\n",
      "Lenght of the negative test set 10\n"
     ]
    }
   ],
   "source": [
    "print (\"Lenght of the negative train set\", len(train_set_neg))\n",
    "print (\"Lenght of the negative test set\", len(test_set_neg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pos = len(docs_pos)\n",
    "\n",
    "num_train_pos = int(0.9*num_pos)\n",
    "\n",
    "train_set_pos = []\n",
    "for i in range(0, num_train_pos):\n",
    "    train_set_pos.append(docs_pos[i])\n",
    "\n",
    "test_set_pos = []\n",
    "for i in range(num_train_pos, num_pos):\n",
    "    test_set_pos.append(docs_pos[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Lenght of the positive train set 90\n",
      "Lenght of the positive train set 10\n"
     ]
    }
   ],
   "source": [
    "print (\"Lenght of the positive train set\", len(train_set_pos))\n",
    "print (\"Lenght of the positive train set\", len(test_set_pos))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set = train_set_pos + train_set_neg\n",
    "test_set = test_set_pos + test_set_neg\n",
    "y_train = [1 for k in range(len(train_set_pos))] + [0 for k in range(len(train_set_neg))]\n",
    "y_test = [1 for k in range(len(test_set_pos))] + [0 for k in range(len(test_set_neg))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_feat(data):\n",
    "    feat = []\n",
    "    for i in range(len(data)):\n",
    "        dat = (data[i]).split(\" \")\n",
    "        temp = []\n",
    "        for w in dat:\n",
    "            temp.append(word2vec(w))\n",
    "        temp = (np.array(temp)).mean(0)\n",
    "        feat.append(temp)\n",
    "    return feat\n",
    "    \n",
    "train_len = len(train_set)\n",
    "test_len = len(test_set)\n",
    "sets = get_feat(train_set + test_set)\n",
    "train_set = sets[ : train_len]\n",
    "test_set = sets[train_len : ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test set accuracy: 0.7000\n"
     ]
    }
   ],
   "source": [
    "def model_tr(train_X, train_Y):\n",
    "    model = LogisticRegression(solver='lbfgs', multi_class='multinomial')\n",
    "    model.fit(train_X, train_Y)\n",
    "    return model\n",
    "\n",
    "log_model = model_tr(train_set, y_train)\n",
    "predict = log_model.predict(test_set)\n",
    "test_acc = np.mean(predict == y_test)\n",
    "print(\"Test set accuracy: %.4f\" % test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}