Seq2Seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.init as init\n",
    "from torch import optim\n",
    "import numpy as np\n",
    "\n",
    "import random\n",
    "import os\n",
    "\n",
    "seed = 10\n",
    "\n",
    "random.seed(seed)\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_FILE = \"data/test.txt\"\n",
    "TRAIN_FILE = \"data/train.txt\"\n",
    "WHOLE_FILE = \"data/whole.txt\"\n",
    "F_VOCAB_FILE = \"data/vocab.f.txt\"\n",
    "Q_VOCAB_FILE = \"data/vocab.q.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Options:\n",
    "    def __init__(self):\n",
    "        self.rnn_size = 50\n",
    "        self.init_weight = 0.08\n",
    "        self.decay_rate = 0.985\n",
    "        self.learning_rate = 0.01\n",
    "        self.plot_every = 10\n",
    "        self.print_every = 50\n",
    "        self.grad_clip = 5\n",
    "        self.dropout = 0.2\n",
    "        self.dropoutrec = 0.2\n",
    "        self.learning_rate_decay = 1 #0.985\n",
    "        self.learning_rate_decay_after = 5\n",
    "        \n",
    "        \n",
    "opt = Options()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LSTM(nn.Module):\n",
    "    def __init__(self, opt):\n",
    "        super(LSTM, self).__init__()\n",
    "        self.opt = opt\n",
    "        self.i2h = nn.Linear(opt.rnn_size, 4 * opt.rnn_size)\n",
    "        self.h2h = nn.Linear(opt.rnn_size, 4 * opt.rnn_size)\n",
    "        if opt.dropoutrec > 0:\n",
    "            self.dropout = nn.Dropout(opt.dropoutrec)\n",
    "            \n",
    "    def forward(self, x, prev_c, prev_h):\n",
    "        gates = self.i2h(x) + self.h2h(prev_h)\n",
    "        ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)\n",
    "        ingate = torch.sigmoid(ingate)\n",
    "        forgetgate = torch.sigmoid(forgetgate)\n",
    "        cellgate = torch.tanh(cellgate)\n",
    "        outgate = torch.sigmoid(outgate)\n",
    "        if self.opt.dropoutrec > 0:\n",
    "            cellgate = self.dropout(cellgate)\n",
    "        cy = (forgetgate * prev_c) + (ingate * cellgate)\n",
    "        hy = outgate * torch.tanh(cy)  # n_b x hidden_dim\n",
    "        return cy, hy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, opt, input_size):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.opt = opt\n",
    "        self.hidden_size = opt.rnn_size\n",
    "        self.embedding = nn.Embedding(input_size, self.hidden_size)\n",
    "        self.lstm = LSTM(self.opt)\n",
    "        if opt.dropout > 0:\n",
    "            self.dropout = nn.Dropout(opt.dropout)\n",
    "        self.__initParameters()\n",
    "\n",
    "    def __initParameters(self):\n",
    "        for name, param in self.named_parameters():\n",
    "            if param.requires_grad:\n",
    "                init.uniform_(param, -opt.init_weight, opt.init_weight)\n",
    "                \n",
    "    def forward(self, input_src, prev_c, prev_h):\n",
    "        src_emb = self.embedding(input_src) # batch_size x src_length x emb_size\n",
    "        if self.opt.dropout > 0:\n",
    "            src_emb = self.dropout(src_emb)\n",
    "        prev_cy, prev_hy = self.lstm(src_emb, prev_c, prev_h)\n",
    "        return prev_cy, prev_hy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, opt, output_size):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.opt = opt\n",
    "        self.hidden_size = opt.rnn_size\n",
    "\n",
    "        self.embedding = nn.Embedding(output_size, self.hidden_size)\n",
    "        self.lstm = LSTM(self.opt)\n",
    "        self.linear = nn.Linear(self.hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "        if opt.dropout > 0:\n",
    "            self.dropout = nn.Dropout(opt.dropout)\n",
    "        self.__initParameters()\n",
    "\n",
    "    def __initParameters(self):\n",
    "        for name, param in self.named_parameters():\n",
    "            if param.requires_grad:\n",
    "                init.uniform_(param, -opt.init_weight, opt.init_weight)\n",
    "                \n",
    "    def forward(self, input, prev_c, prev_h):\n",
    "        output = self.embedding(input)\n",
    "        if self.opt.dropout > 0:\n",
    "            output = self.dropout(output)\n",
    "        next_c, next_h = self.lstm(output, prev_c, prev_h)\n",
    "        if self.opt.dropout > 0:\n",
    "            next_h = self.dropout(next_h)\n",
    "        h2y = self.linear(next_h)\n",
    "        pred = self.softmax(h2y)\n",
    "        return pred, next_c, next_h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_data(fh):\n",
    "    for line in fh:\n",
    "        sentence, lf = line.strip().split(\"\\t\")\n",
    "        sentence = sentence.split()\n",
    "        lf = lf.split()\n",
    "        yield sentence, lf\n",
    "\n",
    "def read_vocab(filename):\n",
    "    t2i = {\"<s>\": 0, \"</s>\":1, \"UNK\": 2}\n",
    "    with open(filename) as target:\n",
    "        for line in target:\n",
    "            token = line.strip().split()[0]\n",
    "            if token not in t2i:\n",
    "                t2i[token] = len(t2i)\n",
    "    return t2i\n",
    "\n",
    "def is_equal(gold, predictions):\n",
    "    total_correct = 0.0\n",
    "    if len(gold) == len(predictions):\n",
    "        equal = True\n",
    "        for g, p in zip(gold, predictions):\n",
    "            if g != p:\n",
    "                equal = False\n",
    "        return equal\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprare_data(file_name):\n",
    "    shuffledData = None\n",
    "    with open(TRAIN_FILE, 'r') as train:\n",
    "        shuffledData = list(read_data(train))\n",
    "        random.shuffle(shuffledData)\n",
    "    sentence_index_tensors = []\n",
    "    form_index_tensors = []\n",
    "    for sentence in shuffledData:\n",
    "        text_tensor = torch.zeros((1, len(sentence[0]) + 2), dtype=torch.long)\n",
    "        text_tensor[0][0] = w2i[\"<s>\"]\n",
    "        for idx, word in enumerate(sentence[0]):\n",
    "            word_index = w2i[word] if word in w2i else w2i[\"UNK\"]\n",
    "            text_tensor[0][idx+1] = word_index\n",
    "        text_tensor[0][-1] = w2i[\"</s>\"]\n",
    "        sentence_index_tensors.append(text_tensor)\n",
    "        form_tensor = torch.zeros((1, len(sentence[1]) + 2), dtype=torch.long)\n",
    "        form_tensor[0][0] = lf2i[\"<s>\"]\n",
    "        for idx, form in enumerate(sentence[1]):\n",
    "            form_index = lf2i[form] if form in lf2i else lf2i[\"UNK\"]\n",
    "            form_tensor[0][idx+1] = form_index\n",
    "        form_tensor[0][-1] = lf2i[\"</s>\"]\n",
    "        form_index_tensors.append(form_tensor)\n",
    "    return shuffledData, sentence_index_tensors, form_index_tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(opt, criterion, encoder_optimizer, decoder_optimizer, encoder, decoder, s1, f1):\n",
    "    encoder_optimizer.zero_grad()\n",
    "    decoder_optimizer.zero_grad()\n",
    "\n",
    "    c = torch.zeros((1, opt.rnn_size), dtype=torch.float, requires_grad=True)\n",
    "    h = torch.zeros((1, opt.rnn_size), dtype=torch.float, requires_grad=True)\n",
    "    for i in range(s1.size(1)):\n",
    "        c, h = encoder(s1[:, i], c, h)\n",
    "\n",
    "    #for dec_in in f1:\n",
    "    loss = 0\n",
    "    for i in range(f1.size(1)-1):\n",
    "        pred, c, h = decoder(f1[:, i], c, h)\n",
    "        loss += criterion(pred, f1[:, i+1])\n",
    "    loss.backward()\n",
    "    if opt.grad_clip != -1:\n",
    "        torch.nn.utils.clip_grad_value_(encoder.parameters(),opt.grad_clip)\n",
    "        torch.nn.utils.clip_grad_value_(decoder.parameters(),opt.grad_clip)\n",
    "    encoder_optimizer.step()\n",
    "    decoder_optimizer.step()\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(opt, s1, lf2i, encoder, decoder):\n",
    "    c = torch.zeros((1, opt.rnn_size), dtype=torch.float, requires_grad=True)\n",
    "    h = torch.zeros((1, opt.rnn_size), dtype=torch.float, requires_grad=True)\n",
    "\n",
    "    for i in range(s1.size(1)):\n",
    "        c, h = encoder(s1[:, i], c, h)\n",
    "\n",
    "    prev = torch.tensor([lf2i['<s>']], dtype=torch.long)\n",
    "    predicted_form = []\n",
    "    counter = 0\n",
    "    while True:\n",
    "        counter += 1\n",
    "        pred, c, h = decoder(prev, c, h)\n",
    "        form_id = pred.argmax().item()\n",
    "        prev = torch.tensor([form_id], dtype=torch.long)\n",
    "        if form_id == lf2i[\"</s>\"] or counter >= 100:\n",
    "            break\n",
    "        predicted_form.append(form_id)\n",
    "    return predicted_form"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.switch_backend('agg')\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "\n",
    "def showPlot(points, fig_name, extra_info):\n",
    "    plt.figure()\n",
    "    fig, ax = plt.subplots()\n",
    "    # this locator puts ticks at regular intervals\n",
    "    loc = ticker.MultipleLocator(base=0.2)\n",
    "    ax.yaxis.set_major_locator(loc)\n",
    "    plt.title(extra_info) \n",
    "    plt.plot(points)\n",
    "    plt.savefig(\"{}.png\".format(fig_name))\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_and_test(epoch_num, directory):\n",
    "    train_data, sentence_index_tensors_train, form_index_tensors_train = preprare_data(TRAIN_FILE)\n",
    "    test_data, sentence_index_tensors_test, form_index_tensors_test = preprare_data(TEST_FILE)\n",
    "    \n",
    "    encoder = Encoder(opt, len(w2i))\n",
    "    decoder = Decoder(opt, len(lf2i))\n",
    "\n",
    "    optim_state = {\"learningRate\" : opt.learning_rate, \"alpha\" :  opt.decay_rate}\n",
    "    encoder_optimizer = optim.RMSprop(encoder.parameters(),  lr=optim_state[\"learningRate\"], alpha=optim_state[\"alpha\"])\n",
    "    decoder_optimizer = optim.RMSprop(decoder.parameters(),  lr=optim_state[\"learningRate\"], alpha=optim_state[\"alpha\"])\n",
    "    criterion = nn.NLLLoss(ignore_index=0)\n",
    "\n",
    "    losses = []\n",
    "    max_acc = 0\n",
    "    maxAccEpochId = 0\n",
    "    accuracies = []\n",
    "    chunkend_losses = []\n",
    "    for epoch in range(epoch_num):\n",
    "        print(\"---Epoch {}---\\n\".format(epoch+1))\n",
    "        print(\"Training...\")\n",
    "        encoder.train()\n",
    "        decoder.train()\n",
    "        plot_data = []\n",
    "        for index, (sentence, form) in enumerate(zip(sentence_index_tensors_train, form_index_tensors_train)):\n",
    "            loss = train(opt, criterion, encoder_optimizer, decoder_optimizer, encoder, decoder, sentence, form)\n",
    "            if index != 0:\n",
    "                if index % opt.plot_every == 0:     \n",
    "                    plot_data.append(np.mean(losses[epoch*len(train_data)+index-opt.plot_every:]))\n",
    "                    chunkend_losses.append(np.mean(plot_data))\n",
    "                if index % opt.print_every == 0:\n",
    "                    print(\"Index {} Loss {}\".format(index, np.mean(losses[epoch*len(train_data)+index-opt.print_every:])))\n",
    "            losses.append(loss.item())\n",
    "        \n",
    "        if opt.learning_rate_decay < 1:\n",
    "            if epoch >= opt.learning_rate_decay_after:\n",
    "                decay_factor = opt.learning_rate_decay\n",
    "                optim_state[\"learningRate\"] = optim_state[\"learningRate\"] * decay_factor #decay it\n",
    "                for param_group in encoder_optimizer.param_groups:\n",
    "                    param_group['lr'] = optim_state[\"learningRate\"]\n",
    "                for param_group in decoder_optimizer.param_groups:\n",
    "                    param_group['lr'] = optim_state[\"learningRate\"]\n",
    "        \n",
    "        print(\"Predicting..\")\n",
    "        encoder.eval()\n",
    "        decoder.eval()\n",
    "        correct = 0.0\n",
    "        with torch.no_grad():\n",
    "            for index, (sentence, form) in enumerate(zip(sentence_index_tensors_test, form_index_tensors_test)):\n",
    "                prediction = predict(opt, sentence, lf2i, encoder, decoder)\n",
    "                prediction = [i2lf[p] for p in prediction]\n",
    "                #print(test_data[index][1])\n",
    "                #print(prediction)\n",
    "                if len(test_data[index][1]) == len(prediction):\n",
    "                    same = True\n",
    "                    for g, p in zip(test_data[index][1], prediction):\n",
    "                        if g != p:\n",
    "                            same = False\n",
    "                    if same:\n",
    "                        correct += 1\n",
    "                    \n",
    "        accuracy = 100*(correct/len(test_data))\n",
    "        accuracies.append(accuracy)\n",
    "        if accuracy > max_acc:\n",
    "            max_acc = accuracy\n",
    "            maxAccEpochId = epoch\n",
    "            \n",
    "        print(\"Accuracy: {} Max Accuracy {}\".format(accuracy, max_acc))\n",
    "        \n",
    "        if not os.path.exists(directory):\n",
    "            os.makedirs(directory)\n",
    "            \n",
    "        file_name = \"{}/epoch.{}\".format(directory, epoch)\n",
    "        extra = \"Mean Loss {0:.2f}\".format(np.mean(plot_data))\n",
    "        showPlot(plot_data, file_name, extra)\n",
    "        \n",
    "    file_name = \"{}/{}\".format(directory, \"accuracies\")\n",
    "    extra = \"Maximum Accuracy {0:.2f} at epoch {1}\".format(np.max(accuracies), maxAccEpochId)\n",
    "    showPlot(accuracies, file_name, extra)\n",
    "    file_name = \"{}/{}\".format(directory, \"all_losses\") \n",
    "    extra = \"Mean Loss {0:.2f}\".format(np.mean(chunkend_losses))\n",
    "    showPlot(chunkend_losses, file_name, extra)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2i = read_vocab(Q_VOCAB_FILE)\n",
    "lf2i = read_vocab(F_VOCAB_FILE)\n",
    "i2lf = {lf2i[i] : i for i in lf2i}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_and_test(100, \"out/WithGradientClippingAndDropout\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt.dropout = 0\n",
    "opt.dropoutrec = 0\n",
    "opt.grad_clip = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_and_test(100, \"out/BaseExperiment\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}