ch04-word2vec高效化.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Negative Sampling Loss实现\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch04/negative_sampling_layer.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from common.np import *  # import numpy as np\n",
    "from common.layers import Embedding, SigmoidWithLoss\n",
    "import collections\n",
    "\n",
    "\n",
    "class EmbeddingDot:\n",
    "    def __init__(self, W):\n",
    "        self.embed = Embedding(W)\n",
    "        self.params = self.embed.params\n",
    "        self.grads = self.embed.grads\n",
    "        self.cache = None\n",
    "\n",
    "    def forward(self, h, idx):\n",
    "        target_W = self.embed.forward(idx)\n",
    "        out = np.sum(target_W * h, axis=1)\n",
    "\n",
    "        self.cache = (h, target_W)\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        h, target_W = self.cache\n",
    "        dout = dout.reshape(dout.shape[0], 1)\n",
    "\n",
    "        dtarget_W = dout * h\n",
    "        self.embed.backward(dtarget_W)\n",
    "        dh = dout * target_W\n",
    "        return dh\n",
    "\n",
    "\n",
    "class UnigramSampler:\n",
    "    def __init__(self, corpus, power, sample_size):\n",
    "        self.sample_size = sample_size\n",
    "        self.vocab_size = None\n",
    "        self.word_p = None\n",
    "\n",
    "        counts = collections.Counter()\n",
    "        for word_id in corpus:\n",
    "            counts[word_id] += 1\n",
    "\n",
    "        vocab_size = len(counts)\n",
    "        self.vocab_size = vocab_size\n",
    "\n",
    "        self.word_p = np.zeros(vocab_size)\n",
    "        for i in range(vocab_size):\n",
    "            self.word_p[i] = counts[i]\n",
    "\n",
    "        self.word_p = np.power(self.word_p, power)\n",
    "        self.word_p /= np.sum(self.word_p)\n",
    "\n",
    "    def get_negative_sample(self, target):\n",
    "        batch_size = target.shape[0]\n",
    "\n",
    "        if not GPU:\n",
    "            negative_sample = np.zeros((batch_size, self.sample_size), dtype=np.int32)\n",
    "\n",
    "            for i in range(batch_size):\n",
    "                p = self.word_p.copy()\n",
    "                target_idx = target[i]\n",
    "                p[target_idx] = 0\n",
    "                p /= p.sum()\n",
    "                negative_sample[i, :] = np.random.choice(self.vocab_size, size=self.sample_size, replace=False, p=p)\n",
    "        else:\n",
    "            # 使用GPU(cupy）计算时，速度优先\n",
    "            # 目标中可能包含负样本\n",
    "            negative_sample = np.random.choice(self.vocab_size, size=(batch_size, self.sample_size),\n",
    "                                               replace=True, p=self.word_p)\n",
    "\n",
    "        return negative_sample\n",
    "\n",
    "\n",
    "class NegativeSamplingLoss:\n",
    "    def __init__(self, W, corpus, power=0.75, sample_size=5):\n",
    "        self.sample_size = sample_size\n",
    "        self.sampler = UnigramSampler(corpus, power, sample_size)\n",
    "        self.loss_layers = [SigmoidWithLoss() for _ in range(sample_size + 1)]\n",
    "        self.embed_dot_layers = [EmbeddingDot(W) for _ in range(sample_size + 1)]\n",
    "\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in self.embed_dot_layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "\n",
    "    def forward(self, h, target):\n",
    "        batch_size = target.shape[0]\n",
    "        negative_sample = self.sampler.get_negative_sample(target)\n",
    "\n",
    "        # 正样本前向传播\n",
    "        score = self.embed_dot_layers[0].forward(h, target)\n",
    "        correct_label = np.ones(batch_size, dtype=np.int32)\n",
    "        loss = self.loss_layers[0].forward(score, correct_label)\n",
    "\n",
    "        # 负样本前向传播\n",
    "        negative_label = np.zeros(batch_size, dtype=np.int32)\n",
    "        for i in range(self.sample_size):\n",
    "            negative_target = negative_sample[:, i]\n",
    "            score = self.embed_dot_layers[1 + i].forward(h, negative_target)\n",
    "            loss += self.loss_layers[1 + i].forward(score, negative_label)\n",
    "\n",
    "        return loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        dh = 0\n",
    "        for l0, l1 in zip(self.loss_layers, self.embed_dot_layers):\n",
    "            dscore = l0.backward(dout)\n",
    "            dh += l1.backward(dscore)\n",
    "\n",
    "        return dh"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# cbow实现\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch04/cbow.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from common.np import *  # import numpy as np\n",
    "from common.layers import Embedding\n",
    "#from ch04.negative_sampling_layer import NegativeSamplingLoss\n",
    "\n",
    "\n",
    "class CBOW:\n",
    "    def __init__(self, vocab_size, hidden_size, window_size, corpus):\n",
    "        V, H = vocab_size, hidden_size\n",
    "\n",
    "        # 权重初始化\n",
    "        W_in = 0.01 * np.random.randn(V, H).astype('f')\n",
    "        W_out = 0.01 * np.random.randn(V, H).astype('f')\n",
    "\n",
    "        # 生成层\n",
    "        self.in_layers = []\n",
    "        for i in range(2 * window_size):\n",
    "            layer = Embedding(W_in)  # 使用Embedding层\n",
    "            self.in_layers.append(layer)\n",
    "        self.ns_loss = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5)\n",
    "\n",
    "        # 将所有的权重和梯度汇总到列表\n",
    "        layers = self.in_layers + [self.ns_loss]\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "\n",
    "        # 设置单词的离散表示成员变量\n",
    "        self.word_vecs = W_in\n",
    "\n",
    "    def forward(self, contexts, target):\n",
    "        h = 0\n",
    "        for i, layer in enumerate(self.in_layers):\n",
    "            h += layer.forward(contexts[:, i])\n",
    "        h *= 1 / len(self.in_layers)\n",
    "        loss = self.ns_loss.forward(h, target)\n",
    "        return loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        dout = self.ns_loss.backward(dout)\n",
    "        dout *= 1 / len(self.in_layers)\n",
    "        for layer in self.in_layers:\n",
    "            layer.backward(dout)\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# skip gram实现\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch04/skip_gram.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')  # 为导入父目录中的文件而进行的设置\n",
    "from common.layers import *\n",
    "#from ch04.negative_sampling_layer import NegativeSamplingLoss\n",
    "\n",
    "\n",
    "class SkipGram:\n",
    "    def __init__(self, vocab_size, hidden_size, window_size, corpus):\n",
    "        V, H = vocab_size, hidden_size\n",
    "        rn = np.random.randn\n",
    "\n",
    "        # 权重初始化\n",
    "        W_in = 0.01 * rn(V, H).astype('f')\n",
    "        W_out = 0.01 * rn(V, H).astype('f')\n",
    "\n",
    "        # 生成层\n",
    "        self.in_layer = Embedding(W_in)\n",
    "        self.loss_layers = []\n",
    "        for i in range(2 * window_size):\n",
    "            layer = NegativeSamplingLoss(W_out, corpus, power=0.75, sample_size=5)\n",
    "            self.loss_layers.append(layer)\n",
    "\n",
    "        # 将所有的权重和梯度汇总到列表\n",
    "        layers = [self.in_layer] + self.loss_layers\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "\n",
    "        # 设置单词的离散表示成员变量\n",
    "        self.word_vecs = W_in\n",
    "\n",
    "    def forward(self, contexts, target):\n",
    "        h = self.in_layer.forward(target)\n",
    "\n",
    "        loss = 0\n",
    "        for i, layer in enumerate(self.loss_layers):\n",
    "            loss += layer.forward(h, contexts[:, i])\n",
    "        return loss\n",
    "\n",
    "    def backward(self, dout=1):\n",
    "        dh = 0\n",
    "        for i, layer in enumerate(self.loss_layers):\n",
    "            dh += layer.backward(dout)\n",
    "        self.in_layer.backward(dh)\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch04/train.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch 1 |  iter 1 / 9295 | time 0[s] | loss 4.16\n",
      "| epoch 1 |  iter 21 / 9295 | time 1[s] | loss 4.16\n",
      "| epoch 1 |  iter 41 / 9295 | time 3[s] | loss 4.15\n",
      "| epoch 1 |  iter 61 / 9295 | time 4[s] | loss 4.13\n",
      "| epoch 1 |  iter 81 / 9295 | time 6[s] | loss 4.05\n",
      "| epoch 1 |  iter 101 / 9295 | time 7[s] | loss 3.93\n",
      "| epoch 1 |  iter 121 / 9295 | time 9[s] | loss 3.78\n",
      "| epoch 1 |  iter 141 / 9295 | time 10[s] | loss 3.62\n",
      "| epoch 1 |  iter 161 / 9295 | time 12[s] | loss 3.51\n",
      "| epoch 1 |  iter 181 / 9295 | time 13[s] | loss 3.37\n",
      "| epoch 1 |  iter 201 / 9295 | time 15[s] | loss 3.25\n",
      "| epoch 1 |  iter 221 / 9295 | time 17[s] | loss 3.16\n",
      "| epoch 1 |  iter 241 / 9295 | time 19[s] | loss 3.08\n",
      "| epoch 1 |  iter 261 / 9295 | time 21[s] | loss 3.04\n",
      "| epoch 1 |  iter 281 / 9295 | time 23[s] | loss 2.96\n",
      "| epoch 1 |  iter 301 / 9295 | time 24[s] | loss 2.92\n",
      "| epoch 1 |  iter 321 / 9295 | time 26[s] | loss 2.90\n",
      "| epoch 1 |  iter 341 / 9295 | time 28[s] | loss 2.83\n",
      "| epoch 1 |  iter 361 / 9295 | time 30[s] | loss 2.82\n",
      "| epoch 1 |  iter 381 / 9295 | time 32[s] | loss 2.78\n",
      "| epoch 1 |  iter 401 / 9295 | time 34[s] | loss 2.76\n",
      "| epoch 1 |  iter 421 / 9295 | time 36[s] | loss 2.74\n",
      "| epoch 1 |  iter 441 / 9295 | time 38[s] | loss 2.70\n",
      "| epoch 1 |  iter 461 / 9295 | time 40[s] | loss 2.71\n",
      "| epoch 1 |  iter 481 / 9295 | time 42[s] | loss 2.68\n",
      "| epoch 1 |  iter 501 / 9295 | time 44[s] | loss 2.70\n",
      "| epoch 1 |  iter 521 / 9295 | time 45[s] | loss 2.67\n",
      "| epoch 1 |  iter 541 / 9295 | time 48[s] | loss 2.64\n",
      "| epoch 1 |  iter 561 / 9295 | time 50[s] | loss 2.65\n",
      "| epoch 1 |  iter 581 / 9295 | time 52[s] | loss 2.67\n",
      "| epoch 1 |  iter 601 / 9295 | time 53[s] | loss 2.61\n",
      "| epoch 1 |  iter 621 / 9295 | time 55[s] | loss 2.63\n",
      "| epoch 1 |  iter 641 / 9295 | time 57[s] | loss 2.65\n",
      "| epoch 1 |  iter 661 / 9295 | time 59[s] | loss 2.60\n",
      "| epoch 1 |  iter 681 / 9295 | time 61[s] | loss 2.62\n",
      "| epoch 1 |  iter 701 / 9295 | time 63[s] | loss 2.59\n",
      "| epoch 1 |  iter 721 / 9295 | time 65[s] | loss 2.59\n",
      "| epoch 1 |  iter 741 / 9295 | time 67[s] | loss 2.60\n",
      "| epoch 1 |  iter 761 / 9295 | time 69[s] | loss 2.61\n",
      "| epoch 1 |  iter 781 / 9295 | time 71[s] | loss 2.60\n",
      "| epoch 1 |  iter 801 / 9295 | time 72[s] | loss 2.57\n",
      "| epoch 1 |  iter 821 / 9295 | time 74[s] | loss 2.61\n",
      "| epoch 1 |  iter 841 / 9295 | time 76[s] | loss 2.59\n",
      "| epoch 1 |  iter 861 / 9295 | time 78[s] | loss 2.57\n",
      "| epoch 1 |  iter 881 / 9295 | time 80[s] | loss 2.58\n",
      "| epoch 1 |  iter 901 / 9295 | time 82[s] | loss 2.57\n",
      "| epoch 1 |  iter 921 / 9295 | time 83[s] | loss 2.56\n",
      "| epoch 1 |  iter 941 / 9295 | time 85[s] | loss 2.54\n",
      "| epoch 1 |  iter 961 / 9295 | time 87[s] | loss 2.54\n",
      "| epoch 1 |  iter 981 / 9295 | time 89[s] | loss 2.55\n",
      "| epoch 1 |  iter 1001 / 9295 | time 91[s] | loss 2.53\n",
      "| epoch 1 |  iter 1021 / 9295 | time 92[s] | loss 2.55\n",
      "| epoch 1 |  iter 1041 / 9295 | time 95[s] | loss 2.55\n",
      "| epoch 1 |  iter 1061 / 9295 | time 97[s] | loss 2.53\n",
      "| epoch 1 |  iter 1081 / 9295 | time 99[s] | loss 2.53\n",
      "| epoch 1 |  iter 1101 / 9295 | time 101[s] | loss 2.54\n",
      "| epoch 1 |  iter 1121 / 9295 | time 103[s] | loss 2.55\n",
      "| epoch 1 |  iter 1141 / 9295 | time 105[s] | loss 2.54\n",
      "| epoch 1 |  iter 1161 / 9295 | time 106[s] | loss 2.50\n",
      "| epoch 1 |  iter 1181 / 9295 | time 108[s] | loss 2.52\n",
      "| epoch 1 |  iter 1201 / 9295 | time 110[s] | loss 2.53\n",
      "| epoch 1 |  iter 1221 / 9295 | time 112[s] | loss 2.52\n",
      "| epoch 1 |  iter 1241 / 9295 | time 114[s] | loss 2.54\n",
      "| epoch 1 |  iter 1261 / 9295 | time 116[s] | loss 2.55\n",
      "| epoch 1 |  iter 1281 / 9295 | time 118[s] | loss 2.55\n",
      "| epoch 1 |  iter 1301 / 9295 | time 120[s] | loss 2.54\n",
      "| epoch 1 |  iter 1321 / 9295 | time 122[s] | loss 2.54\n",
      "| epoch 1 |  iter 1341 / 9295 | time 123[s] | loss 2.53\n",
      "| epoch 1 |  iter 1361 / 9295 | time 125[s] | loss 2.50\n",
      "| epoch 1 |  iter 1381 / 9295 | time 127[s] | loss 2.52\n",
      "| epoch 1 |  iter 1401 / 9295 | time 129[s] | loss 2.49\n",
      "| epoch 1 |  iter 1421 / 9295 | time 131[s] | loss 2.56\n",
      "| epoch 1 |  iter 1441 / 9295 | time 132[s] | loss 2.49\n",
      "| epoch 1 |  iter 1461 / 9295 | time 134[s] | loss 2.51\n",
      "| epoch 1 |  iter 1481 / 9295 | time 136[s] | loss 2.50\n",
      "| epoch 1 |  iter 1501 / 9295 | time 138[s] | loss 2.52\n",
      "| epoch 1 |  iter 1521 / 9295 | time 140[s] | loss 2.53\n",
      "| epoch 1 |  iter 1541 / 9295 | time 141[s] | loss 2.49\n",
      "| epoch 1 |  iter 1561 / 9295 | time 143[s] | loss 2.53\n",
      "| epoch 1 |  iter 1581 / 9295 | time 145[s] | loss 2.54\n",
      "| epoch 1 |  iter 1601 / 9295 | time 147[s] | loss 2.52\n",
      "| epoch 1 |  iter 1621 / 9295 | time 149[s] | loss 2.50\n",
      "| epoch 1 |  iter 1641 / 9295 | time 151[s] | loss 2.50\n",
      "| epoch 1 |  iter 1661 / 9295 | time 152[s] | loss 2.47\n",
      "| epoch 1 |  iter 1681 / 9295 | time 154[s] | loss 2.52\n",
      "| epoch 1 |  iter 1701 / 9295 | time 156[s] | loss 2.48\n",
      "| epoch 1 |  iter 1721 / 9295 | time 158[s] | loss 2.50\n",
      "| epoch 1 |  iter 1741 / 9295 | time 160[s] | loss 2.49\n",
      "| epoch 1 |  iter 1761 / 9295 | time 162[s] | loss 2.50\n",
      "| epoch 1 |  iter 1781 / 9295 | time 163[s] | loss 2.47\n",
      "| epoch 1 |  iter 1801 / 9295 | time 165[s] | loss 2.52\n",
      "| epoch 1 |  iter 1821 / 9295 | time 167[s] | loss 2.50\n",
      "| epoch 1 |  iter 1841 / 9295 | time 169[s] | loss 2.49\n",
      "| epoch 1 |  iter 1861 / 9295 | time 171[s] | loss 2.47\n",
      "| epoch 1 |  iter 1881 / 9295 | time 173[s] | loss 2.51\n",
      "| epoch 1 |  iter 1901 / 9295 | time 175[s] | loss 2.51\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-9e6cfe114534>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     38\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[1;31m# 开始学习\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontexts\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmax_epoch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\deep-learning-from-scratch-2-nlp-notebook\\common\\trainer.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, x, t, max_epoch, batch_size, max_grad, eval_interval)\u001b[0m\n\u001b[0;32m     38\u001b[0m                 \u001b[1;31m# 勾配を求め、パラメータを更新\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch_x\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbatch_t\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 40\u001b[1;33m                 \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     41\u001b[0m                 \u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrads\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mremove_duplicate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgrads\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# 共有された重みを1つに集約\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     42\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mmax_grad\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-2-6d6311a932b3>\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, dout)\u001b[0m\n\u001b[0;32m     44\u001b[0m         \u001b[0mdout\u001b[0m \u001b[1;33m*=\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0min_layers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mlayer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0min_layers\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m             \u001b[0mlayer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\deep-learning-from-scratch-2-nlp-notebook\\common\\layers.py\u001b[0m in \u001b[0;36mbackward\u001b[1;34m(self, dout)\u001b[0m\n\u001b[0;32m    164\u001b[0m         \u001b[0mdW\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgrads\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    165\u001b[0m         \u001b[0mdW\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m...\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 166\u001b[1;33m         \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdW\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0midx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    167\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "import numpy as np\n",
    "from common import config\n",
    "# 如果使用GPU运行，要取消下面的注释（需要cupy）\n",
    "# ===============================================\n",
    "# config.GPU = True\n",
    "# ===============================================\n",
    "import pickle\n",
    "from common.trainer import Trainer\n",
    "from common.optimizer import Adam\n",
    "#from cbow import CBOW\n",
    "#from skip_gram import SkipGram\n",
    "from common.util import create_contexts_target, to_cpu, to_gpu\n",
    "from dataset import ptb\n",
    "\n",
    "\n",
    "# 超参数设置\n",
    "window_size = 5\n",
    "hidden_size = 100\n",
    "batch_size = 100\n",
    "max_epoch = 10\n",
    "\n",
    "# 读入数据\n",
    "corpus, word_to_id, id_to_word = ptb.load_data('train')\n",
    "vocab_size = len(word_to_id)\n",
    "\n",
    "contexts, target = create_contexts_target(corpus, window_size)\n",
    "if config.GPU:\n",
    "    contexts, target = to_gpu(contexts), to_gpu(target)\n",
    "\n",
    "# 生成模型等\n",
    "model = CBOW(vocab_size, hidden_size, window_size, corpus)\n",
    "# model = SkipGram(vocab_size, hidden_size, window_size, corpus)\n",
    "optimizer = Adam()\n",
    "trainer = Trainer(model, optimizer)\n",
    "\n",
    "# 开始学习\n",
    "trainer.fit(contexts, target, max_epoch, batch_size)\n",
    "trainer.plot()\n",
    "\n",
    "# 为了之后也能使用，保存必要的数据\n",
    "word_vecs = model.word_vecs\n",
    "if config.GPU:\n",
    "    word_vecs = to_cpu(word_vecs)\n",
    "params = {}\n",
    "params['word_vecs'] = word_vecs.astype(np.float16)\n",
    "params['word_to_id'] = word_to_id\n",
    "params['id_to_word'] = id_to_word\n",
    "pkl_file = 'cbow_params.pkl'  # or 'skipgram_params.pkl'\n",
    "with open(pkl_file, 'wb') as f:\n",
    "    pickle.dump(params, f, -1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 评估模型\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch-2/blob/master/ch04/eval.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[query] you\n",
      " we: 0.6103515625\n",
      " someone: 0.59130859375\n",
      " i: 0.55419921875\n",
      " something: 0.48974609375\n",
      " anyone: 0.47314453125\n",
      "\n",
      "[query] year\n",
      " month: 0.71875\n",
      " week: 0.65234375\n",
      " spring: 0.62744140625\n",
      " summer: 0.6259765625\n",
      " decade: 0.603515625\n",
      "\n",
      "[query] car\n",
      " luxury: 0.497314453125\n",
      " arabia: 0.47802734375\n",
      " auto: 0.47119140625\n",
      " disk-drive: 0.450927734375\n",
      " travel: 0.4091796875\n",
      "\n",
      "[query] toyota\n",
      " ford: 0.55078125\n",
      " instrumentation: 0.509765625\n",
      " mazda: 0.49365234375\n",
      " bethlehem: 0.47509765625\n",
      " nissan: 0.474853515625\n",
      "--------------------------------------------------\n",
      "\n",
      "[analogy] king:man = queen:?\n",
      " woman: 5.16015625\n",
      " veto: 4.9296875\n",
      " ounce: 4.69140625\n",
      " earthquake: 4.6328125\n",
      " successor: 4.609375\n",
      "\n",
      "[analogy] take:took = go:?\n",
      " went: 4.55078125\n",
      " points: 4.25\n",
      " began: 4.09375\n",
      " comes: 3.98046875\n",
      " oct.: 3.90625\n",
      "\n",
      "[analogy] car:cars = child:?\n",
      " children: 5.21875\n",
      " average: 4.7265625\n",
      " yield: 4.20703125\n",
      " cattle: 4.1875\n",
      " priced: 4.1796875\n",
      "\n",
      "[analogy] good:better = bad:?\n",
      " more: 6.6484375\n",
      " less: 6.0625\n",
      " rather: 5.21875\n",
      " slower: 4.734375\n",
      " greater: 4.671875\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys\n",
    "sys.path.append('..')\n",
    "from common.util import most_similar, analogy\n",
    "import pickle\n",
    "\n",
    "\n",
    "pkl_file = 'cbow_params.pkl'\n",
    "# pkl_file = 'skipgram_params.pkl'\n",
    "\n",
    "with open(pkl_file, 'rb') as f:\n",
    "    params = pickle.load(f)\n",
    "    word_vecs = params['word_vecs']\n",
    "    word_to_id = params['word_to_id']\n",
    "    id_to_word = params['id_to_word']\n",
    "\n",
    "# most similar task\n",
    "querys = ['you', 'year', 'car', 'toyota']\n",
    "for query in querys:\n",
    "    most_similar(query, word_to_id, id_to_word, word_vecs, top=5)\n",
    "\n",
    "# analogy task\n",
    "print('-'*50)\n",
    "analogy('king', 'man', 'queen',  word_to_id, id_to_word, word_vecs)\n",
    "analogy('take', 'took', 'go',  word_to_id, id_to_word, word_vecs)\n",
    "analogy('car', 'cars', 'child',  word_to_id, id_to_word, word_vecs)\n",
    "analogy('good', 'better', 'bad', word_to_id, id_to_word, word_vecs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}