ch05-误差反向传播法.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 乘法层实现\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/layer_naive.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "\n",
    "\n",
    "class MulLayer:\n",
    "    def __init__(self):\n",
    "        self.x = None\n",
    "        self.y = None\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        self.x = x\n",
    "        self.y = y                \n",
    "        out = x * y\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * self.y\n",
    "        dy = dout * self.x\n",
    "\n",
    "        return dx, dy\n",
    "\n",
    "\n",
    "class AddLayer:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def forward(self, x, y):\n",
    "        out = x + y\n",
    "\n",
    "        return out\n",
    "\n",
    "    def backward(self, dout):\n",
    "        dx = dout * 1\n",
    "        dy = dout * 1\n",
    "\n",
    "        return dx, dy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 买苹果\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/buy_apple.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "price: 220\n",
      "dApple: 2.2\n",
      "dApple_num: 110\n",
      "dTax: 200\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "# from layer_naive import *\n",
    "\n",
    "\n",
    "apple = 100\n",
    "apple_num = 2\n",
    "tax = 1.1\n",
    "\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num)\n",
    "price = mul_tax_layer.forward(apple_price, tax)\n",
    "\n",
    "# backward\n",
    "dprice = 1\n",
    "dapple_price, dtax = mul_tax_layer.backward(dprice)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)\n",
    "\n",
    "print(\"price:\", int(price))\n",
    "print(\"dApple:\", dapple)\n",
    "print(\"dApple_num:\", int(dapple_num))\n",
    "print(\"dTax:\", dtax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 买苹果和橘子\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/buy_apple_orange.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "price: 715\n",
      "dApple: 2.2\n",
      "dApple_num: 110\n",
      "dOrange: 3.3000000000000003\n",
      "dOrange_num: 165\n",
      "dTax: 650\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "#from layer_naive import *\n",
    "\n",
    "apple = 100\n",
    "apple_num = 2\n",
    "orange = 150\n",
    "orange_num = 3\n",
    "tax = 1.1\n",
    "\n",
    "# layer\n",
    "mul_apple_layer = MulLayer()\n",
    "mul_orange_layer = MulLayer()\n",
    "add_apple_orange_layer = AddLayer()\n",
    "mul_tax_layer = MulLayer()\n",
    "\n",
    "# forward\n",
    "apple_price = mul_apple_layer.forward(apple, apple_num)  # (1)\n",
    "orange_price = mul_orange_layer.forward(orange, orange_num)  # (2)\n",
    "all_price = add_apple_orange_layer.forward(apple_price, orange_price)  # (3)\n",
    "price = mul_tax_layer.forward(all_price, tax)  # (4)\n",
    "\n",
    "# backward\n",
    "dprice = 1\n",
    "dall_price, dtax = mul_tax_layer.backward(dprice)  # (4)\n",
    "dapple_price, dorange_price = add_apple_orange_layer.backward(dall_price)  # (3)\n",
    "dorange, dorange_num = mul_orange_layer.backward(dorange_price)  # (2)\n",
    "dapple, dapple_num = mul_apple_layer.backward(dapple_price)  # (1)\n",
    "\n",
    "print(\"price:\", int(price))\n",
    "print(\"dApple:\", dapple)\n",
    "print(\"dApple_num:\", int(dapple_num))\n",
    "print(\"dOrange:\", dorange)\n",
    "print(\"dOrange_num:\", int(dorange_num))\n",
    "print(\"dTax:\", dtax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2层神经网络\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/two_layer_net.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)   # 为导入父目录中的文件而进行的设置\n",
    "import numpy as np\n",
    "from common.layers import *\n",
    "from common.gradient import numerical_gradient\n",
    "from collections import OrderedDict\n",
    "\n",
    "\n",
    "class TwoLayerNet:\n",
    "\n",
    "    def __init__(self, input_size, hidden_size, output_size, weight_init_std = 0.01):\n",
    "        # 初始化权重\n",
    "        self.params = {}\n",
    "        self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)\n",
    "        self.params['b1'] = np.zeros(hidden_size)\n",
    "        self.params['W2'] = weight_init_std * np.random.randn(hidden_size, output_size) \n",
    "        self.params['b2'] = np.zeros(output_size)\n",
    "\n",
    "        # 生成层\n",
    "        self.layers = OrderedDict()\n",
    "        self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])\n",
    "        self.layers['Relu1'] = Relu()\n",
    "        self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])\n",
    "\n",
    "        self.lastLayer = SoftmaxWithLoss()\n",
    "        \n",
    "    def predict(self, x):\n",
    "        for layer in self.layers.values():\n",
    "            x = layer.forward(x)\n",
    "        \n",
    "        return x\n",
    "        \n",
    "    # x:输入数据, t:监督数据\n",
    "    def loss(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        return self.lastLayer.forward(y, t)\n",
    "    \n",
    "    def accuracy(self, x, t):\n",
    "        y = self.predict(x)\n",
    "        y = np.argmax(y, axis=1)\n",
    "        if t.ndim != 1 : t = np.argmax(t, axis=1)\n",
    "        \n",
    "        accuracy = np.sum(y == t) / float(x.shape[0])\n",
    "        return accuracy\n",
    "        \n",
    "    # x:输入数据, t:监督数据\n",
    "    def numerical_gradient(self, x, t):\n",
    "        loss_W = lambda W: self.loss(x, t)\n",
    "        \n",
    "        grads = {}\n",
    "        grads['W1'] = numerical_gradient(loss_W, self.params['W1'])\n",
    "        grads['b1'] = numerical_gradient(loss_W, self.params['b1'])\n",
    "        grads['W2'] = numerical_gradient(loss_W, self.params['W2'])\n",
    "        grads['b2'] = numerical_gradient(loss_W, self.params['b2'])\n",
    "        \n",
    "        return grads\n",
    "        \n",
    "    def gradient(self, x, t):\n",
    "        # forward\n",
    "        self.loss(x, t)\n",
    "\n",
    "        # backward\n",
    "        dout = 1\n",
    "        dout = self.lastLayer.backward(dout)\n",
    "        \n",
    "        layers = list(self.layers.values())\n",
    "        layers.reverse()\n",
    "        for layer in layers:\n",
    "            dout = layer.backward(dout)\n",
    "\n",
    "        # 设置\n",
    "        grads = {}\n",
    "        grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db\n",
    "        grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db\n",
    "\n",
    "        return grads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练2层神经网络\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/train_neuralnet.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1353 0.1387\n",
      "0.9069666666666667 0.9085\n",
      "0.9254833333333333 0.9288\n",
      "0.9349833333333334 0.9334\n",
      "0.9458166666666666 0.9426\n",
      "0.95155 0.9492\n",
      "0.95745 0.9521\n",
      "0.9609 0.9553\n",
      "0.9640666666666666 0.9574\n",
      "0.9661 0.9595\n",
      "0.9688833333333333 0.9629\n",
      "0.9714666666666667 0.9632\n",
      "0.9721 0.9625\n",
      "0.9749833333333333 0.9664\n",
      "0.9767333333333333 0.9666\n",
      "0.97865 0.9678\n",
      "0.9793666666666667 0.9682\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)\n",
    "\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "#from two_layer_net import TwoLayerNet\n",
    "\n",
    "# 读入数据\n",
    "(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)\n",
    "\n",
    "iters_num = 10000\n",
    "train_size = x_train.shape[0]\n",
    "batch_size = 100\n",
    "learning_rate = 0.1\n",
    "\n",
    "train_loss_list = []\n",
    "train_acc_list = []\n",
    "test_acc_list = []\n",
    "\n",
    "iter_per_epoch = max(train_size / batch_size, 1)\n",
    "\n",
    "for i in range(iters_num):\n",
    "    batch_mask = np.random.choice(train_size, batch_size)\n",
    "    x_batch = x_train[batch_mask]\n",
    "    t_batch = t_train[batch_mask]\n",
    "    \n",
    "    # 梯度\n",
    "    #grad = network.numerical_gradient(x_batch, t_batch)\n",
    "    grad = network.gradient(x_batch, t_batch)\n",
    "    \n",
    "    # 更新\n",
    "    for key in ('W1', 'b1', 'W2', 'b2'):\n",
    "        network.params[key] -= learning_rate * grad[key]\n",
    "    \n",
    "    loss = network.loss(x_batch, t_batch)\n",
    "    train_loss_list.append(loss)\n",
    "    \n",
    "    if i % iter_per_epoch == 0:\n",
    "        train_acc = network.accuracy(x_train, t_train)\n",
    "        test_acc = network.accuracy(x_test, t_test)\n",
    "        train_acc_list.append(train_acc)\n",
    "        test_acc_list.append(test_acc)\n",
    "        print(train_acc, test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查梯度\n",
    "\n",
    "https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/ch05/gradient_check.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b2:1.4009796528990925e-07\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import sys, os\n",
    "sys.path.append(os.pardir)   # 为导入父目录中的文件而进行的设置\n",
    "import numpy as np\n",
    "from dataset.mnist import load_mnist\n",
    "#from two_layer_net import TwoLayerNet\n",
    "\n",
    "# 读入数据\n",
    "(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)\n",
    "\n",
    "network = TwoLayerNet(input_size=784, hidden_size=50, output_size=10)\n",
    "\n",
    "x_batch = x_train[:3]\n",
    "t_batch = t_train[:3]\n",
    "\n",
    "grad_numerical = network.numerical_gradient(x_batch, t_batch)\n",
    "grad_backprop = network.gradient(x_batch, t_batch)\n",
    "\n",
    "for key in grad_numerical.keys():\n",
    "    diff = np.average( np.abs(grad_backprop[key] - grad_numerical[key]) )\n",
    "print(key + \":\" + str(diff))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}