playground.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import Value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = 2.0\n",
    "b = -3.0\n",
    "c = 10.0 \n",
    "fn = lambda a,b,c: a*b+c \n",
    "print(fn(a,b,c)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://en.wikipedia.org/wiki/Derivative \n",
    "# df/da = (f(a+h)-f(a))/h for infinitessimally small h \n",
    "h = 0.0001 \n",
    "print('slope:', (fn(a+h,b,c) - fn(a,b,c)) / h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Value(2.0, label='a')\n",
    "b = Value(-3.0, label='b')\n",
    "c = Value(10.0, label='c')\n",
    "d = fn(a,b,c); d.label = 'd'\n",
    "print(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d._op, d._prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import trace, draw_dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes, edges = trace(d)\n",
    "print(f'{len(nodes)=} {len(edges)=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_dot(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = Value(2.0, label='a')\n",
    "b = Value(-3.0, label='b')\n",
    "c = Value(10.0, label='c')\n",
    "d = fn(a,b,c); d.label = 'd'\n",
    "f = Value(-2.0, label='f')\n",
    "L = d*f; \n",
    "L.label = 'L'\n",
    "print(L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_dot(L)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward_pass(a,b,c,d,f):\n",
    "    d = fn(a,b,c); d.label = 'd'\n",
    "    L = d*f; \n",
    "    L.label = 'L'\n",
    "    return L\n",
    "L = forward_pass(a,b,c,d,f)\n",
    "Ld = forward_pass(a, b, c, d+h, f)\n",
    "La = forward_pass(a+h, b, c, d, f)\n",
    "Lb = forward_pass(a, b+h, c, d, f)\n",
    "Lc = forward_pass(a, b, c+h, d, f)\n",
    "Lf = forward_pass(a, b, c, d, f+h)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dL_wrt_dx = lambda L2, L1: (L2 - L1) / h\n",
    "print(f'{dL_wrt_dx(L+h, L)=}')\n",
    "print(f'{dL_wrt_dx(Ld, L)=}')\n",
    "print(f'{dL_wrt_dx(Lf, L)=}')\n",
    "print(f'{dL_wrt_dx(La, L)=}')\n",
    "print(f'{dL_wrt_dx(Lb, L)=}')\n",
    "print(f'{dL_wrt_dx(Lc, L)=}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "dL/dL = ((L+h)-L) / h = 1 \n",
    "dL/dd = ((d+h)*f - d*f) / h = f \n",
    "dL/df = d \n",
    "\n",
    "dL/dc = dL/dd * dd/dc = f * d/dc (a*b+c) = f*1 = f = -2\n",
    "\n",
    "let e = a*b\n",
    "dL/de = dL/dd * dd/de = f * d/de (e+c) = f*1 = f = -2 \n",
    "\n",
    "dL/da = dL/dd * dd/de * de/da = f * 1 * d/da (a*b) = f*b = -2*-3 = 6\n",
    "dL/db  = f*a = -4 \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "((d+h)*f - L)/h "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "xs = np.arange(-5,5, 0.2)\n",
    "ys = np.tanh(xs)\n",
    "plt.plot(xs, ys); plt.grid(); "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = Value(2.0, label='x1')\n",
    "x2 = Value(0.0, label='x2')\n",
    "\n",
    "w1 = Value(-3.0, label='w1')\n",
    "w2 = Value(1.0, label='w2')\n",
    "b = Value(6.8813735878195432, label='b')\n",
    "\n",
    "n  = x1*w1 + x2*w2  + b \n",
    "n.label = 'n'\n",
    "o = n.tanh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_dot(o)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "o.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from main import Value \n",
    "from helpers import draw_dot\n",
    "\n",
    "2*Value(2)\n",
    "2+Value(2)\n",
    "Value(2)-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "2-Value(2)\n",
    "Value(4) / Value(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Value(4) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = 4 / Value(2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "draw_dot(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random \n",
    "from main import Value\n",
    "from helpers import draw_dot\n",
    "class Module: \n",
    "    def zero_grad(self): \n",
    "        for p in self.parameters(): \n",
    "            p.grad = 0 \n",
    "\n",
    "    def parameters(self): \n",
    "        return []\n",
    "    \n",
    "    def parameters_grad(self): \n",
    "        return [p.grad for p in self.parameters()]\n",
    "    \n",
    "class Neuron(Module): \n",
    "    def __init__(self, nin): \n",
    "        self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]\n",
    "        self.b = Value(random.uniform(-1,1))\n",
    "\n",
    "    def __repr__(self): \n",
    "        return f'Neuron({len(self.w)})'\n",
    "    \n",
    "    def __call__(self,x): \n",
    "        out = (sum((xi*wi for (wi, xi) in zip(self.w, x)), self.b))\n",
    "        return out.tanh()\n",
    "    \n",
    "    @property\n",
    "    def grad(self): \n",
    "        return ([x.grad for x in self.w], self.b.grad)\n",
    "    \n",
    "    def parameters(self): \n",
    "        return self.w + [self.b]\n",
    "\n",
    "class Layer(Module): \n",
    "    def __init__(self, nin: int, nout: list[int]): \n",
    "        self.neurons = [Neuron(nin) for _ in range(nout)]\n",
    "    \n",
    "    def __call__(self, x): \n",
    "        outs = [n(x) for n in self.neurons]\n",
    "        return outs[0] if len(outs) == 1 else outs \n",
    "    \n",
    "    def parameters(self):\n",
    "        return [p for neuron in self.neurons for p in neuron.parameters()]\n",
    "\n",
    "class MLP(Module): \n",
    "    def __init__(self, nin, nouts): \n",
    "        sz = [nin]+nouts \n",
    "        self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))]\n",
    "    \n",
    "    def __call__(self, x): \n",
    "        for layer in self.layers: \n",
    "            x = layer(x) \n",
    "        return x\n",
    "    \n",
    "    def parameters(self): \n",
    "        return [p for layer in self.layers for p in layer.parameters()]\n",
    "    \n",
    "def mse(ytrue, ypred): \n",
    "    out = sum((yout-ygt)**2 for ygt, yout in zip(ytrue, ypred))\n",
    "    out.label = 'mse_loss'\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n"
     ]
    }
   ],
   "source": [
    "x = [2.0, 3.0, -1.0]\n",
    "model = MLP(3, [4,4,1])\n",
    "model.zero_grad()\n",
    "print(len(model.parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = [\n",
    "    [2.0,3.0,-1.0],\n",
    "    [3.0,-1.0,0.5],\n",
    "    [1.0,1.0,-1.0]\n",
    "]\n",
    "ys = [1.0,-1.0,-1.0,1.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Value(8.0989E-02)\n",
      "0 0.0809891227550811\n",
      "Value(7.9475E-02)\n",
      "1 0.07947466914833257\n",
      "Value(7.8010E-02)\n",
      "2 0.07800990879997755\n",
      "Value(7.6593E-02)\n",
      "3 0.07659257234409468\n",
      "Value(7.5221E-02)\n",
      "4 0.07522052130326581\n",
      "Value(7.3892E-02)\n",
      "5 0.07389173906041406\n",
      "Value(7.2604E-02)\n",
      "6 0.07260432254937296\n",
      "Value(7.1356E-02)\n",
      "7 0.0713564745998019\n",
      "Value(7.0146E-02)\n",
      "8 0.07014649687843598\n",
      "Value(6.8973E-02)\n",
      "9 0.06897278337433158\n"
     ]
    }
   ],
   "source": [
    "for k in range(10): \n",
    "    ypred = [model(x) for x in xs]\n",
    "    model.zero_grad()\n",
    "    loss = mse(ys, ypred)\n",
    "    topos = loss.backward()\n",
    "    lr = 0.01 \n",
    "\n",
    "    for p in model.parameters(): \n",
    "        p.data += -lr * p.grad\n",
    "    print(k, loss.data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Value(7.7487E-01), Value(-9.4917E-01), Value(-8.2886E-01)]"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ypred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08255567969305228"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loss.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "transformer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}