ch12_tensorflow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 12. custom models and training with tensorflow\n",
    "\n",
    "- Lower-lever Python API\n",
    "\n",
    "- for exact control to write custom\n",
    "    - loss functions\n",
    "    - metrics\n",
    "    - layers\n",
    "    - models\n",
    "    - initializers\n",
    "    - regularizers\n",
    "    - weight constrains\n",
    "    - ...\n",
    "    \n",
    "- fully control training loop, e.g.\n",
    "    - apply special transformations or constraints to the gradients beyond just clippig them\n",
    "    - use multiple optimizers for different parts of the network\n",
    "\n",
    "- automatic graph generation feature"
   ]
  },
  {
   "source": [
    "## A quick tour of TensorFlow\n",
    "\n",
    "- similar to NumPy, with GPU support\n",
    "- support distributed computing\n",
    "- computation graph optimize computations \n",
    "- model is portable across different platform\n",
    "- autodiff and excellent optimizers"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Setup Complete.\n"
     ]
    }
   ],
   "source": [
    "# Python ≥3.5 is required\n",
    "import sys\n",
    "assert sys.version_info >= (3, 5)\n",
    "\n",
    "# Scikit-Learn ≥0.20 is required\n",
    "import sklearn\n",
    "assert sklearn.__version__ >= \"0.20\"\n",
    "\n",
    "try:\n",
    "    # %tensorflow_version only exists in Colab.\n",
    "    %tensorflow_version 2.x\n",
    "    !pip install -U tqdm\n",
    "except Exception:\n",
    "    pass\n",
    "\n",
    "# TensorFlow ≥2.0 is required\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "assert tf.__version__ >= \"2.0\"\n",
    "\n",
    "# Common imports\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "# to make this notebook's output stable across runs\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)\n",
    "\n",
    "# To plot pretty figures\n",
    "%matplotlib inline\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)\n",
    "\n",
    "# Where to save the figures\n",
    "PROJECT_ROOT_DIR = \".\"\n",
    "CHAPTER_ID = \"deep\"\n",
    "IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, \"images\", CHAPTER_ID)\n",
    "os.makedirs(IMAGES_PATH, exist_ok=True)\n",
    "\n",
    "def save_fig(fig_id, tight_layout=True, fig_extension=\"png\", resolution=300):\n",
    "    path = os.path.join(IMAGES_PATH, fig_id + \".\" + fig_extension)\n",
    "    print(\"Saving figure\", fig_id)\n",
    "    if tight_layout:\n",
    "        plt.tight_layout()\n",
    "    plt.savefig(path, format=fig_extension, dpi=resolution)\n",
    "\n",
    "print(\"Setup Complete.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## using tensorflow like numpy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tensors and operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 3
    }
   ],
   "source": [
    "tf_const_eg = tf.constant([[1., 2., 3.], \n",
    "                           [4., 5., 6.]]) # matrix\n",
    "tf_const_eg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "TensorShape([2, 3])"
      ]
     },
     "metadata": {},
     "execution_count": 4
    }
   ],
   "source": [
    "tf_const_eg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "metadata": {},
     "execution_count": 5
    }
   ],
   "source": [
    "tf_const_eg.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=int32, numpy=42>"
      ]
     },
     "metadata": {},
     "execution_count": 6
    }
   ],
   "source": [
    "tf_const_num = tf.constant(42) # scalar\n",
    "tf_const_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "42\n"
     ]
    }
   ],
   "source": [
    "print(tf_const_num.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 2. 3.]\n [4. 5. 6.]]\n"
     ]
    }
   ],
   "source": [
    "print(tf_const_eg.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "print(type(tf_const_eg.numpy()))"
   ]
  },
  {
   "source": [
    "indexing tf is similar to that in NumPy"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 2), dtype=float32, numpy=\n",
       "array([[2., 3.],\n",
       "       [5., 6.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 13
    }
   ],
   "source": [
    "t = tf_const_eg\n",
    "t[:, 1:]"
   ]
  },
  {
   "source": [
    "### ??? **tf.newaxis**"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 1), dtype=float32, numpy=\n",
       "array([[2.],\n",
       "       [5.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 14
    }
   ],
   "source": [
    "t[..., 1, tf.newaxis]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 17
    }
   ],
   "source": [
    "# addition by broadcasting\n",
    "t + 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[11., 12., 13.],\n",
       "       [14., 15., 16.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 18
    }
   ],
   "source": [
    "# adding via .add() function\n",
    "tf.add(t, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 19
    }
   ],
   "source": [
    "# element-wise square\n",
    "tf.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The **@** operator in Python is for **matrix multiplication**, equvalent to calling **tf.matmul()** function. \n",
    "\n",
    "**tf.reduce_mean()** = **np.mean()**\n",
    "\n",
    "**tf.reduce_sum()** = **np.sum()**\n",
    "\n",
    "**tf.reduce_max()** = **np.max()**\n",
    "\n",
    "**tf.math.log()** = **np.log()**"
   ]
  },
  {
   "source": [
    "However,... \n",
    "\n",
    "**tf.transpose(a)** is not the same as **a.T** in numpy, because it creates a new copy of the transposed data"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = tf.constant([[1., 2., 3.], \n",
    "                 [4., 5., 6.]]) # tf matrix\n",
    "\n",
    "n = np.array([[1., 2., 3.], \n",
    "              [4., 5., 6.]]) # np matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(2, 3), dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 23
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]])"
      ]
     },
     "metadata": {},
     "execution_count": 24
    }
   ],
   "source": [
    "n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 4.]\n [2. 5.]\n [3. 6.]]\n"
     ]
    }
   ],
   "source": [
    "print(tf.transpose(t).numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "tf.float32"
      ]
     },
     "metadata": {},
     "execution_count": 31
    }
   ],
   "source": [
    "tf.transpose(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tf.Tensor(\n[[1. 2. 3.]\n [4. 5. 6.]], shape=(2, 3), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "tf.Tensor(\n[[1. 4.]\n [2. 5.]\n [3. 6.]], shape=(3, 2), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(tf.transpose(t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 2. 3.]\n [4. 5. 6.]]\n"
     ]
    }
   ],
   "source": [
    "print(t.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 4.]\n [2. 5.]\n [3. 6.]]\n"
     ]
    }
   ],
   "source": [
    "print(n.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "[[1. 2. 3.]\n [4. 5. 6.]]\n"
     ]
    }
   ],
   "source": [
    "print(n)"
   ]
  },
  {
   "source": [
    "## Keras' low-level API\n",
    "\n",
    "For code to be **portable** to other Keras implementations, **low level Keras functions** should be used, instead of low level TensofFlow functions. "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3, 2), dtype=float32, numpy=\n",
       "array([[11., 26.],\n",
       "       [14., 35.],\n",
       "       [19., 46.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 34
    }
   ],
   "source": [
    "from tensorflow import keras\n",
    "K = keras.backend\n",
    "K.square(K.transpose(t)) + 10"
   ]
  },
  {
   "source": [
    "## Tensors and NumPy"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([2., 4., 5.])>"
      ]
     },
     "metadata": {},
     "execution_count": 35
    }
   ],
   "source": [
    "# create tensor from np array\n",
    "a = np.array([2., 4., 5.])\n",
    "tf.constant(a)"
   ]
  },
  {
   "source": [
    "**_.numpy()_** : convert a **tensor** to **numpy array**"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "metadata": {},
     "execution_count": 38
    }
   ],
   "source": [
    "# convert tensor to np array\n",
    "t.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)"
      ]
     },
     "metadata": {},
     "execution_count": 39
    }
   ],
   "source": [
    "np.array(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(3,), dtype=float64, numpy=array([ 4., 16., 25.])>"
      ]
     },
     "metadata": {},
     "execution_count": 40
    }
   ],
   "source": [
    "# apply tf function on np array\n",
    "tf.square(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "array([[ 1.,  4.,  9.],\n",
       "       [16., 25., 36.]], dtype=float32)"
      ]
     },
     "metadata": {},
     "execution_count": 43
    }
   ],
   "source": [
    "# apply np function on tensor\n",
    "np.square(t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## type conversions\n",
    "\n",
    "- TF doesn't perform type conversion automatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a int32 tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "cannot compute AddV2 as input #1(zero-based) was expected to be a float tensor but is a double tensor [Op:AddV2]\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    tf.constant(2.0) + tf.constant(40., dtype=tf.float64)\n",
    "except tf.errors.InvalidArgumentError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "source": [
    "use **_tf.cast()_** to convert type"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=42.0>"
      ]
     },
     "metadata": {},
     "execution_count": 46
    }
   ],
   "source": [
    "t2 = tf.constant(40., dtype=tf.float64)\n",
    "tf.constant(2.0) + tf.cast(t2, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'hello world'>"
      ]
     },
     "metadata": {},
     "execution_count": 47
    }
   ],
   "source": [
    "tf.constant(b'hello world')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=string, numpy=b'cafe'>"
      ]
     },
     "metadata": {},
     "execution_count": 48
    }
   ],
   "source": [
    "tf.constant('cafe')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(4,), dtype=int32, numpy=array([ 99,  97, 102, 101], dtype=int32)>"
      ]
     },
     "metadata": {},
     "execution_count": 49
    }
   ],
   "source": [
    "u = tf.constant([ord(c) for c in 'cafe'])\n",
    "u"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'Variable:0' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1., 2., 3.],\n",
       "       [4., 5., 6.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 50
    }
   ],
   "source": [
    "v = tf.Variable([[1., 2., 3.], [4., 5., 6.]])\n",
    "v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2.,  4.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 51
    }
   ],
   "source": [
    "v.assign(2*v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2., 42.,  6.],\n",
       "       [ 8., 10., 12.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 52
    }
   ],
   "source": [
    "v[0, 1].assign(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[ 2. , 42. ,  0.1],\n",
       "       [ 8. , 10. ,  1. ]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 53
    }
   ],
   "source": [
    "v[:, 2].assign([0.1, 1.])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "'ResourceVariable' object does not support item assignment\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    v[1] = [7., 8., 9.]\n",
    "except TypeError as ex:\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[1.0e+02, 4.2e+01, 1.0e-01],\n",
       "       [8.0e+00, 1.0e+01, 2.0e+02]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 55
    }
   ],
   "source": [
    "v.scatter_nd_update(\n",
    "    indices=[[0, 0], \n",
    "             [1, 2]],\n",
    "    updates=[100., 200.]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=(2, 3) dtype=float32, numpy=\n",
       "array([[4., 5., 6.],\n",
       "       [1., 2., 3.]], dtype=float32)>"
      ]
     },
     "metadata": {},
     "execution_count": 56
    }
   ],
   "source": [
    "sparse_delta = tf.IndexedSlices(\n",
    "    values=[[1., 2., 3.], [4., 5., 6.]], \n",
    "    indices=[1, 0]\n",
    ")\n",
    "v.scatter_update(sparse_delta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## other data structures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# customizing models & training algorithms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## custom loss functions\n",
    "\n",
    "customize a **Huber loss** function "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets           import fetch_california_housing\n",
    "from sklearn.model_selection    import train_test_split\n",
    "from sklearn.preprocessing      import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "X_train_full, X_test, y_train_full, y_test \\\n",
    "= train_test_split(\n",
    "    housing.data, \n",
    "    housing.target.reshape(-1, 1), \n",
    "    random_state=42\n",
    "    )\n",
    "X_train, X_valid, y_train, y_valid \\\n",
    "= train_test_split(\n",
    "    X_train_full, \n",
    "    y_train_full, \n",
    "    random_state=42\n",
    "    )\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_valid_scaled = scaler.transform(X_valid)\n",
    "X_test_scaled = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "def huber_fn(y_true, y_pred):\n",
    "    error = y_true - y_pred\n",
    "    is_small_error  = tf.abs(error) < 1\n",
    "    squared_loss    = tf.square(error) / 2\n",
    "    linear_loss     = tf.abs(error) - 0.5\n",
    "    return tf.where(is_small_error, squared_loss, linear_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 576x252 with 1 Axes>",
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Created with matplotlib (https://matplotlib.org/) -->\n<svg height=\"257.460937pt\" version=\"1.1\" viewBox=\"0 0 479.2525 257.460937\" width=\"479.2525pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n <defs>\n  <style type=\"text/css\">\n*{stroke-linecap:butt;stroke-linejoin:round;}\n  </style>\n </defs>\n <g id=\"figure_1\">\n  <g id=\"patch_1\">\n   <path d=\"M 0 257.460937 \nL 479.2525 257.460937 \nL 479.2525 0 \nL 0 0 \nz\n\" style=\"fill:none;\"/>\n  </g>\n  <g id=\"axes_1\">\n   <g id=\"patch_2\">\n    <path d=\"M 21.835 214.097812 \nL 468.235 214.097812 \nL 468.235 23.837812 \nL 21.835 23.837812 \nz\n\" style=\"fill:#ffffff;\"/>\n   </g>\n   <g id=\"matplotlib.axis_1\">\n    <g id=\"xtick_1\">\n     <g id=\"line2d_1\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 214.097812 \nL 21.835 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_2\">\n      <defs>\n       <path d=\"M 0 0 \nL 0 3.5 \n\" id=\"m6d01d30c49\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_1\">\n      <!-- −4 -->\n      <defs>\n       <path d=\"M 10.59375 35.5 \nL 73.1875 35.5 \nL 73.1875 27.203125 \nL 10.59375 27.203125 \nz\n\" id=\"DejaVuSans-8722\"/>\n       <path d=\"M 37.796875 64.3125 \nL 12.890625 25.390625 \nL 37.796875 25.390625 \nz\nM 35.203125 72.90625 \nL 47.609375 72.90625 \nL 47.609375 25.390625 \nL 58.015625 25.390625 \nL 58.015625 17.1875 \nL 47.609375 17.1875 \nL 47.609375 0 \nL 37.796875 0 \nL 37.796875 17.1875 \nL 4.890625 17.1875 \nL 4.890625 26.703125 \nz\n\" id=\"DejaVuSans-52\"/>\n      </defs>\n      <g transform=\"translate(12.989688 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_2\">\n     <g id=\"line2d_3\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 77.635 214.097812 \nL 77.635 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_4\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"77.635\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_2\">\n      <!-- −3 -->\n      <defs>\n       <path d=\"M 40.578125 39.3125 \nQ 47.65625 37.796875 51.625 33 \nQ 55.609375 28.21875 55.609375 21.1875 \nQ 55.609375 10.40625 48.1875 4.484375 \nQ 40.765625 -1.421875 27.09375 -1.421875 \nQ 22.515625 -1.421875 17.65625 -0.515625 \nQ 12.796875 0.390625 7.625 2.203125 \nL 7.625 11.71875 \nQ 11.71875 9.328125 16.59375 8.109375 \nQ 21.484375 6.890625 26.8125 6.890625 \nQ 36.078125 6.890625 40.9375 10.546875 \nQ 45.796875 14.203125 45.796875 21.1875 \nQ 45.796875 27.640625 41.28125 31.265625 \nQ 36.765625 34.90625 28.71875 34.90625 \nL 20.21875 34.90625 \nL 20.21875 43.015625 \nL 29.109375 43.015625 \nQ 36.375 43.015625 40.234375 45.921875 \nQ 44.09375 48.828125 44.09375 54.296875 \nQ 44.09375 59.90625 40.109375 62.90625 \nQ 36.140625 65.921875 28.71875 65.921875 \nQ 24.65625 65.921875 20.015625 65.03125 \nQ 15.375 64.15625 9.8125 62.3125 \nL 9.8125 71.09375 \nQ 15.4375 72.65625 20.34375 73.4375 \nQ 25.25 74.21875 29.59375 74.21875 \nQ 40.828125 74.21875 47.359375 69.109375 \nQ 53.90625 64.015625 53.90625 55.328125 \nQ 53.90625 49.265625 50.4375 45.09375 \nQ 46.96875 40.921875 40.578125 39.3125 \nz\n\" id=\"DejaVuSans-51\"/>\n      </defs>\n      <g transform=\"translate(68.789688 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_3\">\n     <g id=\"line2d_5\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 133.435 214.097812 \nL 133.435 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_6\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"133.435\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_3\">\n      <!-- −2 -->\n      <defs>\n       <path d=\"M 19.1875 8.296875 \nL 53.609375 8.296875 \nL 53.609375 0 \nL 7.328125 0 \nL 7.328125 8.296875 \nQ 12.9375 14.109375 22.625 23.890625 \nQ 32.328125 33.6875 34.8125 36.53125 \nQ 39.546875 41.84375 41.421875 45.53125 \nQ 43.3125 49.21875 43.3125 52.78125 \nQ 43.3125 58.59375 39.234375 62.25 \nQ 35.15625 65.921875 28.609375 65.921875 \nQ 23.96875 65.921875 18.8125 64.3125 \nQ 13.671875 62.703125 7.8125 59.421875 \nL 7.8125 69.390625 \nQ 13.765625 71.78125 18.9375 73 \nQ 24.125 74.21875 28.421875 74.21875 \nQ 39.75 74.21875 46.484375 68.546875 \nQ 53.21875 62.890625 53.21875 53.421875 \nQ 53.21875 48.921875 51.53125 44.890625 \nQ 49.859375 40.875 45.40625 35.40625 \nQ 44.1875 33.984375 37.640625 27.21875 \nQ 31.109375 20.453125 19.1875 8.296875 \nz\n\" id=\"DejaVuSans-50\"/>\n      </defs>\n      <g transform=\"translate(124.589687 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_4\">\n     <g id=\"line2d_7\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 189.235 214.097812 \nL 189.235 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_8\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"189.235\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_4\">\n      <!-- −1 -->\n      <defs>\n       <path d=\"M 12.40625 8.296875 \nL 28.515625 8.296875 \nL 28.515625 63.921875 \nL 10.984375 60.40625 \nL 10.984375 69.390625 \nL 28.421875 72.90625 \nL 38.28125 72.90625 \nL 38.28125 8.296875 \nL 54.390625 8.296875 \nL 54.390625 0 \nL 12.40625 0 \nz\n\" id=\"DejaVuSans-49\"/>\n      </defs>\n      <g transform=\"translate(180.389688 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-8722\"/>\n       <use x=\"83.789062\" xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_5\">\n     <g id=\"line2d_9\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 245.035 214.097812 \nL 245.035 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_10\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"245.035\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_5\">\n      <!-- 0 -->\n      <defs>\n       <path d=\"M 31.78125 66.40625 \nQ 24.171875 66.40625 20.328125 58.90625 \nQ 16.5 51.421875 16.5 36.375 \nQ 16.5 21.390625 20.328125 13.890625 \nQ 24.171875 6.390625 31.78125 6.390625 \nQ 39.453125 6.390625 43.28125 13.890625 \nQ 47.125 21.390625 47.125 36.375 \nQ 47.125 51.421875 43.28125 58.90625 \nQ 39.453125 66.40625 31.78125 66.40625 \nz\nM 31.78125 74.21875 \nQ 44.046875 74.21875 50.515625 64.515625 \nQ 56.984375 54.828125 56.984375 36.375 \nQ 56.984375 17.96875 50.515625 8.265625 \nQ 44.046875 -1.421875 31.78125 -1.421875 \nQ 19.53125 -1.421875 13.0625 8.265625 \nQ 6.59375 17.96875 6.59375 36.375 \nQ 6.59375 54.828125 13.0625 64.515625 \nQ 19.53125 74.21875 31.78125 74.21875 \nz\n\" id=\"DejaVuSans-48\"/>\n      </defs>\n      <g transform=\"translate(241.2175 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_6\">\n     <g id=\"line2d_11\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 300.835 214.097812 \nL 300.835 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_12\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"300.835\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_6\">\n      <!-- 1 -->\n      <g transform=\"translate(297.0175 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_7\">\n     <g id=\"line2d_13\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 356.635 214.097812 \nL 356.635 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_14\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"356.635\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_7\">\n      <!-- 2 -->\n      <g transform=\"translate(352.8175 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_8\">\n     <g id=\"line2d_15\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 412.435 214.097812 \nL 412.435 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_16\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"412.435\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_8\">\n      <!-- 3 -->\n      <g transform=\"translate(408.6175 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"xtick_9\">\n     <g id=\"line2d_17\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 468.235 214.097812 \nL 468.235 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_18\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"468.235\" xlink:href=\"#m6d01d30c49\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_9\">\n      <!-- 4 -->\n      <g transform=\"translate(464.4175 230.215937)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"text_10\">\n     <!-- $z$ -->\n     <defs>\n      <path d=\"M 11.625 54.6875 \nL 54.296875 54.6875 \nL 52.6875 46.484375 \nL 11.53125 7.171875 \nL 45.515625 7.171875 \nL 44.09375 0 \nL -0.296875 0 \nL 1.3125 8.203125 \nL 42.484375 47.515625 \nL 10.203125 47.515625 \nz\n\" id=\"DejaVuSans-Oblique-122\"/>\n     </defs>\n     <g transform=\"translate(241.325 247.349375)scale(0.14 -0.14)\">\n      <use transform=\"translate(0 0.3125)\" xlink:href=\"#DejaVuSans-Oblique-122\"/>\n     </g>\n    </g>\n   </g>\n   <g id=\"matplotlib.axis_2\">\n    <g id=\"ytick_1\">\n     <g id=\"line2d_19\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 214.097812 \nL 468.235 214.097812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_20\">\n      <defs>\n       <path d=\"M 0 0 \nL -3.5 0 \n\" id=\"m2334e7858f\" style=\"stroke:#000000;stroke-width:0.8;\"/>\n      </defs>\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m2334e7858f\" y=\"214.097812\"/>\n      </g>\n     </g>\n     <g id=\"text_11\">\n      <!-- 0 -->\n      <g transform=\"translate(7.2 218.656875)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-48\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_2\">\n     <g id=\"line2d_21\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 166.532812 \nL 468.235 166.532812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_22\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m2334e7858f\" y=\"166.532812\"/>\n      </g>\n     </g>\n     <g id=\"text_12\">\n      <!-- 1 -->\n      <g transform=\"translate(7.2 171.091875)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-49\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_3\">\n     <g id=\"line2d_23\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 118.967812 \nL 468.235 118.967812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_24\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m2334e7858f\" y=\"118.967812\"/>\n      </g>\n     </g>\n     <g id=\"text_13\">\n      <!-- 2 -->\n      <g transform=\"translate(7.2 123.526875)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-50\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_4\">\n     <g id=\"line2d_25\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 71.402812 \nL 468.235 71.402812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_26\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m2334e7858f\" y=\"71.402812\"/>\n      </g>\n     </g>\n     <g id=\"text_14\">\n      <!-- 3 -->\n      <g transform=\"translate(7.2 75.961875)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-51\"/>\n      </g>\n     </g>\n    </g>\n    <g id=\"ytick_5\">\n     <g id=\"line2d_27\">\n      <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 23.837812 \nL 468.235 23.837812 \n\" style=\"fill:none;stroke:#b0b0b0;stroke-linecap:square;stroke-width:0.8;\"/>\n     </g>\n     <g id=\"line2d_28\">\n      <g>\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"21.835\" xlink:href=\"#m2334e7858f\" y=\"23.837812\"/>\n      </g>\n     </g>\n     <g id=\"text_15\">\n      <!-- 4 -->\n      <g transform=\"translate(7.2 28.396875)scale(0.12 -0.12)\">\n       <use xlink:href=\"#DejaVuSans-52\"/>\n      </g>\n     </g>\n    </g>\n   </g>\n   <g id=\"line2d_29\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 47.620312 \nL 192.319422 192.871867 \nL 196.805854 196.331052 \nL 201.292286 199.482754 \nL 205.778719 202.326973 \nL 210.265151 204.863709 \nL 214.751583 207.092962 \nL 219.238015 209.014732 \nL 223.724447 210.629018 \nL 228.210879 211.935822 \nL 232.697312 212.935142 \nL 237.183744 213.626979 \nL 241.670176 214.011333 \nL 246.156608 214.088204 \nL 250.64304 213.857591 \nL 255.129472 213.319496 \nL 259.615905 212.473917 \nL 264.102337 211.320855 \nL 268.588769 209.86031 \nL 273.075201 208.092282 \nL 277.561633 206.016771 \nL 282.048065 203.633777 \nL 286.534497 200.943299 \nL 291.02093 197.945339 \nL 295.507362 194.639895 \nL 299.993794 191.026968 \nL 360.560628 139.404031 \nL 468.235 47.620312 \nL 468.235 47.620312 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:2;\"/>\n   </g>\n   <g id=\"line2d_30\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 77.226026 -1 \nL 82.401834 12.071778 \nL 89.131482 28.445255 \nL 95.861131 44.126896 \nL 102.590779 59.116699 \nL 109.320427 73.414665 \nL 116.050075 87.020794 \nL 122.779724 99.935086 \nL 129.509372 112.157541 \nL 133.995804 119.921491 \nL 138.482236 127.377957 \nL 142.968668 134.52694 \nL 147.455101 141.36844 \nL 151.941533 147.902457 \nL 156.427965 154.128991 \nL 160.914397 160.048041 \nL 165.400829 165.659608 \nL 169.887261 170.963693 \nL 174.373693 175.960294 \nL 178.860126 180.649412 \nL 183.346558 185.031047 \nL 187.83299 189.105198 \nL 192.319422 192.871867 \nL 196.805854 196.331052 \nL 201.292286 199.482754 \nL 205.778719 202.326973 \nL 210.265151 204.863709 \nL 214.751583 207.092962 \nL 219.238015 209.014732 \nL 223.724447 210.629018 \nL 228.210879 211.935822 \nL 232.697312 212.935142 \nL 237.183744 213.626979 \nL 241.670176 214.011333 \nL 246.156608 214.088204 \nL 250.64304 213.857591 \nL 255.129472 213.319496 \nL 259.615905 212.473917 \nL 264.102337 211.320855 \nL 268.588769 209.86031 \nL 273.075201 208.092282 \nL 277.561633 206.016771 \nL 282.048065 203.633777 \nL 286.534497 200.943299 \nL 291.02093 197.945339 \nL 295.507362 194.639895 \nL 299.993794 191.026968 \nL 304.480226 187.106558 \nL 308.966658 182.878664 \nL 313.45309 178.343288 \nL 317.939523 173.500429 \nL 322.425955 168.350086 \nL 326.912387 162.89226 \nL 331.398819 157.126951 \nL 335.885251 151.054159 \nL 340.371683 144.673884 \nL 344.858116 137.986125 \nL 349.344548 130.990884 \nL 353.83098 123.688159 \nL 358.317412 116.077951 \nL 362.803844 108.16026 \nL 367.290276 99.935086 \nL 374.019925 87.020794 \nL 380.749573 73.414665 \nL 387.479221 59.116699 \nL 394.208869 44.126896 \nL 400.938518 28.445255 \nL 407.668166 12.071778 \nL 412.843974 -1 \nL 412.843974 -1 \n\" style=\"fill:none;stroke:#0000ff;stroke-dasharray:1,1.65;stroke-dashoffset:0;\"/>\n   </g>\n   <g id=\"line2d_31\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 189.235 214.097812 \nL 189.235 190.315312 \n\" style=\"fill:none;stroke:#ff0000;stroke-dasharray:5.55,2.4;stroke-dashoffset:0;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_32\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 300.835 214.097812 \nL 300.835 190.315312 \n\" style=\"fill:none;stroke:#ff0000;stroke-dasharray:5.55,2.4;stroke-dashoffset:0;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_33\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 21.835 214.097812 \nL 468.235 214.097812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"line2d_34\">\n    <path clip-path=\"url(#p2efeac0d6b)\" d=\"M 245.035 214.097812 \nL 245.035 23.837812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-width:1.5;\"/>\n   </g>\n   <g id=\"patch_3\">\n    <path d=\"M 21.835 214.097812 \nL 21.835 23.837812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_4\">\n    <path d=\"M 468.235 214.097812 \nL 468.235 23.837812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_5\">\n    <path d=\"M 21.835 214.097812 \nL 468.235 214.097812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"patch_6\">\n    <path d=\"M 21.835 23.837812 \nL 468.235 23.837812 \n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\n   </g>\n   <g id=\"text_16\">\n    <!-- Huber loss -->\n    <defs>\n     <path d=\"M 9.8125 72.90625 \nL 19.671875 72.90625 \nL 19.671875 43.015625 \nL 55.515625 43.015625 \nL 55.515625 72.90625 \nL 65.375 72.90625 \nL 65.375 0 \nL 55.515625 0 \nL 55.515625 34.71875 \nL 19.671875 34.71875 \nL 19.671875 0 \nL 9.8125 0 \nz\n\" id=\"DejaVuSans-72\"/>\n     <path d=\"M 8.5 21.578125 \nL 8.5 54.6875 \nL 17.484375 54.6875 \nL 17.484375 21.921875 \nQ 17.484375 14.15625 20.5 10.265625 \nQ 23.53125 6.390625 29.59375 6.390625 \nQ 36.859375 6.390625 41.078125 11.03125 \nQ 45.3125 15.671875 45.3125 23.6875 \nL 45.3125 54.6875 \nL 54.296875 54.6875 \nL 54.296875 0 \nL 45.3125 0 \nL 45.3125 8.40625 \nQ 42.046875 3.421875 37.71875 1 \nQ 33.40625 -1.421875 27.6875 -1.421875 \nQ 18.265625 -1.421875 13.375 4.4375 \nQ 8.5 10.296875 8.5 21.578125 \nz\nM 31.109375 56 \nz\n\" id=\"DejaVuSans-117\"/>\n     <path d=\"M 48.6875 27.296875 \nQ 48.6875 37.203125 44.609375 42.84375 \nQ 40.53125 48.484375 33.40625 48.484375 \nQ 26.265625 48.484375 22.1875 42.84375 \nQ 18.109375 37.203125 18.109375 27.296875 \nQ 18.109375 17.390625 22.1875 11.75 \nQ 26.265625 6.109375 33.40625 6.109375 \nQ 40.53125 6.109375 44.609375 11.75 \nQ 48.6875 17.390625 48.6875 27.296875 \nz\nM 18.109375 46.390625 \nQ 20.953125 51.265625 25.265625 53.625 \nQ 29.59375 56 35.59375 56 \nQ 45.5625 56 51.78125 48.09375 \nQ 58.015625 40.1875 58.015625 27.296875 \nQ 58.015625 14.40625 51.78125 6.484375 \nQ 45.5625 -1.421875 35.59375 -1.421875 \nQ 29.59375 -1.421875 25.265625 0.953125 \nQ 20.953125 3.328125 18.109375 8.203125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nz\n\" id=\"DejaVuSans-98\"/>\n     <path d=\"M 56.203125 29.59375 \nL 56.203125 25.203125 \nL 14.890625 25.203125 \nQ 15.484375 15.921875 20.484375 11.0625 \nQ 25.484375 6.203125 34.421875 6.203125 \nQ 39.59375 6.203125 44.453125 7.46875 \nQ 49.3125 8.734375 54.109375 11.28125 \nL 54.109375 2.78125 \nQ 49.265625 0.734375 44.1875 -0.34375 \nQ 39.109375 -1.421875 33.890625 -1.421875 \nQ 20.796875 -1.421875 13.15625 6.1875 \nQ 5.515625 13.8125 5.515625 26.8125 \nQ 5.515625 40.234375 12.765625 48.109375 \nQ 20.015625 56 32.328125 56 \nQ 43.359375 56 49.78125 48.890625 \nQ 56.203125 41.796875 56.203125 29.59375 \nz\nM 47.21875 32.234375 \nQ 47.125 39.59375 43.09375 43.984375 \nQ 39.0625 48.390625 32.421875 48.390625 \nQ 24.90625 48.390625 20.390625 44.140625 \nQ 15.875 39.890625 15.1875 32.171875 \nz\n\" id=\"DejaVuSans-101\"/>\n     <path d=\"M 41.109375 46.296875 \nQ 39.59375 47.171875 37.8125 47.578125 \nQ 36.03125 48 33.890625 48 \nQ 26.265625 48 22.1875 43.046875 \nQ 18.109375 38.09375 18.109375 28.8125 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 54.6875 \nL 18.109375 54.6875 \nL 18.109375 46.1875 \nQ 20.953125 51.171875 25.484375 53.578125 \nQ 30.03125 56 36.53125 56 \nQ 37.453125 56 38.578125 55.875 \nQ 39.703125 55.765625 41.0625 55.515625 \nz\n\" id=\"DejaVuSans-114\"/>\n     <path id=\"DejaVuSans-32\"/>\n     <path d=\"M 9.421875 75.984375 \nL 18.40625 75.984375 \nL 18.40625 0 \nL 9.421875 0 \nz\n\" id=\"DejaVuSans-108\"/>\n     <path d=\"M 30.609375 48.390625 \nQ 23.390625 48.390625 19.1875 42.75 \nQ 14.984375 37.109375 14.984375 27.296875 \nQ 14.984375 17.484375 19.15625 11.84375 \nQ 23.34375 6.203125 30.609375 6.203125 \nQ 37.796875 6.203125 41.984375 11.859375 \nQ 46.1875 17.53125 46.1875 27.296875 \nQ 46.1875 37.015625 41.984375 42.703125 \nQ 37.796875 48.390625 30.609375 48.390625 \nz\nM 30.609375 56 \nQ 42.328125 56 49.015625 48.375 \nQ 55.71875 40.765625 55.71875 27.296875 \nQ 55.71875 13.875 49.015625 6.21875 \nQ 42.328125 -1.421875 30.609375 -1.421875 \nQ 18.84375 -1.421875 12.171875 6.21875 \nQ 5.515625 13.875 5.515625 27.296875 \nQ 5.515625 40.765625 12.171875 48.375 \nQ 18.84375 56 30.609375 56 \nz\n\" id=\"DejaVuSans-111\"/>\n     <path d=\"M 44.28125 53.078125 \nL 44.28125 44.578125 \nQ 40.484375 46.53125 36.375 47.5 \nQ 32.28125 48.484375 27.875 48.484375 \nQ 21.1875 48.484375 17.84375 46.4375 \nQ 14.5 44.390625 14.5 40.28125 \nQ 14.5 37.15625 16.890625 35.375 \nQ 19.28125 33.59375 26.515625 31.984375 \nL 29.59375 31.296875 \nQ 39.15625 29.25 43.1875 25.515625 \nQ 47.21875 21.78125 47.21875 15.09375 \nQ 47.21875 7.46875 41.1875 3.015625 \nQ 35.15625 -1.421875 24.609375 -1.421875 \nQ 20.21875 -1.421875 15.453125 -0.5625 \nQ 10.6875 0.296875 5.421875 2 \nL 5.421875 11.28125 \nQ 10.40625 8.6875 15.234375 7.390625 \nQ 20.0625 6.109375 24.8125 6.109375 \nQ 31.15625 6.109375 34.5625 8.28125 \nQ 37.984375 10.453125 37.984375 14.40625 \nQ 37.984375 18.0625 35.515625 20.015625 \nQ 33.0625 21.96875 24.703125 23.78125 \nL 21.578125 24.515625 \nQ 13.234375 26.265625 9.515625 29.90625 \nQ 5.8125 33.546875 5.8125 39.890625 \nQ 5.8125 47.609375 11.28125 51.796875 \nQ 16.75 56 26.8125 56 \nQ 31.78125 56 36.171875 55.265625 \nQ 40.578125 54.546875 44.28125 53.078125 \nz\n\" id=\"DejaVuSans-115\"/>\n    </defs>\n    <g transform=\"translate(207.960156 17.837812)scale(0.14 -0.14)\">\n     <use xlink:href=\"#DejaVuSans-72\"/>\n     <use x=\"75.195312\" xlink:href=\"#DejaVuSans-117\"/>\n     <use x=\"138.574219\" xlink:href=\"#DejaVuSans-98\"/>\n     <use x=\"202.050781\" xlink:href=\"#DejaVuSans-101\"/>\n     <use x=\"263.574219\" xlink:href=\"#DejaVuSans-114\"/>\n     <use x=\"304.6875\" xlink:href=\"#DejaVuSans-32\"/>\n     <use x=\"336.474609\" xlink:href=\"#DejaVuSans-108\"/>\n     <use x=\"364.257812\" xlink:href=\"#DejaVuSans-111\"/>\n     <use x=\"425.439453\" xlink:href=\"#DejaVuSans-115\"/>\n     <use x=\"477.539062\" xlink:href=\"#DejaVuSans-115\"/>\n    </g>\n   </g>\n   <g id=\"legend_1\">\n    <g id=\"patch_7\">\n     <path d=\"M 31.635 207.097812 \nL 135.795 207.097812 \nQ 138.595 207.097812 138.595 204.297812 \nL 138.595 159.248437 \nQ 138.595 156.448437 135.795 156.448437 \nL 31.635 156.448437 \nQ 28.835 156.448437 28.835 159.248437 \nL 28.835 204.297812 \nQ 28.835 207.097812 31.635 207.097812 \nz\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\n    </g>\n    <g id=\"line2d_35\">\n     <path d=\"M 34.435 167.78625 \nL 62.435 167.78625 \n\" style=\"fill:none;stroke:#0000ff;stroke-linecap:square;stroke-width:2;\"/>\n    </g>\n    <g id=\"line2d_36\"/>\n    <g id=\"text_17\">\n     <!-- huber($z$) -->\n     <defs>\n      <path d=\"M 54.890625 33.015625 \nL 54.890625 0 \nL 45.90625 0 \nL 45.90625 32.71875 \nQ 45.90625 40.484375 42.875 44.328125 \nQ 39.84375 48.1875 33.796875 48.1875 \nQ 26.515625 48.1875 22.3125 43.546875 \nQ 18.109375 38.921875 18.109375 30.90625 \nL 18.109375 0 \nL 9.078125 0 \nL 9.078125 75.984375 \nL 18.109375 75.984375 \nL 18.109375 46.1875 \nQ 21.34375 51.125 25.703125 53.5625 \nQ 30.078125 56 35.796875 56 \nQ 45.21875 56 50.046875 50.171875 \nQ 54.890625 44.34375 54.890625 33.015625 \nz\n\" id=\"DejaVuSans-104\"/>\n      <path d=\"M 31 75.875 \nQ 24.46875 64.65625 21.28125 53.65625 \nQ 18.109375 42.671875 18.109375 31.390625 \nQ 18.109375 20.125 21.3125 9.0625 \nQ 24.515625 -2 31 -13.1875 \nL 23.1875 -13.1875 \nQ 15.875 -1.703125 12.234375 9.375 \nQ 8.59375 20.453125 8.59375 31.390625 \nQ 8.59375 42.28125 12.203125 53.3125 \nQ 15.828125 64.359375 23.1875 75.875 \nz\n\" id=\"DejaVuSans-40\"/>\n      <path d=\"M 8.015625 75.875 \nL 15.828125 75.875 \nQ 23.140625 64.359375 26.78125 53.3125 \nQ 30.421875 42.28125 30.421875 31.390625 \nQ 30.421875 20.453125 26.78125 9.375 \nQ 23.140625 -1.703125 15.828125 -13.1875 \nL 8.015625 -13.1875 \nQ 14.5 -2 17.703125 9.0625 \nQ 20.90625 20.125 20.90625 31.390625 \nQ 20.90625 42.671875 17.703125 53.65625 \nQ 14.5 64.65625 8.015625 75.875 \nz\n\" id=\"DejaVuSans-41\"/>\n     </defs>\n     <g transform=\"translate(73.635 172.68625)scale(0.14 -0.14)\">\n      <use transform=\"translate(0 0.015625)\" xlink:href=\"#DejaVuSans-104\"/>\n      <use transform=\"translate(63.378906 0.015625)\" xlink:href=\"#DejaVuSans-117\"/>\n      <use transform=\"translate(126.757812 0.015625)\" xlink:href=\"#DejaVuSans-98\"/>\n      <use transform=\"translate(190.234375 0.015625)\" xlink:href=\"#DejaVuSans-101\"/>\n      <use transform=\"translate(251.757812 0.015625)\" xlink:href=\"#DejaVuSans-114\"/>\n      <use transform=\"translate(292.871094 0.015625)\" xlink:href=\"#DejaVuSans-40\"/>\n      <use transform=\"translate(331.884766 0.015625)\" xlink:href=\"#DejaVuSans-Oblique-122\"/>\n      <use transform=\"translate(384.375 0.015625)\" xlink:href=\"#DejaVuSans-41\"/>\n     </g>\n    </g>\n    <g id=\"line2d_37\">\n     <path d=\"M 34.435 190.997812 \nL 62.435 190.997812 \n\" style=\"fill:none;stroke:#0000ff;stroke-dasharray:1,1.65;stroke-dashoffset:0;\"/>\n    </g>\n    <g id=\"line2d_38\"/>\n    <g id=\"text_18\">\n     <!-- $\\frac{1}{2}z^2$ -->\n     <g transform=\"translate(73.635 195.897812)scale(0.14 -0.14)\">\n      <use transform=\"translate(0 43.965625)scale(0.7)\" xlink:href=\"#DejaVuSans-49\"/>\n      <use transform=\"translate(0 -39.2375)scale(0.7)\" xlink:href=\"#DejaVuSans-50\"/>\n      <use transform=\"translate(57.036133 0.16875)\" xlink:href=\"#DejaVuSans-Oblique-122\"/>\n      <use transform=\"translate(113.992513 38.45)scale(0.7)\" xlink:href=\"#DejaVuSans-50\"/>\n      <path d=\"M 0 18.965625 \nL 0 25.215625 \nL 44.536133 25.215625 \nL 44.536133 18.965625 \nL 0 18.965625 \nz\n\"/>\n     </g>\n    </g>\n   </g>\n  </g>\n </g>\n <defs>\n  <clipPath id=\"p2efeac0d6b\">\n   <rect height=\"190.26\" width=\"446.4\" x=\"21.835\" y=\"23.837812\"/>\n  </clipPath>\n </defs>\n</svg>\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd8AAAEDCAYAAAB0/A4MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZyN5fvA8c89Y2KGsWXfIhkiWypJMomoJNVXe6hEql+WVJTWLxVSIkQbUpZKElkiI2QpxVcpsu+7MTNmMcv9++OaYSxjzsycc56zXO/X67xm5swz57meOTPnOs9zX/d1G2stSimllPKeEKcDUEoppYKNJl+llFLKyzT5KqWUUl6myVcppZTyMk2+SimllJdp8lVKKaW8TJOvUj7EGBNtjLHGmDJe2l9XY0yCN/allDpNk69SBWSMmWCMmX2e+6/KTKTVvR+VUsqXafJVKggYYy5yOgal1GmafJXykvNdUjbGVM+876qzNr/WGLPWGJNsjFljjGly1mNdZ4xZYoxJNMbsMcaMNcYUz/b9mMz73jHGHAKW5yHOHsaYzcaYk5kfHz/P9zdlxnbYGDPfGFMo83v1jTGLjDFxxpgEY8w6Y8yNefk9KRUMNPkq5ZveAV4ArgK2ArONMREgCQ5YAMwCGgJ3AY2AT896jIcAA7QAOruyU2PMncAHwAjgCuB9YIwx5vbM718FjAZeB2oDNwHzsj3El8A+4JrMmF4Dkl0+aqWCRCGnA1AqQLQ7T+FSQd7c/tdaOx/AGPMIsBt4APgYeA6YZq0dnrWxMaYn8Icxppy19mDm3dustc/mcb/9gM+ttR9kfr0p86z7BeB7oBpwAphlrY0HdgDrsv38JcA71tp/Mr/enMf9KxUU9MxXKff4GTnTy357oACPtyLrE2ttArAeqJt5VxPgoczLugmZST/rsnLNbI+xJh/7vZxzL1Evy7bvH5GEu80Y84UxposxJjLbtu8CHxtjfjLGvGSMqZOPGJQKeJp8lXKPRGvt5uw35Gw1u4zMjybbfWH52FcIcgacPdE3BGoBa7NtdyIfj50TC5B5tnslcA+wExgA/GOMqZT5/deQRD0TuA74nzHmUTfGoVRA0OSrlPccyvxYMdt9jXLY9tqsT4wxRZHx178z7/odqHd2ss+8JRUwxr+B5mfddz2wIesLa22atfYna+0AoAFQFGif7fv/WmtHWmtvAz4BuhUwJqUCjo75KuU9m4FdwGvGmP5AdWBgDtsOzKxS3gu8ApxEipkAhgArjTEfAuOAeKAOcLu1tkcBYxwGfGWMWYMUdbUDHkSKujDGtEcubf8MHAVuBCKBv40x4Uih2FfAdqA8krhXFTAmpQKOJl+lvMRam2qMuQ8YgxQprQVeBM5p0AH0B4YjFcV/Ae2ttScyH+d/xpgbgEHAEiAUqYj+1g0xzjTG/B9SeDUCGd990lr7feYmsUBH5A1BBLAF6GatXZo5l7gUMAE5uz+SeWz9ChqXUoHGWGudjkEppZQKKjrmq5RSSnlZnpKvMaZWZlebyZ4KSCmllAp0eT3zHQ386olAlFJKqWDhcvLNLBSJBRZ5LhyllFIq8LmUfDMbtr8B9PVsOEoppVTgc3Wq0X+BT6y1u40xOW5kjOkOdAcoUqRIk2rVqhU8Qh+VkZFBSEjO712shSNHClOmTIoXo3KP3I7N3wXy8e3atQtrLcH8v+fv/PH4kpNDKVw4A2MuPHvGH48tLzZt2nTYWlvWlW1zTb7GmEZAa6Bxbttaa8cD4wFq165tN27c6EoMfikmJobo6Ohct0tMhIgIz8fjTq4em78K5OOLjo4mNjaWtWvX5r6xnwrk5w/87/hWroR69SAyMvdt/e3Y8soYs8PVbV15CxKNdOLZaYzZj0yYv9sY83u+ogsix45BgwaQluZ0JEop5RkTJ8LevU5H4X9cuew8Hpia7et+SDLu6YmAAkmpUvDnn1BI+4gppQLU2LFOR+Cfcj3ztdYmWmv3Z92ABCDZWnsot59VkJoKL74oY8BKKRVI7rgD/vrL6Sj8U57PyTKXDFMuKlYMqlaVS89h+Vk8TimlfNR770EA1/Z5VOCWnfkIY6BnT9i/3+lIlFLKfb75BkqW1GG1/NLk6wVJSXDbbfJRKaUCwapVOpxWEPqexQvCw2HdOjkLVkopf5eaCkOHOh2Ff9MzXy9JT4d775V5v0op5a+shcaNYc8epyPxb3rm6yWFCkG3bjo+opTyb8bAL79A8eJOR+Lf9MzXi1q3htWrdZxEKeW/3npLX8PcQZOvl733HhzSGdJKKT+Ung4XXQRFizodif/Ti6BeZIyU5yullD86fBiefdbpKAKDnvl6mbXQpg3s3Ol0JEop5bqUFGjZEhISnI4kMOiZr5cZA6NHQ5UqTkeilFKuK1wYNmyAAF4R0Kv01+iAqCj46iuddqSU8g8nT0KXLjK/V7mHJl+HbNgABw86HYVSSrnmrrvk7Fe5h152dsjrr8tiC9Zq5yullG9bvx46dHA6isCiZ74Ouu02WLPG6SiUUipnR4/KsqgZGU5HElj0zNdB06bJqiBKKeWrSpeG+fOdjiLw6Jmvg0qWhI8/ho0bnY5EKaXOtWsX3HKLdrTyBI8l39hYXTneFaVKaem+Uso3VaoE77yjdSmu+OyzvG3vsZf9gweL0KuXtCNTObv7bqhQAeLinI5EKaVOO34cZs2CevWcjsS3ZWTAgAHw6KN5+zmPnnONHAl33AHx8Z7ci/8bOBDmzXM6CqWUOu3AAZkSqXKWmChLxb79NoSG5u1nPZZ8q1ZNpHRpmDMHWrSQsQN1fiNGwD33OB2FUkqJ9HSoWRNeesnpSHzX/v1w443w9deyvOIPP+Tt5z2WfMPD01m1Sro5rVsHTZvqtJqcGAOTJsEXXzgdiVJKwYIF0Lmz01H4rj//lJy2ejVUry7rG998c94ew6OXnS+7DFasgOho2LdPzoBnzvTkHv3X1VfDddc5HYVSSkmF89ixTkfhm+bNk9fqnTvh2mth5cr8jYt7vM42a45Y166QlCQtyt55R0vXz3b55acblyullFNWrZIeBMWLOx2J7xkzRpojxcfLUOFPP0H58vl7LK9McrnoIvj0U3jzTUm6zz0HPXpok+6zLV8OMTFOR6GUCmYRETIFUp2Wng59+sBTT0l188CBMGUKhIfn/zG91uHKGCnHvuwyGUv46CPYulUGq7XLk+jUST5qv2ellBMOH5ZCq/r1nY7EdyQkwP33w+zZEBYmuatLl4I/rtfbO3TqJGd35crBokXQrJkkYSV++AF69nQ6CqVUMPrqK3jvPaej8B27d0ut0uzZcjXgxx/dk3jBod7OTZvKuEL79vDXX/L1d99pwRHIE33ttU5HoZQKRj17aj1Olt9/h9tvh717oVYtScBRUe57fMcaG1avLmOcbdvKpY5WreQaerCLjJTLHN9843QkSqlgMnKknATpkJf8Hlq0kMR7ww0ya8ediRccXlihRAl5N9GzJ6SkwAMPwBtv6Duv9HS53KGUUt7Srh00auR0FM6yFt59F+68U7pXde4sc54vvtj9+3K8pX+hQjB6tIwzGAOvvioHnJLidGTOqVEDevWS3qpKKeVpq1fLmOYllzgdiXNSU+VE8NlnJQkPGgQTJsgUUE9wPPmCJN3eveVUv2hRmDwZWreWy9HBats2uRQf7FcBlFKeN38+/Puv01E45/hxqUEaN06S7dSp0lrTk5fgfSL5Zrn9dli6FCpXhmXLpPAoWNe6rVFDxhl0/EUp5UkZGfDyy8Fb8Lp9uxz7ggVQtiwsXiyLJXiaTyVfgMaNpRK6cWPYskUS8OLFTkfljJMn4fHHdVlGpZTn3HSTzDoJRitXymybDRugbl3JPc2aeWffPpd8Qc58ly6FDh0gNlYaVud1oeJAULSoFEFkZDgdiVIqUE2bJokn2EyfLqsSHTwIbdrI7JsaNby3f59MviCJZ8YMGfxOS5OFigcMCK5EZIz0wl6zRsd+lVLuN3SojHEG0/CWtdLq+N57ITkZuneXpW+93WnRZ5MvyOLE77wDH34on7/9tvzCEhOdjsy7hg6VtSOVUspd0tIkERUt6nQk3nPyJDzyyOliquHDJb+EhXk/Fp9Ovll69IC5c2WVja+/lksFwZKMjJErABUqOB2JUiqQHDwIL7wg0z2DwdGjMoQ5caIsHjFjBvTt69xZv18kX5Br8itWSGes1atlkHz9eqej8p577oHffnM6CqVUIDh6VNbsTUtzOhLv+PdfKd5dsgQqVoSff4aOHZ2NyaXka4yZbIzZZ4yJM8ZsMsZ083Rg51O3rlSnXXutLGTcvLksbBwMhg2DK690OgqlVCAoXRrWrg2Os96ff5ac8e+/0LChnLw1aeJ0VK6f+b4FVLfWFgc6AIOMMY6EX768LGB8772yoPFtt8kCx4GuenVZ8ShY5z0rpdxj61bo1i04iqwmTZKGTUePShONZcugShWnoxIuJV9r7V/W2qyGjzbzVtNjUeUiPBy+/FIWNM7IkAWOe/cO/Pmwx4/LGw6llMqvChXgscecjsKzshqHdOkibSN79YKZM6FYMacjO81YF+ewGGPGAF2BcOAP4AZrbcJZ23QHugOULVu2yfTp090a7PnMn1+ed96pTVpaCM2aHWbgwL+JiPB8Fk5ISKCYA89kerohJSXEo8fo1LF5SyAfX+/evUlPT2fUqFFOh+Ixgfz8gWeP79ixMPbtC6du3TiPPH5uvPHcnTwZwttv12Hx4nKEhFiefvpf7rxzr0f3meXGG29cY629yqWNrbUu34BQ4HpgIBB2oW2joqKstyxZYm3p0taCtQ0bWrtrl+f3uXjxYs/v5DwGDbJ2+HDP7sOpY/OWQD6+li1b2oYNGzodhkcF8vNnrWePb8UKeQ1xiqefuwMHrG3WTHJBZKS1c+d6dHfnAH6zLubTPFU7W2vTrbXLgCpAzzy9JfCgG26QQqxatWDdOrjmGmlMEYief17K45VSKi/S0qTw6KWXnI7EMzZskFkwK1ZAtWrSsapdO6ejyll+pxoVwsEx3/OpVUsScMuWsG+fJOSZM52Oyv3CwuSPasgQpyNRSvmT99+XRkWB6McfpSfz9u1w9dXSo7l+faejurBck68xppwx5j5jTDFjTKgxpi1wP7DI8+HlTenSsjJFly7SBeuuu6SDSaC1Zrz0UoiOdjoKpZQ/6dVL1qsNNB99JHOW4+Lg7rshJsY/mhK5cuZrkUvMu4FjwDtAb2vtLE8Gll8XXSSLMAweLEm3Xz944gmpeAsUFSvKu7qlS52ORCnlDyZOhP/9D0qUcDoS90lPh+eek97M6enQv78slhAR4XRkrsl1irW19hDQ0guxuI0x8OKLcNllchY8frzMbfvqK+83z/aUI0fg00+hRQunI1FK+brSpQMr8Z44AQ89JEOLhQpJf2Z/mz7lN+0l8+Oee2Qt4HLlYOFCWTB52zano3KPqlXlDD/Q5zYrpQpm40ZpRlTTp6p08m/vXqntmTlTTqbmz/e/xAsBnnxBqvtWrYJ69eDvv09XwwWCpCS44orgW+VJKeW6Z5+FLVucjsI91q2T1/A1a6T2ZcUKaNXK6ajyJ+CTL0hrxuXLZUWLQ4dkVaSpU52OquDCw2Xc11/GOJRS3jd7tswG8Xdz5kg//9275eOqVVCnjtNR5V9QJF+Q8Y45c6T4KiUF7r8fBg3y/0roMmXkOI4edToSpZQvSUyE668PjCtjI0dChw4y1vvAAzKMWKaM01EVTNAkX5CB+TFj4L33pCgrq/dnSkruP+vLqlULnqXBlFKuiYiQYlN/vjKWlgZPPy3TpDIy4LXXYPJkKFLE6cgKLqiSL0jS7d1bBuuLFoXPP5e1go8ccTqy/OvcWf4wjx93OhKllC84cQI++USWYfVXcXFytjt6tEwhnTwZXn01cFZjCrrkm6VDBxkvrVRJPl57rX8v1/fmm/DLL05HoZTyBUePwuHDTkeRfzt3yiXzuXPl8vKiRfDgg05H5V5Bm3wBGjeWhZUbN4bNm6U9WUyM01Hlz8iR0uVFKRXckpPh4ovhhRecjiR/fv1V+vOvXw+1a0vb4Ouvdzoq9wvq5AtQuTL8/LOcCR87JhXRn33mdFT5M2kSDB3qdBRKKSctWCDjpP7om29kDu+BAzIrZcWKwJmffLagT74gCyzPmCGrBaWmwqOPSoesjAynI8ub1q0ldqVU8OrQAcaNczqKvLFWThz+8x/pX/DYYzBvHpQq5XRknqPJN1NoqCzCMHasfP7WW3DvvfKH4C8qVZLK7a++cjoSpZQTRo6EWbNk9TN/cfIkPP746cvkQ4bIYgkXXeRsXJ6Wa2/nYPPEE9I5pVMn+PprGfifNQvKl3c6MtdkZMiyWkqp4NO2rX9Nwzl2TM52f/pJ4p48WVYmCgZ65nseN98slcOXXCIFWU2bwp9/Oh2Va6pWlZU+DhxwOhKllDfNny8nCZdc4nQkrtmyRYpcf/pJ4l6yJHgSL2jyzVG9etK+rGlT2LFDFmWYP9/pqFyTkAA33SRVj0qp4LBggcyN9QfLl5+e3lm/vpzkXHON01F5lybfCyhfXlZFuuceiI+XlUHGjnU6qtwVKyZrd/rT5SelVP6dOCE1K9WqOR1J7r78UhZDOHwY2rWDZcv8I2530+Sbi/BwmDIFXnpJlu978kno08f3l/ILCYGuXf27cYhSKnfx8dCoke9f6bIWJk68hAcflCKrp56C77+H4sWdjswZmnxdEBIiixdMmCBVhCNGwCuvXEFCgtORXdjTT8uKTkqpwBUZKUvt+fKVrpQUePhhmDChBiEh8P778MEH0m8/WGnyzYMuXeDHH2Xu2S+/lKFFC1neyldddZWse/n3305HopTyhE2boH9/31484fBh6UHwxRdQpEg6330HzzzjdFTO0+SbRy1bSruzKlUSWbtWCrJ+/93pqHK2dSvs3et0FEopTyhTRmZn+Kp//pHXyGXLoEoVGDXqD9q3dzoq36DJNx+iouCDD37nhhsksbVoAd9953RU5/fQQ1LcoCseKRVYNm+W1dhatXI6kvP76SeZSrR1K1x5pcweuewyHx+r8yJNvvlUokQaCxbIpejERLjzTnj3XSkq8DUzZ8KzzzodhVLKndau9d2FYD79VBp+xMbCHXdI//xKlZyOyrcE8XB3wRUuLIsw1KoFAwdKgtu0CUaN8q32bh06yE0pFRhOnpTOUL4mI0P64g8ZIl/36wdvvy0te9WZ9My3gIyRaUjTpkkyHjdO5gP70mXe0FApenjgAf9bLEIpda777/e9s97EROmJMGSIvOaMGwfDhmnizYkmXze55x75ZyhbViqir7sOtm1zOqrTypWDbt3kzYJSyr9NnCi1Jr5i/36IjpYlAYsXh7lzoXt3p6PybZp83ejaa6WooG5d2LBBqvxWrHA6KmGMFGZMn+5fKzUppU5LS4MePeRzXzmjXL9eXut+/VX6CqxYAW3aOB2V79Pk62Y1asiiDG3awKFDsiD0tGlOR3Xan3/qogtK+bM2baBoUaejEPPmQfPmsvpb9pMPlTtNvh5QogTMmSPvUFNS4L77pEOWL1RC//e/UnUYH+90JEqpvDhxQt7Y/+c/vjF8NGaM1LfEx8va5z/9JMNbyjWafD0kLEwWYRg+XP5RXn5Zei2npDgdmSTgyZOdjkIplRdbt0ovZKelp0Pv3tKbOSNDZnp8+aX0wVeu06lGHmQM9O0LNWtKpfGkSbLQ/YwZcPHFzsX16qvB3VNVKX+TnAxXXCHVw05KSJBK69mz5QTj44+hc2dnY/JXeubrBXfcAUuXyuXen3+WsZFNm5yLp1AhaYn51FPOxaCUct0778iCLk7avVsqrGfPhtKlYeFCTbwFocnXS7LaqzVqJG3hmjWDJUuci6dOHZl6pJTyfQMGnK5ydsKaNbLY/dq10lRo5Uq44Qbn4gkEmny9qEoVOQO+/XY4elSqFidOdCaWiAioV08uG/lCIZhS6vyeeQZ27HBu5aLvvpNEu2+ffFyxQhKwKhhNvl5WrBh8+y306QOpqVKENXCgM52nQkPl8ndiovf3rZRyza23yht3b7NWCkbvvFNeI7KWVHWyXiWQaPJ1QGioLMIwZox8PniwFDF4u/lFaCgMHSr7PXnSu/tWSl1YWpo0xWnbFi66yLv7Tk2Fnj2lN7O1MlXys8+8H0cg0+TroJ49ZT5wZKT8k914ozMNMHr3huXLvb9fpVTODh2SsVVvO35c5u+OGyf96qdNk/71vjC3OJBo8nVY27Yycf6SS6Qgq2lT+Osv78YwaZIkfqWUb0hIgFKl5AqZN5Petm3Sl/7HH6VPfUyM9K1X7qfJ1wdcccXpxLtjh/zxz5/vvf2HhMgZeN++3tunUipn06fLfHxvWrFCXoM2bJAWkatWybRI5Rm5Jl9jTGFjzCfGmB3GmHhjzFpjzC3eCC6YlC8PixdDp04QFyeXfT780Hv7v/56Gd9RSjnv0UdlnNVbpk2Tq1+HDsksjF9+kT71ynNcOfMtBOwCWgIlgIHAdGNMdc+FFZzCw2HqVFmMOj1dxoT79pXPPa1ECRl7fvNNnXqklJOGDq3N2rXSQcrTrJWCz/vuk9a3PXrIVbASJTy/72CXa/K11p6w1r5mrd1urc2w1s4GtgFNPB9e8AkJkX+Gzz6Tf7733oO77pIxIE+LiJACi7Q0raxQyil3373bKysDpaScnupojEwrGjvWO0lf5WPM1xhTHogCvFwWFFy6doUFC6ToYtYsaeu2e7dn9xkaCs8+C/v2hevcX6W8LCVFEmD16ic8PqXnyBG4+WYptoyIkN4DfftqRbM35am9vjEmDPgCmGit/ec83+8OdAcoW7YsMTEx7ojRJyUkJHjl+N5/P5wBA+qzdm0EjRun8Oab66lVy7OnwVOm1CAhYQ116wbmuoPeeu6cEBsbS3p6esAeHwTu8xcXV4gtWypQu7Znj2/3bnlN2b07gjJlUhg8eD0lSiTgjV9poD53+WKtdemGnCVPBX4AwnLbPioqygayxYsXe21fhw5Z26KFtWBtRIS1333n2f1lHVtKimf34xRvPnfe1rJlS9uwYUOnw/CoQHz+9u+39sAB+dyTxxcTY23p0vJa0rChtbt2eWxX5xWIz112wG/WxZzq0mVnY4wBPgHKA3dba1M99F5AnUeZMjLv7uGHpc1bx44yFuzJwqiZM+HJJz33+Eqp0xYtgo8+8uw+Jk2SSuajR6F9e1i2zJm2lUq4etl5LHA50Npa6+UmiAqkEGriRIiKgpdflvGZTZtg1CjPrM17660yJqSU8qy0NFnv21MyMmTOcNbUpd69ZYnC0FDP7VPlzpV5vpcAPYBGwH5jTELm7UGPR6fOYIxUJk6ZIsn4ww9lPvDx4+7fV1bBx4MPer/ntFLBpG1bWL/eM4+dlCSJfdAgmUkxerRcNdPE67xcz5mstTsArYHzIffdJ+0o77hDKqKbN5cFrqtXd+9+IiLkUrc2U1fKc6ZOlaEldzt4UF4jVq483T++XTv370flj7aX9FPNmkn7t8svl17QTZt6pgl7u3bS33XrVvc/tlLBbNcueOopSbzunuKzYcPp14Rq1WThFE28vkWTrx+rUUPawLVpI+9yb7xR3t262/btzqy2pFQgK11a1sp1d+L98Ud5c759O1x9tbxJr1/fvftQBafJ18+VLCnt4Lp3h+RkuPde97eIfOwxeRe9bZv7HlOpYLZsGezcCa1bu/dxx4+HW26R/vD/+Y9ctapQwb37UO6hyTcAhIVJ8dXw4fIu+qWX4JFH4ORJ9+1j7VpdeEEpd9m5E/btc9/jpafL/2ePHvJ5//6yWEJEhPv2odzLA5NUlBOMkelHl14qFcoTJ8plp2++gYsvLvjjX3klfP21/GNrpaRS+bdtm3unFp04If/z330n0w7HjZNVkZRv0zPfANOxIyxdCpUqwZIlMvbz77/ue/xWrWR+sVIq706elMXpY2Pd83h798INN0jiLVlSZj9o4vUPmnwD0JVXSpFFw4aSeK+9VhJxQRkjBV1RUQV/LKWCTUaGnJmuXi2JsqDWroVrroHff4eaNWHFCim6VP5Bk2+AqlJFijrat5d2cm3aSHu5gipfHubNk8n6SinXTZ8uq4a5o7p59my4/nrYs0fm+a9cCXXqFPxxlfdo8g1gxYpJj+bevSE1Fbp0kQ5ZGRkFe9w6deQfXynluk6dpBiyIKyF99+X5hlZY72LFnmmSYfyLE2+AS40VNrJjR4tnw8eLMUeBWkZWb26NPf4+GPPLu6gVCCwVpppbNtWsCSZlgb/93/yZjojA15/HT7/XFrNKv+jyTdIPPmkXKqKjJQpCK1aSWOO/AoJgc2bZZUlpVTOjJFmGtWq5f8x4uLg9tvlTfRFF8EXX8Arr7i/QYfyHk2+QaRdO2kzV62ajBE1bSqtKfOjUCF4+21ISID9+90bp1KB4sgRmfbXunX+e6Tv2CHjuvPmyZnzTz95dhUk5R2afINM/fpSCX3NNTIP+LrrpB1dfk2cWLCfVyqQxcbKWWt+/fqrvEn+80+ptVi5UhKx8n+afINQhQqweLG0n4uLk3Z048bl77Gef15WPjpxwr0xKuXv1q+X+fb/93/5+/lvvoGWLaWveqtW0se9Zk33xqico8k3SEVEyNjvgAHSteqJJ2QaRHp63h/r4EGZS5yW5v44lfJXH30Ef/yR95+zFoYMkTfHSUnSW33ePChVyv0xKudoe8kgFhIiizDUqiULM7z7LmzZAj165O09WblycjmskP41KQXI5eaRI/P+cydPwjvv1OaHH+TrIUPguee0sCoQ6Zmv4pFHpC1dqVLSpq5Xr8bs2ZO3xyhaVKovZ8zwTIxK+YuNG6W5TV6n4R07JkWRP/xQkfBwuez8/POaeAOVJl8FSFu6FStkTOnffyNp2jTvl8y6dIG2bT0Tn1L+wFqoXVsaX+QlaW7ZIn3YFy+G0qVTWLIE7rrLc3Eq52nyVafUri2Xjxs0iGXPHmjRAr7/3vWfr1lTph49/7w231DB6cknYf78vDW+WLZMKpo3bpTZCGPG/M7VV3suRuUbNPmqM5QpA8OGreOhh6SC+Y47YMQI15Np6X1LvrcAAB2FSURBVNLQqJFnY1TKV738sqwy5KovvoCbbpL5wLfcIom4fPkUzwWofIYmX3WOiy6yTJoEb7whSbdPH2mP50o1c1iYNABYuFA6YCkVDHbvhp49oWJFCA/PfXtr4bXX4KGHpMjq6adh1iwoXtzjoSofoclXnZcx8i7+yy/lEtrYsVJEcvy4az+/ezccOuTZGJXyFRdfLC0kXRnnTU6WpPv66zLjYORIGDVKZwsEG02+6oLuv1/a2ZUtK2NZzZtLZ6zcPPKIjGP9+qvHQ1TKUV9+Cbt2wc03577toUPSavLLL2XVsVmz8t+EQ/k3x95rxcXFcfDgQVJTU50KoUBKlCjB33//7XQYHnH2sZUrF8bSpeW4887i/PWXJNVZs+TjhRw8CIMGyfSj0FAPB62UQ5KTZbglN//8A7fdBlu3ynrbs2dDw4aej0/5JkeSb1xcHAcOHKBy5cqEh4dj/HAiW3x8PJGRkU6H4RHZj81aS1JSEnv27GHhQujSpTgLF0J0NEyaJGuU5qRCBZk3nJAgY1wB+utSQSo5WRYqefTR3Lf96Se4+25pvtGkicwiqFjR8zEq3+XIZeeDBw9SuXJlIiIi/DLxBhNjDBEREVSuXJnExIP88AM8/ri88NxzD7z1Vu6V0EOGwPTp3olXKW/ZsUPOXnPzyScy/z02Fjp2hCVLNPEqh5Jvamoq4a6UBCqfER4eTmpqKmFhsgjDO+9IccmLL8o7/5Mnc/7Z116T/rQX2kYpf7Jpk7Rlfe+9nLfJyID+/aFbN5kp8Nxz0rWqaFHvxal8l2MFV3rG61+yP1/GyCIMM2bIAg0TJkixydGj5//Z0FC59Ny4sa5+pALDSy/JMn85SUyUK0NDhsjf//jxMHSoVDcrBVrtrAqgY0f4+We5hLZkibTH+/ff829brJg0EChaVLtfKf+VlibLcE6fDg0anH+b/fulJuKbb6BECVmR6PHHvRqm8gOafFWBNGkCq1dL1eamTbK04NKl59+2VCmZYvHGG96NUSl3+eEH6Ns35/m869efnmJXo4aswdu6tXdjVP5Bk68qsCpVJOHedptcer7pJvj88/Nv266drB2slL9JS4MOHWDMmPN/f+5cmQe/c6dcBVq5EurW9W6Myn9o8s2j6Ohonn76accfIzfHjh2jfPnybNmyJddtO3XqxPDhwwu0v8jIrOUIITUVOneWJQbPvsRcurSs//vII1ItqpQ/SEuDq6+Gw4fhoovO/f7o0dIBLj4e7rtPphaVK+f9OJX/0OQboN58801uvfVWatasmeu2r7zyCoMHD+a4q70jcxAaKoswfPCBFJb897/S5zk5+cztjJHkW6lSgXanlFdYK60f58+XhUeyS0+H3r2lN3NGhrRk/eILKFLEmViV/9DkG0BOZs7lSUxM5OOPP+axxx5z6efq16/PpZdeyuTJk90Sx1NPyfzHyEiYOhVatZJuV9ndcIOc+b75plt2qZTHDB4sFf1nn8nGx8uqX++/Lx2ushYj0Ypm5Qr9M8mHjIwMXn/9dcqUKUO5cuXo168fGRkZwPkvKXft2pX27dufcV9aWhq9evWiVKlSlCpViueee+7UY4B0lho6dCg1a9YkPDyc+vXrn5Mco6Oj6dmzJ/369aNs2bI0b94cgB9++AFjzKmvAYYOHYox5pzbK6+8AkCHDh2YMmWK235Ht9wi3X+qVYMVK6QIZcOGM7cpW1bmSirly554QsZ6s9u9W9a7njNHhlIWLoSHH3YmPuWffCL5GuPMLb+++OILQkND+eWXX/jggw8YMWIE06ZNy/NjZGRksGLFCsaNG8f48eMZMWLEqe8PHDiQTz75hNGjR7NhwwYGDBhAjx49mDNnzhmPM3nyZKy1LF26lEmTJgGwdOlSmjRpcsbc3J49e7Jv375Tt2effZYKFSrQuXNnAK655hpWr15NUlJSfn8t56hfH1atkrGy7dulCOXHH09/v0QJaU85YwasXeu23SrlFlu3woMPyopFpUufvn/NGrjmGli3DqKipLAqL2v4KgUOLqzgz+rWrcvAgQOJjIwkKiqKjz76iEWLFnH//fe7/BgVK1Zk5MiRGGOoU6cOmzZt4t1336Vv376cOHGCd999lwULFtCiRQsAatSowerVqxk9ejS33XbbqcepUaPGOcVSO3bsoNJZA6qRkZGn+jUPGTKEKVOmEBMTw2WXXQZApUqVSE1NZe/evZRzY6VIhQoQEyMFWN98I2fEY8ZA9+6ntwkN1bm/yvdUqyYFhNnfqM+cKQk5MfH0XN7siVkpV/nEma+1ztzyq8FZs+srVarEwbMHNXNx7bXXnnFm2qxZM/bs2UNcXBwbNmwgOTmZdu3aUaxYsVO3sWPHnlO93KRJk3MeOykpiSI5VHy89dZbjBo1isWLF1O7du1T92e1+3TnmW+WiAhpStC/vxSo9OgB/frJ5yDjZg0awEcfSVWpUk6yVuby7tghZ7hZ9w0fDnfdJYm3SxcpwNLEq/LLpTNfY8zTQFegPjDFWtvVgzH5vLCz1g8zxpwarw0JCcGeldnzumxi1mN9//33VKtW7YL7LnqeRrFlypTh2LFj59w/aNAgPvzwwzPOeLMczewNWbZs2TzF6qqQEFmEoVYtSb7Dh8OWLTB58umuV9u3S/vJEiU8EoJSLjEG2rSBypXl69RUqWYeP16+fvNNeSOpHXJVQbh65rsXGAR86sFYAkLZsmXZt2/fGfetW7funO1WrVp1RpJeuXIllSpVonjx4tStW5fChQuzY8cOLrvssjNul1xySa4xNG7cmA1nVTe98cYbjB8/niVLlpyTeAH+/PNPKleuTPny5V091Hx59FFYsABKlpRLeDfcAHv3ylSOwYPlzHfRIo+GoFSOZs6UGoRbbpHpQrGxcOutkniLFJErOAMGaOJVBedS8rXWzrDWzgSOeDgev9eqVSvmzp3LrFmz2LhxI3379mXXrl3nbLd371569+7Nxo0b+frrrxk2bBh9+vQBZHy2X79+9OvXj08//ZTNmzezdu1aPvzwQ8Znvf2+gLZt2/L3339z5Ig8XYMGDWLkyJFMnTqVokWLsn//fvbv309ytgm4S5cupW3btm76LVzYjTdKkUrNmvD773JpL6vgas8e6QGtlBMuvRSqV5fPt22TjlULF8o0o5iYC69frVReuLXgyhjTHegOcgYYExNz3u1KlChBfHy8O3ftNenp6Zw8eZL09PRTx5CamkpaWhrx8fF06tSJ3377jUceeQSAxx9/nPbt23PkyJFT26enp3PPPfeQlJRE06ZNMcbw8MMP061bt1PbPP/885QoUYKhQ4fSs2dPIiMjadCgAb169TrjcU6ePHnO77J69eo0adKECRMm8PjjjzNs2DDi4uLOmHoEMGvWLKKjo0lOTubbb79lxowZxMfHn3Fs2SUnJ+f4nObH8OFhvPxyPdavL0mzZum88soGmjU7QsuW8PHHxShePJVy5VLctr8sCQkJbj0OXxIbG0t6enrAHh945vmLjQ3ju+8q0bnzDoyB0aOLM3DgFcTGXkT16id46631JCUl441fayD/fQbyseWZtdblG3LpeYIr20ZFRdmcbNiwIcfv+Yu4uDinQ7iguXPn2qioKJuWlpbrth988IFt06bNqa9zOjZPPG/JydY+9JCUwIWEWDtihLUZGdaOHGnt3Llu35211trFixd75oF9QMuWLW3Dhg2dDsOjPPH8HT1q7cSJ8vnUqdYWLix/kzffbG1srNt3d0GB/PcZyMdmrbXAb9bFfOoT1c7K/dq1a8dTTz3F7t27c902LCyMUaNGeSGqcxUuLJ2BXn9d2vNlterr2VMWYVi06HRVtFLuZq0UAlorTTIGDZLezCkp0lxjzhwtAFSeofN8A9gzzzzj0nbds0+6dYAxsghDrVrQtavMA966FaZMkWroOnVOV54q5U7WShtUY+Rvb9Ik+fzdd8+d46uUO7k61ahQ5rahQKgxpgiQZq3VWZnKbe6/XxobdOwoC5C3aCE9osuXl2KX6GinI1SB5Isv4PLL5e+uY0f4+WeZkz5lyrntJJVyN1cvOw8EkoD+wEOZnw/0VFAqeDVvLi0p69SBP/+UntBz50pje+2CpdypWDFZ8OPaayXxVqok61Jr4lXe4OpUo9esteas22sejk0FqUsvlcUYbroJDhyAe+6RtVIPHYJff3U6OuXv/vc/ubxcsqS0ity8GRo3htWr4cornY5OBQstuFI+qWRJOePt1k3WA+7UCV56CRYvdjoy5e+KFJHFEdq0gaNH4fbb5cxX6wqUN2nyVT4rLEw6Cw0dKoUvH38M//wjTThcKOJW6gz798OLL8LEiTBypLSN7NMHvv1WLkEr5U2afJVPMwaee05WjwkPh88+g0cegd9+czoy5W8KFZIrJ2++KStpjRkjVc2hoU5HpoKRJl/lF+68Uy4NVqggY3QvvCAvnMePOx2Z8nUpKbKkZbt20tY0MlLm7/bs6XRkKphp8lV+46qrpCimQQPYtAkGDoQff3Q6KuXr/v1Xpq6tWQOXXAK//AJeamOuVI40+Sq/UrWqjPneeiskJcEDD0g1tAeWIVYB4JZbZCrRoUMybW3VKrjiCqejUkqTb57deeedlCpViocfftjpUIJWZCR89x0884wUzXz1lbQF1HnAKou1MG6cLF954oRUyy9eLA1blPIFmnzzqFevXkyaNCnPP7dr1y6io6OpW7cuDRo04KuvvvJAdMGjUCF4/30YNQpCQqSIpk4deaFVwS09XebrPvGE9At/8UWYOlUK9pTyFZp88yg6OprIyMg8/1yhQoUYMWIEGzZsYMGCBfTu3ZsTmikK7Omn4fvvoWhRGQe++Wa5xKiC0/HjcNddsj50oUJSHT94sLxBU8qX6J+kl1SsWJFGjRoBUKFCBcqUKcPRo0cdjiow3HqrFNFUrSofa9SAv/92OirlbXv2SK/mWbOgVCkpxuva1emolDo/Tb4OWLNmDenp6VStWtXpUAJGgwZSTHPVVXLpuVkzmD/f6aiUt6xYIc/9vn1Qs6ZMKdKFOJQv0+TrZUePHqVz586MHz/e6VACTsWKsGQJ3H23XH685Rb46COno1Ke9v33kmj375eVsFauhKgop6NS6sI0+brR0KFDMcacc3vllVcASElJoWPHjvTv35/rrrvO4WgDU0QETJ8Ozz8vFa/du8OTT0rhjQos1kqVe4cOcPIkPPSQXGouU8bpyJTKnSbfPGrdujWdOnViwYIFVKlShRUrVpz6Xs+ePdm3b9+p27PPPkuFChXo3Lkz1lq6du1Kq1atdJqSh4WEwJAh0gs6JATGjpWzYa1vCxxpaVJs9/LL8vV//ysrFRUu7GxcSrmqkNMB+JuFCxcCEB8ff07Vc2Rk5Kn7hgwZwpQpU4iJieGyyy5j2bJlTJs2jQYNGjBz5kwAPv/8c+rXr+/dAwgijz0mxVd33QUzZ8qY4KJFTkelCurEiVCio2H5ckm2n30G99/vdFRK5Y0mXw946623GD16NIsXLyYqc/Dp+uuvJ0OvfXpdq1YyBtiqlayI1LQpvPpqUS3G8VM7dsDTTzdm+3YoXlyWndQRHOWPfOay82uvyQ2kWGLTJunF2qSJ3PfsszB8uHxeqRLs3QsxMacrGrt3l+XnQDogxcdLIcbtt8t9DzwAX34pnxuTvxizj+MWL178nLFdgEGDBjF69GhiYmJOJV7lrDp1ZAH15s1lKcInn7ySOXOcjkrl1erV0KgRbN9ejMsvhz/+0MSr/Ji11iO3qKgom5MNGzbk+D1ftnPnTtuyZUt7+eWX23r16tnp06ef8f3XX3/dVq1a1W7evNmhCN0jLi7uvPf76/OWJSnJ2sdu3mmrsNMaY+277zodkfu1bNnSNmzY0Okw3G7CBGsLF7a2CjvtTVF/2mPHnI7IcxYvXux0CB4TyMdmrbXAb9bFHOkzZ77+IHuXqpkzZ57RpWrQoEGMHDmSqVOnUrRoUfbv38/+/ftJTk52OGqVpUgR+GheVVp3TcNa6NtXinbS0pyOTOUkLU2ep65dZWnAW7tXpf/ow5Qs6XRkShWMJt88yN6lqnz58qe6VFlrGTZsGEeOHKF58+ZUrFjx1G358uUOR62yM9On0a/qZ0yaBGFhMHq0rHqzb5/TkamzHT4M118P770nrSJHj4ZxraZR8WetmlP+Twuu8umPP/44o0vVcV3V3T+MHUvl2FjqrX2DmjVljuiaNdC4MUybBi1bOh2gAvj9d5kellVYNWeOJGKi5fnjjTecDlGpAtEz33w4evQoPXr00C5Vfu666+Cvv+DGG+HAAfk4dKguTegka2W1qquvlsR79dXyHF1/vdORKeVemnzzKKtLVZ8+fbRLVQAoX17WfO3TR174X3hB2lIePOh0ZMHnyBFo3Rp695aOZI89Bj//DFWqOB2ZUu6nyTcPbLYuVffrrP6AUagQvPsufPedXOKcPx+uuAKdjuRFS5fKNKKffpLlIb/5RjqUFSnidGRKeYYm3zxYvnw506ZNY+bMmTRv3pxGjRqxfv16p8NSbtKhA6xfL835Dx2C9u3l7Csx0enIAldSkszhv+EGmYPdrBn8+ad0JVMqkGnBVR5k71J1vvaSyg98/TV/LV9O8xy+Xa2aNG8ZNgxefBE+/VQufX72mY47utsvv8CDD8rYrjGShN98U6rQc5TL86eUv9AzXxVcypQhtUSJC24SEiJjv7//DvXqwebNcjb8xBMQG+ulOANYYiL06ycdx7Zvh9q1ZS3mYcNySbzg0vOnlD/Q5KuCy4QJVJg3z6VNGzaE336DAQNkXHjcOLj0UhmP1IrovLNWxtVr1ZJWscbIm5y1a6Wq2SV5eP6U8mWafFVwyeOLd5Eicil07Vpo0ACOHYP//Ed6iv/9t+fCDDSbN8sc6o4dpS/7ZZfJghdvv53HoipNvipAaPJVygX16kkj/7FjpRr355+hfn1pT3n4sNPR+a7jx+H556FuXaloDg+HkSPljcs11zgdnVLO0eSrlItCQmTcd+tW6NFD5qKOHg01a8pl1KQkpyP0HUlJ8ju59FIZy01NhS5dYNs2+L//k8v4SgUzx5Kv1UEzv6LP12nlysGHH8K6dVKIFRcnBURVqsCIEcGdhFNT4ZNP5A1Jv35w9KgUVv3yC0yYIE1NlFIOJd+wsDCSgvkVyg8lJSURlmspanCpXx+WLJFmHLVqSaLp0wcqVJCmHZkLXgWFhAR541G5MnTrJgtVREXJYvdLl8r8XaXUaY4k33LlyrFnzx4SExP1jMrHWWtJTExkz549lCtXzulwCu6HH/jf22+77eGMgVtvhY0bYdYsGduMi5M5qxUqQK9eMp0mUB04AAMHyhltnz7SnKRqVZgyRcZ127WT35HbuPn5U8opjoy8FC9eHIC9e/eSmprqRAgFlpycTJEA7X139rGFhYVRvnz5U8+bX4uIIMMDz5sxcPvt0hVrzhwYPFiqeUeOhFGjoFUr6N9fPob4eaVFRgYsXCjHNm8epKfL/VddBS+/LL8Djx2jh54/pbzNsbKH4sWL+/WLeUxMDI0bN3Y6DI8I5GNjzBgqbdokc4U8wBhJPu3bw+rVkninTIFFi+RWtix07iyLw19xhUdC8JjNm2HqVEm6hw7JfcbAHXfAc8/J2K7Hefj5U8pbtOZQBZfp0ynnpTZV11wDn38u1b7jx0sh0s6dUgU8fLjMde3SRS5bN27s5suzbrJhA3z9tbyB+Oef0/dXqgQ9e8Ijj8g4r9d48flTypNcujhkjCltjPnWGHPCGLPDGPOApwNTKlBUqACvvCLTbJYulWlKxYrJmeTLL0OTJlCxoizi8NVX0oTCKfv3S6J97DF5c1CvHrz6qiTeIkWkF/P8+bBrl4z1ejXxKhVAXD3zHQ2cBMoDjYA5xph11tq/PBaZUgEmJEQWZ7j+elkwfv58mD1bziwPHJBFHD79VLYtV04u47ZsKZ216tSRJO7Os+MjR2S61B9/SAevxYthz54zt4mMlLHsBx+Em26CwoXdt3+lglmuydcYUxS4G7jCWpsALDPGzAIeBvp7OD6lAlLhwrKEYYcO0jP6f/+TaumFCyURHjwI334rtyxFi0KNGjJWXLKkzCsuX17GkcPDpXHF8eNw4kQhli2D+PjTt+PHpTnI7t2wYwds2SL3ny08XN4cREdDmzZyOVwbYijlfia3qT7GmMbAcmttRLb7+gEtrbW35/RzERER9poA7h8XGxtLyZIlnQ7DIwL52Fi7lrS0NApddZXTkeTIWln5Jy5OkmZCAqSkQFqaKz+9NvNjo1y3DAmBiAhJ6sWLy6XwyEjfHHs+xQ+ev4IK5P+/QD42gCVLlqyx1rr0x+lK8m0BfGWtrZDtvseBB6210Wdt2x3onvnlFcCfeYjb35QBArWrbyAfG+jx+Ts9Pv8VyMcGUNta69JC765cUEoAzp4TVBw456KVtXY8MB7AGPObq+8A/FEgH18gHxvo8fk7PT7/FcjHBnJ8rm7rSrXzJqCQMaZWtvsaAlpspZRSSuVDrsnXWnsCmAG8YYwpaoxpDtwBfO7p4JRSSqlA5GoTuCeBcOAgMAXo6cI0o/EFCcwPBPLxBfKxgR6fv9Pj81+BfGyQh+PLteBKKaWUUu7l5y3elVJKKf+jyVcppZTyMq8kX2NMLWNMsjFmsjf25y3GmMnGmH3GmDhjzCZjTDenY3IXY0xhY8wnmb28440xa40xtzgdlzsZY542xvxmjEkxxkxwOp6CCuQe7IH2XJ0t0P/fAvm1Mru85DpvnfmOBn710r686S2gurW2ONABGGSMaeJwTO5SCNgFtARKAAOB6caY6g7G5G57gUHAp04H4ibZe7A/CIw1xtRzNiS3CbTn6myB/v8WyK+V2bmc6zyefI0x9wGxwCJP78vbrLV/WWtTsr7MvNV0MCS3sdaesNa+Zq3dbq3NsNbOBrYBAfMPY62dYa2dCRxxOpaCytaD/WVrbYK1dhmQ1YPd7wXSc3U+gf7/FsivlVnymus8mnyNMcWBN4C+ntyPk4wxY4wxicA/wD7gB4dD8ghjTHkgCm2u4quigDRr7aZs960DAuXMN6gE4v9bIL9W5ifXefrM97/AJ9ba3R7ej2OstU8CkUALpBlJyoV/wv8YY8KAL4CJ1tp/ctteOaIYEHfWfceRv03lRwL1/y3AXyvznOvynXyNMTHGGJvDbZkxphHQGngvv/twUm7Hl31ba2165mW+KkBPZyLOG1ePzxgTgnQzOwk87VjAeZSX5y9AuNyDXfkuf/1/c5U/vlbmJr+5Lt8rdZ69otF5AuoNVAd2GlmjrBgQaoypa629Mr/79Zbcji8HhfCTcQxXjs/IE/cJUsBzq7U21dNxuUs+nz9/dqoHu7X238z7tAe7H/Hn/7d88JvXShdEk49c58nLzuORX26jzNuHwBygrQf36TXGmHLGmPuMMcWMMaHGmLbA/QRWYdlY4HLgdmttktPBuJsxppAxpggQivyzFDHG+OXS8YHegz2QnqsLCMj/tyB4rcxXrvNY8rXWJlpr92fdkMtiydbaQ57ap5dZ5LLJbuAY8A7Q21o7y9Go3MQYcwnQA/lj2m+MSci8PehwaO40EEgC+gMPZX4+0NGICiY/Pdj9RaA9V2cI8P+3gH6tzG+u097OSimllJdpe0mllFLKyzT5KqWUUl6myVcppZTyMk2+SimllJdp8lVKKaW8TJOvUkop5WWafJVSSikv0+SrlFJKeZkmX6WUUsrLNPkqFQCMMc/nsILTG07HppQ6l7aXVCoAGGMigaLZ7uoHPAi0sNZudiYqpVRONPkqFWCMMS8AzwCtrLUbnY5HKXWuQFuSS6mgZowZADwF3Git3eR0PEqp89Pkq1SAMMYMBJ4AovVSs1K+TZOvUgHAGPMK0A1oaa3d4nQ8SqkL0+SrlJ/LPON9BugAnDDGVMj8Vqy1Ntm5yJRSOdGCK6X8mDHGALFA8fN8u7W1dpGXQ1JKuUCTr1JKKeVl2mRDKaWU8jJNvkoppZSXafJVSimlvEyTr1JKKeVlmnyVUkopL9Pkq5RSSnmZJl+llFLKyzT5KqWUUl6myVcppZTysv8HMk22K5kLKGQAAAAASUVORK5CYII=\n"
     },
     "metadata": {
      "needs_background": "light"
     }
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 3.5))\n",
    "z = np.linspace(-4, 4, 200)\n",
    "plt.plot(z, huber_fn(0, z), \"b-\", linewidth=2, label=\"huber($z$)\")\n",
    "plt.plot(z, z**2 / 2, \"b:\", linewidth=1, label=r\"$\\frac{1}{2}z^2$\")\n",
    "plt.plot([-1, -1], [0, huber_fn(0., -1.)], \"r--\")\n",
    "plt.plot([1, 1], [0, huber_fn(0., 1.)], \"r--\")\n",
    "plt.gca().axhline(y=0, color='k')\n",
    "plt.gca().axvline(x=0, color='k')\n",
    "plt.axis([-4, 4, 0, 4])\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"$z$\")\n",
    "plt.legend(fontsize=14)\n",
    "plt.title(\"Huber loss\", fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_shape = X_train.shape[1:]\n",
    "\n",
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        30, \n",
    "        activation='selu', \n",
    "        kernel_initializer='lecun_normal', \n",
    "        input_shape=input_shape),\n",
    "    keras.layers.Dense(1), \n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=huber_fn, \n",
    "    optimizer='nadam', \n",
    "    metrics=['mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/10\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.6235 - mae: 0.9953 - val_loss: 0.2862 - val_mae: 0.5866\n",
      "Epoch 2/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2197 - mae: 0.5177 - val_loss: 0.2382 - val_mae: 0.5281\n",
      "Epoch 3/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2051 - mae: 0.4975 - val_loss: 0.2196 - val_mae: 0.5040\n",
      "Epoch 4/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2002 - mae: 0.4908 - val_loss: 0.2069 - val_mae: 0.4917\n",
      "Epoch 5/10\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.1964 - mae: 0.4843 - val_loss: 0.1940 - val_mae: 0.4751\n",
      "Epoch 6/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1935 - mae: 0.4795 - val_loss: 0.1893 - val_mae: 0.4705\n",
      "Epoch 7/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1912 - mae: 0.4761 - val_loss: 0.1851 - val_mae: 0.4627\n",
      "Epoch 8/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1887 - mae: 0.4722 - val_loss: 0.1801 - val_mae: 0.4567\n",
      "Epoch 9/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1866 - mae: 0.4689 - val_loss: 0.1788 - val_mae: 0.4553\n",
      "Epoch 10/10\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1845 - mae: 0.4647 - val_loss: 0.1837 - val_mae: 0.4563\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaaa09a750>"
      ]
     },
     "metadata": {},
     "execution_count": 62
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled,\n",
    "    y_train, \n",
    "    epochs=10,\n",
    "    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save/load models with custom objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('my_model_with_a_custom_loss.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    'my_model_with_a_custom_loss.h5', \n",
    "    custom_objects={'huber_fn': huber_fn}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 2ms/step - loss: 0.1832 - mean_absolute_error: 0.4630 - val_loss: 0.1742 - val_mean_absolute_error: 0.4477\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1818 - mean_absolute_error: 0.4603 - val_loss: 0.1699 - val_mean_absolute_error: 0.4465\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaaa010e90>"
      ]
     },
     "metadata": {},
     "execution_count": 65
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled,\n",
    "    y_train, \n",
    "    epochs=2,\n",
    "    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_huber(threshold=1.0):\n",
    "    def huber_fn(y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss = threshold * tf.abs(error) - threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    return huber_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=create_huber(2.0),\n",
    "    optimizer='nadam', \n",
    "    metrics=['mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.2019 - mae: 0.4613 - val_loss: 0.1876 - val_mae: 0.4421\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1997 - mae: 0.4585 - val_loss: 0.2105 - val_mae: 0.4589\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaa81a5850>"
      ]
     },
     "metadata": {},
     "execution_count": 68
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled,\n",
    "    y_train, \n",
    "    epochs=2,\n",
    "    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('my_model_with_a_custom_loss_threshold_2.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    'my_model_with_a_custom_loss_threshold_2.h5',\n",
    "    custom_objects={'huber_fn': create_huber(2.0)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1987 - mean_absolute_error: 0.4570 - val_loss: 0.1849 - val_mean_absolute_error: 0.4389\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1966 - mean_absolute_error: 0.4541 - val_loss: 0.2027 - val_mean_absolute_error: 0.4524\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaa70eb550>"
      ]
     },
     "metadata": {},
     "execution_count": 71
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled,\n",
    "    y_train, \n",
    "    epochs=2,\n",
    "    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberLoss(keras.losses.Loss):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        self.threshold = threshold\n",
    "        super().__init__(**kwargs)\n",
    "    def call(self, y_true, y_pred):\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=HuberLoss(2.), optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 0.7095 - mae: 0.8863 - val_loss: 0.3378 - val_mae: 0.5485\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2416 - mae: 0.5083 - val_loss: 0.2660 - val_mae: 0.5089\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaa6fb3c90>"
      ]
     },
     "metadata": {},
     "execution_count": 75
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_loss_class.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_a_custom_loss_class.h5\", \n",
    "    custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2286 - mean_absolute_error: 0.4970 - val_loss: 0.2120 - val_mean_absolute_error: 0.4723\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2216 - mean_absolute_error: 0.4904 - val_loss: 0.2045 - val_mean_absolute_error: 0.4725\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaac47f550>"
      ]
     },
     "metadata": {},
     "execution_count": 78
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled, \n",
    "    y_train, \n",
    "    epochs=2,\n",
    "    validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_a_custom_loss_class.h5\",\n",
    "    custom_objects={\"HuberLoss\": HuberLoss})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "metadata": {},
     "execution_count": 80
    }
   ],
   "source": [
    "model.loss.threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## other custom functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_softplus(z): # return value is just tf.nn.softplus(z)\n",
    "    return tf.math.log(tf.exp(z) + 1.0)\n",
    "\n",
    "def my_glorot_initializer(shape, dtype=tf.float32):\n",
    "    stddev = tf.sqrt(2. / (shape[0] + shape[1]))\n",
    "    return tf.random.normal(shape, stddev=stddev, dtype=dtype)\n",
    "\n",
    "def my_l1_regularizer(weights):\n",
    "    return tf.reduce_sum(tf.abs(0.01 * weights))\n",
    "\n",
    "def my_positive_weights(weights): # return value is just tf.nn.relu(weights)\n",
    "    return tf.where(weights < 0., tf.zeros_like(weights), weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "layer = keras.layers.Dense(1, activation=my_softplus,\n",
    "                           kernel_initializer=my_glorot_initializer,\n",
    "                           kernel_regularizer=my_l1_regularizer,\n",
    "                           kernel_constraint=my_positive_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, \n",
    "                       activation=\"selu\", \n",
    "                       kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, \n",
    "                       activation=my_softplus,\n",
    "                       kernel_regularizer=my_l1_regularizer,\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=\"mse\", \n",
    "    optimizer=\"nadam\", \n",
    "    metrics=[\"mae\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 1s 1ms/step - loss: 1.5542 - mae: 0.8962 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.5943 - mae: 0.5256 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaacc97510>"
      ]
     },
     "metadata": {},
     "execution_count": 87
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"my_l1_regularizer\": my_l1_regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyL1Regularizer(keras.regularizers.Regularizer):\n",
    "    def __init__(self, factor):\n",
    "        self.factor = factor\n",
    "    def __call__(self, weights):\n",
    "        return tf.reduce_sum(tf.abs(self.factor * weights))\n",
    "    def get_config(self):\n",
    "        return {\"factor\": self.factor}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1, activation=my_softplus,\n",
    "                       kernel_regularizer=MyL1Regularizer(0.01),\n",
    "                       kernel_constraint=my_positive_weights,\n",
    "                       kernel_initializer=my_glorot_initializer),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"mse\", optimizer=\"nadam\", metrics=[\"mae\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 1.5542 - mae: 0.8962 - val_loss: 1.4154 - val_mae: 0.5607\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.5943 - mae: 0.5256 - val_loss: 1.4399 - val_mae: 0.5137\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaadc11c50>"
      ]
     },
     "metadata": {},
     "execution_count": 94
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2,\n",
    "          validation_data=(X_valid_scaled, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_many_custom_parts.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.load_model(\n",
    "    \"my_model_with_many_custom_parts.h5\",\n",
    "    custom_objects={\n",
    "       \"MyL1Regularizer\": MyL1Regularizer,\n",
    "       \"my_positive_weights\": my_positive_weights,\n",
    "       \"my_glorot_initializer\": my_glorot_initializer,\n",
    "       \"my_softplus\": my_softplus,\n",
    "    })"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## custom metrics"
   ]
  },
  {
   "source": [
    "## Losses\n",
    "- e.g. **cross entropy**\n",
    "- used by **Gradient Descent** to **_train_** a model, so they must be **differentiable**\n",
    "- it's ok if they're not easily interpretable by humams.\n",
    "\n",
    "## Metrics\n",
    "- e.g. **accuracy**\n",
    "- used to **_evaluate_** a model\n",
    "- can be non-differnetiable\n",
    "- must be easily interpretable by humans. "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(\n",
    "        30, \n",
    "        activation=\"selu\", \n",
    "        kernel_initializer=\"lecun_normal\",\n",
    "        input_shape=input_shape\n",
    "    ),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=\"mse\", \n",
    "    optimizer=\"nadam\", \n",
    "    metrics=[create_huber(2.0)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 930us/step - loss: 2.0982 - huber_fn: 0.9192\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 886us/step - loss: 0.6052 - huber_fn: 0.2733\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbaadcf2850>"
      ]
     },
     "metadata": {},
     "execution_count": 100
    }
   ],
   "source": [
    "model.fit(X_train_scaled, y_train, epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=create_huber(2.0), \n",
    "    optimizer=\"nadam\", \n",
    "    metrics=[create_huber(2.0)]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_weight = np.random.rand(len(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 960us/step - loss: 0.1175 - huber_fn: 0.2399\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 906us/step - loss: 0.1131 - huber_fn: 0.2297\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train_scaled, y_train, epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(0.11749906837940216, 0.11906625573138947)"
      ]
     },
     "metadata": {},
     "execution_count": 104
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"huber_fn\"][0] * sample_weight.mean()"
   ]
  },
  {
   "source": [
    "### streaming metrics    "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision = keras.metrics.Precision()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.8>"
      ]
     },
     "metadata": {},
     "execution_count": 106
    }
   ],
   "source": [
    "precision([0,1,1,1,0,1,0,1], [1,1,0,1,0,1,0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "metadata": {},
     "execution_count": 107
    }
   ],
   "source": [
    "precision([0,1,0,0,1,0,1,1], [1,0,1,1,0,0,0,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=0.5>"
      ]
     },
     "metadata": {},
     "execution_count": 108
    }
   ],
   "source": [
    "precision.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[<tf.Variable 'true_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>,\n",
       " <tf.Variable 'false_positives:0' shape=(1,) dtype=float32, numpy=array([4.], dtype=float32)>]"
      ]
     },
     "metadata": {},
     "execution_count": 109
    }
   ],
   "source": [
    "precision.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "precision.reset_states()"
   ]
  },
  {
   "source": [
    "create a streaming metric :    "
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Metric):\n",
    "    def __init__(self, threshold=1.0, **kwargs):\n",
    "        super().__init__(**kwargs) # handles base args (e.g., dtype)\n",
    "        self.threshold = threshold\n",
    "        #self.huber_fn = create_huber(threshold) # TODO: investigate why this fails\n",
    "        self.total = self.add_weight(\"total\", initializer=\"zeros\")\n",
    "        self.count = self.add_weight(\"count\", initializer=\"zeros\")\n",
    "    def huber_fn(self, y_true, y_pred): # workaround\n",
    "        error = y_true - y_pred\n",
    "        is_small_error = tf.abs(error) < self.threshold\n",
    "        squared_loss = tf.square(error) / 2\n",
    "        linear_loss  = self.threshold * tf.abs(error) - self.threshold**2 / 2\n",
    "        return tf.where(is_small_error, squared_loss, linear_loss)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        self.total.assign_add(tf.reduce_sum(metric))\n",
    "        self.count.assign_add(tf.cast(tf.size(y_true), tf.float32))\n",
    "    def result(self):\n",
    "        return self.total / self.count\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=14.0>"
      ]
     },
     "metadata": {},
     "execution_count": 112
    }
   ],
   "source": [
    "m = HuberMetric(2.)\n",
    "\n",
    "# total = 2 * |10 - 2| - 2²/2 = 14\n",
    "# count = 1\n",
    "# result = 14 / 1 = 14\n",
    "m(tf.constant([[2.]]), tf.constant([[10.]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(), dtype=float32, numpy=7.0>"
      ]
     },
     "metadata": {},
     "execution_count": 113
    }
   ],
   "source": [
    "# total = total + (|1 - 0|² / 2) + (2 * |9.25 - 5| - 2² / 2) = 14 + 7 = 21\n",
    "# count = count + 2 = 3\n",
    "# result = total / count = 21 / 3 = 7\n",
    "m(tf.constant([[0.], [5.]]), tf.constant([[1.], [9.25]]))\n",
    "\n",
    "m.result()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=21.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=3.0>]"
      ]
     },
     "metadata": {},
     "execution_count": 114
    }
   ],
   "source": [
    "m.variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "[<tf.Variable 'total:0' shape=() dtype=float32, numpy=0.0>,\n",
       " <tf.Variable 'count:0' shape=() dtype=float32, numpy=0.0>]"
      ]
     },
     "metadata": {},
     "execution_count": 115
    }
   ],
   "source": [
    "m.reset_states()\n",
    "m.variables"
   ]
  },
  {
   "source": [
    "check if the HuberMetric class works well :"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(\n",
    "    loss=create_huber(2.0), \n",
    "    optimizer=\"nadam\", \n",
    "    metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 952us/step - loss: 0.8707 - huber_metric: 0.8707\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 923us/step - loss: 0.2595 - huber_metric: 0.2595\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbab00956d0>"
      ]
     },
     "metadata": {},
     "execution_count": 119
    }
   ],
   "source": [
    "model.fit(\n",
    "    X_train_scaled.astype(np.float32), \n",
    "    y_train.astype(np.float32), \n",
    "    epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 913us/step - loss: 0.2350 - huber_metric: 0.2350\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 907us/step - loss: 0.2278 - huber_metric: 0.2278\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbab0847f50>"
      ]
     },
     "metadata": {},
     "execution_count": 121
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "metadata": {},
     "execution_count": 122
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "class HuberMetric(keras.metrics.Mean):\n",
    "    def __init__(self, threshold=1.0, name='HuberMetric', dtype=None):\n",
    "        self.threshold = threshold\n",
    "        self.huber_fn = create_huber(threshold)\n",
    "        super().__init__(name=name, dtype=dtype)\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        metric = self.huber_fn(y_true, y_pred)\n",
    "        super(HuberMetric, self).update_state(metric, sample_weight)\n",
    "    def get_config(self):\n",
    "        base_config = super().get_config()\n",
    "        return {**base_config, \"threshold\": self.threshold}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras.backend.clear_session()\n",
    "np.random.seed(42)\n",
    "tf.random.set_seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"selu\", kernel_initializer=\"lecun_normal\",\n",
    "                       input_shape=input_shape),\n",
    "    keras.layers.Dense(1),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.Huber(2.0), optimizer=\"nadam\", weighted_metrics=[HuberMetric(2.0)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.4455 - HuberMetric: 0.8978\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.1305 - HuberMetric: 0.2631\n"
     ]
    }
   ],
   "source": [
    "sample_weight = np.random.rand(len(y_train))\n",
    "history = model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32),\n",
    "                    epochs=2, sample_weight=sample_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "(0.44554394483566284, 0.44554404180100277)"
      ]
     },
     "metadata": {},
     "execution_count": 128
    }
   ],
   "source": [
    "history.history[\"loss\"][0], history.history[\"HuberMetric\"][0] * sample_weight.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"my_model_with_a_custom_metric_v2.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Epoch 1/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2377 - HuberMetric: 0.2377\n",
      "Epoch 2/2\n",
      "363/363 [==============================] - 0s 1ms/step - loss: 0.2279 - HuberMetric: 0.2279\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fbab0f78090>"
      ]
     },
     "metadata": {},
     "execution_count": 130
    }
   ],
   "source": [
    "model.fit(X_train_scaled.astype(np.float32), y_train.astype(np.float32), epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "2.0"
      ]
     },
     "metadata": {},
     "execution_count": 131
    }
   ],
   "source": [
    "model.metrics[-1].threshold"
   ]
  },
  {
   "source": [
    "### Custom Layers"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5-final"
  },
  "orig_nbformat": 2,
  "kernelspec": {
   "name": "python37564bit42731fe29a1c4072b88558cb14c14b00",
   "display_name": "Python 3.7.5 64-bit"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}