uwb-nn-train-transformer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(779, 4)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_34752/1221680123.py:49: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.\n",
      "  X_train, y_train = np.array(X_train), np.array(y_train)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "sequence_length = 30\n",
    "n_steps = sequence_length\n",
    "input_dim = 3  \n",
    "num_heads = 4\n",
    "# Example usage:\n",
    "num_layers = 4\n",
    "d_model = 128\n",
    "num_heads = 8\n",
    "dff = 512\n",
    "num_scales=3\n",
    "\n",
    "# # Generate random synthetic data\n",
    "# synthetic_data = np.random.rand(num_samples, sequence_length, input_dim)\n",
    "\n",
    "# # Split the data into inputs and outputs\n",
    "# X = synthetic_data[:, :-1]  # Input data (sequence_length - 1 time steps)\n",
    "# y = synthetic_data[:, -1]   # Output data (1 time step ahead)\n",
    "\n",
    "dataset = pd.read_csv(\"data/train_newdata.csv\", delimiter=',')\n",
    "# print(dataset.head)\n",
    "\n",
    "dataset_selected = dataset[[\"node1\", \"node2\", \"uwb_range\", \"tb_node1_yaw\", \"tb_node2_yaw\",  \"error\"]] #\"mocap_range\",\n",
    "\n",
    "uwb_pair = [(1,4), (2,4), (3,4),(4,5),(1,2), (1,3), (1,5), (2,3), (2,4), (2,5), (3,5)]\n",
    "uwb_data = []\n",
    "for up in uwb_pair:\n",
    "    rslt_df = dataset_selected.loc[dataset_selected['node1'] == up[0]]\n",
    "    rslt_df_new = rslt_df.loc[rslt_df['node2'] == up[1]]\n",
    "    uwb_data.append(rslt_df_new[[\"uwb_range\", \"tb_node1_yaw\", \"tb_node2_yaw\",  \"error\"]].iloc[:,:].values)\n",
    "\n",
    "print(uwb_data[1].shape)\n",
    "\n",
    "X_train = []\n",
    "y_train = []\n",
    "for inx, dataset_train in enumerate(uwb_data):\n",
    "    x_temp = []\n",
    "    y_temp = []\n",
    "    for i in range(n_steps,dataset_train.shape[0]):\n",
    "        x_temp.append(dataset_train[i-n_steps:i,0:3])\n",
    "        y_temp.append(dataset_train[i,3:])\n",
    "    # print(np.array(x_temp).shape, np.array(y_temp).shape)\n",
    "    X_train.append(np.array(x_temp))\n",
    "    y_train.append(np.array(y_temp))\n",
    "X_train, y_train = np.array(X_train), np.array(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the Multi-Scale Multi-Head Attention model\n",
    "class MultiScaleMultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, num_heads, d_model, num_scales, name=\"multi_scale_multi_head_attention\"):\n",
    "        super(MultiScaleMultiHeadAttention, self).__init__(name=name)\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "        self.num_scales = num_scales\n",
    "        self.head_size = d_model // num_heads\n",
    "\n",
    "        assert self.head_size * num_heads == d_model, \"d_model must be divisible by num_heads\"\n",
    "\n",
    "        # Define multi-head attention layers for each scale\n",
    "        self.attention_layers = [tf.keras.layers.MultiHeadAttention(\n",
    "            num_heads=num_heads,\n",
    "            key_dim=self.head_size,\n",
    "            value_dim=self.head_size,\n",
    "        ) for _ in range(num_scales)]\n",
    "\n",
    "        # Define a final linear layer to combine the scaled attentions\n",
    "        self.linear_layer = tf.keras.layers.Dense(d_model)\n",
    "\n",
    "    def call(self, inputs, mask=None):\n",
    "        # Apply multi-head attention for each scale\n",
    "        attention_outputs = []\n",
    "        for i in range(self.num_scales):\n",
    "            attention_output = self.attention_layers[i](inputs, inputs, attention_mask=mask)\n",
    "            attention_outputs.append(attention_output)\n",
    "\n",
    "        # Concatenate attention outputs from different scales\n",
    "        concatenated_attention = tf.concat(attention_outputs, axis=-1)\n",
    "\n",
    "        # Apply a linear layer to combine scaled attentions\n",
    "        combined_attention = self.linear_layer(concatenated_attention)xz xc zxx\n",
    "\n",
    "        return combined_attention\n",
    "\n",
    "# Example usage of the MultiScaleMultiHeadAttention layer within a Transformer\n",
    "class TransformerBlock(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, num_scales, dff, rate=0.1):\n",
    "        super(TransformerBlock, self).__init__()\n",
    "\n",
    "        self.mha = MultiScaleMultiHeadAttention(num_heads, d_model, num_scales)\n",
    "        self.ffn = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(dff, activation='relu'),\n",
    "            tf.keras.layers.Dense(d_model)\n",
    "        ])\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "\n",
    "    def call(self, x, training, mask=None):\n",
    "        attn_output = self.mha(x, mask=mask)  # Multi-Scale Multi-Head Attention\n",
    "        out1 = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(x + out1)  # Residual Connection and Layer Normalization\n",
    "\n",
    "        ffn_output = self.ffn(out1)  # Feed Forward Network\n",
    "        out2 = self.dropout2(ffn_output, training=training)\n",
    "        out2 = self.layernorm2(out1 + out2)  # Residual Connection and Layer Normalization\n",
    "\n",
    "        return out2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-04 14:27:20.088299: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Exception encountered when calling layer \"transformer_block\" (type TransformerBlock).\n\nin user code:\n\n    File \"/tmp/ipykernel_34752/3654125400.py\", line 57, in call  *\n        out1 = self.layernorm1(x + out1)  # Residual Connection and Layer Normalization\n\n    ValueError: Dimensions must be equal, but are 3 and 32 for '{{node transformer_block/add}} = AddV2[T=DT_FLOAT](Placeholder, transformer_block/dropout/Identity)' with input shapes: [?,30,3], [?,30,32].\n\n\nCall arguments received by layer \"transformer_block\" (type TransformerBlock):\n  • x=tf.Tensor(shape=(None, 30, 3), dtype=float32)\n  • training=None\n  • mask=None",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 13\u001b[0m\n\u001b[1;32m     10\u001b[0m lstm_models \u001b[39m=\u001b[39m []\n\u001b[1;32m     11\u001b[0m \u001b[39mfor\u001b[39;00m inx \u001b[39min\u001b[39;00m \u001b[39mrange\u001b[39m(X_train\u001b[39m.\u001b[39mshape[\u001b[39m0\u001b[39m]):\n\u001b[1;32m     12\u001b[0m     \u001b[39m# Create a simple forecasting model using Multi-Scale Multi-Head Attention\u001b[39;00m\n\u001b[0;32m---> 13\u001b[0m     model \u001b[39m=\u001b[39m tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mSequential([\n\u001b[1;32m     14\u001b[0m         tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mlayers\u001b[39m.\u001b[39mInput(shape\u001b[39m=\u001b[39m(sequence_length, input_dim)),\n\u001b[1;32m     15\u001b[0m         TransformerBlock(d_model\u001b[39m=\u001b[39m\u001b[39m32\u001b[39m, num_heads\u001b[39m=\u001b[39m\u001b[39m4\u001b[39m, num_scales\u001b[39m=\u001b[39m\u001b[39m3\u001b[39m, dff\u001b[39m=\u001b[39m\u001b[39m512\u001b[39m),\n\u001b[1;32m     16\u001b[0m         tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mlayers\u001b[39m.\u001b[39mFlatten(),\n\u001b[1;32m     17\u001b[0m         tf\u001b[39m.\u001b[39mkeras\u001b[39m.\u001b[39mlayers\u001b[39m.\u001b[39mDense(\u001b[39m1\u001b[39m)  \u001b[39m# Output layer for forecasting\u001b[39;00m\n\u001b[1;32m     18\u001b[0m     ])\n\u001b[1;32m     20\u001b[0m     \u001b[39m# Compile the model\u001b[39;00m\n\u001b[1;32m     21\u001b[0m     model\u001b[39m.\u001b[39mcompile(optimizer\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39madam\u001b[39m\u001b[39m'\u001b[39m, loss\u001b[39m=\u001b[39m\u001b[39m'\u001b[39m\u001b[39mmse\u001b[39m\u001b[39m'\u001b[39m)\n",
      "File \u001b[0;32m~/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/tensorflow/python/trackable/base.py:205\u001b[0m, in \u001b[0;36mno_automatic_dependency_tracking.<locals>._method_wrapper\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    203\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_self_setattr_tracking \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m  \u001b[39m# pylint: disable=protected-access\u001b[39;00m\n\u001b[1;32m    204\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 205\u001b[0m   result \u001b[39m=\u001b[39m method(\u001b[39mself\u001b[39m, \u001b[39m*\u001b[39margs, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mkwargs)\n\u001b[1;32m    206\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m    207\u001b[0m   \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_self_setattr_tracking \u001b[39m=\u001b[39m previous_value  \u001b[39m# pylint: disable=protected-access\u001b[39;00m\n",
      "File \u001b[0;32m~/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[39m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[39m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[39mraise\u001b[39;00m e\u001b[39m.\u001b[39mwith_traceback(filtered_tb) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[39mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m/tmp/__autograph_generated_filehezu72au.py:12\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__call\u001b[0;34m(self, x, training, mask)\u001b[0m\n\u001b[1;32m     10\u001b[0m attn_output \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mmha, (ag__\u001b[39m.\u001b[39mld(x),), \u001b[39mdict\u001b[39m(mask\u001b[39m=\u001b[39mag__\u001b[39m.\u001b[39mld(mask)), fscope)\n\u001b[1;32m     11\u001b[0m out1 \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mdropout1, (ag__\u001b[39m.\u001b[39mld(attn_output),), \u001b[39mdict\u001b[39m(training\u001b[39m=\u001b[39mag__\u001b[39m.\u001b[39mld(training)), fscope)\n\u001b[0;32m---> 12\u001b[0m out1 \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mlayernorm1, (ag__\u001b[39m.\u001b[39mld(x) \u001b[39m+\u001b[39m ag__\u001b[39m.\u001b[39mld(out1),), \u001b[39mNone\u001b[39;00m, fscope)\n\u001b[1;32m     13\u001b[0m ffn_output \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mffn, (ag__\u001b[39m.\u001b[39mld(out1),), \u001b[39mNone\u001b[39;00m, fscope)\n\u001b[1;32m     14\u001b[0m out2 \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mdropout2, (ag__\u001b[39m.\u001b[39mld(ffn_output),), \u001b[39mdict\u001b[39m(training\u001b[39m=\u001b[39mag__\u001b[39m.\u001b[39mld(training)), fscope)\n",
      "\u001b[0;31mValueError\u001b[0m: Exception encountered when calling layer \"transformer_block\" (type TransformerBlock).\n\nin user code:\n\n    File \"/tmp/ipykernel_34752/3654125400.py\", line 57, in call  *\n        out1 = self.layernorm1(x + out1)  # Residual Connection and Layer Normalization\n\n    ValueError: Dimensions must be equal, but are 3 and 32 for '{{node transformer_block/add}} = AddV2[T=DT_FLOAT](Placeholder, transformer_block/dropout/Identity)' with input shapes: [?,30,3], [?,30,32].\n\n\nCall arguments received by layer \"transformer_block\" (type TransformerBlock):\n  • x=tf.Tensor(shape=(None, 30, 3), dtype=float32)\n  • training=None\n  • mask=None"
     ]
    }
   ],
   "source": [
    "# Create a Transformer model\n",
    "num_layers = 4\n",
    "d_model = 128\n",
    "num_heads = 8\n",
    "dff = 512\n",
    "input_vocab_size = 10000\n",
    "maximum_position_encoding = 10000\n",
    "\n",
    "\n",
    "lstm_models = []\n",
    "for inx in range(X_train.shape[0]):\n",
    "    # Create a simple forecasting model using Multi-Scale Multi-Head Attention\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Input(shape=(sequence_length, input_dim)),\n",
    "        TransformerBlock(d_model=32, num_heads=4, num_scales=3, dff=512),\n",
    "        tf.keras.layers.Flatten(),\n",
    "        tf.keras.layers.Dense(1)  # Output layer for forecasting\n",
    "    ])\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(optimizer='adam', loss='mse')\n",
    "    lstm_models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "history_list = []\n",
    "earlyStopCallBack = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=8)\n",
    "for inx, lm in enumerate(lstm_models):\n",
    "    print(f\"------------ In data pair {inx} --------------\")\n",
    "    history = lm.fit(X_train[inx], y_train[inx], epochs=5, batch_size=16, validation_split=0.3, callbacks=[earlyStopCallBack] )\n",
    "    history_list.append(history)\n",
    "    \n",
    "print(f\"------------ Training Ended --------------\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for history in history_list:\n",
    "    # list all data in history\n",
    "    print(history.history.keys())\n",
    "    # summarize history for accuracy\n",
    "    plt.plot(history.history['loss'])\n",
    "    plt.plot(history.history['val_loss'])\n",
    "    plt.title('model mean_squared_error')\n",
    "    plt.ylabel('mean_squared_error')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.legend(['train', 'test'], loc='upper left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"data/test_newdata.csv\", delimiter=',')\n",
    "# print(dataset.head)\n",
    "\n",
    "uwb_pair = [(1,4), (2,4), (3,4),(4,5),(1,2), (1,3), (1,5), (2,3), (2,4), (2,5), (3,5)]\n",
    "uwb_data_test = []\n",
    "for up in uwb_pair:\n",
    "    rslt_df = dataset.loc[dataset['node1'] == up[0]]\n",
    "    rslt_df_new = rslt_df.loc[rslt_df['node2'] == up[1]]\n",
    "    uwb_data_test.append(rslt_df_new[[\"uwb_range\", \"tb_node1_yaw\", \"tb_node2_yaw\",  \"error\"]].iloc[:200,:].values)\n",
    "\n",
    "print(uwb_data_test[1].shape)\n",
    "\n",
    "\n",
    "X_test = []\n",
    "y_test = []\n",
    "for inx, dataset_test in enumerate(uwb_data_test):\n",
    "    x_temp = []\n",
    "    y_temp = []\n",
    "    for i in range(n_steps,dataset_test.shape[0]):\n",
    "        x_temp.append(dataset_test[i-n_steps:i,0:3])\n",
    "        y_temp.append(dataset_test[i,3:])\n",
    "    print(np.array(x_temp).shape, np.array(y_temp).shape)\n",
    "    X_test.append(np.array(x_temp))\n",
    "    y_test.append(np.array(y_temp))\n",
    "X_test, y_test = np.array(X_test), np.array(y_test)\n",
    "\n",
    "print(X_test[1].shape, y_test[1].shape)\n",
    "\n",
    "Predicted_X = []\n",
    "for inx in range(len(lstm_models)):\n",
    "    predict_x = lstm_models[inx].predict(X_test[inx], verbose=0)\n",
    "    Predicted_X.append(predict_x)\n",
    "    mse = tf.keras.losses.MeanSquaredError()\n",
    "    print(mse(y_test[inx], predict_x).numpy())\n",
    "\n",
    "\n",
    "for inx in range(len(lstm_models)):\n",
    "    lstm_models[inx].save('models/lstm_uwb_{}'.format(inx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for p in range(len(Predicted_X)):\n",
    "    plt.plot(y_test[p], color='red', label= 'Real UWB Error')\n",
    "    plt.plot(Predicted_X[p], color='blue', label='Estimated UWB Error')\n",
    "    plt.title(\"UWB Error Estimation\")\n",
    "    plt.xlabel('Time')\n",
    "    plt.ylabel('UWB Error')\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "uwb-lstm-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}