example-11.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the TensorBoard notebook extension.\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-04 09:35:15.527122: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow version:  2.12.0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.12.1'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from packaging import version\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "    \n",
    "print(\"TensorFlow version: \", tf.__version__)\n",
    "assert version.parse(tf.__version__).release[0] >= 2, \\\n",
    "    \"This notebook requires TensorFlow 2.0 or above.\"\n",
    "\n",
    "import tensorboard\n",
    "tensorboard.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Simulated weather data (replace with your own dataset)\n",
    "num_samples = 1000\n",
    "sequence_length = 47  # Time steps per sequence\n",
    "num_features = 5     # Weather features (e.g., temperature, humidity, pressure, etc.)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weather_data shape: (1000, 47, 5)\n"
     ]
    }
   ],
   "source": [
    "# Generate synthetic weather data (replace with actual data loading)\n",
    "weather_data = np.random.rand(num_samples, sequence_length, num_features)\n",
    "print(f\"weather_data shape: {weather_data.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Define a hybrid model with multi-scale attention\n",
    "class MultiScaleAttentionModel(tf.keras.Model):\n",
    "    def __init__(self, num_heads, num_layers, num_filters):\n",
    "        super(MultiScaleAttentionModel, self).__init__()\n",
    "        self.conv1d = layers.Conv1D(num_filters, kernel_size=3, padding='same', activation='relu')\n",
    "        self.multi_head_attention = layers.MultiHeadAttention(num_heads=num_heads, key_dim=num_features, value_dim=num_features)  # Adjust key_dim and value_dim\n",
    "        self.feed_forward = layers.Dense(num_filters, activation='relu')\n",
    "        self.layer_norm1 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layer_norm2 = layers.LayerNormalization(epsilon=1e-6)\n",
    "        \n",
    "    def call(self, x):\n",
    "        # print(x.shape)\n",
    "        # Convolutional layer to capture local information\n",
    "        x_conv = self.conv1d(x)\n",
    "        print(\"1------\")\n",
    "        # Multi-head attention to capture global information\n",
    "        x_attention = self.multi_head_attention(x_conv, x_conv, x_conv)\n",
    "        print(f\"{x_attention.shape}\")\n",
    "        x_attention = self.conv1d(x_attention)  # Additional convolution to match dimensions\n",
    "        print(f\"{x_attention.shape}\")\n",
    "        print(f\"2--------------:{x.shape}, {x_attention.shape}\")\n",
    "        # Residual connection and layer normalization\n",
    "        x_residual = self.layer_norm1(x + x_attention)\n",
    "        print(\"3-------------------------\")\n",
    "        # Feed-forward network\n",
    "        x_ffn = self.feed_forward(x_residual)\n",
    "        print(\"4------------------------------\")\n",
    "        # Residual connection and layer normalization\n",
    "        x_out = self.layer_norm2(x_residual + x_ffn)\n",
    "        print(\"5------------------------------\")\n",
    "        print(x_out.shape)\n",
    "        return x_out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-09-04 09:35:16.926880: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n"
     ]
    }
   ],
   "source": [
    "# Build the multi-scale attention model\n",
    "num_heads = 8\n",
    "num_layers = 4\n",
    "num_filters = 64\n",
    "\n",
    "model = MultiScaleAttentionModel(num_heads, num_layers, num_filters)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x train shape: (1000, 46, 5)\n",
      "y train shape: (1000, 46, 5)\n",
      "Epoch 1/10\n",
      "1------\n",
      "(None, 46, 64)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "in user code:\n\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1284, in train_function  *\n        return step_function(self, iterator)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1268, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1249, in run_step  **\n        outputs = model.train_step(data)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1050, in train_step\n        y_pred = self(x, training=True)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/tmp/__autograph_generated_file9zm7mv8t.py\", line 14, in tf__call\n        x_attention = ag__.converted_call(ag__.ld(self).conv1d, (ag__.ld(x_attention),), None, fscope)\n\n    ValueError: Exception encountered when calling layer 'multi_scale_attention_model' (type MultiScaleAttentionModel).\n    \n    in user code:\n    \n        File \"/tmp/ipykernel_3400/35417886.py\", line 19, in call  *\n            x_attention = self.conv1d(x_attention)  # Additional convolution to match dimensions\n        File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py\", line 70, in error_handler  **\n            raise e.with_traceback(filtered_tb) from None\n        File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/input_spec.py\", line 280, in assert_input_compatibility\n            raise ValueError(\n    \n        ValueError: Input 0 of layer \"conv1d\" is incompatible with the layer: expected axis -1 of input shape to have value 5, but received input with shape (None, 46, 64)\n    \n    \n    Call arguments received by layer 'multi_scale_attention_model' (type MultiScaleAttentionModel):\n      • x=tf.Tensor(shape=(None, 46, 5), dtype=float32)\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 18\u001b[0m\n\u001b[1;32m     13\u001b[0m logdir\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mlogs/fit/\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m+\u001b[39m datetime\u001b[39m.\u001b[39mnow()\u001b[39m.\u001b[39mstrftime(\u001b[39m\"\u001b[39m\u001b[39m%\u001b[39m\u001b[39mY\u001b[39m\u001b[39m%\u001b[39m\u001b[39mm\u001b[39m\u001b[39m%d\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m%\u001b[39m\u001b[39mH\u001b[39m\u001b[39m%\u001b[39m\u001b[39mM\u001b[39m\u001b[39m%\u001b[39m\u001b[39mS\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     14\u001b[0m tensorboard_callback \u001b[39m=\u001b[39m keras\u001b[39m.\u001b[39mcallbacks\u001b[39m.\u001b[39mTensorBoard(log_dir\u001b[39m=\u001b[39mlogdir)\n\u001b[0;32m---> 18\u001b[0m model\u001b[39m.\u001b[39mfit(x_train, y_train, epochs\u001b[39m=\u001b[39m\u001b[39m10\u001b[39m, batch_size\u001b[39m=\u001b[39m\u001b[39m32\u001b[39m)\n\u001b[1;32m     19\u001b[0m  \u001b[39m#, callbacks=[tensorboard_callback])\u001b[39;00m\n\u001b[1;32m     20\u001b[0m \n\u001b[1;32m     21\u001b[0m \n\u001b[1;32m     22\u001b[0m \u001b[39m# Use the trained model for weather forecasting\u001b[39;00m\n\u001b[1;32m     23\u001b[0m forecast \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mpredict(x_train[:\u001b[39m1\u001b[39m])  \u001b[39m# Make a forecast for the first sequence\u001b[39;00m\n",
      "File \u001b[0;32m~/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py:70\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[39m=\u001b[39m _process_traceback_frames(e\u001b[39m.\u001b[39m__traceback__)\n\u001b[1;32m     68\u001b[0m     \u001b[39m# To get the full stack trace, call:\u001b[39;00m\n\u001b[1;32m     69\u001b[0m     \u001b[39m# `tf.debugging.disable_traceback_filtering()`\u001b[39;00m\n\u001b[0;32m---> 70\u001b[0m     \u001b[39mraise\u001b[39;00m e\u001b[39m.\u001b[39mwith_traceback(filtered_tb) \u001b[39mfrom\u001b[39;00m \u001b[39mNone\u001b[39;00m\n\u001b[1;32m     71\u001b[0m \u001b[39mfinally\u001b[39;00m:\n\u001b[1;32m     72\u001b[0m     \u001b[39mdel\u001b[39;00m filtered_tb\n",
      "File \u001b[0;32m/tmp/__autograph_generated_file__yk8xpq.py:15\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__train_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m     14\u001b[0m     do_return \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m---> 15\u001b[0m     retval_ \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(step_function), (ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m), ag__\u001b[39m.\u001b[39mld(iterator)), \u001b[39mNone\u001b[39;00m, fscope)\n\u001b[1;32m     16\u001b[0m \u001b[39mexcept\u001b[39;00m:\n\u001b[1;32m     17\u001b[0m     do_return \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m\n",
      "File \u001b[0;32m/tmp/__autograph_generated_file9zm7mv8t.py:14\u001b[0m, in \u001b[0;36mouter_factory.<locals>.inner_factory.<locals>.tf__call\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     12\u001b[0m x_attention \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mmulti_head_attention, (ag__\u001b[39m.\u001b[39mld(x_conv), ag__\u001b[39m.\u001b[39mld(x_conv), ag__\u001b[39m.\u001b[39mld(x_conv)), \u001b[39mNone\u001b[39;00m, fscope)\n\u001b[1;32m     13\u001b[0m ag__\u001b[39m.\u001b[39mld(\u001b[39mprint\u001b[39m)(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mag__\u001b[39m.\u001b[39mld(x_attention)\u001b[39m.\u001b[39mshape\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[0;32m---> 14\u001b[0m x_attention \u001b[39m=\u001b[39m ag__\u001b[39m.\u001b[39mconverted_call(ag__\u001b[39m.\u001b[39mld(\u001b[39mself\u001b[39m)\u001b[39m.\u001b[39mconv1d, (ag__\u001b[39m.\u001b[39mld(x_attention),), \u001b[39mNone\u001b[39;00m, fscope)\n\u001b[1;32m     15\u001b[0m ag__\u001b[39m.\u001b[39mld(\u001b[39mprint\u001b[39m)(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m{\u001b[39;00mag__\u001b[39m.\u001b[39mld(x_attention)\u001b[39m.\u001b[39mshape\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n\u001b[1;32m     16\u001b[0m ag__\u001b[39m.\u001b[39mld(\u001b[39mprint\u001b[39m)(\u001b[39mf\u001b[39m\u001b[39m'\u001b[39m\u001b[39m2--------------:\u001b[39m\u001b[39m{\u001b[39;00mag__\u001b[39m.\u001b[39mld(x)\u001b[39m.\u001b[39mshape\u001b[39m}\u001b[39;00m\u001b[39m, \u001b[39m\u001b[39m{\u001b[39;00mag__\u001b[39m.\u001b[39mld(x_attention)\u001b[39m.\u001b[39mshape\u001b[39m}\u001b[39;00m\u001b[39m'\u001b[39m)\n",
      "\u001b[0;31mValueError\u001b[0m: in user code:\n\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1284, in train_function  *\n        return step_function(self, iterator)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1268, in step_function  **\n        outputs = model.distribute_strategy.run(run_step, args=(data,))\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1249, in run_step  **\n        outputs = model.train_step(data)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/training.py\", line 1050, in train_step\n        y_pred = self(x, training=True)\n    File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py\", line 70, in error_handler\n        raise e.with_traceback(filtered_tb) from None\n    File \"/tmp/__autograph_generated_file9zm7mv8t.py\", line 14, in tf__call\n        x_attention = ag__.converted_call(ag__.ld(self).conv1d, (ag__.ld(x_attention),), None, fscope)\n\n    ValueError: Exception encountered when calling layer 'multi_scale_attention_model' (type MultiScaleAttentionModel).\n    \n    in user code:\n    \n        File \"/tmp/ipykernel_3400/35417886.py\", line 19, in call  *\n            x_attention = self.conv1d(x_attention)  # Additional convolution to match dimensions\n        File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/utils/traceback_utils.py\", line 70, in error_handler  **\n            raise e.with_traceback(filtered_tb) from None\n        File \"/home/xianjia/Workspace/miniconda3/envs/uwb-lstm-env/lib/python3.11/site-packages/keras/engine/input_spec.py\", line 280, in assert_input_compatibility\n            raise ValueError(\n    \n        ValueError: Input 0 of layer \"conv1d\" is incompatible with the layer: expected axis -1 of input shape to have value 5, but received input with shape (None, 46, 64)\n    \n    \n    Call arguments received by layer 'multi_scale_attention_model' (type MultiScaleAttentionModel):\n      • x=tf.Tensor(shape=(None, 46, 5), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# Compile the model (customize as needed)\n",
    "model.compile(optimizer='adam', loss='mse')\n",
    "\n",
    "# Train the model (replace with your own training data and settings)\n",
    "x_train = weather_data[:, :-1, :]\n",
    "y_train = weather_data[:, 1:, :]  # Predict the next time step\n",
    "\n",
    "print(f\"x train shape: {x_train.shape}\")\n",
    "print(f\"y train shape: {y_train.shape}\")\n",
    "\n",
    "\n",
    "# Define the Keras TensorBoard callback.\n",
    "logdir=\"logs/fit/\" + datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)\n",
    "\n",
    "\n",
    "\n",
    "model.fit(x_train, y_train, epochs=10, batch_size=32)\n",
    " #, callbacks=[tensorboard_callback])\n",
    "\n",
    "\n",
    "# Use the trained model for weather forecasting\n",
    "forecast = model.predict(x_train[:1])  # Make a forecast for the first sequence\n",
    "\n",
    "print(\"Forecasted Weather:\")\n",
    "print(forecast)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}