02-feature-extraction-vggish_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pre-compute Embeddings from `vggish`\n",
    "\n",
    "\n",
    "> **NOTE**\n",
    ">\n",
    "> You **do not** have the source data to run this notebook\n",
    "> and you don't have to!\n",
    ">\n",
    "> You have the pre-computed embeddings already.\n",
    "> "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Requirements\n",
    "\n",
    "- All data has been split into `trn`, `val` and `tst` sets.\n",
    "- All audio data have corresponding labels with the same filename (except extension).\n",
    "- Running `audioset/vggish_smoke_test.py` is successful."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:50:11.946233Z",
     "start_time": "2018-09-29T01:50:03.990593Z"
    }
   },
   "outputs": [],
   "source": [
    "# imports\n",
    "from pathlib import Path\n",
    "import tensorflow as tf\n",
    "\n",
    "import feat_ext as fx\n",
    "from audioset import vggish_slim\n",
    "from rennet.datasets.ka3 import ActiveSpeakers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Input Filepaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:50:12.652743Z",
     "start_time": "2018-09-29T01:50:12.642746Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "splits_root:\n",
      "/ml-school/data/fx\n"
     ]
    }
   ],
   "source": [
    "# Where to look for splits?\n",
    "\n",
    "dir_splits_root = Path.cwd().joinpath(\"data/fx\")\n",
    "\n",
    "if not dir_splits_root.exists():\n",
    "    raise RuntimeError(f\"splits_root does not exist at: {dir_splits_root}\")\n",
    "    \n",
    "print(f'splits_root:\\n{dir_splits_root}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:50:14.844016Z",
     "start_time": "2018-09-29T01:50:14.835988Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "splits:\n",
      "/ml-school/data/fx/trn\n",
      "/ml-school/data/fx/val\n",
      "/ml-school/data/fx/tst\n"
     ]
    }
   ],
   "source": [
    "dir_trn = dir_splits_root.joinpath('trn')\n",
    "dir_val = dir_splits_root.joinpath('val')\n",
    "dir_tst = dir_splits_root.joinpath('tst')\n",
    "\n",
    "for split in [dir_trn, dir_val, dir_tst]:\n",
    "    if not split.exists():\n",
    "        raise RuntimeError(f'split directory does not exist: {split}')\n",
    "        \n",
    "print('splits:', dir_trn, dir_val, dir_tst, sep='\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:50:23.510791Z",
     "start_time": "2018-09-29T01:50:16.230908Z"
    }
   },
   "outputs": [],
   "source": [
    "def get_ActiveSpeakers_labels(filepath):\n",
    "    return ActiveSpeakers.from_file(\n",
    "        filepath, \n",
    "        use_tags='ns', \n",
    "        tiers=lambda tn: \"x@\" in tn or tn.startswith(\"sp\"), \n",
    "        warn_duplicates=False\n",
    "    )\n",
    "\n",
    "pairs_trn = fx.AudioLabelPair.all_in_dir(dir_trn, \"**/*.wav\", [\"**/*.eaf\", \"**/*.xml\"], labels_parser=get_ActiveSpeakers_labels)\n",
    "pairs_val = fx.AudioLabelPair.all_in_dir(dir_val, \"**/*.wav\", [\"**/*.eaf\", \"**/*.xml\"], labels_parser=get_ActiveSpeakers_labels)\n",
    "pairs_tst = fx.AudioLabelPair.all_in_dir(dir_tst, \"**/*.wav\", [\"**/*.eaf\", \"**/*.xml\"], labels_parser=get_ActiveSpeakers_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:50:59.612337Z",
     "start_time": "2018-09-29T01:50:59.607335Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trn audio-label-pairs: 160\t 77162.04 sec\n",
      "val audio-label-pairs: 8\t  4262.33 sec\n",
      "tst audio-label-pairs: 62\t 33460.37 sec\n"
     ]
    }
   ],
   "source": [
    "print(f'trn audio-label-pairs: {len(pairs_trn)}\\t{sum(p.audio.seconds for p in pairs_trn):9.2f} sec')\n",
    "print(f'val audio-label-pairs: {len(pairs_val)}\\t{sum(p.audio.seconds for p in pairs_val):9.2f} sec')\n",
    "print(f'tst audio-label-pairs: {len(pairs_tst)}\\t{sum(p.audio.seconds for p in pairs_tst):9.2f} sec')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-29T01:51:15.899305Z",
     "start_time": "2018-09-29T01:51:15.762183Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percentage of [silence, speech] per split:\n",
      "\n",
      "val [31.28018978 68.71981022]\n",
      "tst [35.41842402 64.58157598]\n",
      "trn [35.81076051 64.18923949]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "pairs_splits = [\n",
    "        ('val', pairs_val), \n",
    "        ('tst', pairs_tst), \n",
    "        ('trn', pairs_trn)\n",
    "]\n",
    "\n",
    "print(\"Percentage of [silence, speech] per split:\\n\")\n",
    "for name, pairs in pairs_splits:\n",
    "    labels = np.zeros(2)\n",
    "    for pair in pairs:\n",
    "        labels += pair._get_label_examples().sum(axis=0)\n",
    "        \n",
    "    print(name, 100 * labels / labels.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Output Filepaths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:07:47.737903Z",
     "start_time": "2018-09-28T14:07:47.732900Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pickles for each split will be saved at:\n",
      "/ml-school/data/fx/pickles2/20190909-vggish_embedding\n"
     ]
    }
   ],
   "source": [
    "# Where to output\n",
    "\n",
    "dir_pickles_root = dir_splits_root.joinpath(\"pickles2\")\n",
    "\n",
    "dir_this_pickles = dir_pickles_root.joinpath(\"20190909-vggish_embedding\")\n",
    "dir_this_pickles.mkdir(exist_ok=True, parents=True)\n",
    "\n",
    "print(f'pickles for each split will be saved at:\\n{dir_this_pickles}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Write Pickles (`tfrecord`) for each split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VGGish model files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:07:47.781900Z",
     "start_time": "2018-09-28T14:07:47.740906Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vggish_model:\n",
      "/ml-school/data/models/vggish/vggish_model.ckpt\n",
      "\n",
      "vggish_pca_params:\n",
      "/ml-school/data/models/vggish/vggish_pca_params.npz\n"
     ]
    }
   ],
   "source": [
    "dir_vggish = Path.cwd().joinpath('data/models/vggish')\n",
    "fp_vggish_model = dir_vggish.joinpath('vggish_model.ckpt')\n",
    "fp_vggish_pca_params = dir_vggish.joinpath('vggish_pca_params.npz')\n",
    "\n",
    "for fp in [fp_vggish_model, fp_vggish_pca_params]:\n",
    "    if not fp.exists():\n",
    "        raise RuntimeError(\"model file {fp} not found.\")\n",
    "        \n",
    "print(f'vggish_model:\\n{fp_vggish_model}\\n')\n",
    "print(f'vggish_pca_params:\\n{fp_vggish_pca_params}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T14:11:54.898105Z",
     "start_time": "2018-09-28T14:11:54.889103Z"
    }
   },
   "outputs": [],
   "source": [
    "def parse(proto):\n",
    "    context_features = {k: v.Feature_type for k, v in fx.AudioLabelPair.context_features.items()}\n",
    "    \n",
    "    sequence_features = {\n",
    "        \"embedding\": tf.FixedLenSequenceFeature([3], dtype=tf.float32),\n",
    "        \"speechact\": tf.FixedLenSequenceFeature([2], dtype=tf.float32),\n",
    "    }\n",
    "    \n",
    "    _, feat_labels = tf.parse_single_sequence_example(\n",
    "        proto,\n",
    "        context_features=context_features,\n",
    "        sequence_features=sequence_features,\n",
    "    )\n",
    "    e = feat_labels[\"embedding\"]\n",
    "    \n",
    "    return {\"embedding\": e}, feat_labels['speechact']\n",
    "\n",
    "def input_fn():\n",
    "    dataset = tf.data.TFRecordDataset(\"t.tfrecord\")\n",
    "    dataset = dataset.map(parse)\n",
    "    dataset = dataset.repeat()\n",
    "    dataset = dataset.batch(32)\n",
    "    iterator = dataset.make_one_shot_iterator()\n",
    "    return iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:52.034644Z",
     "start_time": "2018-09-28T13:54:52.016645Z"
    }
   },
   "outputs": [],
   "source": [
    "post_processor = fx.get_post_processor(fp_vggish_pca_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-09-28T13:54:52.071631Z",
     "start_time": "2018-09-28T13:54:52.036620Z"
    }
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2018-09-28T13:54:35.802Z"
    }
   },
   "outputs": [],
   "source": [
    "with tf.Graph().as_default(), tf.Session() as sess:\n",
    "    vggish_slim.define_vggish_slim(training=False)\n",
    "    vggish_slim.load_vggish_slim_checkpoint(sess, str(fp_vggish_model.absolute()))\n",
    "    \n",
    "    pairs_splits = [\n",
    "            ('val', pairs_val), \n",
    "#             ('tst', pairs_tst),  # Skip exporting test split for the workshop\n",
    "            ('trn', pairs_trn)\n",
    "    ]\n",
    "    seconds = [sum(p.audio.nsamples for p in ps) for (_, ps) in pairs_splits]\n",
    "    main_pbar = tqdm(total=sum(seconds))\n",
    "    for ((name, pairs), sec) in zip(pairs_splits, seconds):\n",
    "        split_pbar = tqdm(total=sec)\n",
    "        main_pbar.set_description(name)\n",
    "        with tf.python_io.TFRecordWriter(str(dir_this_pickles.joinpath(f'{name}.tfrecord'))) as writer:\n",
    "            for (i, pair) in enumerate(pairs):\n",
    "                try:\n",
    "                    ex = pair.to_vggish_SequenceExample(sess, post_processor)\n",
    "                    writer.write(ex.SerializeToString())\n",
    "                except (KeyboardInterrupt, SystemExit):\n",
    "                    raise\n",
    "                except:\n",
    "                    print(pair.audio)\n",
    "                    print(pair._get_audio_examples().shape)\n",
    "                    print(pair._get_label_examples().shape)\n",
    "                \n",
    "                main_pbar.update(pair.audio.nsamples)\n",
    "                split_pbar.update(pair.audio.nsamples)\n",
    "            split_pbar.close()\n",
    "    main_pbar.close()\n",
    "    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}