Scratch Cells.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook inspired from: http://www.austintaylor.io/d3/python/pandas/2016/02/01/create-d3-chart-python-force-directed/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "import re\n",
    "import numpy as np\n",
    "\n",
    "def opposite(value, choices):\n",
    "    return set(choices).difference(set([value])).pop()\n",
    "\n",
    "### Generate some data to work with\n",
    "\n",
    "Time = np.random.rand(100) * np.random.rand(100) * 2\n",
    "Source = np.random.choice(['10.25.22.253','10.25.22.250'],100)\n",
    "Src_Port = np.random.choice([2546,80],100)\n",
    "df = pd.DataFrame({'Time':Time,'Source':Source,'Src_Port':Src_Port})\n",
    "df['Destination'] = df['Source'].apply(lambda x: opposite(x, ['10.25.22.253','10.25.22.250']))\n",
    "df['Dst_Port'] = df['Src_Port'].apply(lambda x: opposite(x, [2546,80]))\n",
    "df['Protocol'] = np.random.choice(['TCP','HTTP'],100)\n",
    "df['Length'] = np.random.randint(60,14654,100)\n",
    "df\n",
    "\n",
    "src_dst = df[[\"Source\",\"Destination\"]]\n",
    "\n",
    "src_dst.sample(10)\n",
    "\n",
    "def ip_matcher(address):\n",
    "    # Used to validate if string is an ipaddress\n",
    "    ip = re.match(\n",
    "        '^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$', address)\n",
    "    if ip:\n",
    "        return True\n",
    "    else:\n",
    "        return False\n",
    "\n",
    "src_dst.rename(columns={\"Source\":\"source\",\"Destination\":\"target\"}, inplace=True)\n",
    "src_dst['valid_src'] = src_dst.source.apply(ip_matcher)\n",
    "src_dst['valid_target'] = src_dst.target.apply(ip_matcher)\n",
    "\n",
    "valid_src_dest = src_dst[(src_dst.valid_src==True) & (src_dst.valid_target==True)]\n",
    "\n",
    "grouped_src_dst = valid_src_dest.groupby([\"source\",\"target\"]).size().reset_index()\n",
    "\n",
    "unique_ips = pd.Index(grouped_src_dst['source']\n",
    "                      .append(grouped_src_dst['target'])\n",
    "                      .reset_index(drop=True).unique())\n",
    "\n",
    "group_dict = {}\n",
    "counter = 0\n",
    "for ip in unique_ips:\n",
    "    breakout_ip = re.match(\"^(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$\", ip)\n",
    "    if breakout_ip:\n",
    "        net_id = '.'.join(breakout_ip.group(1,2,3))\n",
    "        if net_id not in group_dict:\n",
    "            counter += 1\n",
    "            group_dict[net_id] = counter\n",
    "        else:\n",
    "            pass\n",
    "\n",
    "grouped_src_dst.rename(columns={0:'count'}, inplace=True)\n",
    "temp_links_list = list(grouped_src_dst.apply(lambda row: {\"source\": row['source'], \"target\": row['target'], \"value\": row['count']}, axis=1))\n",
    "\n",
    "temp_links_list\n",
    "\n",
    "links_list = []\n",
    "for link in temp_links_list:\n",
    "    record = {\"value\":link['value'], \"source\":unique_ips.get_loc(link['source']),\n",
    "     \"target\": unique_ips.get_loc(link['target'])}\n",
    "    links_list.append(record)\n",
    "\n",
    "nodes_list = []\n",
    "\n",
    "for ip in unique_ips:\n",
    "    breakout_ip = re.match(\"^(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})$\", ip)\n",
    "    if breakout_ip:\n",
    "        net_id = '.'.join(breakout_ip.group(1,2,3))\n",
    "        nodes_list.append({\"name\":ip, \"group\": group_dict.get(net_id)})\n",
    "\n",
    "nodes_list\n",
    "\n",
    "links_list"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}