Keras - LSTM (KDDCup1999).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset : KDDCup 1999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/abulbasar/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.layers import InputLayer, Dense, Dropout\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "\n",
    "from time import time\n",
    "\n",
    "pd.options.display.max_columns = 1000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"/data/kddcup.data\", header=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The file kddcup.data does not have a header. In the description file, you can find the column names. Parse the columns names."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['duration', 'protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes', 'land', 'wrong_fragment', 'urgent', 'hot', 'num_failed_logins', 'logged_in', 'num_compromised', 'root_shell', 'su_attempted', 'num_root', 'num_file_creations', 'num_shells', 'num_access_files', 'num_outbound_cmds', 'is_host_login', 'is_guest_login', 'count', 'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate', 'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate', 'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count', 'dst_host_same_srv_rate', 'dst_host_diff_srv_rate', 'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate', 'dst_host_serror_rate', 'dst_host_srv_serror_rate', 'dst_host_rerror_rate', 'dst_host_srv_rerror_rate', 'Category']\n"
     ]
    }
   ],
   "source": [
    "columns = [f.split(\":\")[0] for f in \"\"\"\n",
    "duration: continuous.\n",
    "protocol_type: symbolic.\n",
    "service: symbolic.\n",
    "flag: symbolic.\n",
    "src_bytes: continuous.\n",
    "dst_bytes: continuous.\n",
    "land: symbolic.\n",
    "wrong_fragment: continuous.\n",
    "urgent: continuous.\n",
    "hot: continuous.\n",
    "num_failed_logins: continuous.\n",
    "logged_in: symbolic.\n",
    "num_compromised: continuous.\n",
    "root_shell: continuous.\n",
    "su_attempted: continuous.\n",
    "num_root: continuous.\n",
    "num_file_creations: continuous.\n",
    "num_shells: continuous.\n",
    "num_access_files: continuous.\n",
    "num_outbound_cmds: continuous.\n",
    "is_host_login: symbolic.\n",
    "is_guest_login: symbolic.\n",
    "count: continuous.\n",
    "srv_count: continuous.\n",
    "serror_rate: continuous.\n",
    "srv_serror_rate: continuous.\n",
    "rerror_rate: continuous.\n",
    "srv_rerror_rate: continuous.\n",
    "same_srv_rate: continuous.\n",
    "diff_srv_rate: continuous.\n",
    "srv_diff_host_rate: continuous.\n",
    "dst_host_count: continuous.\n",
    "dst_host_srv_count: continuous.\n",
    "dst_host_same_srv_rate: continuous.\n",
    "dst_host_diff_srv_rate: continuous.\n",
    "dst_host_same_src_port_rate: continuous.\n",
    "dst_host_srv_diff_host_rate: continuous.\n",
    "dst_host_serror_rate: continuous.\n",
    "dst_host_srv_serror_rate: continuous.\n",
    "dst_host_rerror_rate: continuous.\n",
    "dst_host_srv_rerror_rate: continuous.\n",
    "\"\"\".split(\"\\n\") if len(f)>0]\n",
    "\n",
    "columns.append(\"Category\")\n",
    "print(columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the column names in the dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>num_root</th>\n",
       "      <th>num_file_creations</th>\n",
       "      <th>num_shells</th>\n",
       "      <th>num_access_files</th>\n",
       "      <th>num_outbound_cmds</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>count</th>\n",
       "      <th>srv_count</th>\n",
       "      <th>serror_rate</th>\n",
       "      <th>srv_serror_rate</th>\n",
       "      <th>rerror_rate</th>\n",
       "      <th>srv_rerror_rate</th>\n",
       "      <th>same_srv_rate</th>\n",
       "      <th>diff_srv_rate</th>\n",
       "      <th>srv_diff_host_rate</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>Category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>215</td>\n",
       "      <td>45076</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>162</td>\n",
       "      <td>4528</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>236</td>\n",
       "      <td>1228</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>233</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>tcp</td>\n",
       "      <td>http</td>\n",
       "      <td>SF</td>\n",
       "      <td>239</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.25</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>normal.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration protocol_type service flag  src_bytes  dst_bytes  land  \\\n",
       "0         0           tcp    http   SF        215      45076     0   \n",
       "1         0           tcp    http   SF        162       4528     0   \n",
       "2         0           tcp    http   SF        236       1228     0   \n",
       "3         0           tcp    http   SF        233       2032     0   \n",
       "4         0           tcp    http   SF        239        486     0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  num_failed_logins  logged_in  num_compromised  \\\n",
       "0               0       0    0                  0          1                0   \n",
       "1               0       0    0                  0          1                0   \n",
       "2               0       0    0                  0          1                0   \n",
       "3               0       0    0                  0          1                0   \n",
       "4               0       0    0                  0          1                0   \n",
       "\n",
       "   root_shell  su_attempted  num_root  num_file_creations  num_shells  \\\n",
       "0           0             0         0                   0           0   \n",
       "1           0             0         0                   0           0   \n",
       "2           0             0         0                   0           0   \n",
       "3           0             0         0                   0           0   \n",
       "4           0             0         0                   0           0   \n",
       "\n",
       "   num_access_files  num_outbound_cmds  is_host_login  is_guest_login  count  \\\n",
       "0                 0                  0              0               0      1   \n",
       "1                 0                  0              0               0      2   \n",
       "2                 0                  0              0               0      1   \n",
       "3                 0                  0              0               0      2   \n",
       "4                 0                  0              0               0      3   \n",
       "\n",
       "   srv_count  serror_rate  srv_serror_rate  rerror_rate  srv_rerror_rate  \\\n",
       "0          1          0.0              0.0          0.0              0.0   \n",
       "1          2          0.0              0.0          0.0              0.0   \n",
       "2          1          0.0              0.0          0.0              0.0   \n",
       "3          2          0.0              0.0          0.0              0.0   \n",
       "4          3          0.0              0.0          0.0              0.0   \n",
       "\n",
       "   same_srv_rate  diff_srv_rate  srv_diff_host_rate  dst_host_count  \\\n",
       "0            1.0            0.0                 0.0               0   \n",
       "1            1.0            0.0                 0.0               1   \n",
       "2            1.0            0.0                 0.0               2   \n",
       "3            1.0            0.0                 0.0               3   \n",
       "4            1.0            0.0                 0.0               4   \n",
       "\n",
       "   dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                   0                     0.0                     0.0   \n",
       "1                   1                     1.0                     0.0   \n",
       "2                   2                     1.0                     0.0   \n",
       "3                   3                     1.0                     0.0   \n",
       "4                   4                     1.0                     0.0   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.00                          0.0   \n",
       "1                         1.00                          0.0   \n",
       "2                         0.50                          0.0   \n",
       "3                         0.33                          0.0   \n",
       "4                         0.25                          0.0   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                   0.0                       0.0                   0.0   \n",
       "1                   0.0                       0.0                   0.0   \n",
       "2                   0.0                       0.0                   0.0   \n",
       "3                   0.0                       0.0                   0.0   \n",
       "4                   0.0                       0.0                   0.0   \n",
       "\n",
       "   dst_host_srv_rerror_rate Category  \n",
       "0                       0.0  normal.  \n",
       "1                       0.0  normal.  \n",
       "2                       0.0  normal.  \n",
       "3                       0.0  normal.  \n",
       "4                       0.0  normal.  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Take 70% data into training keeping aside the rest for testing. Do not shuffle the data since it is a chornological dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_size = int(df.shape[0] * 0.7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find any columns that have 0 standard deviation in the training set and eliminate it from analysis. 0 standard deviation will create devide by zero problem when calculating the Z-score for the columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>num_root</th>\n",
       "      <th>num_file_creations</th>\n",
       "      <th>num_shells</th>\n",
       "      <th>num_access_files</th>\n",
       "      <th>num_outbound_cmds</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>count</th>\n",
       "      <th>srv_count</th>\n",
       "      <th>serror_rate</th>\n",
       "      <th>srv_serror_rate</th>\n",
       "      <th>rerror_rate</th>\n",
       "      <th>srv_rerror_rate</th>\n",
       "      <th>same_srv_rate</th>\n",
       "      <th>diff_srv_rate</th>\n",
       "      <th>srv_diff_host_rate</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3428901.0</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "      <td>3.428901e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.798946e+01</td>\n",
       "      <td>1.336918e+03</td>\n",
       "      <td>1.438338e+03</td>\n",
       "      <td>4.957857e-06</td>\n",
       "      <td>3.858379e-04</td>\n",
       "      <td>7.290966e-06</td>\n",
       "      <td>1.484645e-02</td>\n",
       "      <td>4.112105e-05</td>\n",
       "      <td>1.762547e-01</td>\n",
       "      <td>9.721191e-03</td>\n",
       "      <td>8.778323e-05</td>\n",
       "      <td>3.995449e-05</td>\n",
       "      <td>1.470529e-02</td>\n",
       "      <td>1.266295e-03</td>\n",
       "      <td>8.457520e-05</td>\n",
       "      <td>1.073522e-03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.916386e-07</td>\n",
       "      <td>9.624075e-04</td>\n",
       "      <td>3.590027e+02</td>\n",
       "      <td>3.367904e+02</td>\n",
       "      <td>1.210919e-01</td>\n",
       "      <td>1.211769e-01</td>\n",
       "      <td>1.769130e-02</td>\n",
       "      <td>1.774800e-02</td>\n",
       "      <td>8.816183e-01</td>\n",
       "      <td>1.436075e-02</td>\n",
       "      <td>3.370236e-02</td>\n",
       "      <td>2.285571e+02</td>\n",
       "      <td>2.148149e+02</td>\n",
       "      <td>8.554925e-01</td>\n",
       "      <td>1.907444e-02</td>\n",
       "      <td>6.781845e-01</td>\n",
       "      <td>7.311792e-03</td>\n",
       "      <td>1.211629e-01</td>\n",
       "      <td>1.209890e-01</td>\n",
       "      <td>1.787890e-02</td>\n",
       "      <td>1.766435e-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>5.524748e+02</td>\n",
       "      <td>3.796818e+05</td>\n",
       "      <td>7.709141e+05</td>\n",
       "      <td>2.226619e-03</td>\n",
       "      <td>3.276238e-02</td>\n",
       "      <td>4.148088e-03</td>\n",
       "      <td>5.147092e-01</td>\n",
       "      <td>8.383510e-03</td>\n",
       "      <td>3.810368e-01</td>\n",
       "      <td>4.518491e+00</td>\n",
       "      <td>9.368861e-03</td>\n",
       "      <td>8.383516e-03</td>\n",
       "      <td>4.594144e+00</td>\n",
       "      <td>1.244340e-01</td>\n",
       "      <td>9.322081e-03</td>\n",
       "      <td>3.639156e-02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.400358e-04</td>\n",
       "      <td>3.100777e-02</td>\n",
       "      <td>2.162727e+02</td>\n",
       "      <td>2.380757e+02</td>\n",
       "      <td>3.253301e-01</td>\n",
       "      <td>3.259454e-01</td>\n",
       "      <td>1.297890e-01</td>\n",
       "      <td>1.309576e-01</td>\n",
       "      <td>3.111246e-01</td>\n",
       "      <td>8.070478e-02</td>\n",
       "      <td>1.513811e-01</td>\n",
       "      <td>6.941384e+01</td>\n",
       "      <td>8.754385e+01</td>\n",
       "      <td>3.338119e-01</td>\n",
       "      <td>8.883612e-02</td>\n",
       "      <td>4.579518e-01</td>\n",
       "      <td>4.097323e-02</td>\n",
       "      <td>3.252498e-01</td>\n",
       "      <td>3.258422e-01</td>\n",
       "      <td>1.276196e-01</td>\n",
       "      <td>1.286350e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.220000e+02</td>\n",
       "      <td>1.400000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.000000e-02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>5.832900e+04</td>\n",
       "      <td>6.933756e+08</td>\n",
       "      <td>1.309937e+09</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>7.700000e+01</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>7.479000e+03</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>7.468000e+03</td>\n",
       "      <td>4.000000e+01</td>\n",
       "      <td>2.000000e+00</td>\n",
       "      <td>9.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>5.110000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>2.550000e+02</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           duration     src_bytes     dst_bytes          land  wrong_fragment  \\\n",
       "count  3.428901e+06  3.428901e+06  3.428901e+06  3.428901e+06    3.428901e+06   \n",
       "mean   1.798946e+01  1.336918e+03  1.438338e+03  4.957857e-06    3.858379e-04   \n",
       "std    5.524748e+02  3.796818e+05  7.709141e+05  2.226619e-03    3.276238e-02   \n",
       "min    0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00    0.000000e+00   \n",
       "25%    0.000000e+00  2.550000e+02  0.000000e+00  0.000000e+00    0.000000e+00   \n",
       "50%    0.000000e+00  1.032000e+03  0.000000e+00  0.000000e+00    0.000000e+00   \n",
       "75%    0.000000e+00  1.032000e+03  0.000000e+00  0.000000e+00    0.000000e+00   \n",
       "max    5.832900e+04  6.933756e+08  1.309937e+09  1.000000e+00    3.000000e+00   \n",
       "\n",
       "             urgent           hot  num_failed_logins     logged_in  \\\n",
       "count  3.428901e+06  3.428901e+06       3.428901e+06  3.428901e+06   \n",
       "mean   7.290966e-06  1.484645e-02       4.112105e-05  1.762547e-01   \n",
       "std    4.148088e-03  5.147092e-01       8.383510e-03  3.810368e-01   \n",
       "min    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n",
       "25%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n",
       "50%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n",
       "75%    0.000000e+00  0.000000e+00       0.000000e+00  0.000000e+00   \n",
       "max    5.000000e+00  7.700000e+01       5.000000e+00  1.000000e+00   \n",
       "\n",
       "       num_compromised    root_shell  su_attempted      num_root  \\\n",
       "count     3.428901e+06  3.428901e+06  3.428901e+06  3.428901e+06   \n",
       "mean      9.721191e-03  8.778323e-05  3.995449e-05  1.470529e-02   \n",
       "std       4.518491e+00  9.368861e-03  8.383516e-03  4.594144e+00   \n",
       "min       0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%       0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "50%       0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "75%       0.000000e+00  0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "max       7.479000e+03  1.000000e+00  2.000000e+00  7.468000e+03   \n",
       "\n",
       "       num_file_creations    num_shells  num_access_files  num_outbound_cmds  \\\n",
       "count        3.428901e+06  3.428901e+06      3.428901e+06          3428901.0   \n",
       "mean         1.266295e-03  8.457520e-05      1.073522e-03                0.0   \n",
       "std          1.244340e-01  9.322081e-03      3.639156e-02                0.0   \n",
       "min          0.000000e+00  0.000000e+00      0.000000e+00                0.0   \n",
       "25%          0.000000e+00  0.000000e+00      0.000000e+00                0.0   \n",
       "50%          0.000000e+00  0.000000e+00      0.000000e+00                0.0   \n",
       "75%          0.000000e+00  0.000000e+00      0.000000e+00                0.0   \n",
       "max          4.000000e+01  2.000000e+00      9.000000e+00                0.0   \n",
       "\n",
       "       is_host_login  is_guest_login         count     srv_count  \\\n",
       "count   3.428901e+06    3.428901e+06  3.428901e+06  3.428901e+06   \n",
       "mean    2.916386e-07    9.624075e-04  3.590027e+02  3.367904e+02   \n",
       "std     5.400358e-04    3.100777e-02  2.162727e+02  2.380757e+02   \n",
       "min     0.000000e+00    0.000000e+00  0.000000e+00  0.000000e+00   \n",
       "25%     0.000000e+00    0.000000e+00  1.220000e+02  1.400000e+01   \n",
       "50%     0.000000e+00    0.000000e+00  5.110000e+02  5.110000e+02   \n",
       "75%     0.000000e+00    0.000000e+00  5.110000e+02  5.110000e+02   \n",
       "max     1.000000e+00    1.000000e+00  5.110000e+02  5.110000e+02   \n",
       "\n",
       "        serror_rate  srv_serror_rate   rerror_rate  srv_rerror_rate  \\\n",
       "count  3.428901e+06     3.428901e+06  3.428901e+06     3.428901e+06   \n",
       "mean   1.210919e-01     1.211769e-01  1.769130e-02     1.774800e-02   \n",
       "std    3.253301e-01     3.259454e-01  1.297890e-01     1.309576e-01   \n",
       "min    0.000000e+00     0.000000e+00  0.000000e+00     0.000000e+00   \n",
       "25%    0.000000e+00     0.000000e+00  0.000000e+00     0.000000e+00   \n",
       "50%    0.000000e+00     0.000000e+00  0.000000e+00     0.000000e+00   \n",
       "75%    0.000000e+00     0.000000e+00  0.000000e+00     0.000000e+00   \n",
       "max    1.000000e+00     1.000000e+00  1.000000e+00     1.000000e+00   \n",
       "\n",
       "       same_srv_rate  diff_srv_rate  srv_diff_host_rate  dst_host_count  \\\n",
       "count   3.428901e+06   3.428901e+06        3.428901e+06    3.428901e+06   \n",
       "mean    8.816183e-01   1.436075e-02        3.370236e-02    2.285571e+02   \n",
       "std     3.111246e-01   8.070478e-02        1.513811e-01    6.941384e+01   \n",
       "min     0.000000e+00   0.000000e+00        0.000000e+00    0.000000e+00   \n",
       "25%     1.000000e+00   0.000000e+00        0.000000e+00    2.550000e+02   \n",
       "50%     1.000000e+00   0.000000e+00        0.000000e+00    2.550000e+02   \n",
       "75%     1.000000e+00   0.000000e+00        0.000000e+00    2.550000e+02   \n",
       "max     1.000000e+00   1.000000e+00        1.000000e+00    2.550000e+02   \n",
       "\n",
       "       dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "count        3.428901e+06            3.428901e+06            3.428901e+06   \n",
       "mean         2.148149e+02            8.554925e-01            1.907444e-02   \n",
       "std          8.754385e+01            3.338119e-01            8.883612e-02   \n",
       "min          0.000000e+00            0.000000e+00            0.000000e+00   \n",
       "25%          2.550000e+02            1.000000e+00            0.000000e+00   \n",
       "50%          2.550000e+02            1.000000e+00            0.000000e+00   \n",
       "75%          2.550000e+02            1.000000e+00            0.000000e+00   \n",
       "max          2.550000e+02            1.000000e+00            1.000000e+00   \n",
       "\n",
       "       dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "count                 3.428901e+06                 3.428901e+06   \n",
       "mean                  6.781845e-01                 7.311792e-03   \n",
       "std                   4.579518e-01                 4.097323e-02   \n",
       "min                   0.000000e+00                 0.000000e+00   \n",
       "25%                   2.000000e-02                 0.000000e+00   \n",
       "50%                   1.000000e+00                 0.000000e+00   \n",
       "75%                   1.000000e+00                 0.000000e+00   \n",
       "max                   1.000000e+00                 1.000000e+00   \n",
       "\n",
       "       dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "count          3.428901e+06              3.428901e+06          3.428901e+06   \n",
       "mean           1.211629e-01              1.209890e-01          1.787890e-02   \n",
       "std            3.252498e-01              3.258422e-01          1.276196e-01   \n",
       "min            0.000000e+00              0.000000e+00          0.000000e+00   \n",
       "25%            0.000000e+00              0.000000e+00          0.000000e+00   \n",
       "50%            0.000000e+00              0.000000e+00          0.000000e+00   \n",
       "75%            0.000000e+00              0.000000e+00          0.000000e+00   \n",
       "max            1.000000e+00              1.000000e+00          1.000000e+00   \n",
       "\n",
       "       dst_host_srv_rerror_rate  \n",
       "count              3.428901e+06  \n",
       "mean               1.766435e-02  \n",
       "std                1.286350e-01  \n",
       "min                0.000000e+00  \n",
       "25%                0.000000e+00  \n",
       "50%                0.000000e+00  \n",
       "75%                0.000000e+00  \n",
       "max                1.000000e+00  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[:training_size, :].describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "num_outbound_cmds has 0 standard deviation, so eliminate it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(columns=[\"num_outbound_cmds\"], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Look at the column types. We would focus on numeric columns. For 3 categorical feature columns, you can do one hot encoding."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4898431 entries, 0 to 4898430\n",
      "Data columns (total 41 columns):\n",
      "duration                       int64\n",
      "protocol_type                  object\n",
      "service                        object\n",
      "flag                           object\n",
      "src_bytes                      int64\n",
      "dst_bytes                      int64\n",
      "land                           int64\n",
      "wrong_fragment                 int64\n",
      "urgent                         int64\n",
      "hot                            int64\n",
      "num_failed_logins              int64\n",
      "logged_in                      int64\n",
      "num_compromised                int64\n",
      "root_shell                     int64\n",
      "su_attempted                   int64\n",
      "num_root                       int64\n",
      "num_file_creations             int64\n",
      "num_shells                     int64\n",
      "num_access_files               int64\n",
      "is_host_login                  int64\n",
      "is_guest_login                 int64\n",
      "count                          int64\n",
      "srv_count                      int64\n",
      "serror_rate                    float64\n",
      "srv_serror_rate                float64\n",
      "rerror_rate                    float64\n",
      "srv_rerror_rate                float64\n",
      "same_srv_rate                  float64\n",
      "diff_srv_rate                  float64\n",
      "srv_diff_host_rate             float64\n",
      "dst_host_count                 int64\n",
      "dst_host_srv_count             int64\n",
      "dst_host_same_srv_rate         float64\n",
      "dst_host_diff_srv_rate         float64\n",
      "dst_host_same_src_port_rate    float64\n",
      "dst_host_srv_diff_host_rate    float64\n",
      "dst_host_serror_rate           float64\n",
      "dst_host_srv_serror_rate       float64\n",
      "dst_host_rerror_rate           float64\n",
      "dst_host_srv_rerror_rate       float64\n",
      "Category                       object\n",
      "dtypes: float64(15), int64(22), object(4)\n",
      "memory usage: 1.5+ GB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find only numeric columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4898431, 37)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_numeric = df.select_dtypes(include=[np.float64, np.int64])\n",
    "df_numeric.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Column names for the numeric columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['duration', 'src_bytes', 'dst_bytes', 'land', 'wrong_fragment',\n",
       "       'urgent', 'hot', 'num_failed_logins', 'logged_in', 'num_compromised',\n",
       "       'root_shell', 'su_attempted', 'num_root', 'num_file_creations',\n",
       "       'num_shells', 'num_access_files', 'is_host_login', 'is_guest_login',\n",
       "       'count', 'srv_count', 'serror_rate', 'srv_serror_rate', 'rerror_rate',\n",
       "       'srv_rerror_rate', 'same_srv_rate', 'diff_srv_rate',\n",
       "       'srv_diff_host_rate', 'dst_host_count', 'dst_host_srv_count',\n",
       "       'dst_host_same_srv_rate', 'dst_host_diff_srv_rate',\n",
       "       'dst_host_same_src_port_rate', 'dst_host_srv_diff_host_rate',\n",
       "       'dst_host_serror_rate', 'dst_host_srv_serror_rate',\n",
       "       'dst_host_rerror_rate', 'dst_host_srv_rerror_rate'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_columns = df_numeric.columns\n",
    "num_columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Form training and test dataframes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = df.iloc[:training_size, :]\n",
    "df_test = df.iloc[training_size:, :]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find mean and standard deviation for training dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_mean = df_numeric.values[:training_size].mean(axis = 0)\n",
    "feature_std = df_numeric.values[:training_size].std(axis = 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Verify that there is 0 standard deviation for any column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.52474763e+02, 3.79681720e+05, 7.70913959e+05, 2.22661900e-03,\n",
       "       3.27623707e-02, 4.14808705e-03, 5.14709113e-01, 8.38350879e-03,\n",
       "       3.81036739e-01, 4.51849039e+00, 9.36885919e-03, 8.38351443e-03,\n",
       "       4.59414297e+00, 1.24434017e-01, 9.32207915e-03, 3.63915531e-02,\n",
       "       5.40035691e-04, 3.10077613e-02, 2.16272661e+02, 2.38075635e+02,\n",
       "       3.25330051e-01, 3.25945337e-01, 1.29788998e-01, 1.30957597e-01,\n",
       "       3.11124558e-01, 8.07047659e-02, 1.51381121e-01, 6.94138253e+01,\n",
       "       8.75438367e+01, 3.33811862e-01, 8.88361104e-02, 4.57951771e-01,\n",
       "       4.09732266e-02, 3.25249721e-01, 3.25842135e-01, 1.27619624e-01,\n",
       "       1.28634967e-01])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Build the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import LSTM, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 10, 37)            0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 12)                2400      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 12)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 13        \n",
      "=================================================================\n",
      "Total params: 2,413\n",
      "Trainable params: 2,413\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "n_x = 37# Dimension of each transactional record\n",
    "sequence_length = 10 # number of time steps that we look into to make prediction\n",
    "\n",
    "size_of_hidden_state_within_cell = 12 \n",
    "\n",
    "model = keras.Sequential()\n",
    "model.add(InputLayer((sequence_length, n_x)))\n",
    "model.add(LSTM(size_of_hidden_state_within_cell, activation=\"tanh\"))\n",
    "model.add(Dropout(0.8))\n",
    "model.add(Dense(1, activation=\"sigmoid\"))\n",
    "\n",
    "model.compile(loss=\"binary_crossentropy\", metrics=[\"accuracy\"], optimizer=\"adam\")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Find baseline accuracy. As it shows below, the baseline accuracy is 0.84. So, our model should give better accuracy thatn 0.84."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False    0.80141\n",
       "True     0.19859\n",
       "Name: Category, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(df.Category == \"normal.\").value_counts()/df.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Refresh the numpy slicing behavior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([2, 3, 4, 5, 6, 7], 8)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[0, 1, 2, 3, 4, 5, 6, 7, 8][2:8], [0, 1, 2, 3, 4, 5, 6, 7, 8][8] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batching of records\n",
    "\n",
    "```\n",
    "0, 1, 2, 3, ... 100 - these are the index of the record. Each record of dim 37\n",
    "```\n",
    "Goal is to predict the label for record i\n",
    "\n",
    "Logistic regression\n",
    "- to do this we look at features of only record i\n",
    "\n",
    "In RNN\n",
    "- to do this we look at the features of record, i, i - 1, i - 2, ... i - sequence_len\n",
    "\n",
    "\n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "Batch size: 16, and sequence length: 10\n",
    "\n",
    "\n",
    "Batch 0: \n",
    "    0: X: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] y: [9]\n",
    "    1: X: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] y: [10]\n",
    "    ...\n",
    "    15: X: [15, 16, 17, 18, 19, 20, 21, 22, 23, 24] y: [24]\n",
    "\n",
    "Batch 1: \n",
    "    0: X: [16, 17, 18, 19, 20, 21, 22, 23, 24, 25], y: [25]\n",
    "    1: X: [17, 18, 19, 20, 21, 22, 23, 24, 25, 26], y: [26]\n",
    "    ...\n",
    "    15: X: [31, 32, 33, 34, 35, 36, 37, 38, 39, 40], y: [40]\n",
    "....\n",
    "\n",
    "Total number of batches: (100 - 10)/16 [(total_count - sequence_len)/batch_size]\n",
    "\n",
    "Last Batch:\n",
    "    0: X: [80, 81, 82, 83, 84, 85, 86, 87, 88, 89], y: [89]\n",
    "    ...\n",
    "    9: X: [89, 90, 91, 92, 93, 94, 95, 96, 97, 98], y: [98]\n",
    "    10: X: [90, 91, 92, 93, 94, 95, 96, 97, 98, 99], y: [99]\n",
    "\n",
    "```\n",
    "Last batch has 11 ... fewer than the batch size\n",
    "\n",
    "\n",
    "\n",
    "Let's build out generator function, that takes a dataframe and create batches of sequential data that is ready for RNN models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n",
      "(16, 10, 37) (16,)\n"
     ]
    }
   ],
   "source": [
    "def generator(df, batch_size = 128, sequence_length = 10):\n",
    "    X = df[num_columns].values\n",
    "    X = (X - feature_mean)/feature_std\n",
    "    y = np.where(df.Category == \"normal.\", 0, 1)\n",
    "    size = X.shape[0]\n",
    "    i = sequence_length - 1\n",
    "    X_batch = []\n",
    "    y_batch = []\n",
    "    num_batches = 0\n",
    "    while True:\n",
    "        X_batch.append(X[i-sequence_length+1:i+1])\n",
    "        y_batch.append(y[i])\n",
    "        \n",
    "        if (len(X_batch) == batch_size) or (i == size - 1):\n",
    "            X_batch = np.array(X_batch)\n",
    "            y_batch = np.array(y_batch)\n",
    "            num_batches += 1\n",
    "            #print(\"i\", i, \"batch size\", len(X_batch), \"num_batches\", num_batches,)\n",
    "            yield X_batch, y_batch\n",
    "            X_batch = []\n",
    "            y_batch = []\n",
    "            if i == size - 1:\n",
    "                i = sequence_length - 1\n",
    "        else:    \n",
    "            i += 1\n",
    "        \n",
    "count = 0\n",
    "df = df_train.iloc[:100, :]\n",
    "for batch in generator(df, 16):\n",
    "    print(batch[0].shape, batch[1].shape)\n",
    "    count += 1\n",
    "    # count > math.ceil((df.shape[0] - sequence_length)/16)\n",
    "    if count >= 20:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use the generator to fit the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch count 26789\n",
      "Epoch 1/10\n",
      "26789/26789 [==============================] - 364s 14ms/step - loss: 0.0771 - acc: 0.9772\n",
      "Epoch 2/10\n",
      "26789/26789 [==============================] - 361s 13ms/step - loss: 0.0891 - acc: 0.9805\n",
      "Epoch 3/10\n",
      "26789/26789 [==============================] - 357s 13ms/step - loss: 0.0869 - acc: 0.9812\n",
      "Epoch 4/10\n",
      "26789/26789 [==============================] - 358s 13ms/step - loss: 0.0830 - acc: 0.9820\n",
      "Epoch 5/10\n",
      "26789/26789 [==============================] - 362s 14ms/step - loss: 0.0814 - acc: 0.9822\n",
      "Epoch 6/10\n",
      "26789/26789 [==============================] - 365s 14ms/step - loss: 0.0818 - acc: 0.9822\n",
      "Epoch 7/10\n",
      "26789/26789 [==============================] - 368s 14ms/step - loss: 0.0795 - acc: 0.9826\n",
      "Epoch 8/10\n",
      "26789/26789 [==============================] - 371s 14ms/step - loss: 0.0803 - acc: 0.9827\n",
      "Epoch 9/10\n",
      "26789/26789 [==============================] - 334s 12ms/step - loss: 0.0797 - acc: 0.9826\n",
      "Epoch 10/10\n",
      "26789/26789 [==============================] - 319s 12ms/step - loss: 0.0803 - acc: 0.9827\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x181aa4f710>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tbaord = keras.callbacks.TensorBoard(log_dir=\"/tmp/tf/logs/%d\" % time())\n",
    "df = df_train #.iloc[:100, :]\n",
    "batch_size = 128\n",
    "batch_count = math.ceil((df.shape[0] - sequence_length)/batch_size)\n",
    "print(\"Batch count\", batch_count)\n",
    "model.fit_generator(generator(df, batch_size)\n",
    "                    , steps_per_epoch=batch_count\n",
    "                    , epochs=10\n",
    "                    , verbose=1\n",
    "                    , callbacks=[tbaord])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Evaluate the performance of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.018972053827949795, 0.9980383698397491]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "df = df_test\n",
    "\n",
    "batch_count = math.ceil((df.shape[0] - sequence_length)/batch_size)\n",
    "model.evaluate_generator(generator(df, batch_size), batch_count)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate prediction probabilities, classes and confusion matrix."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((1469696,), (1469696,))"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 256\n",
    "df = df_test\n",
    "batch_count = math.ceil((df.shape[0] - sequence_length)/batch_size)\n",
    "\n",
    "y_test, y_test_prob = [], []\n",
    "count = 0 \n",
    "for batch in generator(df, batch_size):\n",
    "    count += 1\n",
    "    y_test += list(batch[1])\n",
    "    y_test_prob += list(model.predict(batch[0]).flatten())\n",
    "    if count >= batch_count:\n",
    "        break\n",
    "y_test = np.array(y_test)\n",
    "y_test_prob = np.array(y_test_prob)\n",
    "y_test.shape, y_test_prob.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 222089,    2683],\n",
       "       [    200, 1244724]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn \n",
    "y_test_pred = np.where(y_test_prob > 0.5, 1, 0)\n",
    "sklearn.metrics.confusion_matrix(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results after 2 epochs\n",
    "```\n",
    "array([[ 221800,    2972],\n",
    "       [    453, 1244471]])\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 37)                0         \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                380       \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 5)                 55        \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 5)                 0         \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 1)                 6         \n",
      "=================================================================\n",
      "Total params: 441\n",
      "Trainable params: 441\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "n_x = len(num_columns)\n",
    "\n",
    "tf.set_random_seed(1)\n",
    "np.random.seed(1)\n",
    "\n",
    "model = keras.Sequential()\n",
    "model.add(InputLayer((n_x, )))\n",
    "model.add(Dense(units=10, activation=\"relu\"))\n",
    "model.add(Dropout(rate=0.5))\n",
    "model.add(Dense(units=5, activation=\"relu\"))\n",
    "model.add(Dropout(rate=0.5))\n",
    "model.add(Dense(units=1, activation=\"sigmoid\"))\n",
    "\n",
    "model.compile(loss=keras.losses.binary_crossentropy\n",
    "            , metrics=[\"accuracy\"], optimizer=\"adam\")\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(df):\n",
    "    X = df[num_columns].values\n",
    "    X = (X - feature_mean)/feature_std\n",
    "    y = np.where(df.Category == \"normal.\", 0, 1)\n",
    "    return X, y\n",
    "\n",
    "X_train, y_train = build_dataset(df_train)\n",
    "X_test, y_test = build_dataset(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3428901 samples, validate on 1469530 samples\n",
      "Epoch 1/10\n",
      "3428901/3428901 [==============================] - 40s 12us/step - loss: 0.1300 - acc: 0.9641 - val_loss: 0.0125 - val_acc: 0.9980\n",
      "Epoch 2/10\n",
      "3428901/3428901 [==============================] - 40s 12us/step - loss: 0.0966 - acc: 0.9697 - val_loss: 0.0094 - val_acc: 0.9985\n",
      "Epoch 3/10\n",
      "3428901/3428901 [==============================] - 43s 12us/step - loss: 0.0964 - acc: 0.9698 - val_loss: 0.0117 - val_acc: 0.9970\n",
      "Epoch 4/10\n",
      "3428901/3428901 [==============================] - 39s 12us/step - loss: 0.0959 - acc: 0.9698 - val_loss: 0.0122 - val_acc: 0.9973\n",
      "Epoch 5/10\n",
      "3428901/3428901 [==============================] - 42s 12us/step - loss: 0.0957 - acc: 0.9699 - val_loss: 0.0191 - val_acc: 0.9954\n",
      "Epoch 6/10\n",
      "3428901/3428901 [==============================] - 40s 12us/step - loss: 0.0955 - acc: 0.9700 - val_loss: 0.0132 - val_acc: 0.9969\n",
      "Epoch 7/10\n",
      "3428901/3428901 [==============================] - 38s 11us/step - loss: 0.0950 - acc: 0.9703 - val_loss: 0.0185 - val_acc: 0.9954\n",
      "Epoch 8/10\n",
      "3428901/3428901 [==============================] - 41s 12us/step - loss: 0.0949 - acc: 0.9703 - val_loss: 0.0157 - val_acc: 0.9963\n",
      "Epoch 9/10\n",
      "3428901/3428901 [==============================] - 42s 12us/step - loss: 0.0955 - acc: 0.9701 - val_loss: 0.0156 - val_acc: 0.9963\n",
      "Epoch 10/10\n",
      "3428901/3428901 [==============================] - 43s 12us/step - loss: 0.0953 - acc: 0.9702 - val_loss: 0.0204 - val_acc: 0.9942\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1833bb8c88>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor_board = keras.callbacks.TensorBoard(log_dir=\"/tmp/tf/logs/%d\" % time())\n",
    "\n",
    "model.fit(X_train, y_train, batch_size=256\n",
    "          , validation_data = (X_test, y_test)\n",
    "          , epochs=10\n",
    "          , verbose = 1\n",
    "          , callbacks=[tensor_board])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 221769,    7699],\n",
       "       [    770, 1239292]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test_pred = model.predict_classes(X_test)\n",
    "sklearn.metrics.confusion_matrix(y_test, y_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}