Deep Learning Grid Search.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning Grid Search\n",
    "\n",
    "In this project, we will learn how to use the scikit-learn grid search capability.\n",
    "\n",
    "We are going to learn the following topics:\n",
    "\n",
    "* How to use Keras models in scikit-learn.\n",
    "* How to use grid search in scikit-learn.\n",
    "* How to tune batch size and training epochs.\n",
    "* How to tune learning rate\n",
    "* How to tune network weight initialization.\n",
    "* How to tune activation functions.\n",
    "* How to tune dropout regularization.\n",
    "* How to tune the number of neurons in the hidden layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "C:\\Programdata\\anaconda2\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python: 2.7.13 |Continuum Analytics, Inc.| (default, May 11 2017, 13:17:26) [MSC v.1500 64 bit (AMD64)]\n",
      "Pandas: 0.21.0\n",
      "Numpy: 1.14.0\n",
      "Sklearn: 0.19.1\n",
      "Keras: 2.0.5\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import pandas\n",
    "import numpy\n",
    "import sklearn\n",
    "import keras\n",
    "\n",
    "print('Python: {}'.format(sys.version))\n",
    "print('Pandas: {}'.format(pandas.__version__))\n",
    "print('Numpy: {}'.format(numpy.__version__))\n",
    "print('Sklearn: {}'.format(sklearn.__version__))\n",
    "print('Keras: {}'.format(keras.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# import the uci pima indians diabetes dataset\n",
    "url = \"http://archive.ics.uci.edu/ml/machine-learning-databases/pima-indians-diabetes/pima-indians-diabetes.data\"\n",
    "names = ['n_pregnant', 'glucose_concentration', 'blood_pressuer (mm Hg)', 'skin_thickness (mm)', 'serum_insulin (mu U/ml)',\n",
    "        'BMI', 'pedigree_function', 'age', 'class']\n",
    "df = pd.read_csv(url, names = names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_pregnant</th>\n",
       "      <th>glucose_concentration</th>\n",
       "      <th>blood_pressuer (mm Hg)</th>\n",
       "      <th>skin_thickness (mm)</th>\n",
       "      <th>serum_insulin (mu U/ml)</th>\n",
       "      <th>BMI</th>\n",
       "      <th>pedigree_function</th>\n",
       "      <th>age</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>120.894531</td>\n",
       "      <td>69.105469</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>31.992578</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>31.972618</td>\n",
       "      <td>19.355807</td>\n",
       "      <td>15.952218</td>\n",
       "      <td>115.244002</td>\n",
       "      <td>7.884160</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       n_pregnant  glucose_concentration  blood_pressuer (mm Hg)  \\\n",
       "count  768.000000             768.000000              768.000000   \n",
       "mean     3.845052             120.894531               69.105469   \n",
       "std      3.369578              31.972618               19.355807   \n",
       "min      0.000000               0.000000                0.000000   \n",
       "25%      1.000000              99.000000               62.000000   \n",
       "50%      3.000000             117.000000               72.000000   \n",
       "75%      6.000000             140.250000               80.000000   \n",
       "max     17.000000             199.000000              122.000000   \n",
       "\n",
       "       skin_thickness (mm)  serum_insulin (mu U/ml)         BMI  \\\n",
       "count           768.000000               768.000000  768.000000   \n",
       "mean             20.536458                79.799479   31.992578   \n",
       "std              15.952218               115.244002    7.884160   \n",
       "min               0.000000                 0.000000    0.000000   \n",
       "25%               0.000000                 0.000000   27.300000   \n",
       "50%              23.000000                30.500000   32.000000   \n",
       "75%              32.000000               127.250000   36.600000   \n",
       "max              99.000000               846.000000   67.100000   \n",
       "\n",
       "       pedigree_function         age       class  \n",
       "count         768.000000  768.000000  768.000000  \n",
       "mean            0.471876   33.240885    0.348958  \n",
       "std             0.331329   11.760232    0.476951  \n",
       "min             0.078000   21.000000    0.000000  \n",
       "25%             0.243750   24.000000    0.000000  \n",
       "50%             0.372500   29.000000    0.000000  \n",
       "75%             0.626250   41.000000    1.000000  \n",
       "max             2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Describe the dataset\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_pregnant</th>\n",
       "      <th>glucose_concentration</th>\n",
       "      <th>blood_pressuer (mm Hg)</th>\n",
       "      <th>skin_thickness (mm)</th>\n",
       "      <th>serum_insulin (mu U/ml)</th>\n",
       "      <th>BMI</th>\n",
       "      <th>pedigree_function</th>\n",
       "      <th>age</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>48</td>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>24.7</td>\n",
       "      <td>0.140</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>74</td>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>27.7</td>\n",
       "      <td>0.299</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.389</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>349</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>80</td>\n",
       "      <td>32</td>\n",
       "      <td>0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>0.346</td>\n",
       "      <td>37</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>68</td>\n",
       "      <td>41</td>\n",
       "      <td>0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0.727</td>\n",
       "      <td>41</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     n_pregnant  glucose_concentration  blood_pressuer (mm Hg)  \\\n",
       "75            1                      0                      48   \n",
       "182           1                      0                      74   \n",
       "342           1                      0                      68   \n",
       "349           5                      0                      80   \n",
       "502           6                      0                      68   \n",
       "\n",
       "     skin_thickness (mm)  serum_insulin (mu U/ml)   BMI  pedigree_function  \\\n",
       "75                    20                        0  24.7              0.140   \n",
       "182                   20                       23  27.7              0.299   \n",
       "342                   35                        0  32.0              0.389   \n",
       "349                   32                        0  41.0              0.346   \n",
       "502                   41                        0  39.0              0.727   \n",
       "\n",
       "     age  class  \n",
       "75    22      0  \n",
       "182   21      0  \n",
       "342   22      0  \n",
       "349   37      1  \n",
       "502   41      1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['glucose_concentration'] == 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_pregnant</th>\n",
       "      <th>glucose_concentration</th>\n",
       "      <th>blood_pressuer (mm Hg)</th>\n",
       "      <th>skin_thickness (mm)</th>\n",
       "      <th>serum_insulin (mu U/ml)</th>\n",
       "      <th>BMI</th>\n",
       "      <th>pedigree_function</th>\n",
       "      <th>age</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>763.000000</td>\n",
       "      <td>733.000000</td>\n",
       "      <td>541.000000</td>\n",
       "      <td>394.000000</td>\n",
       "      <td>757.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>121.686763</td>\n",
       "      <td>72.405184</td>\n",
       "      <td>29.153420</td>\n",
       "      <td>155.548223</td>\n",
       "      <td>32.457464</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "      <td>0.348958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>30.535641</td>\n",
       "      <td>12.382158</td>\n",
       "      <td>10.476982</td>\n",
       "      <td>118.775855</td>\n",
       "      <td>6.924988</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "      <td>0.476951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>18.200000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>64.000000</td>\n",
       "      <td>22.000000</td>\n",
       "      <td>76.250000</td>\n",
       "      <td>27.500000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>125.000000</td>\n",
       "      <td>32.300000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>141.000000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>190.000000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       n_pregnant  glucose_concentration  blood_pressuer (mm Hg)  \\\n",
       "count  768.000000             763.000000              733.000000   \n",
       "mean     3.845052             121.686763               72.405184   \n",
       "std      3.369578              30.535641               12.382158   \n",
       "min      0.000000              44.000000               24.000000   \n",
       "25%      1.000000              99.000000               64.000000   \n",
       "50%      3.000000             117.000000               72.000000   \n",
       "75%      6.000000             141.000000               80.000000   \n",
       "max     17.000000             199.000000              122.000000   \n",
       "\n",
       "       skin_thickness (mm)  serum_insulin (mu U/ml)         BMI  \\\n",
       "count           541.000000               394.000000  757.000000   \n",
       "mean             29.153420               155.548223   32.457464   \n",
       "std              10.476982               118.775855    6.924988   \n",
       "min               7.000000                14.000000   18.200000   \n",
       "25%              22.000000                76.250000   27.500000   \n",
       "50%              29.000000               125.000000   32.300000   \n",
       "75%              36.000000               190.000000   36.600000   \n",
       "max              99.000000               846.000000   67.100000   \n",
       "\n",
       "       pedigree_function         age       class  \n",
       "count         768.000000  768.000000  768.000000  \n",
       "mean            0.471876   33.240885    0.348958  \n",
       "std             0.331329   11.760232    0.476951  \n",
       "min             0.078000   21.000000    0.000000  \n",
       "25%             0.243750   24.000000    0.000000  \n",
       "50%             0.372500   29.000000    0.000000  \n",
       "75%             0.626250   41.000000    1.000000  \n",
       "max             2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Preprocess the data, mark zero values as NaN and drop\n",
    "columns = ['glucose_concentration', 'blood_pressuer (mm Hg)', 'skin_thickness (mm)', 'serum_insulin (mu U/ml)', 'BMI']\n",
    "\n",
    "for col in columns:\n",
    "    df[col].replace(0, np.NaN, inplace=True)\n",
    "    \n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>n_pregnant</th>\n",
       "      <th>glucose_concentration</th>\n",
       "      <th>blood_pressuer (mm Hg)</th>\n",
       "      <th>skin_thickness (mm)</th>\n",
       "      <th>serum_insulin (mu U/ml)</th>\n",
       "      <th>BMI</th>\n",
       "      <th>pedigree_function</th>\n",
       "      <th>age</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "      <td>392.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.301020</td>\n",
       "      <td>122.627551</td>\n",
       "      <td>70.663265</td>\n",
       "      <td>29.145408</td>\n",
       "      <td>156.056122</td>\n",
       "      <td>33.086224</td>\n",
       "      <td>0.523046</td>\n",
       "      <td>30.864796</td>\n",
       "      <td>0.331633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.211424</td>\n",
       "      <td>30.860781</td>\n",
       "      <td>12.496092</td>\n",
       "      <td>10.516424</td>\n",
       "      <td>118.841690</td>\n",
       "      <td>7.027659</td>\n",
       "      <td>0.345488</td>\n",
       "      <td>10.200777</td>\n",
       "      <td>0.471401</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>56.000000</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>14.000000</td>\n",
       "      <td>18.200000</td>\n",
       "      <td>0.085000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>76.750000</td>\n",
       "      <td>28.400000</td>\n",
       "      <td>0.269750</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>119.000000</td>\n",
       "      <td>70.000000</td>\n",
       "      <td>29.000000</td>\n",
       "      <td>125.500000</td>\n",
       "      <td>33.200000</td>\n",
       "      <td>0.449500</td>\n",
       "      <td>27.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>143.000000</td>\n",
       "      <td>78.000000</td>\n",
       "      <td>37.000000</td>\n",
       "      <td>190.000000</td>\n",
       "      <td>37.100000</td>\n",
       "      <td>0.687000</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>198.000000</td>\n",
       "      <td>110.000000</td>\n",
       "      <td>63.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       n_pregnant  glucose_concentration  blood_pressuer (mm Hg)  \\\n",
       "count  392.000000             392.000000              392.000000   \n",
       "mean     3.301020             122.627551               70.663265   \n",
       "std      3.211424              30.860781               12.496092   \n",
       "min      0.000000              56.000000               24.000000   \n",
       "25%      1.000000              99.000000               62.000000   \n",
       "50%      2.000000             119.000000               70.000000   \n",
       "75%      5.000000             143.000000               78.000000   \n",
       "max     17.000000             198.000000              110.000000   \n",
       "\n",
       "       skin_thickness (mm)  serum_insulin (mu U/ml)         BMI  \\\n",
       "count           392.000000               392.000000  392.000000   \n",
       "mean             29.145408               156.056122   33.086224   \n",
       "std              10.516424               118.841690    7.027659   \n",
       "min               7.000000                14.000000   18.200000   \n",
       "25%              21.000000                76.750000   28.400000   \n",
       "50%              29.000000               125.500000   33.200000   \n",
       "75%              37.000000               190.000000   37.100000   \n",
       "max              63.000000               846.000000   67.100000   \n",
       "\n",
       "       pedigree_function         age       class  \n",
       "count         392.000000  392.000000  392.000000  \n",
       "mean            0.523046   30.864796    0.331633  \n",
       "std             0.345488   10.200777    0.471401  \n",
       "min             0.085000   21.000000    0.000000  \n",
       "25%             0.269750   23.000000    0.000000  \n",
       "50%             0.449500   27.000000    0.000000  \n",
       "75%             0.687000   36.000000    1.000000  \n",
       "max             2.420000   81.000000    1.000000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Drop rows with missing values\n",
    "df.dropna(inplace=True)\n",
    "\n",
    "# summarize the number of rows and columns in df\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(392L, 9L)\n"
     ]
    }
   ],
   "source": [
    "# Convert dataframe to numpy array\n",
    "dataset = df.values\n",
    "print(dataset.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split into input (X) and an output (Y)\n",
    "X = dataset[:,0:8]\n",
    "Y = dataset[:, 8].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(392L, 8L)\n",
      "(392L,)\n",
      "[0 1 1 1 1]\n"
     ]
    }
   ],
   "source": [
    "print(X.shape)\n",
    "print(Y.shape)\n",
    "print(Y[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Normalize the data using sklearn StandardScaler\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler().fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "StandardScaler(copy=True, with_mean=True, with_std=True)\n"
     ]
    }
   ],
   "source": [
    "print(scaler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "      <td>3.920000e+02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-9.063045e-18</td>\n",
       "      <td>1.132881e-17</td>\n",
       "      <td>-4.531523e-16</td>\n",
       "      <td>1.087565e-16</td>\n",
       "      <td>1.064908e-16</td>\n",
       "      <td>1.631348e-16</td>\n",
       "      <td>1.812609e-17</td>\n",
       "      <td>1.110223e-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "      <td>1.001278e+00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.029213e+00</td>\n",
       "      <td>-2.161731e+00</td>\n",
       "      <td>-3.739001e+00</td>\n",
       "      <td>-2.108484e+00</td>\n",
       "      <td>-1.196867e+00</td>\n",
       "      <td>-2.120941e+00</td>\n",
       "      <td>-1.269525e+00</td>\n",
       "      <td>-9.682991e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-7.174265e-01</td>\n",
       "      <td>-7.665958e-01</td>\n",
       "      <td>-6.941640e-01</td>\n",
       "      <td>-7.755315e-01</td>\n",
       "      <td>-6.681786e-01</td>\n",
       "      <td>-6.676780e-01</td>\n",
       "      <td>-7.340909e-01</td>\n",
       "      <td>-7.719850e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-4.056403e-01</td>\n",
       "      <td>-1.176959e-01</td>\n",
       "      <td>-5.314565e-02</td>\n",
       "      <td>-1.384444e-02</td>\n",
       "      <td>-2.574448e-01</td>\n",
       "      <td>1.621036e-02</td>\n",
       "      <td>-2.131475e-01</td>\n",
       "      <td>-3.793569e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.297185e-01</td>\n",
       "      <td>6.609841e-01</td>\n",
       "      <td>5.878727e-01</td>\n",
       "      <td>7.478426e-01</td>\n",
       "      <td>2.859877e-01</td>\n",
       "      <td>5.718696e-01</td>\n",
       "      <td>4.751644e-01</td>\n",
       "      <td>5.040564e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>4.271153e+00</td>\n",
       "      <td>2.445459e+00</td>\n",
       "      <td>3.151946e+00</td>\n",
       "      <td>3.223325e+00</td>\n",
       "      <td>5.812990e+00</td>\n",
       "      <td>4.846172e+00</td>\n",
       "      <td>5.497667e+00</td>\n",
       "      <td>4.921123e+00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  0             1             2             3             4  \\\n",
       "count  3.920000e+02  3.920000e+02  3.920000e+02  3.920000e+02  3.920000e+02   \n",
       "mean  -9.063045e-18  1.132881e-17 -4.531523e-16  1.087565e-16  1.064908e-16   \n",
       "std    1.001278e+00  1.001278e+00  1.001278e+00  1.001278e+00  1.001278e+00   \n",
       "min   -1.029213e+00 -2.161731e+00 -3.739001e+00 -2.108484e+00 -1.196867e+00   \n",
       "25%   -7.174265e-01 -7.665958e-01 -6.941640e-01 -7.755315e-01 -6.681786e-01   \n",
       "50%   -4.056403e-01 -1.176959e-01 -5.314565e-02 -1.384444e-02 -2.574448e-01   \n",
       "75%    5.297185e-01  6.609841e-01  5.878727e-01  7.478426e-01  2.859877e-01   \n",
       "max    4.271153e+00  2.445459e+00  3.151946e+00  3.223325e+00  5.812990e+00   \n",
       "\n",
       "                  5             6             7  \n",
       "count  3.920000e+02  3.920000e+02  3.920000e+02  \n",
       "mean   1.631348e-16  1.812609e-17  1.110223e-16  \n",
       "std    1.001278e+00  1.001278e+00  1.001278e+00  \n",
       "min   -2.120941e+00 -1.269525e+00 -9.682991e-01  \n",
       "25%   -6.676780e-01 -7.340909e-01 -7.719850e-01  \n",
       "50%    1.621036e-02 -2.131475e-01 -3.793569e-01  \n",
       "75%    5.718696e-01  4.751644e-01  5.040564e-01  \n",
       "max    4.846172e+00  5.497667e+00  4.921123e+00  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform and display the training data\n",
    "X_standardized = scaler.transform(X)\n",
    "\n",
    "data = pd.DataFrame(X_standardized)\n",
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import necessary sklearn and keras packages\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 8)                 72        \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 36        \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1)                 5         \n",
      "=================================================================\n",
      "Total params: 113\n",
      "Trainable params: 113\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Start defining the model\n",
    "def create_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(4, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # compile the model\n",
    "    adam = Adam(lr = 0.01)\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "model = create_model()\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 9 candidates, totalling 27 fits\n",
      "[CV] epochs=10, batch_size=10 ........................................\n",
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6333 - acc: 0.6897     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.4842 - acc: 0.6973     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.4422 - acc: 0.6973     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.4247 - acc: 0.6973     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.4139 - acc: 0.8084     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.4024 - acc: 0.8238     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.3962 - acc: 0.8391     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.3886 - acc: 0.8276     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.3891 - acc: 0.8161     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.3763 - acc: 0.8238     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] ... epochs=10, batch_size=10, score=0.717557250543, total=   6.6s\n",
      "[CV] epochs=10, batch_size=10 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    6.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6607 - acc: 0.7280     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.6058 - acc: 0.7816     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.5757 - acc: 0.7854     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.5311 - acc: 0.8123     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.5161 - acc: 0.8008     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.5176 - acc: 0.7969     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.5040 - acc: 0.7969     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.4814 - acc: 0.7931     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.4738 - acc: 0.8008     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.4682 - acc: 0.8084     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] ... epochs=10, batch_size=10, score=0.770992366412, total=   4.6s\n",
      "[CV] epochs=10, batch_size=10 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   11.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "262/262 [==============================] - 0s - loss: 0.6352 - acc: 0.6450     \n",
      "Epoch 2/10\n",
      "262/262 [==============================] - 0s - loss: 0.5273 - acc: 0.6527     \n",
      "Epoch 3/10\n",
      "262/262 [==============================] - 0s - loss: 0.5015 - acc: 0.6870     \n",
      "Epoch 4/10\n",
      "262/262 [==============================] - 0s - loss: 0.4985 - acc: 0.7519     \n",
      "Epoch 5/10\n",
      "262/262 [==============================] - 0s - loss: 0.4887 - acc: 0.7748     \n",
      "Epoch 6/10\n",
      "262/262 [==============================] - 0s - loss: 0.4835 - acc: 0.7710     \n",
      "Epoch 7/10\n",
      "262/262 [==============================] - 0s - loss: 0.4791 - acc: 0.7824     \n",
      "Epoch 8/10\n",
      "262/262 [==============================] - 0s - loss: 0.4739 - acc: 0.7901     \n",
      "Epoch 9/10\n",
      "262/262 [==============================] - 0s - loss: 0.4707 - acc: 0.7901     \n",
      "Epoch 10/10\n",
      "262/262 [==============================] - 0s - loss: 0.4750 - acc: 0.7901     \n",
      " 10/262 [>.............................] - ETA: 0s[CV] ... epochs=10, batch_size=10, score=0.838461532043, total=   3.8s\n",
      "[CV] epochs=50, batch_size=10 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   15.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6134 - acc: 0.6935     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.4622 - acc: 0.6973     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.4310 - acc: 0.6973     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4156 - acc: 0.7893     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4094 - acc: 0.8391     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - 0s - loss: 0.3978 - acc: 0.8161     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - 0s - loss: 0.3857 - acc: 0.8276     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3780 - acc: 0.853 - 0s - loss: 0.3772 - acc: 0.8544     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.3789 - acc: 0.8314     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.3593 - acc: 0.8506     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.3637 - acc: 0.8429     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.3395 - acc: 0.8544     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - 0s - loss: 0.3306 - acc: 0.8582     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.3207 - acc: 0.8506     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3131 - acc: 0.8582     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3164 - acc: 0.8621     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - 0s - loss: 0.2957 - acc: 0.8697     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.2972 - acc: 0.8697     - ETA: 0s - loss: 0.2803 - acc: 0.888\n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.2893 - acc: 0.8736     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.2776 - acc: 0.8889     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.2766 - acc: 0.8927     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.2840 - acc: 0.8774     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.2744 - acc: 0.8774     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.2743 - acc: 0.8851     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.2613 - acc: 0.8851     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.2816 - acc: 0.8812     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.2729 - acc: 0.8851     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.2702 - acc: 0.8812     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.2674 - acc: 0.8812     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.2608 - acc: 0.8659     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.2561 - acc: 0.8774     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.2492 - acc: 0.8966     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.2457 - acc: 0.8927     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.2467 - acc: 0.8927     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.2545 - acc: 0.8927     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.2654 - acc: 0.8812     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - 0s - loss: 0.2445 - acc: 0.8774     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.2453 - acc: 0.8927     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - 0s - loss: 0.2379 - acc: 0.8889     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.2393 - acc: 0.8927     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.2293 - acc: 0.9004     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.2317 - acc: 0.8927     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.2331 - acc: 0.8927     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.2220 - acc: 0.9042     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.2313 - acc: 0.9004     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - 0s - loss: 0.2202 - acc: 0.8966     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.2304 - acc: 0.8927     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.2271 - acc: 0.9004     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.2274 - acc: 0.9004     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.2186 - acc: 0.9042     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] ... epochs=50, batch_size=10, score=0.732824422931, total=   7.0s\n",
      "[CV] epochs=50, batch_size=10 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   22.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6351 - acc: 0.6398     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.4993 - acc: 0.7625     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.4590 - acc: 0.7854     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4415 - acc: 0.8008     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4258 - acc: 0.8199     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - 0s - loss: 0.4231 - acc: 0.8161     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - 0s - loss: 0.4100 - acc: 0.8084     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - 0s - loss: 0.4047 - acc: 0.8199     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.3970 - acc: 0.8238     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.3907 - acc: 0.8352     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.3910 - acc: 0.8238     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.3771 - acc: 0.8429     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.1752 - acc: 1.000 - 0s - loss: 0.3838 - acc: 0.8276     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.3866 - acc: 0.8238     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3780 - acc: 0.8429     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3738 - acc: 0.8314     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - 0s - loss: 0.3853 - acc: 0.8391     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.3623 - acc: 0.8391     \n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.3621 - acc: 0.8506     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.3639 - acc: 0.8161     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.3629 - acc: 0.8314     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.3491 - acc: 0.8429     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.3654 - acc: 0.8467     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.3667 - acc: 0.8467     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.3488 - acc: 0.8467     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.3362 - acc: 0.8544     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.3369 - acc: 0.8429     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.3351 - acc: 0.8506     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.3394 - acc: 0.8582     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.3413 - acc: 0.8429     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.3285 - acc: 0.8582     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.3214 - acc: 0.8697     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.3304 - acc: 0.8621     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.3172 - acc: 0.8582     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.3099 - acc: 0.8774     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.3085 - acc: 0.8659     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3094 - acc: 0.876 - 0s - loss: 0.3082 - acc: 0.8736     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.3088 - acc: 0.8697     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - 0s - loss: 0.2950 - acc: 0.8889     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.3002 - acc: 0.8812     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.2992 - acc: 0.8659     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.2886 - acc: 0.8851     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.2860 - acc: 0.8927     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.2927 - acc: 0.8736     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.3003 - acc: 0.8889     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - 0s - loss: 0.3008 - acc: 0.8774     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.2795 - acc: 0.8966     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.2717 - acc: 0.8927     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.2708 - acc: 0.9080     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.2784 - acc: 0.8812     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] ... epochs=50, batch_size=10, score=0.793893126131, total=   7.0s\n",
      "[CV] epochs=50, batch_size=10 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   29.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/50\n",
      "262/262 [==============================] - 0s - loss: 0.6278 - acc: 0.6450     \n",
      "Epoch 2/50\n",
      "262/262 [==============================] - 0s - loss: 0.4938 - acc: 0.7481     \n",
      "Epoch 3/50\n",
      "262/262 [==============================] - 0s - loss: 0.4830 - acc: 0.7366     \n",
      "Epoch 4/50\n",
      "262/262 [==============================] - 0s - loss: 0.4683 - acc: 0.7634     \n",
      "Epoch 5/50\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3783 - acc: 0.800 - 0s - loss: 0.4621 - acc: 0.7519     \n",
      "Epoch 6/50\n",
      "262/262 [==============================] - 0s - loss: 0.4593 - acc: 0.7672     \n",
      "Epoch 7/50\n",
      "262/262 [==============================] - 0s - loss: 0.4508 - acc: 0.7748     \n",
      "Epoch 8/50\n",
      "262/262 [==============================] - 0s - loss: 0.4437 - acc: 0.7824     \n",
      "Epoch 9/50\n",
      "262/262 [==============================] - 0s - loss: 0.4459 - acc: 0.7710     \n",
      "Epoch 10/50\n",
      "262/262 [==============================] - 0s - loss: 0.4418 - acc: 0.7710     \n",
      "Epoch 11/50\n",
      "262/262 [==============================] - 0s - loss: 0.4367 - acc: 0.7824     \n",
      "Epoch 12/50\n",
      "262/262 [==============================] - 0s - loss: 0.4300 - acc: 0.7901     \n",
      "Epoch 13/50\n",
      "262/262 [==============================] - 0s - loss: 0.4377 - acc: 0.7595     \n",
      "Epoch 14/50\n",
      "262/262 [==============================] - 0s - loss: 0.4248 - acc: 0.7863     \n",
      "Epoch 15/50\n",
      "262/262 [==============================] - 0s - loss: 0.4261 - acc: 0.7863     \n",
      "Epoch 16/50\n",
      "262/262 [==============================] - 0s - loss: 0.4195 - acc: 0.7977     \n",
      "Epoch 17/50\n",
      "262/262 [==============================] - 0s - loss: 0.4177 - acc: 0.7977     \n",
      "Epoch 18/50\n",
      "262/262 [==============================] - 0s - loss: 0.4087 - acc: 0.7939     \n",
      "Epoch 19/50\n",
      "262/262 [==============================] - 0s - loss: 0.4073 - acc: 0.8015     \n",
      "Epoch 20/50\n",
      "262/262 [==============================] - 0s - loss: 0.4099 - acc: 0.8130     \n",
      "Epoch 21/50\n",
      "262/262 [==============================] - 0s - loss: 0.4064 - acc: 0.8206     \n",
      "Epoch 22/50\n",
      "262/262 [==============================] - 0s - loss: 0.3977 - acc: 0.8053     \n",
      "Epoch 23/50\n",
      "262/262 [==============================] - 0s - loss: 0.3970 - acc: 0.8244     \n",
      "Epoch 24/50\n",
      "262/262 [==============================] - 0s - loss: 0.3995 - acc: 0.8321     \n",
      "Epoch 25/50\n",
      "262/262 [==============================] - 0s - loss: 0.4084 - acc: 0.8206     \n",
      "Epoch 26/50\n",
      "262/262 [==============================] - 0s - loss: 0.3990 - acc: 0.8435     \n",
      "Epoch 27/50\n",
      "262/262 [==============================] - 0s - loss: 0.3940 - acc: 0.8206     \n",
      "Epoch 28/50\n",
      "262/262 [==============================] - 0s - loss: 0.3873 - acc: 0.8397     \n",
      "Epoch 29/50\n",
      "262/262 [==============================] - 0s - loss: 0.3849 - acc: 0.8435     \n",
      "Epoch 30/50\n",
      "262/262 [==============================] - 0s - loss: 0.3762 - acc: 0.8397     \n",
      "Epoch 31/50\n",
      "262/262 [==============================] - 0s - loss: 0.3755 - acc: 0.8359     \n",
      "Epoch 32/50\n",
      "262/262 [==============================] - 0s - loss: 0.3777 - acc: 0.8473     \n",
      "Epoch 33/50\n",
      "262/262 [==============================] - 0s - loss: 0.3696 - acc: 0.8511     \n",
      "Epoch 34/50\n",
      "262/262 [==============================] - 0s - loss: 0.3704 - acc: 0.8359     \n",
      "Epoch 35/50\n",
      "262/262 [==============================] - 0s - loss: 0.3782 - acc: 0.8321     \n",
      "Epoch 36/50\n",
      "262/262 [==============================] - 0s - loss: 0.3711 - acc: 0.8397     \n",
      "Epoch 37/50\n",
      "262/262 [==============================] - 0s - loss: 0.3657 - acc: 0.8550     \n",
      "Epoch 38/50\n",
      "262/262 [==============================] - 0s - loss: 0.3656 - acc: 0.8321     \n",
      "Epoch 39/50\n",
      "262/262 [==============================] - 0s - loss: 0.3658 - acc: 0.8473     \n",
      "Epoch 40/50\n",
      "262/262 [==============================] - 0s - loss: 0.3548 - acc: 0.8664     \n",
      "Epoch 41/50\n",
      "262/262 [==============================] - 0s - loss: 0.3531 - acc: 0.8550     \n",
      "Epoch 42/50\n",
      "262/262 [==============================] - 0s - loss: 0.3592 - acc: 0.8550     \n",
      "Epoch 43/50\n",
      "262/262 [==============================] - 0s - loss: 0.3474 - acc: 0.8588     \n",
      "Epoch 44/50\n",
      "262/262 [==============================] - 0s - loss: 0.3495 - acc: 0.8588     \n",
      "Epoch 45/50\n",
      "262/262 [==============================] - 0s - loss: 0.3486 - acc: 0.8588     \n",
      "Epoch 46/50\n",
      "262/262 [==============================] - 0s - loss: 0.3441 - acc: 0.8664     \n",
      "Epoch 47/50\n",
      "262/262 [==============================] - 0s - loss: 0.3339 - acc: 0.8702     \n",
      "Epoch 48/50\n",
      "262/262 [==============================] - 0s - loss: 0.3391 - acc: 0.8511     \n",
      "Epoch 49/50\n",
      "262/262 [==============================] - 0s - loss: 0.3505 - acc: 0.8511     \n",
      "Epoch 50/50\n",
      "262/262 [==============================] - 0s - loss: 0.3393 - acc: 0.8740     \n",
      " 10/262 [>.............................] - ETA: 0s[CV] ... epochs=50, batch_size=10, score=0.838461541213, total=   6.0s\n",
      "[CV] epochs=100, batch_size=10 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   35.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6011 - acc: 0.6858     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.4654 - acc: 0.6973     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - 0s - loss: 0.4275 - acc: 0.7126     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.4149 - acc: 0.8084     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.4071 - acc: 0.8161     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.3969 - acc: 0.8161     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.3885 - acc: 0.8314     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.3758 - acc: 0.8391     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.3699 - acc: 0.8352     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.3624 - acc: 0.8276     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.3537 - acc: 0.8352     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.3566 - acc: 0.8314     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.3423 - acc: 0.8391     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.3579 - acc: 0.8544     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.3475 - acc: 0.8391     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.3326 - acc: 0.8467     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.3230 - acc: 0.8544     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.3181 - acc: 0.8736     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.3196 - acc: 0.8659     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.3098 - acc: 0.8736     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.3057 - acc: 0.8774     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.3071 - acc: 0.8812     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.3336 - acc: 0.8621     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.3059 - acc: 0.8697     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.2998 - acc: 0.8851     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - 0s - loss: 0.2924 - acc: 0.8889     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.2900 - acc: 0.9004     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.2863 - acc: 0.8927     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.2890 - acc: 0.8851     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.2846 - acc: 0.8889     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.2821 - acc: 0.8966     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.2819 - acc: 0.8927     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.2909 - acc: 0.8697     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.2752 - acc: 0.8966     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.2801 - acc: 0.8966     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - 0s - loss: 0.2746 - acc: 0.9080     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.2716 - acc: 0.8966     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.2699 - acc: 0.8851     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.2673 - acc: 0.9080     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.2670 - acc: 0.8966     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.2690 - acc: 0.9004     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.2648 - acc: 0.9042     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.2638 - acc: 0.9042     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - 0s - loss: 0.2662 - acc: 0.9042     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.2999 - acc: 0.8812     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.2822 - acc: 0.8889     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.2805 - acc: 0.8851     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.2778 - acc: 0.8812     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.2724 - acc: 0.8927     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.2578 - acc: 0.9004     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.2558 - acc: 0.9004     - ETA: 0s - loss: 0.2568 - acc: 0.900\n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.2544 - acc: 0.9119     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.2512 - acc: 0.9080     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.2491 - acc: 0.9080     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.2486 - acc: 0.9157     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.2575 - acc: 0.9004     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.3078 - acc: 0.8736     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.2815 - acc: 0.8812     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.2667 - acc: 0.8851     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.2560 - acc: 0.8927     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.2512 - acc: 0.9042     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - 0s - loss: 0.2542 - acc: 0.9080     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.2520 - acc: 0.9042     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.2787 - acc: 0.8659     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.2544 - acc: 0.8927     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.2480 - acc: 0.9042     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.2470 - acc: 0.9195     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.2424 - acc: 0.9119     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.2442 - acc: 0.9080     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.2425 - acc: 0.9119     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.2484 - acc: 0.9080     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2683 - acc: 0.8889     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2400 - acc: 0.9157     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.2382 - acc: 0.9157     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.2421 - acc: 0.9195     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - 0s - loss: 0.2361 - acc: 0.9195     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2460 - acc: 0.9080     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2443 - acc: 0.9080     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2562 - acc: 0.8966     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2359 - acc: 0.9195     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2329 - acc: 0.9195     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.2363 - acc: 0.9157     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.2355 - acc: 0.9157     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.2386 - acc: 0.9080     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.2752 - acc: 0.8966     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2473 - acc: 0.9080     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.2279 - acc: 0.9234     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - 0s - loss: 0.2330 - acc: 0.9157     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2267 - acc: 0.9195     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2316 - acc: 0.9234     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2252 - acc: 0.9195     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2242 - acc: 0.9195     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.2251 - acc: 0.9272     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2279 - acc: 0.9157     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2220 - acc: 0.9272     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.2187 - acc: 0.9310     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2260 - acc: 0.9157     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2197 - acc: 0.9195     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2270 - acc: 0.9272     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2219 - acc: 0.9272     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] .. epochs=100, batch_size=10, score=0.725190851525, total=  10.2s\n",
      "[CV] epochs=100, batch_size=10 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   45.8s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6231 - acc: 0.7203     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.4737 - acc: 0.7739     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.4542 - acc: 0.782 - 0s - loss: 0.4470 - acc: 0.7931     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.4371 - acc: 0.7931     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.4250 - acc: 0.8084     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.4268 - acc: 0.8046     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.4125 - acc: 0.8161     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.4188 - acc: 0.8199     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.3976 - acc: 0.8238     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.3981 - acc: 0.8314     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.3897 - acc: 0.8391     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.3885 - acc: 0.8352     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.3962 - acc: 0.8199     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.3865 - acc: 0.8276     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.3902 - acc: 0.8314     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.3824 - acc: 0.8391     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.3799 - acc: 0.8276     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.3859 - acc: 0.8352     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.3803 - acc: 0.8276     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.3770 - acc: 0.8276     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.3747 - acc: 0.8276     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.3688 - acc: 0.8467     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.4085 - acc: 0.8161     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.3791 - acc: 0.8314     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.3702 - acc: 0.8467     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - 0s - loss: 0.3609 - acc: 0.8429     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.3619 - acc: 0.8391     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.3611 - acc: 0.8391     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.3584 - acc: 0.8544     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.3515 - acc: 0.8467     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.3587 - acc: 0.8621     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.3490 - acc: 0.8467     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.3460 - acc: 0.8467     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.3533 - acc: 0.8352     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.3504 - acc: 0.8506     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.4364 - acc: 0.800 - 0s - loss: 0.3439 - acc: 0.8506     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.3373 - acc: 0.8582     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.3390 - acc: 0.8659     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.3278 - acc: 0.8697     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.3457 - acc: 0.8391     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.3342 - acc: 0.8467     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.3352 - acc: 0.8544     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.3388 - acc: 0.8774     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - 0s - loss: 0.3253 - acc: 0.8506     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.3376 - acc: 0.8582     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.3643 - acc: 0.8352     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.3624 - acc: 0.8314     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.3335 - acc: 0.8391     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.3166 - acc: 0.8429     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.3071 - acc: 0.8544     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.2997 - acc: 0.8582     \n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.3020 - acc: 0.8467     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.2978 - acc: 0.8506     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.2992 - acc: 0.8659     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.2895 - acc: 0.8506     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.2878 - acc: 0.8429     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.2854 - acc: 0.8736     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.2886 - acc: 0.8582     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.2815 - acc: 0.8621     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.2812 - acc: 0.8582     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.2758 - acc: 0.8582     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - 0s - loss: 0.2731 - acc: 0.8659     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.3155 - acc: 0.8506     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.3057 - acc: 0.8582     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.2874 - acc: 0.8659     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.2783 - acc: 0.8736     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.2814 - acc: 0.8659     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.2762 - acc: 0.8659     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.2706 - acc: 0.8736     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.2678 - acc: 0.8621     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.2588 - acc: 0.8736     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2620 - acc: 0.8736     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2532 - acc: 0.8774     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.2525 - acc: 0.8697     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.2499 - acc: 0.8774     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - 0s - loss: 0.2546 - acc: 0.8659     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2527 - acc: 0.8697     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2732 - acc: 0.8697     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2701 - acc: 0.8697     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2660 - acc: 0.8736     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2516 - acc: 0.8774     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.2603 - acc: 0.8582     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.2627 - acc: 0.8544     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.2537 - acc: 0.8659     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.2429 - acc: 0.8851     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2432 - acc: 0.8774     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.3496 - acc: 0.8659     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - 0s - loss: 0.2760 - acc: 0.8621     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2466 - acc: 0.8774     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2601 - acc: 0.8659     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2458 - acc: 0.8697     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2324 - acc: 0.8851     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.2262 - acc: 0.8697     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2321 - acc: 0.8889     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2419 - acc: 0.8621     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.2277 - acc: 0.8851     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2294 - acc: 0.8851     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2431 - acc: 0.8621     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2419 - acc: 0.8812     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2437 - acc: 0.8697     \n",
      " 10/261 [>.............................] - ETA: 0s[CV] .. epochs=100, batch_size=10, score=0.740458017087, total=  11.1s\n",
      "[CV] epochs=100, batch_size=10 .......................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   57.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "262/262 [==============================] - 0s - loss: 0.5967 - acc: 0.6412     \n",
      "Epoch 2/100\n",
      "262/262 [==============================] - 0s - loss: 0.5242 - acc: 0.6527     \n",
      "Epoch 3/100\n",
      "262/262 [==============================] - 0s - loss: 0.5020 - acc: 0.7023     \n",
      "Epoch 4/100\n",
      "262/262 [==============================] - 0s - loss: 0.4930 - acc: 0.7443     \n",
      "Epoch 5/100\n",
      "262/262 [==============================] - 0s - loss: 0.4898 - acc: 0.7519     \n",
      "Epoch 6/100\n",
      "262/262 [==============================] - 0s - loss: 0.4829 - acc: 0.7710     \n",
      "Epoch 7/100\n",
      "262/262 [==============================] - 0s - loss: 0.4757 - acc: 0.7748     \n",
      "Epoch 8/100\n",
      "262/262 [==============================] - 0s - loss: 0.4720 - acc: 0.7748     \n",
      "Epoch 9/100\n",
      "262/262 [==============================] - 0s - loss: 0.4628 - acc: 0.7863     \n",
      "Epoch 10/100\n",
      "262/262 [==============================] - 0s - loss: 0.4590 - acc: 0.7863     \n",
      "Epoch 11/100\n",
      "262/262 [==============================] - 0s - loss: 0.4533 - acc: 0.7939     \n",
      "Epoch 12/100\n",
      "262/262 [==============================] - 0s - loss: 0.4508 - acc: 0.7786     \n",
      "Epoch 13/100\n",
      "262/262 [==============================] - 0s - loss: 0.4451 - acc: 0.7977     \n",
      "Epoch 14/100\n",
      "262/262 [==============================] - 0s - loss: 0.4415 - acc: 0.8053     \n",
      "Epoch 15/100\n",
      "262/262 [==============================] - 0s - loss: 0.4452 - acc: 0.8015     \n",
      "Epoch 16/100\n",
      "262/262 [==============================] - 0s - loss: 0.4288 - acc: 0.8130     \n",
      "Epoch 17/100\n",
      "262/262 [==============================] - 0s - loss: 0.4334 - acc: 0.7977     \n",
      "Epoch 18/100\n",
      "262/262 [==============================] - 0s - loss: 0.4259 - acc: 0.8244     \n",
      "Epoch 19/100\n",
      "262/262 [==============================] - 0s - loss: 0.4216 - acc: 0.8168     \n",
      "Epoch 20/100\n",
      "262/262 [==============================] - 0s - loss: 0.4242 - acc: 0.8168     \n",
      "Epoch 21/100\n",
      "262/262 [==============================] - 0s - loss: 0.4207 - acc: 0.8015     \n",
      "Epoch 22/100\n",
      "262/262 [==============================] - 0s - loss: 0.4165 - acc: 0.8130     \n",
      "Epoch 23/100\n",
      "262/262 [==============================] - 0s - loss: 0.4225 - acc: 0.8015     \n",
      "Epoch 24/100\n",
      "262/262 [==============================] - 0s - loss: 0.4165 - acc: 0.8053     \n",
      "Epoch 25/100\n",
      "262/262 [==============================] - 0s - loss: 0.4153 - acc: 0.8168     \n",
      "Epoch 26/100\n",
      "262/262 [==============================] - 0s - loss: 0.4158 - acc: 0.8206     \n",
      "Epoch 27/100\n",
      "262/262 [==============================] - 0s - loss: 0.4144 - acc: 0.8092     \n",
      "Epoch 28/100\n",
      "262/262 [==============================] - 0s - loss: 0.4051 - acc: 0.8168     \n",
      "Epoch 29/100\n",
      "262/262 [==============================] - 0s - loss: 0.4074 - acc: 0.8282     \n",
      "Epoch 30/100\n",
      "262/262 [==============================] - 0s - loss: 0.4179 - acc: 0.8092     \n",
      "Epoch 31/100\n",
      "262/262 [==============================] - 0s - loss: 0.4131 - acc: 0.8244     \n",
      "Epoch 32/100\n",
      "262/262 [==============================] - 0s - loss: 0.4064 - acc: 0.8244     \n",
      "Epoch 33/100\n",
      "262/262 [==============================] - 0s - loss: 0.4046 - acc: 0.8130     \n",
      "Epoch 34/100\n",
      "262/262 [==============================] - 0s - loss: 0.4070 - acc: 0.8130     \n",
      "Epoch 35/100\n",
      "262/262 [==============================] - 0s - loss: 0.4002 - acc: 0.8244     \n",
      "Epoch 36/100\n",
      "262/262 [==============================] - 0s - loss: 0.3963 - acc: 0.8168     \n",
      "Epoch 37/100\n",
      "262/262 [==============================] - 0s - loss: 0.3953 - acc: 0.8130     \n",
      "Epoch 38/100\n",
      "262/262 [==============================] - 0s - loss: 0.4063 - acc: 0.8206     \n",
      "Epoch 39/100\n",
      "262/262 [==============================] - 0s - loss: 0.4030 - acc: 0.8321     \n",
      "Epoch 40/100\n",
      "262/262 [==============================] - 0s - loss: 0.3898 - acc: 0.8092     \n",
      "Epoch 41/100\n",
      "262/262 [==============================] - 0s - loss: 0.4008 - acc: 0.8053     \n",
      "Epoch 42/100\n",
      "262/262 [==============================] - 0s - loss: 0.3916 - acc: 0.8282     \n",
      "Epoch 43/100\n",
      "262/262 [==============================] - 0s - loss: 0.3982 - acc: 0.8244     \n",
      "Epoch 44/100\n",
      "262/262 [==============================] - 0s - loss: 0.3939 - acc: 0.8206     \n",
      "Epoch 45/100\n",
      "262/262 [==============================] - 0s - loss: 0.3941 - acc: 0.8282     \n",
      "Epoch 46/100\n",
      "262/262 [==============================] - 0s - loss: 0.3859 - acc: 0.8282     \n",
      "Epoch 47/100\n",
      "262/262 [==============================] - 0s - loss: 0.3867 - acc: 0.8206     \n",
      "Epoch 48/100\n",
      "262/262 [==============================] - 0s - loss: 0.3816 - acc: 0.8397     \n",
      "Epoch 49/100\n",
      "262/262 [==============================] - 0s - loss: 0.3840 - acc: 0.8321     \n",
      "Epoch 50/100\n",
      "262/262 [==============================] - 0s - loss: 0.3833 - acc: 0.8359     \n",
      "Epoch 51/100\n",
      "262/262 [==============================] - 0s - loss: 0.3791 - acc: 0.8206     \n",
      "Epoch 52/100\n",
      "262/262 [==============================] - 0s - loss: 0.3763 - acc: 0.8397     \n",
      "Epoch 53/100\n",
      "262/262 [==============================] - 0s - loss: 0.3775 - acc: 0.8359     \n",
      "Epoch 54/100\n",
      "262/262 [==============================] - 0s - loss: 0.3872 - acc: 0.8321     \n",
      "Epoch 55/100\n",
      "262/262 [==============================] - 0s - loss: 0.3769 - acc: 0.8359     \n",
      "Epoch 56/100\n",
      "262/262 [==============================] - 0s - loss: 0.3738 - acc: 0.8321     \n",
      "Epoch 57/100\n",
      "262/262 [==============================] - 0s - loss: 0.3711 - acc: 0.8359     \n",
      "Epoch 58/100\n",
      "262/262 [==============================] - 0s - loss: 0.3651 - acc: 0.8588     \n",
      "Epoch 59/100\n",
      "262/262 [==============================] - 0s - loss: 0.3704 - acc: 0.8244     \n",
      "Epoch 60/100\n",
      "262/262 [==============================] - 0s - loss: 0.3709 - acc: 0.8206     \n",
      "Epoch 61/100\n",
      "262/262 [==============================] - 0s - loss: 0.3642 - acc: 0.8550     \n",
      "Epoch 62/100\n",
      "262/262 [==============================] - 0s - loss: 0.3680 - acc: 0.8321     \n",
      "Epoch 63/100\n",
      "262/262 [==============================] - 0s - loss: 0.3689 - acc: 0.8359     \n",
      "Epoch 64/100\n",
      "262/262 [==============================] - 0s - loss: 0.3692 - acc: 0.8435     - ETA: 0s - loss: 0.3732 - acc: 0.840\n",
      "Epoch 65/100\n",
      "262/262 [==============================] - 0s - loss: 0.3665 - acc: 0.8244     \n",
      "Epoch 66/100\n",
      "262/262 [==============================] - 0s - loss: 0.3670 - acc: 0.8397     \n",
      "Epoch 67/100\n",
      "262/262 [==============================] - 0s - loss: 0.3638 - acc: 0.8435     \n",
      "Epoch 68/100\n",
      "262/262 [==============================] - 0s - loss: 0.3589 - acc: 0.8473     \n",
      "Epoch 69/100\n",
      "262/262 [==============================] - 0s - loss: 0.3577 - acc: 0.8397     \n",
      "Epoch 70/100\n",
      "262/262 [==============================] - 0s - loss: 0.3638 - acc: 0.8321     \n",
      "Epoch 71/100\n",
      "262/262 [==============================] - 0s - loss: 0.3584 - acc: 0.8588     \n",
      "Epoch 72/100\n",
      "262/262 [==============================] - 0s - loss: 0.3702 - acc: 0.8282     \n",
      "Epoch 73/100\n",
      "262/262 [==============================] - 0s - loss: 0.3563 - acc: 0.8473     \n",
      "Epoch 74/100\n",
      "262/262 [==============================] - 0s - loss: 0.3550 - acc: 0.8473     \n",
      "Epoch 75/100\n",
      "262/262 [==============================] - 0s - loss: 0.3593 - acc: 0.8511     \n",
      "Epoch 76/100\n",
      "262/262 [==============================] - 0s - loss: 0.3611 - acc: 0.8397     \n",
      "Epoch 77/100\n",
      "262/262 [==============================] - 0s - loss: 0.3658 - acc: 0.8550     \n",
      "Epoch 78/100\n",
      "262/262 [==============================] - 0s - loss: 0.3633 - acc: 0.8435     \n",
      "Epoch 79/100\n",
      "262/262 [==============================] - 0s - loss: 0.3511 - acc: 0.8588     \n",
      "Epoch 80/100\n",
      "262/262 [==============================] - 0s - loss: 0.3543 - acc: 0.8626     \n",
      "Epoch 81/100\n",
      "262/262 [==============================] - 0s - loss: 0.3719 - acc: 0.8282     \n",
      "Epoch 82/100\n",
      "262/262 [==============================] - 0s - loss: 0.3601 - acc: 0.8511     \n",
      "Epoch 83/100\n",
      "262/262 [==============================] - 0s - loss: 0.3542 - acc: 0.8550     \n",
      "Epoch 84/100\n",
      "262/262 [==============================] - 0s - loss: 0.3460 - acc: 0.8550     \n",
      "Epoch 85/100\n",
      "262/262 [==============================] - 0s - loss: 0.3541 - acc: 0.8435     \n",
      "Epoch 86/100\n",
      "262/262 [==============================] - 0s - loss: 0.3534 - acc: 0.8397     \n",
      "Epoch 87/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3609 - acc: 0.900 - 0s - loss: 0.3511 - acc: 0.8588     \n",
      "Epoch 88/100\n",
      "262/262 [==============================] - 0s - loss: 0.3505 - acc: 0.8626     \n",
      "Epoch 89/100\n",
      "262/262 [==============================] - 0s - loss: 0.3515 - acc: 0.8473     \n",
      "Epoch 90/100\n",
      "262/262 [==============================] - 0s - loss: 0.3473 - acc: 0.8550     \n",
      "Epoch 91/100\n",
      "262/262 [==============================] - 0s - loss: 0.3493 - acc: 0.8321     \n",
      "Epoch 92/100\n",
      "262/262 [==============================] - 0s - loss: 0.3485 - acc: 0.8550     \n",
      "Epoch 93/100\n",
      "262/262 [==============================] - 0s - loss: 0.3429 - acc: 0.8588     \n",
      "Epoch 94/100\n",
      "262/262 [==============================] - 0s - loss: 0.3446 - acc: 0.8397     \n",
      "Epoch 95/100\n",
      "262/262 [==============================] - 0s - loss: 0.3544 - acc: 0.8473     \n",
      "Epoch 96/100\n",
      "262/262 [==============================] - 0s - loss: 0.3416 - acc: 0.8473     \n",
      "Epoch 97/100\n",
      "262/262 [==============================] - 0s - loss: 0.3441 - acc: 0.8702     \n",
      "Epoch 98/100\n",
      "262/262 [==============================] - 0s - loss: 0.3470 - acc: 0.8550     \n",
      "Epoch 99/100\n",
      "262/262 [==============================] - 0s - loss: 0.3386 - acc: 0.8664     \n",
      "Epoch 100/100\n",
      "262/262 [==============================] - 0s - loss: 0.3460 - acc: 0.8511     \n",
      " 10/262 [>.............................] - ETA: 0s[CV] ... epochs=100, batch_size=10, score=0.85384614651, total=   9.8s\n",
      "[CV] epochs=10, batch_size=20 ........................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:  1.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6705 - acc: 0.6552     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.5194 - acc: 0.6973     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.4662 - acc: 0.6973     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.4340 - acc: 0.6973     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.4213 - acc: 0.6973     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.4118 - acc: 0.7395     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.4079 - acc: 0.8123     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.4010 - acc: 0.8123     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.3990 - acc: 0.8123     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.3912 - acc: 0.8276     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] ... epochs=10, batch_size=20, score=0.755725190385, total=   3.8s\n",
      "[CV] epochs=10, batch_size=20 ........................................\n",
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6750 - acc: 0.7395     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.6166 - acc: 0.7663     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.5294 - acc: 0.7816     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.4573 - acc: 0.7816     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.4464 - acc: 0.7931     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.4300 - acc: 0.8008     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.4490 - acc: 0.8084     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.4393 - acc: 0.8008     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.4288 - acc: 0.7969     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.4219 - acc: 0.8084     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] ... epochs=10, batch_size=20, score=0.763358790001, total=   3.9s\n",
      "[CV] epochs=10, batch_size=20 ........................................\n",
      "Epoch 1/10\n",
      "262/262 [==============================] - 0s - loss: 0.6790 - acc: 0.6412     \n",
      "Epoch 2/10\n",
      "262/262 [==============================] - 0s - loss: 0.6126 - acc: 0.6527     \n",
      "Epoch 3/10\n",
      "262/262 [==============================] - 0s - loss: 0.5122 - acc: 0.7214     \n",
      "Epoch 4/10\n",
      "262/262 [==============================] - 0s - loss: 0.4792 - acc: 0.7634     \n",
      "Epoch 5/10\n",
      "262/262 [==============================] - 0s - loss: 0.4734 - acc: 0.7519     \n",
      "Epoch 6/10\n",
      "262/262 [==============================] - 0s - loss: 0.4653 - acc: 0.7595     \n",
      "Epoch 7/10\n",
      "262/262 [==============================] - 0s - loss: 0.4642 - acc: 0.7481     \n",
      "Epoch 8/10\n",
      "262/262 [==============================] - 0s - loss: 0.4457 - acc: 0.7939     \n",
      "Epoch 9/10\n",
      "262/262 [==============================] - 0s - loss: 0.4426 - acc: 0.7824     \n",
      "Epoch 10/10\n",
      "262/262 [==============================] - 0s - loss: 0.4422 - acc: 0.7901     \n",
      " 20/262 [=>............................] - ETA: 0s[CV] ... epochs=10, batch_size=20, score=0.823076926745, total=   4.7s\n",
      "[CV] epochs=50, batch_size=20 ........................................\n",
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6640 - acc: 0.6935     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.5841 - acc: 0.6973     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.5015 - acc: 0.6973     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4581 - acc: 0.6973     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4376 - acc: 0.6973     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.4468 - acc: 0.600 - 0s - loss: 0.4266 - acc: 0.6973     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - 0s - loss: 0.4161 - acc: 0.6973     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - 0s - loss: 0.4101 - acc: 0.8123     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.4083 - acc: 0.8161     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.4027 - acc: 0.8238     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.3949 - acc: 0.8238     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.3862 - acc: 0.8276     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - 0s - loss: 0.3755 - acc: 0.8352     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.3673 - acc: 0.8391     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3630 - acc: 0.8314     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3655 - acc: 0.8199     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - 0s - loss: 0.3610 - acc: 0.8429     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.3550 - acc: 0.8314     \n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.3515 - acc: 0.8352     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.3449 - acc: 0.8429     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.3434 - acc: 0.8429     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.3404 - acc: 0.8467     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.3372 - acc: 0.8467     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.3345 - acc: 0.8544     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.3323 - acc: 0.8506     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.3297 - acc: 0.8506     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.3248 - acc: 0.8544     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.3232 - acc: 0.8506     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.3365 - acc: 0.8276     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.3346 - acc: 0.8429     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.3233 - acc: 0.8544     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.3214 - acc: 0.8506     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.3198 - acc: 0.8506     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.3184 - acc: 0.8467     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.3148 - acc: 0.8582     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.3104 - acc: 0.8621     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - 0s - loss: 0.3072 - acc: 0.8582     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.3051 - acc: 0.8659     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - 0s - loss: 0.3303 - acc: 0.8391     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.3173 - acc: 0.8506     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.3050 - acc: 0.8582     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.2992 - acc: 0.8582     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.3011 - acc: 0.8736     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.3088 - acc: 0.8582     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.3094 - acc: 0.8544     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - 0s - loss: 0.3045 - acc: 0.8544     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.3006 - acc: 0.8736     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.3155 - acc: 0.8582     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.2980 - acc: 0.8544     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.2890 - acc: 0.8621     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] ... epochs=50, batch_size=20, score=0.740458014357, total=   4.8s\n",
      "[CV] epochs=50, batch_size=20 ........................................\n",
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6597 - acc: 0.6475     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.5476 - acc: 0.6935     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.4877 - acc: 0.7816     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4604 - acc: 0.7816     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4442 - acc: 0.7816     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - 0s - loss: 0.4353 - acc: 0.7931     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - 0s - loss: 0.4250 - acc: 0.8046     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - 0s - loss: 0.4176 - acc: 0.8046     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.4142 - acc: 0.7969     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.4064 - acc: 0.8084     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.4120 - acc: 0.8123     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.4035 - acc: 0.8123     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - 0s - loss: 0.4071 - acc: 0.8008     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.3965 - acc: 0.8276     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3915 - acc: 0.8084     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3908 - acc: 0.8084     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.5197 - acc: 0.750 - 0s - loss: 0.4019 - acc: 0.8123     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.3933 - acc: 0.8314     \n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.3897 - acc: 0.8276     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.3984 - acc: 0.8276     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.3845 - acc: 0.8161     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.3923 - acc: 0.8123     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.3900 - acc: 0.8199     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.3863 - acc: 0.8199     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.3781 - acc: 0.8391     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.3746 - acc: 0.8391     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.3743 - acc: 0.8352     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.3723 - acc: 0.8276     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.3716 - acc: 0.8391     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.3725 - acc: 0.8352     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.3681 - acc: 0.8352     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.3651 - acc: 0.8391     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.3659 - acc: 0.8429     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.3613 - acc: 0.8391     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.3633 - acc: 0.8467     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.3601 - acc: 0.8391     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - 0s - loss: 0.3584 - acc: 0.8352     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.3577 - acc: 0.8429     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - 0s - loss: 0.3552 - acc: 0.8544     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.3497 - acc: 0.8506     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.3529 - acc: 0.8544     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.3631 - acc: 0.8352     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.4059 - acc: 0.8352     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.3911 - acc: 0.8391     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.3667 - acc: 0.8467     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - 0s - loss: 0.3675 - acc: 0.8429     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.3602 - acc: 0.8467     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.3561 - acc: 0.8506     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.3490 - acc: 0.8506     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.3472 - acc: 0.8621     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] .... epochs=50, batch_size=20, score=0.78625954881, total=   5.0s\n",
      "[CV] epochs=50, batch_size=20 ........................................\n",
      "Epoch 1/50\n",
      "262/262 [==============================] - 0s - loss: 0.6864 - acc: 0.6107     \n",
      "Epoch 2/50\n",
      "262/262 [==============================] - 0s - loss: 0.6692 - acc: 0.6565     \n",
      "Epoch 3/50\n",
      "262/262 [==============================] - 0s - loss: 0.6328 - acc: 0.7519     \n",
      "Epoch 4/50\n",
      "262/262 [==============================] - 0s - loss: 0.6079 - acc: 0.7328     \n",
      "Epoch 5/50\n",
      "262/262 [==============================] - 0s - loss: 0.5847 - acc: 0.7481     \n",
      "Epoch 6/50\n",
      "262/262 [==============================] - 0s - loss: 0.5651 - acc: 0.7634     \n",
      "Epoch 7/50\n",
      "262/262 [==============================] - 0s - loss: 0.5541 - acc: 0.7481     \n",
      "Epoch 8/50\n",
      "262/262 [==============================] - 0s - loss: 0.5379 - acc: 0.7557     \n",
      "Epoch 9/50\n",
      "262/262 [==============================] - 0s - loss: 0.5273 - acc: 0.7748     \n",
      "Epoch 10/50\n",
      "262/262 [==============================] - 0s - loss: 0.5198 - acc: 0.7710     \n",
      "Epoch 11/50\n",
      "262/262 [==============================] - 0s - loss: 0.5007 - acc: 0.7939     \n",
      "Epoch 12/50\n",
      "262/262 [==============================] - 0s - loss: 0.4919 - acc: 0.7710     \n",
      "Epoch 13/50\n",
      "262/262 [==============================] - 0s - loss: 0.4920 - acc: 0.7824     \n",
      "Epoch 14/50\n",
      "262/262 [==============================] - 0s - loss: 0.4784 - acc: 0.7939     \n",
      "Epoch 15/50\n",
      "262/262 [==============================] - 0s - loss: 0.4730 - acc: 0.7748     \n",
      "Epoch 16/50\n",
      "262/262 [==============================] - 0s - loss: 0.4584 - acc: 0.7901     \n",
      "Epoch 17/50\n",
      "262/262 [==============================] - 0s - loss: 0.4563 - acc: 0.7863     \n",
      "Epoch 18/50\n",
      "262/262 [==============================] - 0s - loss: 0.4579 - acc: 0.7977     \n",
      "Epoch 19/50\n",
      "262/262 [==============================] - 0s - loss: 0.4448 - acc: 0.8092     \n",
      "Epoch 20/50\n",
      "262/262 [==============================] - 0s - loss: 0.4380 - acc: 0.8130     \n",
      "Epoch 21/50\n",
      "262/262 [==============================] - 0s - loss: 0.4368 - acc: 0.8321     \n",
      "Epoch 22/50\n",
      "262/262 [==============================] - 0s - loss: 0.4254 - acc: 0.8168     \n",
      "Epoch 23/50\n",
      "262/262 [==============================] - 0s - loss: 0.4288 - acc: 0.8130     \n",
      "Epoch 24/50\n",
      "262/262 [==============================] - 0s - loss: 0.4238 - acc: 0.8053     \n",
      "Epoch 25/50\n",
      "262/262 [==============================] - 0s - loss: 0.4179 - acc: 0.8206     \n",
      "Epoch 26/50\n",
      "262/262 [==============================] - 0s - loss: 0.4140 - acc: 0.8206     \n",
      "Epoch 27/50\n",
      "262/262 [==============================] - 0s - loss: 0.4280 - acc: 0.8092     \n",
      "Epoch 28/50\n",
      "262/262 [==============================] - 0s - loss: 0.4098 - acc: 0.8130     \n",
      "Epoch 29/50\n",
      "262/262 [==============================] - 0s - loss: 0.4018 - acc: 0.8321     \n",
      "Epoch 30/50\n",
      "262/262 [==============================] - 0s - loss: 0.4004 - acc: 0.8168     \n",
      "Epoch 31/50\n",
      "262/262 [==============================] - 0s - loss: 0.4034 - acc: 0.8168     \n",
      "Epoch 32/50\n",
      "262/262 [==============================] - 0s - loss: 0.4063 - acc: 0.8168     \n",
      "Epoch 33/50\n",
      "262/262 [==============================] - 0s - loss: 0.4010 - acc: 0.8321     \n",
      "Epoch 34/50\n",
      "262/262 [==============================] - 0s - loss: 0.4214 - acc: 0.8206     \n",
      "Epoch 35/50\n",
      "262/262 [==============================] - 0s - loss: 0.4045 - acc: 0.8321     \n",
      "Epoch 36/50\n",
      "262/262 [==============================] - 0s - loss: 0.3967 - acc: 0.8244     \n",
      "Epoch 37/50\n",
      "262/262 [==============================] - 0s - loss: 0.4214 - acc: 0.8244     \n",
      "Epoch 38/50\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3346 - acc: 0.900 - 0s - loss: 0.4039 - acc: 0.8053     \n",
      "Epoch 39/50\n",
      "262/262 [==============================] - 0s - loss: 0.3927 - acc: 0.8244     \n",
      "Epoch 40/50\n",
      "262/262 [==============================] - 0s - loss: 0.3831 - acc: 0.8321     \n",
      "Epoch 41/50\n",
      "262/262 [==============================] - 0s - loss: 0.3793 - acc: 0.8435     \n",
      "Epoch 42/50\n",
      "262/262 [==============================] - 0s - loss: 0.3768 - acc: 0.8435     \n",
      "Epoch 43/50\n",
      "262/262 [==============================] - 0s - loss: 0.3851 - acc: 0.8282     \n",
      "Epoch 44/50\n",
      "262/262 [==============================] - 0s - loss: 0.4045 - acc: 0.8282     \n",
      "Epoch 45/50\n",
      "262/262 [==============================] - 0s - loss: 0.3798 - acc: 0.8397     \n",
      "Epoch 46/50\n",
      "262/262 [==============================] - 0s - loss: 0.3902 - acc: 0.8359     \n",
      "Epoch 47/50\n",
      "262/262 [==============================] - 0s - loss: 0.3797 - acc: 0.8130     \n",
      "Epoch 48/50\n",
      "262/262 [==============================] - 0s - loss: 0.3733 - acc: 0.8359     \n",
      "Epoch 49/50\n",
      "262/262 [==============================] - 0s - loss: 0.3663 - acc: 0.8321     \n",
      "Epoch 50/50\n",
      "262/262 [==============================] - 0s - loss: 0.3647 - acc: 0.8321     \n",
      " 20/262 [=>............................] - ETA: 0s[CV] .... epochs=50, batch_size=20, score=0.82307693133, total=   5.5s\n",
      "[CV] epochs=100, batch_size=20 .......................................\n",
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6829 - acc: 0.6820     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.6526 - acc: 0.7318     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - 0s - loss: 0.6140 - acc: 0.7778     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.5753 - acc: 0.8046     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.5458 - acc: 0.8123     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.5209 - acc: 0.8161     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.5008 - acc: 0.8276     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.4875 - acc: 0.8238     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.4748 - acc: 0.8276     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.4695 - acc: 0.8199     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.4526 - acc: 0.8314     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.4475 - acc: 0.8352     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.4345 - acc: 0.8467     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.4290 - acc: 0.8429     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.4196 - acc: 0.8506     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.4086 - acc: 0.8467     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.4025 - acc: 0.8506     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.4054 - acc: 0.8506     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.3890 - acc: 0.8659     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.3835 - acc: 0.8697     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.3732 - acc: 0.8697     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.3733 - acc: 0.8582     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.3658 - acc: 0.8621     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.3644 - acc: 0.8697     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.3512 - acc: 0.8697     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - 0s - loss: 0.3468 - acc: 0.8736     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.3477 - acc: 0.8851     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.3649 - acc: 0.8621     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.3530 - acc: 0.8659     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.3377 - acc: 0.8582     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.3541 - acc: 0.8467     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.3768 - acc: 0.8352     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.3383 - acc: 0.8736     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.3222 - acc: 0.8927     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.3150 - acc: 0.8927     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - 0s - loss: 0.3120 - acc: 0.8889     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.3066 - acc: 0.8774     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.3072 - acc: 0.8736     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.2999 - acc: 0.8774     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.2965 - acc: 0.8812     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.2972 - acc: 0.8736     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.2917 - acc: 0.8851     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.2916 - acc: 0.8851     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - 0s - loss: 0.2897 - acc: 0.8889     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.2799 - acc: 0.8889     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.2829 - acc: 0.8889     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.2879 - acc: 0.8889     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.3075 - acc: 0.8851     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.2941 - acc: 0.8851     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.2852 - acc: 0.9042     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.2869 - acc: 0.8966     \n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.2772 - acc: 0.8927     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.2725 - acc: 0.8966     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.2696 - acc: 0.8889     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.2654 - acc: 0.8927     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.2660 - acc: 0.9004     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.2658 - acc: 0.8966     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.2587 - acc: 0.9004     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.2532 - acc: 0.9042     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.2512 - acc: 0.9157     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.2508 - acc: 0.9234     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - 0s - loss: 0.2567 - acc: 0.9080     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.2536 - acc: 0.9119     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.2872 - acc: 0.8889     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.2700 - acc: 0.8889     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.2575 - acc: 0.9004     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.2470 - acc: 0.9119     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.2813 - acc: 0.8812     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.2565 - acc: 0.9004     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.2396 - acc: 0.9195     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.2423 - acc: 0.9080     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2436 - acc: 0.9119     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2746 - acc: 0.9004     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.2738 - acc: 0.8927     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.2533 - acc: 0.8889     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - 0s - loss: 0.2876 - acc: 0.8889     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2765 - acc: 0.9042     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2506 - acc: 0.9042     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2481 - acc: 0.9042     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2360 - acc: 0.9080     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2362 - acc: 0.9080     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.2379 - acc: 0.9042     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.2469 - acc: 0.9042     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.2476 - acc: 0.8966     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.2321 - acc: 0.9195     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2322 - acc: 0.9157     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.2349 - acc: 0.9119     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.2169 - acc: 0.950 - 0s - loss: 0.2293 - acc: 0.9119     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2251 - acc: 0.9004     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2175 - acc: 0.9080     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2190 - acc: 0.9042     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2144 - acc: 0.9119     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.2150 - acc: 0.9080     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2332 - acc: 0.8966     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2305 - acc: 0.9119     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.2132 - acc: 0.9080     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2125 - acc: 0.9119     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2358 - acc: 0.9119     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2424 - acc: 0.9080     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2551 - acc: 0.8966     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] .. epochs=100, batch_size=20, score=0.770992363682, total=   7.2s\n",
      "[CV] epochs=100, batch_size=20 .......................................\n",
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6763 - acc: 0.6284     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.6203 - acc: 0.6552     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - 0s - loss: 0.5354 - acc: 0.6552     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.5034 - acc: 0.6552     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.4914 - acc: 0.6552     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.4831 - acc: 0.6552     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.4755 - acc: 0.7739     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.4685 - acc: 0.7931     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.4605 - acc: 0.7969     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.4559 - acc: 0.8046     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.4507 - acc: 0.8046     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.4448 - acc: 0.8046     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.4397 - acc: 0.8123     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.4341 - acc: 0.8123     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.4315 - acc: 0.8046     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.4291 - acc: 0.8199     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.4285 - acc: 0.8123     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.4247 - acc: 0.8276     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.4186 - acc: 0.8276     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.4159 - acc: 0.8238     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.4207 - acc: 0.8084     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.4131 - acc: 0.8084     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.4107 - acc: 0.8161     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.4081 - acc: 0.8161     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.4043 - acc: 0.8238     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - 0s - loss: 0.3947 - acc: 0.8391     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.4039 - acc: 0.8238     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.4037 - acc: 0.8199     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.4029 - acc: 0.8238     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.3931 - acc: 0.8276     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.3879 - acc: 0.8352     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.3821 - acc: 0.8429     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.3744 - acc: 0.8544     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.3743 - acc: 0.8621     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.3711 - acc: 0.8659     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - 0s - loss: 0.3653 - acc: 0.8582     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.3591 - acc: 0.8621     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.3593 - acc: 0.8544     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.3540 - acc: 0.8582     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.3595 - acc: 0.8659     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.3905 - acc: 0.8352     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.3796 - acc: 0.8506     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.3618 - acc: 0.8506     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - 0s - loss: 0.3547 - acc: 0.8544     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.3536 - acc: 0.8659     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.3422 - acc: 0.8736     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.3393 - acc: 0.8812     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.3350 - acc: 0.8659     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.3302 - acc: 0.8736     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.3283 - acc: 0.8736     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.3258 - acc: 0.8736     \n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.3217 - acc: 0.8812     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.3245 - acc: 0.8774     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.3302 - acc: 0.8736     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.3258 - acc: 0.8659     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.3197 - acc: 0.8812     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.3142 - acc: 0.8851     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.3335 - acc: 0.8582     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.3241 - acc: 0.8774     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.3134 - acc: 0.8851     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.3112 - acc: 0.8889     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - 0s - loss: 0.3063 - acc: 0.8851     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.3127 - acc: 0.8966     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.3068 - acc: 0.8927     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.3063 - acc: 0.8966     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.3070 - acc: 0.8851     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.3252 - acc: 0.8621     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.3359 - acc: 0.8582     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.3131 - acc: 0.8659     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.3040 - acc: 0.8812     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.3033 - acc: 0.8889     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2975 - acc: 0.8889     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2949 - acc: 0.8927     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.3351 - acc: 0.8659     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.3436 - acc: 0.8506     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - 0s - loss: 0.3096 - acc: 0.8812     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2976 - acc: 0.8966     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2996 - acc: 0.8966     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2931 - acc: 0.8966     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2905 - acc: 0.8927     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2883 - acc: 0.9004     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.3278 - acc: 0.8736     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.3399 - acc: 0.8506     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.3177 - acc: 0.8736     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.3117 - acc: 0.8697     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2959 - acc: 0.8927     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.2855 - acc: 0.8966     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - 0s - loss: 0.2872 - acc: 0.9042     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2825 - acc: 0.9004     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2849 - acc: 0.8966     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2805 - acc: 0.9004     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2813 - acc: 0.9004     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.3037 - acc: 0.8851     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2826 - acc: 0.9157     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2878 - acc: 0.9157     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.3016 - acc: 0.8697     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2939 - acc: 0.9080     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2907 - acc: 0.8927     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2842 - acc: 0.9042     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2760 - acc: 0.9080     \n",
      " 20/261 [=>............................] - ETA: 0s[CV] .. epochs=100, batch_size=20, score=0.770992369597, total=   7.3s\n",
      "[CV] epochs=100, batch_size=20 .......................................\n",
      "Epoch 1/100\n",
      "262/262 [==============================] - 0s - loss: 0.6722 - acc: 0.6489     \n",
      "Epoch 2/100\n",
      "262/262 [==============================] - 0s - loss: 0.5920 - acc: 0.7290     \n",
      "Epoch 3/100\n",
      "262/262 [==============================] - 0s - loss: 0.4955 - acc: 0.7481     \n",
      "Epoch 4/100\n",
      "262/262 [==============================] - 0s - loss: 0.4783 - acc: 0.7481     \n",
      "Epoch 5/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3935 - acc: 0.850 - 0s - loss: 0.4722 - acc: 0.7672     \n",
      "Epoch 6/100\n",
      "262/262 [==============================] - 0s - loss: 0.4618 - acc: 0.7672     \n",
      "Epoch 7/100\n",
      "262/262 [==============================] - 0s - loss: 0.4579 - acc: 0.7634     \n",
      "Epoch 8/100\n",
      "262/262 [==============================] - 0s - loss: 0.4565 - acc: 0.7595     \n",
      "Epoch 9/100\n",
      "262/262 [==============================] - 0s - loss: 0.4552 - acc: 0.7672     \n",
      "Epoch 10/100\n",
      "262/262 [==============================] - 0s - loss: 0.4516 - acc: 0.7634     \n",
      "Epoch 11/100\n",
      "262/262 [==============================] - 0s - loss: 0.4515 - acc: 0.7748     \n",
      "Epoch 12/100\n",
      "262/262 [==============================] - 0s - loss: 0.4448 - acc: 0.7519     \n",
      "Epoch 13/100\n",
      "262/262 [==============================] - 0s - loss: 0.4377 - acc: 0.7710     \n",
      "Epoch 14/100\n",
      "262/262 [==============================] - 0s - loss: 0.4442 - acc: 0.7786     \n",
      "Epoch 15/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3318 - acc: 0.950 - 0s - loss: 0.4411 - acc: 0.7748     \n",
      "Epoch 16/100\n",
      "262/262 [==============================] - 0s - loss: 0.4312 - acc: 0.7863     \n",
      "Epoch 17/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.2941 - acc: 0.850 - 0s - loss: 0.4294 - acc: 0.7939     \n",
      "Epoch 18/100\n",
      "262/262 [==============================] - 0s - loss: 0.4245 - acc: 0.7901     \n",
      "Epoch 19/100\n",
      "262/262 [==============================] - 0s - loss: 0.4262 - acc: 0.7863     \n",
      "Epoch 20/100\n",
      "262/262 [==============================] - 0s - loss: 0.4179 - acc: 0.7977     \n",
      "Epoch 21/100\n",
      "262/262 [==============================] - 0s - loss: 0.4165 - acc: 0.8130     \n",
      "Epoch 22/100\n",
      "262/262 [==============================] - 0s - loss: 0.4118 - acc: 0.8092     \n",
      "Epoch 23/100\n",
      "262/262 [==============================] - 0s - loss: 0.4132 - acc: 0.8053     \n",
      "Epoch 24/100\n",
      "262/262 [==============================] - 0s - loss: 0.4078 - acc: 0.8015     \n",
      "Epoch 25/100\n",
      "262/262 [==============================] - 0s - loss: 0.4049 - acc: 0.8206     \n",
      "Epoch 26/100\n",
      "262/262 [==============================] - 0s - loss: 0.4112 - acc: 0.8168     \n",
      "Epoch 27/100\n",
      "262/262 [==============================] - 0s - loss: 0.4024 - acc: 0.8206     \n",
      "Epoch 28/100\n",
      "262/262 [==============================] - 0s - loss: 0.4064 - acc: 0.7863     \n",
      "Epoch 29/100\n",
      "262/262 [==============================] - 0s - loss: 0.4125 - acc: 0.7977     \n",
      "Epoch 30/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3660 - acc: 0.850 - 0s - loss: 0.4083 - acc: 0.8130     \n",
      "Epoch 31/100\n",
      "262/262 [==============================] - 0s - loss: 0.3979 - acc: 0.8053     \n",
      "Epoch 32/100\n",
      "262/262 [==============================] - 0s - loss: 0.3982 - acc: 0.8053     \n",
      "Epoch 33/100\n",
      "262/262 [==============================] - 0s - loss: 0.3980 - acc: 0.8244     \n",
      "Epoch 34/100\n",
      "262/262 [==============================] - 0s - loss: 0.3918 - acc: 0.8053     \n",
      "Epoch 35/100\n",
      "262/262 [==============================] - 0s - loss: 0.3956 - acc: 0.8359     \n",
      "Epoch 36/100\n",
      "262/262 [==============================] - 0s - loss: 0.3911 - acc: 0.8206     \n",
      "Epoch 37/100\n",
      "262/262 [==============================] - 0s - loss: 0.3830 - acc: 0.8206     \n",
      "Epoch 38/100\n",
      "262/262 [==============================] - 0s - loss: 0.3790 - acc: 0.8282     \n",
      "Epoch 39/100\n",
      "262/262 [==============================] - 0s - loss: 0.3767 - acc: 0.8321     \n",
      "Epoch 40/100\n",
      "262/262 [==============================] - 0s - loss: 0.3814 - acc: 0.8130     \n",
      "Epoch 41/100\n",
      "262/262 [==============================] - 0s - loss: 0.3734 - acc: 0.8206     \n",
      "Epoch 42/100\n",
      "262/262 [==============================] - 0s - loss: 0.3773 - acc: 0.8244     \n",
      "Epoch 43/100\n",
      "262/262 [==============================] - 0s - loss: 0.3744 - acc: 0.8244     \n",
      "Epoch 44/100\n",
      "262/262 [==============================] - 0s - loss: 0.3763 - acc: 0.8397     \n",
      "Epoch 45/100\n",
      "262/262 [==============================] - 0s - loss: 0.3833 - acc: 0.8206     \n",
      "Epoch 46/100\n",
      "262/262 [==============================] - 0s - loss: 0.3876 - acc: 0.8397     \n",
      "Epoch 47/100\n",
      "262/262 [==============================] - 0s - loss: 0.3724 - acc: 0.8435     \n",
      "Epoch 48/100\n",
      "262/262 [==============================] - 0s - loss: 0.3725 - acc: 0.8282     \n",
      "Epoch 49/100\n",
      "262/262 [==============================] - 0s - loss: 0.3814 - acc: 0.8359     \n",
      "Epoch 50/100\n",
      "262/262 [==============================] - 0s - loss: 0.3777 - acc: 0.8321     \n",
      "Epoch 51/100\n",
      "262/262 [==============================] - 0s - loss: 0.3673 - acc: 0.8321     \n",
      "Epoch 52/100\n",
      "262/262 [==============================] - 0s - loss: 0.3726 - acc: 0.8664     \n",
      "Epoch 53/100\n",
      "262/262 [==============================] - 0s - loss: 0.3743 - acc: 0.8626     \n",
      "Epoch 54/100\n",
      "262/262 [==============================] - 0s - loss: 0.3636 - acc: 0.8359     \n",
      "Epoch 55/100\n",
      "262/262 [==============================] - 0s - loss: 0.3713 - acc: 0.8359     \n",
      "Epoch 56/100\n",
      "262/262 [==============================] - 0s - loss: 0.3750 - acc: 0.8168     \n",
      "Epoch 57/100\n",
      "262/262 [==============================] - 0s - loss: 0.3649 - acc: 0.8321     \n",
      "Epoch 58/100\n",
      "262/262 [==============================] - 0s - loss: 0.3612 - acc: 0.8473     \n",
      "Epoch 59/100\n",
      "262/262 [==============================] - 0s - loss: 0.3593 - acc: 0.8397     \n",
      "Epoch 60/100\n",
      "262/262 [==============================] - 0s - loss: 0.3637 - acc: 0.8244     \n",
      "Epoch 61/100\n",
      "262/262 [==============================] - 0s - loss: 0.3590 - acc: 0.8359     \n",
      "Epoch 62/100\n",
      "262/262 [==============================] - 0s - loss: 0.3663 - acc: 0.8435     \n",
      "Epoch 63/100\n",
      "262/262 [==============================] - 0s - loss: 0.3692 - acc: 0.8244     \n",
      "Epoch 64/100\n",
      "262/262 [==============================] - 0s - loss: 0.3579 - acc: 0.8359     \n",
      "Epoch 65/100\n",
      "262/262 [==============================] - 0s - loss: 0.3554 - acc: 0.8511     \n",
      "Epoch 66/100\n",
      "262/262 [==============================] - 0s - loss: 0.3732 - acc: 0.8473     \n",
      "Epoch 67/100\n",
      "262/262 [==============================] - 0s - loss: 0.3612 - acc: 0.8397     \n",
      "Epoch 68/100\n",
      "262/262 [==============================] - 0s - loss: 0.4298 - acc: 0.8206     \n",
      "Epoch 69/100\n",
      "262/262 [==============================] - 0s - loss: 0.4367 - acc: 0.8092     \n",
      "Epoch 70/100\n",
      "262/262 [==============================] - 0s - loss: 0.4149 - acc: 0.8321     \n",
      "Epoch 71/100\n",
      "262/262 [==============================] - 0s - loss: 0.3808 - acc: 0.8321     \n",
      "Epoch 72/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3369 - acc: 0.850 - 0s - loss: 0.3674 - acc: 0.8244     \n",
      "Epoch 73/100\n",
      "262/262 [==============================] - 0s - loss: 0.3534 - acc: 0.8435     \n",
      "Epoch 74/100\n",
      "262/262 [==============================] - 0s - loss: 0.3529 - acc: 0.8435     \n",
      "Epoch 75/100\n",
      "262/262 [==============================] - 0s - loss: 0.3479 - acc: 0.8397     \n",
      "Epoch 76/100\n",
      "262/262 [==============================] - 0s - loss: 0.3579 - acc: 0.8550     \n",
      "Epoch 77/100\n",
      "262/262 [==============================] - 0s - loss: 0.3498 - acc: 0.8511     \n",
      "Epoch 78/100\n",
      "262/262 [==============================] - 0s - loss: 0.3496 - acc: 0.8397     \n",
      "Epoch 79/100\n",
      "262/262 [==============================] - 0s - loss: 0.3452 - acc: 0.8511     \n",
      "Epoch 80/100\n",
      "262/262 [==============================] - 0s - loss: 0.3502 - acc: 0.8397     \n",
      "Epoch 81/100\n",
      "262/262 [==============================] - 0s - loss: 0.3488 - acc: 0.8550     \n",
      "Epoch 82/100\n",
      "262/262 [==============================] - 0s - loss: 0.3652 - acc: 0.8511     \n",
      "Epoch 83/100\n",
      "262/262 [==============================] - 0s - loss: 0.3536 - acc: 0.8397     \n",
      "Epoch 84/100\n",
      "262/262 [==============================] - 0s - loss: 0.3488 - acc: 0.8511     \n",
      "Epoch 85/100\n",
      "262/262 [==============================] - 0s - loss: 0.3396 - acc: 0.8511     \n",
      "Epoch 86/100\n",
      "262/262 [==============================] - 0s - loss: 0.3395 - acc: 0.8550     \n",
      "Epoch 87/100\n",
      "262/262 [==============================] - 0s - loss: 0.3318 - acc: 0.8664     \n",
      "Epoch 88/100\n",
      "262/262 [==============================] - 0s - loss: 0.3303 - acc: 0.8626     \n",
      "Epoch 89/100\n",
      "262/262 [==============================] - 0s - loss: 0.3378 - acc: 0.8702     \n",
      "Epoch 90/100\n",
      "262/262 [==============================] - 0s - loss: 0.3459 - acc: 0.8511     \n",
      "Epoch 91/100\n",
      "262/262 [==============================] - 0s - loss: 0.3902 - acc: 0.8359     \n",
      "Epoch 92/100\n",
      "262/262 [==============================] - 0s - loss: 0.4058 - acc: 0.8206     \n",
      "Epoch 93/100\n",
      "262/262 [==============================] - 0s - loss: 0.3570 - acc: 0.8435     \n",
      "Epoch 94/100\n",
      "262/262 [==============================] - 0s - loss: 0.3362 - acc: 0.8550     \n",
      "Epoch 95/100\n",
      "262/262 [==============================] - 0s - loss: 0.3301 - acc: 0.8626     \n",
      "Epoch 96/100\n",
      "262/262 [==============================] - 0s - loss: 0.3303 - acc: 0.8664     \n",
      "Epoch 97/100\n",
      "262/262 [==============================] - 0s - loss: 0.3225 - acc: 0.8740     \n",
      "Epoch 98/100\n",
      "262/262 [==============================] - 0s - loss: 0.3246 - acc: 0.8702     \n",
      "Epoch 99/100\n",
      "262/262 [==============================] - 0s - loss: 0.3190 - acc: 0.8664     \n",
      "Epoch 100/100\n",
      "262/262 [==============================] - 0s - loss: 0.3222 - acc: 0.8664     \n",
      " 20/262 [=>............................] - ETA: 0s[CV] .. epochs=100, batch_size=20, score=0.838461536628, total=   7.2s\n",
      "[CV] epochs=10, batch_size=40 ........................................\n",
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6754 - acc: 0.6667     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.6105 - acc: 0.7816     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.5157 - acc: 0.7893     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.4384 - acc: 0.7969     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.4081 - acc: 0.7931     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.3975 - acc: 0.8046     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.3900 - acc: 0.8123     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.3806 - acc: 0.8161     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.3743 - acc: 0.8314     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.3706 - acc: 0.8391     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] ... epochs=10, batch_size=40, score=0.717557246448, total=   3.1s\n",
      "[CV] epochs=10, batch_size=40 ........................................\n",
      "Epoch 1/10\n",
      "261/261 [==============================] - 0s - loss: 0.6811 - acc: 0.6207     \n",
      "Epoch 2/10\n",
      "261/261 [==============================] - 0s - loss: 0.6336 - acc: 0.7280     \n",
      "Epoch 3/10\n",
      "261/261 [==============================] - 0s - loss: 0.5609 - acc: 0.7625     \n",
      "Epoch 4/10\n",
      "261/261 [==============================] - 0s - loss: 0.4879 - acc: 0.7816     \n",
      "Epoch 5/10\n",
      "261/261 [==============================] - 0s - loss: 0.4574 - acc: 0.7816     \n",
      "Epoch 6/10\n",
      "261/261 [==============================] - 0s - loss: 0.4443 - acc: 0.7854     \n",
      "Epoch 7/10\n",
      "261/261 [==============================] - 0s - loss: 0.4367 - acc: 0.7931     \n",
      "Epoch 8/10\n",
      "261/261 [==============================] - 0s - loss: 0.4317 - acc: 0.8008     \n",
      "Epoch 9/10\n",
      "261/261 [==============================] - 0s - loss: 0.4280 - acc: 0.7969     \n",
      "Epoch 10/10\n",
      "261/261 [==============================] - 0s - loss: 0.4271 - acc: 0.7931     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] ... epochs=10, batch_size=40, score=0.770992361407, total=   3.5s\n",
      "[CV] epochs=10, batch_size=40 ........................................\n",
      "Epoch 1/10\n",
      "262/262 [==============================] - 0s - loss: 0.6841 - acc: 0.6221     \n",
      "Epoch 2/10\n",
      "262/262 [==============================] - 0s - loss: 0.6518 - acc: 0.6527     \n",
      "Epoch 3/10\n",
      "262/262 [==============================] - 0s - loss: 0.6013 - acc: 0.6527     \n",
      "Epoch 4/10\n",
      "262/262 [==============================] - 0s - loss: 0.5499 - acc: 0.6527     \n",
      "Epoch 5/10\n",
      "262/262 [==============================] - 0s - loss: 0.5239 - acc: 0.6527     \n",
      "Epoch 6/10\n",
      "262/262 [==============================] - 0s - loss: 0.5155 - acc: 0.6527     \n",
      "Epoch 7/10\n",
      "262/262 [==============================] - 0s - loss: 0.5078 - acc: 0.6527     \n",
      "Epoch 8/10\n",
      "262/262 [==============================] - 0s - loss: 0.5025 - acc: 0.6527     \n",
      "Epoch 9/10\n",
      "262/262 [==============================] - 0s - loss: 0.4976 - acc: 0.6527     \n",
      "Epoch 10/10\n",
      "262/262 [==============================] - 0s - loss: 0.4945 - acc: 0.6527     \n",
      " 40/262 [===>..........................] - ETA: 0s[CV] ... epochs=10, batch_size=40, score=0.699999988079, total=   2.9s\n",
      "[CV] epochs=50, batch_size=40 ........................................\n",
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6688 - acc: 0.6513     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.5790 - acc: 0.7241     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.4898 - acc: 0.7701     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4486 - acc: 0.7778     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4202 - acc: 0.7931     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - 0s - loss: 0.3996 - acc: 0.8084     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3418 - acc: 0.825 - 0s - loss: 0.3894 - acc: 0.8161     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - 0s - loss: 0.3836 - acc: 0.8123     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.3789 - acc: 0.8161     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.3737 - acc: 0.8199     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.3696 - acc: 0.8276     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.3667 - acc: 0.8276     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - 0s - loss: 0.3622 - acc: 0.8199     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.3586 - acc: 0.8352     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3548 - acc: 0.8429     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3526 - acc: 0.8429     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - 0s - loss: 0.3488 - acc: 0.8429     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.3467 - acc: 0.8467     \n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.3452 - acc: 0.8314     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.3422 - acc: 0.8429     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.3383 - acc: 0.8391     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.3369 - acc: 0.8544     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.3359 - acc: 0.8467     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.3353 - acc: 0.8429     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.3356 - acc: 0.8429     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.3315 - acc: 0.8506     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.3277 - acc: 0.8544     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.3263 - acc: 0.8544     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.3259 - acc: 0.8391     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.3247 - acc: 0.8582     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.3217 - acc: 0.8544     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.3208 - acc: 0.8467     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.3160 - acc: 0.8621     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.3153 - acc: 0.8621     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.3141 - acc: 0.8621     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.3122 - acc: 0.8544     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - 0s - loss: 0.3119 - acc: 0.8582     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.3095 - acc: 0.8544     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.2712 - acc: 0.875 - 0s - loss: 0.3062 - acc: 0.8582     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.3035 - acc: 0.8582     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.3009 - acc: 0.8621     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.2989 - acc: 0.8544     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.2980 - acc: 0.8544     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.2971 - acc: 0.8621     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.2958 - acc: 0.8621     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3716 - acc: 0.800 - 0s - loss: 0.2948 - acc: 0.8659     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.2908 - acc: 0.8659     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.2888 - acc: 0.8659     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.2913 - acc: 0.8659     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.2884 - acc: 0.8582     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] ... epochs=50, batch_size=40, score=0.725190835145, total=   4.0s\n",
      "[CV] epochs=50, batch_size=40 ........................................\n",
      "Epoch 1/50\n",
      "261/261 [==============================] - 0s - loss: 0.6819 - acc: 0.7471     \n",
      "Epoch 2/50\n",
      "261/261 [==============================] - 0s - loss: 0.6401 - acc: 0.7586     \n",
      "Epoch 3/50\n",
      "261/261 [==============================] - 0s - loss: 0.5727 - acc: 0.7931     \n",
      "Epoch 4/50\n",
      "261/261 [==============================] - 0s - loss: 0.4982 - acc: 0.7778     \n",
      "Epoch 5/50\n",
      "261/261 [==============================] - 0s - loss: 0.4563 - acc: 0.7739     \n",
      "Epoch 6/50\n",
      "261/261 [==============================] - 0s - loss: 0.4460 - acc: 0.7778     \n",
      "Epoch 7/50\n",
      "261/261 [==============================] - 0s - loss: 0.4386 - acc: 0.7893     \n",
      "Epoch 8/50\n",
      "261/261 [==============================] - 0s - loss: 0.4307 - acc: 0.8008     \n",
      "Epoch 9/50\n",
      "261/261 [==============================] - 0s - loss: 0.4247 - acc: 0.8123     \n",
      "Epoch 10/50\n",
      "261/261 [==============================] - 0s - loss: 0.4281 - acc: 0.8008     \n",
      "Epoch 11/50\n",
      "261/261 [==============================] - 0s - loss: 0.4178 - acc: 0.7969     \n",
      "Epoch 12/50\n",
      "261/261 [==============================] - 0s - loss: 0.4132 - acc: 0.8123     \n",
      "Epoch 13/50\n",
      "261/261 [==============================] - 0s - loss: 0.4071 - acc: 0.8123     \n",
      "Epoch 14/50\n",
      "261/261 [==============================] - 0s - loss: 0.4042 - acc: 0.8123     \n",
      "Epoch 15/50\n",
      "261/261 [==============================] - 0s - loss: 0.3998 - acc: 0.8123     \n",
      "Epoch 16/50\n",
      "261/261 [==============================] - 0s - loss: 0.3976 - acc: 0.8238     \n",
      "Epoch 17/50\n",
      "261/261 [==============================] - 0s - loss: 0.3935 - acc: 0.8314     \n",
      "Epoch 18/50\n",
      "261/261 [==============================] - 0s - loss: 0.3901 - acc: 0.8352     \n",
      "Epoch 19/50\n",
      "261/261 [==============================] - 0s - loss: 0.3878 - acc: 0.8314     \n",
      "Epoch 20/50\n",
      "261/261 [==============================] - 0s - loss: 0.3850 - acc: 0.8352     \n",
      "Epoch 21/50\n",
      "261/261 [==============================] - 0s - loss: 0.3821 - acc: 0.8391     \n",
      "Epoch 22/50\n",
      "261/261 [==============================] - 0s - loss: 0.3803 - acc: 0.8391     \n",
      "Epoch 23/50\n",
      "261/261 [==============================] - 0s - loss: 0.3790 - acc: 0.8467     \n",
      "Epoch 24/50\n",
      "261/261 [==============================] - 0s - loss: 0.3776 - acc: 0.8506     \n",
      "Epoch 25/50\n",
      "261/261 [==============================] - 0s - loss: 0.3729 - acc: 0.8544     \n",
      "Epoch 26/50\n",
      "261/261 [==============================] - 0s - loss: 0.3719 - acc: 0.8467     \n",
      "Epoch 27/50\n",
      "261/261 [==============================] - 0s - loss: 0.3683 - acc: 0.8582     \n",
      "Epoch 28/50\n",
      "261/261 [==============================] - 0s - loss: 0.3684 - acc: 0.8506     \n",
      "Epoch 29/50\n",
      "261/261 [==============================] - 0s - loss: 0.3643 - acc: 0.8544     \n",
      "Epoch 30/50\n",
      "261/261 [==============================] - 0s - loss: 0.3649 - acc: 0.8582     \n",
      "Epoch 31/50\n",
      "261/261 [==============================] - 0s - loss: 0.3612 - acc: 0.8621     \n",
      "Epoch 32/50\n",
      "261/261 [==============================] - 0s - loss: 0.3637 - acc: 0.8506     \n",
      "Epoch 33/50\n",
      "261/261 [==============================] - 0s - loss: 0.3601 - acc: 0.8506     \n",
      "Epoch 34/50\n",
      "261/261 [==============================] - 0s - loss: 0.3590 - acc: 0.8544     \n",
      "Epoch 35/50\n",
      "261/261 [==============================] - 0s - loss: 0.3581 - acc: 0.8506     \n",
      "Epoch 36/50\n",
      "261/261 [==============================] - 0s - loss: 0.3568 - acc: 0.8506     \n",
      "Epoch 37/50\n",
      "261/261 [==============================] - 0s - loss: 0.3577 - acc: 0.8467     \n",
      "Epoch 38/50\n",
      "261/261 [==============================] - 0s - loss: 0.3596 - acc: 0.8544     \n",
      "Epoch 39/50\n",
      "261/261 [==============================] - 0s - loss: 0.3563 - acc: 0.8506     \n",
      "Epoch 40/50\n",
      "261/261 [==============================] - 0s - loss: 0.3521 - acc: 0.8429     \n",
      "Epoch 41/50\n",
      "261/261 [==============================] - 0s - loss: 0.3527 - acc: 0.8467     \n",
      "Epoch 42/50\n",
      "261/261 [==============================] - 0s - loss: 0.3502 - acc: 0.8506     \n",
      "Epoch 43/50\n",
      "261/261 [==============================] - 0s - loss: 0.3534 - acc: 0.8467     \n",
      "Epoch 44/50\n",
      "261/261 [==============================] - 0s - loss: 0.3550 - acc: 0.8429     \n",
      "Epoch 45/50\n",
      "261/261 [==============================] - 0s - loss: 0.3490 - acc: 0.8429     \n",
      "Epoch 46/50\n",
      "261/261 [==============================] - 0s - loss: 0.3489 - acc: 0.8621     \n",
      "Epoch 47/50\n",
      "261/261 [==============================] - 0s - loss: 0.3442 - acc: 0.8582     \n",
      "Epoch 48/50\n",
      "261/261 [==============================] - 0s - loss: 0.3490 - acc: 0.8506     \n",
      "Epoch 49/50\n",
      "261/261 [==============================] - 0s - loss: 0.3472 - acc: 0.8506     \n",
      "Epoch 50/50\n",
      "261/261 [==============================] - 0s - loss: 0.3426 - acc: 0.8506     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] ... epochs=50, batch_size=40, score=0.770992369597, total=   4.4s\n",
      "[CV] epochs=50, batch_size=40 ........................................\n",
      "Epoch 1/50\n",
      "262/262 [==============================] - 0s - loss: 0.6769 - acc: 0.6489     \n",
      "Epoch 2/50\n",
      "262/262 [==============================] - 0s - loss: 0.6276 - acc: 0.6489     \n",
      "Epoch 3/50\n",
      "262/262 [==============================] - 0s - loss: 0.5430 - acc: 0.7214     \n",
      "Epoch 4/50\n",
      "262/262 [==============================] - 0s - loss: 0.5002 - acc: 0.7519     \n",
      "Epoch 5/50\n",
      "262/262 [==============================] - 0s - loss: 0.4800 - acc: 0.7443     \n",
      "Epoch 6/50\n",
      "262/262 [==============================] - 0s - loss: 0.4746 - acc: 0.7748     \n",
      "Epoch 7/50\n",
      "262/262 [==============================] - 0s - loss: 0.4674 - acc: 0.7786     \n",
      "Epoch 8/50\n",
      "262/262 [==============================] - 0s - loss: 0.4617 - acc: 0.7748     \n",
      "Epoch 9/50\n",
      "262/262 [==============================] - 0s - loss: 0.4601 - acc: 0.7710     \n",
      "Epoch 10/50\n",
      "262/262 [==============================] - 0s - loss: 0.4585 - acc: 0.7824     \n",
      "Epoch 11/50\n",
      "262/262 [==============================] - 0s - loss: 0.4566 - acc: 0.7786     \n",
      "Epoch 12/50\n",
      "262/262 [==============================] - 0s - loss: 0.4537 - acc: 0.7824     \n",
      "Epoch 13/50\n",
      "262/262 [==============================] - 0s - loss: 0.4537 - acc: 0.7672     \n",
      "Epoch 14/50\n",
      "262/262 [==============================] - 0s - loss: 0.4491 - acc: 0.7786     \n",
      "Epoch 15/50\n",
      "262/262 [==============================] - 0s - loss: 0.4481 - acc: 0.7863     \n",
      "Epoch 16/50\n",
      "262/262 [==============================] - 0s - loss: 0.4460 - acc: 0.7786     \n",
      "Epoch 17/50\n",
      "262/262 [==============================] - 0s - loss: 0.4448 - acc: 0.7672     \n",
      "Epoch 18/50\n",
      "262/262 [==============================] - 0s - loss: 0.4463 - acc: 0.7939     \n",
      "Epoch 19/50\n",
      "262/262 [==============================] - 0s - loss: 0.4412 - acc: 0.7863     \n",
      "Epoch 20/50\n",
      "262/262 [==============================] - 0s - loss: 0.4369 - acc: 0.7786     \n",
      "Epoch 21/50\n",
      "262/262 [==============================] - 0s - loss: 0.4332 - acc: 0.7901     \n",
      "Epoch 22/50\n",
      "262/262 [==============================] - 0s - loss: 0.4306 - acc: 0.7939     \n",
      "Epoch 23/50\n",
      "262/262 [==============================] - 0s - loss: 0.4271 - acc: 0.7901     \n",
      "Epoch 24/50\n",
      "262/262 [==============================] - 0s - loss: 0.4238 - acc: 0.8015     \n",
      "Epoch 25/50\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.5338 - acc: 0.700 - 0s - loss: 0.4178 - acc: 0.8092     \n",
      "Epoch 26/50\n",
      "262/262 [==============================] - 0s - loss: 0.4139 - acc: 0.8053     \n",
      "Epoch 27/50\n",
      "262/262 [==============================] - 0s - loss: 0.4105 - acc: 0.7977     \n",
      "Epoch 28/50\n",
      "262/262 [==============================] - 0s - loss: 0.4086 - acc: 0.8130     \n",
      "Epoch 29/50\n",
      "262/262 [==============================] - 0s - loss: 0.4078 - acc: 0.8130     \n",
      "Epoch 30/50\n",
      "262/262 [==============================] - 0s - loss: 0.4013 - acc: 0.8244     \n",
      "Epoch 31/50\n",
      "262/262 [==============================] - 0s - loss: 0.3990 - acc: 0.8206     \n",
      "Epoch 32/50\n",
      "262/262 [==============================] - 0s - loss: 0.3951 - acc: 0.8168     \n",
      "Epoch 33/50\n",
      "262/262 [==============================] - 0s - loss: 0.3928 - acc: 0.8282     \n",
      "Epoch 34/50\n",
      "262/262 [==============================] - 0s - loss: 0.3896 - acc: 0.8359     \n",
      "Epoch 35/50\n",
      "262/262 [==============================] - 0s - loss: 0.3879 - acc: 0.8321     \n",
      "Epoch 36/50\n",
      "262/262 [==============================] - 0s - loss: 0.3868 - acc: 0.8206     \n",
      "Epoch 37/50\n",
      "262/262 [==============================] - 0s - loss: 0.3837 - acc: 0.8282     \n",
      "Epoch 38/50\n",
      "262/262 [==============================] - 0s - loss: 0.3834 - acc: 0.8282     \n",
      "Epoch 39/50\n",
      "262/262 [==============================] - 0s - loss: 0.3798 - acc: 0.8321     \n",
      "Epoch 40/50\n",
      "262/262 [==============================] - 0s - loss: 0.3807 - acc: 0.8168     \n",
      "Epoch 41/50\n",
      "262/262 [==============================] - 0s - loss: 0.3758 - acc: 0.8282     \n",
      "Epoch 42/50\n",
      "262/262 [==============================] - 0s - loss: 0.3761 - acc: 0.8435     \n",
      "Epoch 43/50\n",
      "262/262 [==============================] - 0s - loss: 0.3730 - acc: 0.8397     \n",
      "Epoch 44/50\n",
      "262/262 [==============================] - 0s - loss: 0.3732 - acc: 0.8359     \n",
      "Epoch 45/50\n",
      "262/262 [==============================] - 0s - loss: 0.3715 - acc: 0.8397     \n",
      "Epoch 46/50\n",
      "262/262 [==============================] - 0s - loss: 0.3710 - acc: 0.8473     \n",
      "Epoch 47/50\n",
      "262/262 [==============================] - 0s - loss: 0.3630 - acc: 0.8550     \n",
      "Epoch 48/50\n",
      "262/262 [==============================] - 0s - loss: 0.3659 - acc: 0.8282     \n",
      "Epoch 49/50\n",
      "262/262 [==============================] - 0s - loss: 0.3604 - acc: 0.8397     \n",
      "Epoch 50/50\n",
      "262/262 [==============================] - 0s - loss: 0.3605 - acc: 0.8359     \n",
      " 40/262 [===>..........................] - ETA: 0s[CV] .... epochs=50, batch_size=40, score=0.85384615568, total=   4.5s\n",
      "[CV] epochs=100, batch_size=40 .......................................\n",
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6761 - acc: 0.6743     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.6188 - acc: 0.6973     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - 0s - loss: 0.5401 - acc: 0.6973     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.4767 - acc: 0.6973     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.4461 - acc: 0.7050     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.4258 - acc: 0.7739     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.4049 - acc: 0.8008     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.3862 - acc: 0.8238     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.3801 - acc: 0.8276     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.3752 - acc: 0.8276     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.3683 - acc: 0.8276     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.3627 - acc: 0.8352     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.3584 - acc: 0.8352     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.3537 - acc: 0.8391     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.3486 - acc: 0.8391     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.3455 - acc: 0.8391     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.3434 - acc: 0.8467     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.3401 - acc: 0.8429     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.3385 - acc: 0.8391     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.3344 - acc: 0.8391     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.3319 - acc: 0.8391     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.3300 - acc: 0.8429     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.3276 - acc: 0.8467     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.3245 - acc: 0.8429     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.3221 - acc: 0.8391     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - 0s - loss: 0.3206 - acc: 0.8391     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.3180 - acc: 0.8506     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.3155 - acc: 0.8544     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.3143 - acc: 0.8506     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.3133 - acc: 0.8506     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.3111 - acc: 0.8544     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.3086 - acc: 0.8659     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.3072 - acc: 0.8659     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.3052 - acc: 0.8621     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.3045 - acc: 0.8582     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - 0s - loss: 0.3038 - acc: 0.8621     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.3021 - acc: 0.8659     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.3036 - acc: 0.8582     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.3001 - acc: 0.8736     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.3002 - acc: 0.8659     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.2974 - acc: 0.8621     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.2990 - acc: 0.8659     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.2999 - acc: 0.8621     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - 0s - loss: 0.2972 - acc: 0.8774     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.2976 - acc: 0.8697     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.2962 - acc: 0.8621     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.2964 - acc: 0.8736     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.2980 - acc: 0.8697     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.2952 - acc: 0.8582     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.2933 - acc: 0.8659     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.2943 - acc: 0.8659     \n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.3007 - acc: 0.8659     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.2929 - acc: 0.8736     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.2939 - acc: 0.8774     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.2922 - acc: 0.8621     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.2888 - acc: 0.8774     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.2954 - acc: 0.8774     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.2901 - acc: 0.8774     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.2899 - acc: 0.8774     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.2888 - acc: 0.8736     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.2908 - acc: 0.8736     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - 0s - loss: 0.2854 - acc: 0.8812     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.2863 - acc: 0.8774     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.2889 - acc: 0.8736     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.2862 - acc: 0.8736     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.2848 - acc: 0.8774     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.2848 - acc: 0.8697     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.2859 - acc: 0.8736     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.2836 - acc: 0.8736     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.2830 - acc: 0.8736     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.2822 - acc: 0.8812     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2842 - acc: 0.8697     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2841 - acc: 0.8774     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.2846 - acc: 0.8812     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.2831 - acc: 0.8736     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.2648 - acc: 0.900 - 0s - loss: 0.2837 - acc: 0.8736     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2808 - acc: 0.8697     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2804 - acc: 0.8812     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2787 - acc: 0.8851     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2782 - acc: 0.8812     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2782 - acc: 0.8774     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.2790 - acc: 0.8812     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.2781 - acc: 0.8736     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.2752 - acc: 0.8851     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.2707 - acc: 0.8851     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2724 - acc: 0.8851     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.2681 - acc: 0.8774     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - 0s - loss: 0.2663 - acc: 0.8736     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2628 - acc: 0.8851     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2680 - acc: 0.8812     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2635 - acc: 0.8736     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2590 - acc: 0.8774     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.2582 - acc: 0.8812     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2614 - acc: 0.8774     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2559 - acc: 0.8659     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.2566 - acc: 0.8736     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2535 - acc: 0.8736     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2554 - acc: 0.8736     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2500 - acc: 0.8774     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2519 - acc: 0.8812     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] .. epochs=100, batch_size=40, score=0.717557254638, total=   5.5s\n",
      "[CV] epochs=100, batch_size=40 .......................................\n",
      "Epoch 1/100\n",
      "261/261 [==============================] - 0s - loss: 0.6819 - acc: 0.6054     \n",
      "Epoch 2/100\n",
      "261/261 [==============================] - 0s - loss: 0.6348 - acc: 0.7280     \n",
      "Epoch 3/100\n",
      "261/261 [==============================] - 0s - loss: 0.5609 - acc: 0.7739     \n",
      "Epoch 4/100\n",
      "261/261 [==============================] - 0s - loss: 0.4891 - acc: 0.7778     \n",
      "Epoch 5/100\n",
      "261/261 [==============================] - 0s - loss: 0.4530 - acc: 0.7969     \n",
      "Epoch 6/100\n",
      "261/261 [==============================] - 0s - loss: 0.4523 - acc: 0.7893     \n",
      "Epoch 7/100\n",
      "261/261 [==============================] - 0s - loss: 0.4429 - acc: 0.7816     \n",
      "Epoch 8/100\n",
      "261/261 [==============================] - 0s - loss: 0.4332 - acc: 0.7931     \n",
      "Epoch 9/100\n",
      "261/261 [==============================] - 0s - loss: 0.4292 - acc: 0.7931     \n",
      "Epoch 10/100\n",
      "261/261 [==============================] - 0s - loss: 0.4243 - acc: 0.8008     \n",
      "Epoch 11/100\n",
      "261/261 [==============================] - 0s - loss: 0.4218 - acc: 0.7931     \n",
      "Epoch 12/100\n",
      "261/261 [==============================] - 0s - loss: 0.4173 - acc: 0.8008     \n",
      "Epoch 13/100\n",
      "261/261 [==============================] - 0s - loss: 0.4164 - acc: 0.8199     \n",
      "Epoch 14/100\n",
      "261/261 [==============================] - 0s - loss: 0.4119 - acc: 0.8123     \n",
      "Epoch 15/100\n",
      "261/261 [==============================] - 0s - loss: 0.4096 - acc: 0.8046     \n",
      "Epoch 16/100\n",
      "261/261 [==============================] - 0s - loss: 0.4072 - acc: 0.8199     \n",
      "Epoch 17/100\n",
      "261/261 [==============================] - 0s - loss: 0.4032 - acc: 0.8238     \n",
      "Epoch 18/100\n",
      "261/261 [==============================] - 0s - loss: 0.3982 - acc: 0.8276     \n",
      "Epoch 19/100\n",
      "261/261 [==============================] - 0s - loss: 0.3969 - acc: 0.8199     \n",
      "Epoch 20/100\n",
      "261/261 [==============================] - 0s - loss: 0.3933 - acc: 0.8276     \n",
      "Epoch 21/100\n",
      "261/261 [==============================] - 0s - loss: 0.3913 - acc: 0.8276     \n",
      "Epoch 22/100\n",
      "261/261 [==============================] - 0s - loss: 0.3861 - acc: 0.8276     \n",
      "Epoch 23/100\n",
      "261/261 [==============================] - 0s - loss: 0.3840 - acc: 0.8352     \n",
      "Epoch 24/100\n",
      "261/261 [==============================] - 0s - loss: 0.3817 - acc: 0.8391     \n",
      "Epoch 25/100\n",
      "261/261 [==============================] - 0s - loss: 0.3769 - acc: 0.8429     \n",
      "Epoch 26/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3456 - acc: 0.925 - 0s - loss: 0.3737 - acc: 0.8506     \n",
      "Epoch 27/100\n",
      "261/261 [==============================] - 0s - loss: 0.3704 - acc: 0.8429     \n",
      "Epoch 28/100\n",
      "261/261 [==============================] - 0s - loss: 0.3700 - acc: 0.8429     \n",
      "Epoch 29/100\n",
      "261/261 [==============================] - 0s - loss: 0.3622 - acc: 0.8506     \n",
      "Epoch 30/100\n",
      "261/261 [==============================] - 0s - loss: 0.3606 - acc: 0.8467     \n",
      "Epoch 31/100\n",
      "261/261 [==============================] - 0s - loss: 0.3569 - acc: 0.8506     \n",
      "Epoch 32/100\n",
      "261/261 [==============================] - 0s - loss: 0.3570 - acc: 0.8544     \n",
      "Epoch 33/100\n",
      "261/261 [==============================] - 0s - loss: 0.3543 - acc: 0.8467     \n",
      "Epoch 34/100\n",
      "261/261 [==============================] - 0s - loss: 0.3515 - acc: 0.8429     \n",
      "Epoch 35/100\n",
      "261/261 [==============================] - 0s - loss: 0.3496 - acc: 0.8544     \n",
      "Epoch 36/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3249 - acc: 0.925 - 0s - loss: 0.3493 - acc: 0.8544     \n",
      "Epoch 37/100\n",
      "261/261 [==============================] - 0s - loss: 0.3482 - acc: 0.8582     \n",
      "Epoch 38/100\n",
      "261/261 [==============================] - 0s - loss: 0.3461 - acc: 0.8506     \n",
      "Epoch 39/100\n",
      "261/261 [==============================] - 0s - loss: 0.3404 - acc: 0.8621     \n",
      "Epoch 40/100\n",
      "261/261 [==============================] - 0s - loss: 0.3436 - acc: 0.8582     \n",
      "Epoch 41/100\n",
      "261/261 [==============================] - 0s - loss: 0.3411 - acc: 0.8582     \n",
      "Epoch 42/100\n",
      "261/261 [==============================] - 0s - loss: 0.3346 - acc: 0.8621     \n",
      "Epoch 43/100\n",
      "261/261 [==============================] - 0s - loss: 0.3346 - acc: 0.8544     \n",
      "Epoch 44/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.3152 - acc: 0.900 - 0s - loss: 0.3310 - acc: 0.8506     \n",
      "Epoch 45/100\n",
      "261/261 [==============================] - 0s - loss: 0.3310 - acc: 0.8582     \n",
      "Epoch 46/100\n",
      "261/261 [==============================] - 0s - loss: 0.3224 - acc: 0.8697     \n",
      "Epoch 47/100\n",
      "261/261 [==============================] - 0s - loss: 0.3355 - acc: 0.8621     \n",
      "Epoch 48/100\n",
      "261/261 [==============================] - 0s - loss: 0.3282 - acc: 0.8621     \n",
      "Epoch 49/100\n",
      "261/261 [==============================] - 0s - loss: 0.3182 - acc: 0.8812     \n",
      "Epoch 50/100\n",
      "261/261 [==============================] - 0s - loss: 0.3172 - acc: 0.8736     \n",
      "Epoch 51/100\n",
      "261/261 [==============================] - 0s - loss: 0.3184 - acc: 0.8582     \n",
      "Epoch 52/100\n",
      "261/261 [==============================] - 0s - loss: 0.3124 - acc: 0.8697     \n",
      "Epoch 53/100\n",
      "261/261 [==============================] - 0s - loss: 0.3123 - acc: 0.8736     \n",
      "Epoch 54/100\n",
      "261/261 [==============================] - 0s - loss: 0.3130 - acc: 0.8812     \n",
      "Epoch 55/100\n",
      "261/261 [==============================] - 0s - loss: 0.3181 - acc: 0.8506     \n",
      "Epoch 56/100\n",
      "261/261 [==============================] - 0s - loss: 0.3052 - acc: 0.8659     \n",
      "Epoch 57/100\n",
      "261/261 [==============================] - 0s - loss: 0.2998 - acc: 0.8851     \n",
      "Epoch 58/100\n",
      "261/261 [==============================] - 0s - loss: 0.3010 - acc: 0.8812     \n",
      "Epoch 59/100\n",
      "261/261 [==============================] - 0s - loss: 0.3017 - acc: 0.8851     \n",
      "Epoch 60/100\n",
      "261/261 [==============================] - 0s - loss: 0.2918 - acc: 0.8736     \n",
      "Epoch 61/100\n",
      "261/261 [==============================] - 0s - loss: 0.2942 - acc: 0.8697     \n",
      "Epoch 62/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.2333 - acc: 0.925 - 0s - loss: 0.2931 - acc: 0.8736     \n",
      "Epoch 63/100\n",
      "261/261 [==============================] - 0s - loss: 0.2857 - acc: 0.8774     \n",
      "Epoch 64/100\n",
      "261/261 [==============================] - 0s - loss: 0.2865 - acc: 0.8851     \n",
      "Epoch 65/100\n",
      "261/261 [==============================] - 0s - loss: 0.2791 - acc: 0.8812     \n",
      "Epoch 66/100\n",
      "261/261 [==============================] - 0s - loss: 0.2814 - acc: 0.8659     \n",
      "Epoch 67/100\n",
      "261/261 [==============================] - 0s - loss: 0.2760 - acc: 0.8812     \n",
      "Epoch 68/100\n",
      "261/261 [==============================] - 0s - loss: 0.2804 - acc: 0.8774     \n",
      "Epoch 69/100\n",
      "261/261 [==============================] - 0s - loss: 0.2782 - acc: 0.8889     \n",
      "Epoch 70/100\n",
      "261/261 [==============================] - 0s - loss: 0.2707 - acc: 0.8697     \n",
      "Epoch 71/100\n",
      "261/261 [==============================] - 0s - loss: 0.2720 - acc: 0.8659     \n",
      "Epoch 72/100\n",
      "261/261 [==============================] - 0s - loss: 0.2716 - acc: 0.8889     \n",
      "Epoch 73/100\n",
      "261/261 [==============================] - 0s - loss: 0.2684 - acc: 0.8927     \n",
      "Epoch 74/100\n",
      "261/261 [==============================] - 0s - loss: 0.2700 - acc: 0.8697     \n",
      "Epoch 75/100\n",
      "261/261 [==============================] - 0s - loss: 0.2660 - acc: 0.8851     \n",
      "Epoch 76/100\n",
      "261/261 [==============================] - ETA: 0s - loss: 0.2817 - acc: 0.875 - 0s - loss: 0.2656 - acc: 0.8774     \n",
      "Epoch 77/100\n",
      "261/261 [==============================] - 0s - loss: 0.2669 - acc: 0.8851     \n",
      "Epoch 78/100\n",
      "261/261 [==============================] - 0s - loss: 0.2572 - acc: 0.8851     \n",
      "Epoch 79/100\n",
      "261/261 [==============================] - 0s - loss: 0.2649 - acc: 0.8659     \n",
      "Epoch 80/100\n",
      "261/261 [==============================] - 0s - loss: 0.2584 - acc: 0.8927     \n",
      "Epoch 81/100\n",
      "261/261 [==============================] - 0s - loss: 0.2545 - acc: 0.8927     \n",
      "Epoch 82/100\n",
      "261/261 [==============================] - 0s - loss: 0.2540 - acc: 0.8851     \n",
      "Epoch 83/100\n",
      "261/261 [==============================] - 0s - loss: 0.2535 - acc: 0.8851     \n",
      "Epoch 84/100\n",
      "261/261 [==============================] - 0s - loss: 0.2481 - acc: 0.8889     \n",
      "Epoch 85/100\n",
      "261/261 [==============================] - 0s - loss: 0.2509 - acc: 0.8736     \n",
      "Epoch 86/100\n",
      "261/261 [==============================] - 0s - loss: 0.2534 - acc: 0.8774     \n",
      "Epoch 87/100\n",
      "261/261 [==============================] - 0s - loss: 0.2469 - acc: 0.8851     \n",
      "Epoch 88/100\n",
      "261/261 [==============================] - 0s - loss: 0.2480 - acc: 0.8927     \n",
      "Epoch 89/100\n",
      "261/261 [==============================] - 0s - loss: 0.2405 - acc: 0.8812     \n",
      "Epoch 90/100\n",
      "261/261 [==============================] - 0s - loss: 0.2546 - acc: 0.8966     \n",
      "Epoch 91/100\n",
      "261/261 [==============================] - 0s - loss: 0.2430 - acc: 0.8889     \n",
      "Epoch 92/100\n",
      "261/261 [==============================] - 0s - loss: 0.2442 - acc: 0.8927     \n",
      "Epoch 93/100\n",
      "261/261 [==============================] - 0s - loss: 0.2348 - acc: 0.8927     \n",
      "Epoch 94/100\n",
      "261/261 [==============================] - 0s - loss: 0.2360 - acc: 0.8927     \n",
      "Epoch 95/100\n",
      "261/261 [==============================] - 0s - loss: 0.2316 - acc: 0.8927     \n",
      "Epoch 96/100\n",
      "261/261 [==============================] - 0s - loss: 0.2316 - acc: 0.8927     \n",
      "Epoch 97/100\n",
      "261/261 [==============================] - 0s - loss: 0.2295 - acc: 0.8851     \n",
      "Epoch 98/100\n",
      "261/261 [==============================] - 0s - loss: 0.2280 - acc: 0.8966     \n",
      "Epoch 99/100\n",
      "261/261 [==============================] - 0s - loss: 0.2264 - acc: 0.8927     \n",
      "Epoch 100/100\n",
      "261/261 [==============================] - 0s - loss: 0.2281 - acc: 0.8889     \n",
      " 40/261 [===>..........................] - ETA: 0s[CV] .. epochs=100, batch_size=40, score=0.778625950103, total=   7.5s\n",
      "[CV] epochs=100, batch_size=40 .......................................\n",
      "Epoch 1/100\n",
      "262/262 [==============================] - 0s - loss: 0.6763 - acc: 0.6527     \n",
      "Epoch 2/100\n",
      "262/262 [==============================] - 0s - loss: 0.6274 - acc: 0.6679     \n",
      "Epoch 3/100\n",
      "262/262 [==============================] - 0s - loss: 0.5410 - acc: 0.7366     \n",
      "Epoch 4/100\n",
      "262/262 [==============================] - 0s - loss: 0.4938 - acc: 0.7557     \n",
      "Epoch 5/100\n",
      "262/262 [==============================] - 0s - loss: 0.4832 - acc: 0.7443     \n",
      "Epoch 6/100\n",
      "262/262 [==============================] - 0s - loss: 0.4703 - acc: 0.7672     \n",
      "Epoch 7/100\n",
      "262/262 [==============================] - 0s - loss: 0.4715 - acc: 0.7672     \n",
      "Epoch 8/100\n",
      "262/262 [==============================] - 0s - loss: 0.4627 - acc: 0.7786     \n",
      "Epoch 9/100\n",
      "262/262 [==============================] - 0s - loss: 0.4609 - acc: 0.7595     \n",
      "Epoch 10/100\n",
      "262/262 [==============================] - 0s - loss: 0.4618 - acc: 0.7634     \n",
      "Epoch 11/100\n",
      "262/262 [==============================] - 0s - loss: 0.4542 - acc: 0.7710     \n",
      "Epoch 12/100\n",
      "262/262 [==============================] - 0s - loss: 0.4520 - acc: 0.7672     \n",
      "Epoch 13/100\n",
      "262/262 [==============================] - 0s - loss: 0.4502 - acc: 0.7786     \n",
      "Epoch 14/100\n",
      "262/262 [==============================] - 0s - loss: 0.4460 - acc: 0.7863     \n",
      "Epoch 15/100\n",
      "262/262 [==============================] - 0s - loss: 0.4448 - acc: 0.7748     \n",
      "Epoch 16/100\n",
      "262/262 [==============================] - 0s - loss: 0.4426 - acc: 0.7748     \n",
      "Epoch 17/100\n",
      "262/262 [==============================] - 0s - loss: 0.4405 - acc: 0.7824     \n",
      "Epoch 18/100\n",
      "262/262 [==============================] - 0s - loss: 0.4394 - acc: 0.7786     \n",
      "Epoch 19/100\n",
      "262/262 [==============================] - 0s - loss: 0.4353 - acc: 0.7824     \n",
      "Epoch 20/100\n",
      "262/262 [==============================] - 0s - loss: 0.4356 - acc: 0.7824     \n",
      "Epoch 21/100\n",
      "262/262 [==============================] - 0s - loss: 0.4344 - acc: 0.7748     \n",
      "Epoch 22/100\n",
      "262/262 [==============================] - 0s - loss: 0.4267 - acc: 0.7824     \n",
      "Epoch 23/100\n",
      "262/262 [==============================] - 0s - loss: 0.4274 - acc: 0.7863     \n",
      "Epoch 24/100\n",
      "262/262 [==============================] - 0s - loss: 0.4259 - acc: 0.7786     \n",
      "Epoch 25/100\n",
      "262/262 [==============================] - 0s - loss: 0.4212 - acc: 0.7901     \n",
      "Epoch 26/100\n",
      "262/262 [==============================] - 0s - loss: 0.4201 - acc: 0.7901     \n",
      "Epoch 27/100\n",
      "262/262 [==============================] - 0s - loss: 0.4158 - acc: 0.7863     \n",
      "Epoch 28/100\n",
      "262/262 [==============================] - 0s - loss: 0.4139 - acc: 0.7786     \n",
      "Epoch 29/100\n",
      "262/262 [==============================] - 0s - loss: 0.4129 - acc: 0.7710     \n",
      "Epoch 30/100\n",
      "262/262 [==============================] - 0s - loss: 0.4104 - acc: 0.7824     \n",
      "Epoch 31/100\n",
      "262/262 [==============================] - 0s - loss: 0.4092 - acc: 0.7748     \n",
      "Epoch 32/100\n",
      "262/262 [==============================] - 0s - loss: 0.4066 - acc: 0.7748     \n",
      "Epoch 33/100\n",
      "262/262 [==============================] - 0s - loss: 0.4049 - acc: 0.7824     \n",
      "Epoch 34/100\n",
      "262/262 [==============================] - 0s - loss: 0.4039 - acc: 0.7901     \n",
      "Epoch 35/100\n",
      "262/262 [==============================] - 0s - loss: 0.3987 - acc: 0.7977     \n",
      "Epoch 36/100\n",
      "262/262 [==============================] - 0s - loss: 0.3995 - acc: 0.7939     \n",
      "Epoch 37/100\n",
      "262/262 [==============================] - 0s - loss: 0.3937 - acc: 0.7939     \n",
      "Epoch 38/100\n",
      "262/262 [==============================] - 0s - loss: 0.3950 - acc: 0.8053     \n",
      "Epoch 39/100\n",
      "262/262 [==============================] - 0s - loss: 0.3898 - acc: 0.8092     \n",
      "Epoch 40/100\n",
      "262/262 [==============================] - 0s - loss: 0.3890 - acc: 0.8053     \n",
      "Epoch 41/100\n",
      "262/262 [==============================] - 0s - loss: 0.3852 - acc: 0.8053     \n",
      "Epoch 42/100\n",
      "262/262 [==============================] - 0s - loss: 0.3862 - acc: 0.8053     \n",
      "Epoch 43/100\n",
      "262/262 [==============================] - 0s - loss: 0.3864 - acc: 0.8053     \n",
      "Epoch 44/100\n",
      "262/262 [==============================] - 0s - loss: 0.3799 - acc: 0.8168     \n",
      "Epoch 45/100\n",
      "262/262 [==============================] - 0s - loss: 0.3788 - acc: 0.8282     \n",
      "Epoch 46/100\n",
      "262/262 [==============================] - 0s - loss: 0.3778 - acc: 0.8168     \n",
      "Epoch 47/100\n",
      "262/262 [==============================] - 0s - loss: 0.3767 - acc: 0.8244     \n",
      "Epoch 48/100\n",
      "262/262 [==============================] - 0s - loss: 0.3726 - acc: 0.8206     \n",
      "Epoch 49/100\n",
      "262/262 [==============================] - 0s - loss: 0.3711 - acc: 0.8244     \n",
      "Epoch 50/100\n",
      "262/262 [==============================] - 0s - loss: 0.3718 - acc: 0.8130     \n",
      "Epoch 51/100\n",
      "262/262 [==============================] - 0s - loss: 0.3692 - acc: 0.8282     \n",
      "Epoch 52/100\n",
      "262/262 [==============================] - 0s - loss: 0.3707 - acc: 0.8282     \n",
      "Epoch 53/100\n",
      "262/262 [==============================] - 0s - loss: 0.3661 - acc: 0.8282     \n",
      "Epoch 54/100\n",
      "262/262 [==============================] - 0s - loss: 0.3661 - acc: 0.8397     \n",
      "Epoch 55/100\n",
      "262/262 [==============================] - 0s - loss: 0.3668 - acc: 0.8321     \n",
      "Epoch 56/100\n",
      "262/262 [==============================] - 0s - loss: 0.3669 - acc: 0.8168     \n",
      "Epoch 57/100\n",
      "262/262 [==============================] - 0s - loss: 0.3618 - acc: 0.8206     \n",
      "Epoch 58/100\n",
      "262/262 [==============================] - 0s - loss: 0.3625 - acc: 0.8397     \n",
      "Epoch 59/100\n",
      "262/262 [==============================] - 0s - loss: 0.3611 - acc: 0.8397     \n",
      "Epoch 60/100\n",
      "262/262 [==============================] - 0s - loss: 0.3584 - acc: 0.8206     \n",
      "Epoch 61/100\n",
      "262/262 [==============================] - 0s - loss: 0.3578 - acc: 0.8321     \n",
      "Epoch 62/100\n",
      "262/262 [==============================] - 0s - loss: 0.3553 - acc: 0.8244     \n",
      "Epoch 63/100\n",
      "262/262 [==============================] - 0s - loss: 0.3518 - acc: 0.8359     \n",
      "Epoch 64/100\n",
      "262/262 [==============================] - 0s - loss: 0.3543 - acc: 0.8397     \n",
      "Epoch 65/100\n",
      "262/262 [==============================] - 0s - loss: 0.3507 - acc: 0.8397     \n",
      "Epoch 66/100\n",
      "262/262 [==============================] - 0s - loss: 0.3492 - acc: 0.8321     \n",
      "Epoch 67/100\n",
      "262/262 [==============================] - 0s - loss: 0.3427 - acc: 0.8397     \n",
      "Epoch 68/100\n",
      "262/262 [==============================] - 0s - loss: 0.3438 - acc: 0.8397     \n",
      "Epoch 69/100\n",
      "262/262 [==============================] - 0s - loss: 0.3390 - acc: 0.8550     \n",
      "Epoch 70/100\n",
      "262/262 [==============================] - 0s - loss: 0.3393 - acc: 0.8511     \n",
      "Epoch 71/100\n",
      "262/262 [==============================] - 0s - loss: 0.3354 - acc: 0.8435     \n",
      "Epoch 72/100\n",
      "262/262 [==============================] - 0s - loss: 0.3344 - acc: 0.8435     \n",
      "Epoch 73/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.4804 - acc: 0.750 - 0s - loss: 0.3342 - acc: 0.8359     \n",
      "Epoch 74/100\n",
      "262/262 [==============================] - 0s - loss: 0.3291 - acc: 0.8397     \n",
      "Epoch 75/100\n",
      "262/262 [==============================] - 0s - loss: 0.3320 - acc: 0.8473     \n",
      "Epoch 76/100\n",
      "262/262 [==============================] - 0s - loss: 0.3263 - acc: 0.8511     \n",
      "Epoch 77/100\n",
      "262/262 [==============================] - 0s - loss: 0.3226 - acc: 0.8397     \n",
      "Epoch 78/100\n",
      "262/262 [==============================] - 0s - loss: 0.3209 - acc: 0.8435     \n",
      "Epoch 79/100\n",
      "262/262 [==============================] - ETA: 0s - loss: 0.3244 - acc: 0.850 - 0s - loss: 0.3201 - acc: 0.8435     \n",
      "Epoch 80/100\n",
      "262/262 [==============================] - 0s - loss: 0.3186 - acc: 0.8435     \n",
      "Epoch 81/100\n",
      "262/262 [==============================] - 0s - loss: 0.3171 - acc: 0.8511     \n",
      "Epoch 82/100\n",
      "262/262 [==============================] - 0s - loss: 0.3179 - acc: 0.8397     \n",
      "Epoch 83/100\n",
      "262/262 [==============================] - 0s - loss: 0.3167 - acc: 0.8435     \n",
      "Epoch 84/100\n",
      "262/262 [==============================] - 0s - loss: 0.3144 - acc: 0.8435     \n",
      "Epoch 85/100\n",
      "262/262 [==============================] - 0s - loss: 0.3131 - acc: 0.8550     \n",
      "Epoch 86/100\n",
      "262/262 [==============================] - 0s - loss: 0.3103 - acc: 0.8397     \n",
      "Epoch 87/100\n",
      "262/262 [==============================] - 0s - loss: 0.3111 - acc: 0.8511     \n",
      "Epoch 88/100\n",
      "262/262 [==============================] - 0s - loss: 0.3075 - acc: 0.8435     \n",
      "Epoch 89/100\n",
      "262/262 [==============================] - 0s - loss: 0.3034 - acc: 0.8626     \n",
      "Epoch 90/100\n",
      "262/262 [==============================] - 0s - loss: 0.3103 - acc: 0.8550     \n",
      "Epoch 91/100\n",
      "262/262 [==============================] - 0s - loss: 0.3025 - acc: 0.8550     \n",
      "Epoch 92/100\n",
      "262/262 [==============================] - 0s - loss: 0.3008 - acc: 0.8550     \n",
      "Epoch 93/100\n",
      "262/262 [==============================] - 0s - loss: 0.3007 - acc: 0.8588     \n",
      "Epoch 94/100\n",
      "262/262 [==============================] - 0s - loss: 0.2983 - acc: 0.8550     \n",
      "Epoch 95/100\n",
      "262/262 [==============================] - 0s - loss: 0.3041 - acc: 0.8664     \n",
      "Epoch 96/100\n",
      "262/262 [==============================] - 0s - loss: 0.2919 - acc: 0.8702     \n",
      "Epoch 97/100\n",
      "262/262 [==============================] - 0s - loss: 0.2998 - acc: 0.8473     \n",
      "Epoch 98/100\n",
      "262/262 [==============================] - 0s - loss: 0.2946 - acc: 0.8664     \n",
      "Epoch 99/100\n",
      "262/262 [==============================] - 0s - loss: 0.2911 - acc: 0.8588     \n",
      "Epoch 100/100\n",
      "262/262 [==============================] - 0s - loss: 0.2909 - acc: 0.8626     \n",
      " 40/262 [===>..........................] - ETA: 0s[CV] .. epochs=100, batch_size=40, score=0.838461536628, total=   7.6s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:  2.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "392/392 [==============================] - 0s - loss: 0.6723 - acc: 0.6480     \n",
      "Epoch 2/100\n",
      "392/392 [==============================] - 0s - loss: 0.5787 - acc: 0.6684     \n",
      "Epoch 3/100\n",
      "392/392 [==============================] - 0s - loss: 0.5089 - acc: 0.6684     \n",
      "Epoch 4/100\n",
      "392/392 [==============================] - 0s - loss: 0.4887 - acc: 0.6684     \n",
      "Epoch 5/100\n",
      "392/392 [==============================] - 0s - loss: 0.4738 - acc: 0.6684     \n",
      "Epoch 6/100\n",
      "392/392 [==============================] - 0s - loss: 0.4648 - acc: 0.7628     \n",
      "Epoch 7/100\n",
      "392/392 [==============================] - 0s - loss: 0.4607 - acc: 0.7908     \n",
      "Epoch 8/100\n",
      "392/392 [==============================] - 0s - loss: 0.4533 - acc: 0.7934     \n",
      "Epoch 9/100\n",
      "392/392 [==============================] - 0s - loss: 0.4445 - acc: 0.8061     \n",
      "Epoch 10/100\n",
      "392/392 [==============================] - 0s - loss: 0.4356 - acc: 0.8036     - ETA: 0s - loss: 0.4487 - acc: 0.780\n",
      "Epoch 11/100\n",
      "392/392 [==============================] - 0s - loss: 0.4312 - acc: 0.8061     \n",
      "Epoch 12/100\n",
      "392/392 [==============================] - 0s - loss: 0.4232 - acc: 0.8112     \n",
      "Epoch 13/100\n",
      "392/392 [==============================] - 0s - loss: 0.4148 - acc: 0.8214     \n",
      "Epoch 14/100\n",
      "392/392 [==============================] - 0s - loss: 0.4101 - acc: 0.8240     \n",
      "Epoch 15/100\n",
      "392/392 [==============================] - 0s - loss: 0.4064 - acc: 0.8214     \n",
      "Epoch 16/100\n",
      "392/392 [==============================] - 0s - loss: 0.3955 - acc: 0.8214     \n",
      "Epoch 17/100\n",
      "392/392 [==============================] - 0s - loss: 0.3893 - acc: 0.8316     \n",
      "Epoch 18/100\n",
      "392/392 [==============================] - 0s - loss: 0.3861 - acc: 0.8265     \n",
      "Epoch 19/100\n",
      "392/392 [==============================] - 0s - loss: 0.3813 - acc: 0.8342     \n",
      "Epoch 20/100\n",
      "392/392 [==============================] - 0s - loss: 0.3770 - acc: 0.8291     \n",
      "Epoch 21/100\n",
      "392/392 [==============================] - 0s - loss: 0.3724 - acc: 0.8316     \n",
      "Epoch 22/100\n",
      "392/392 [==============================] - 0s - loss: 0.3684 - acc: 0.8393     \n",
      "Epoch 23/100\n",
      "392/392 [==============================] - 0s - loss: 0.3660 - acc: 0.8367     \n",
      "Epoch 24/100\n",
      "392/392 [==============================] - 0s - loss: 0.3625 - acc: 0.8342     \n",
      "Epoch 25/100\n",
      "392/392 [==============================] - 0s - loss: 0.3585 - acc: 0.8393     \n",
      "Epoch 26/100\n",
      "392/392 [==============================] - 0s - loss: 0.3525 - acc: 0.8520     \n",
      "Epoch 27/100\n",
      "392/392 [==============================] - 0s - loss: 0.3498 - acc: 0.8546     \n",
      "Epoch 28/100\n",
      "392/392 [==============================] - 0s - loss: 0.3481 - acc: 0.8444     \n",
      "Epoch 29/100\n",
      "392/392 [==============================] - 0s - loss: 0.3447 - acc: 0.8520     \n",
      "Epoch 30/100\n",
      "392/392 [==============================] - 0s - loss: 0.3457 - acc: 0.8495     \n",
      "Epoch 31/100\n",
      "392/392 [==============================] - 0s - loss: 0.3414 - acc: 0.8571     \n",
      "Epoch 32/100\n",
      "392/392 [==============================] - 0s - loss: 0.3394 - acc: 0.8546     \n",
      "Epoch 33/100\n",
      "392/392 [==============================] - 0s - loss: 0.3423 - acc: 0.8495     \n",
      "Epoch 34/100\n",
      "392/392 [==============================] - 0s - loss: 0.3392 - acc: 0.8520     \n",
      "Epoch 35/100\n",
      "392/392 [==============================] - 0s - loss: 0.3409 - acc: 0.8469     \n",
      "Epoch 36/100\n",
      "392/392 [==============================] - 0s - loss: 0.3428 - acc: 0.8571     \n",
      "Epoch 37/100\n",
      "392/392 [==============================] - 0s - loss: 0.3345 - acc: 0.8571     \n",
      "Epoch 38/100\n",
      "392/392 [==============================] - 0s - loss: 0.3385 - acc: 0.8546     \n",
      "Epoch 39/100\n",
      "392/392 [==============================] - 0s - loss: 0.3332 - acc: 0.8571     \n",
      "Epoch 40/100\n",
      "392/392 [==============================] - 0s - loss: 0.3257 - acc: 0.8571     \n",
      "Epoch 41/100\n",
      "392/392 [==============================] - 0s - loss: 0.3275 - acc: 0.8520     \n",
      "Epoch 42/100\n",
      "392/392 [==============================] - 0s - loss: 0.3275 - acc: 0.8546     \n",
      "Epoch 43/100\n",
      "392/392 [==============================] - 0s - loss: 0.3215 - acc: 0.8571     \n",
      "Epoch 44/100\n",
      "392/392 [==============================] - 0s - loss: 0.3251 - acc: 0.8571     \n",
      "Epoch 45/100\n",
      "392/392 [==============================] - 0s - loss: 0.3249 - acc: 0.8520     \n",
      "Epoch 46/100\n",
      "392/392 [==============================] - 0s - loss: 0.3343 - acc: 0.8495     \n",
      "Epoch 47/100\n",
      "392/392 [==============================] - 0s - loss: 0.3206 - acc: 0.8597     \n",
      "Epoch 48/100\n",
      "392/392 [==============================] - 0s - loss: 0.3157 - acc: 0.8571     \n",
      "Epoch 49/100\n",
      "392/392 [==============================] - 0s - loss: 0.3169 - acc: 0.8546     \n",
      "Epoch 50/100\n",
      "392/392 [==============================] - 0s - loss: 0.3139 - acc: 0.8622     \n",
      "Epoch 51/100\n",
      "392/392 [==============================] - 0s - loss: 0.3117 - acc: 0.8571     \n",
      "Epoch 52/100\n",
      "392/392 [==============================] - 0s - loss: 0.3142 - acc: 0.8571     \n",
      "Epoch 53/100\n",
      "392/392 [==============================] - 0s - loss: 0.3171 - acc: 0.8597     \n",
      "Epoch 54/100\n",
      "392/392 [==============================] - 0s - loss: 0.3120 - acc: 0.8546     \n",
      "Epoch 55/100\n",
      "392/392 [==============================] - 0s - loss: 0.3139 - acc: 0.8597     \n",
      "Epoch 56/100\n",
      "392/392 [==============================] - 0s - loss: 0.3089 - acc: 0.8622     \n",
      "Epoch 57/100\n",
      "392/392 [==============================] - 0s - loss: 0.3124 - acc: 0.8801     \n",
      "Epoch 58/100\n",
      "392/392 [==============================] - 0s - loss: 0.3081 - acc: 0.8827     \n",
      "Epoch 59/100\n",
      "392/392 [==============================] - 0s - loss: 0.3092 - acc: 0.8673     \n",
      "Epoch 60/100\n",
      "392/392 [==============================] - 0s - loss: 0.3069 - acc: 0.8827     \n",
      "Epoch 61/100\n",
      "392/392 [==============================] - 0s - loss: 0.3063 - acc: 0.8750     \n",
      "Epoch 62/100\n",
      "392/392 [==============================] - 0s - loss: 0.3031 - acc: 0.8776     \n",
      "Epoch 63/100\n",
      "392/392 [==============================] - 0s - loss: 0.3018 - acc: 0.8801     \n",
      "Epoch 64/100\n",
      "392/392 [==============================] - 0s - loss: 0.3026 - acc: 0.8801     \n",
      "Epoch 65/100\n",
      "392/392 [==============================] - 0s - loss: 0.3083 - acc: 0.8699     \n",
      "Epoch 66/100\n",
      "392/392 [==============================] - 0s - loss: 0.2980 - acc: 0.8801     \n",
      "Epoch 67/100\n",
      "392/392 [==============================] - 0s - loss: 0.2997 - acc: 0.8801     \n",
      "Epoch 68/100\n",
      "392/392 [==============================] - 0s - loss: 0.2987 - acc: 0.8750     \n",
      "Epoch 69/100\n",
      "392/392 [==============================] - 0s - loss: 0.2970 - acc: 0.8801     \n",
      "Epoch 70/100\n",
      "392/392 [==============================] - 0s - loss: 0.2967 - acc: 0.8801     \n",
      "Epoch 71/100\n",
      "392/392 [==============================] - 0s - loss: 0.2965 - acc: 0.8801     \n",
      "Epoch 72/100\n",
      "392/392 [==============================] - 0s - loss: 0.2968 - acc: 0.8852     \n",
      "Epoch 73/100\n",
      "392/392 [==============================] - 0s - loss: 0.2996 - acc: 0.8801     \n",
      "Epoch 74/100\n",
      "392/392 [==============================] - 0s - loss: 0.2953 - acc: 0.8801     \n",
      "Epoch 75/100\n",
      "392/392 [==============================] - 0s - loss: 0.2922 - acc: 0.8776     \n",
      "Epoch 76/100\n",
      "392/392 [==============================] - 0s - loss: 0.2935 - acc: 0.8827     \n",
      "Epoch 77/100\n",
      "392/392 [==============================] - 0s - loss: 0.2951 - acc: 0.8776     \n",
      "Epoch 78/100\n",
      "392/392 [==============================] - 0s - loss: 0.2862 - acc: 0.8852     \n",
      "Epoch 79/100\n",
      "392/392 [==============================] - 0s - loss: 0.2866 - acc: 0.8852     \n",
      "Epoch 80/100\n",
      "392/392 [==============================] - 0s - loss: 0.2905 - acc: 0.8776     \n",
      "Epoch 81/100\n",
      "392/392 [==============================] - 0s - loss: 0.2858 - acc: 0.8801     \n",
      "Epoch 82/100\n",
      "392/392 [==============================] - 0s - loss: 0.2921 - acc: 0.8724     \n",
      "Epoch 83/100\n",
      "392/392 [==============================] - 0s - loss: 0.2939 - acc: 0.8776     \n",
      "Epoch 84/100\n",
      "392/392 [==============================] - 0s - loss: 0.2884 - acc: 0.8776     \n",
      "Epoch 85/100\n",
      "392/392 [==============================] - 0s - loss: 0.2837 - acc: 0.8776     \n",
      "Epoch 86/100\n",
      "392/392 [==============================] - 0s - loss: 0.2802 - acc: 0.8801     \n",
      "Epoch 87/100\n",
      "392/392 [==============================] - 0s - loss: 0.2751 - acc: 0.8750     \n",
      "Epoch 88/100\n",
      "392/392 [==============================] - 0s - loss: 0.2772 - acc: 0.8827     \n",
      "Epoch 89/100\n",
      "392/392 [==============================] - 0s - loss: 0.2752 - acc: 0.8801     \n",
      "Epoch 90/100\n",
      "392/392 [==============================] - 0s - loss: 0.2805 - acc: 0.8801     \n",
      "Epoch 91/100\n",
      "392/392 [==============================] - 0s - loss: 0.2741 - acc: 0.8801     \n",
      "Epoch 92/100\n",
      "392/392 [==============================] - 0s - loss: 0.2739 - acc: 0.8776     \n",
      "Epoch 93/100\n",
      "392/392 [==============================] - 0s - loss: 0.2698 - acc: 0.8776     \n",
      "Epoch 94/100\n",
      "392/392 [==============================] - 0s - loss: 0.2709 - acc: 0.8852     \n",
      "Epoch 95/100\n",
      "392/392 [==============================] - 0s - loss: 0.2718 - acc: 0.8750     \n",
      "Epoch 96/100\n",
      "392/392 [==============================] - 0s - loss: 0.2774 - acc: 0.8776     \n",
      "Epoch 97/100\n",
      "392/392 [==============================] - 0s - loss: 0.2711 - acc: 0.8750     \n",
      "Epoch 98/100\n",
      "392/392 [==============================] - 0s - loss: 0.2685 - acc: 0.8827     \n",
      "Epoch 99/100\n",
      "392/392 [==============================] - 0s - loss: 0.2701 - acc: 0.8827     \n",
      "Epoch 100/100\n",
      "392/392 [==============================] - 0s - loss: 0.2683 - acc: 0.8903     \n",
      "Best: 0.793367346483, using {'epochs': 100, 'batch_size': 20}\n",
      "0.775510201497 (0.0494308487858) with: {'epochs': 10, 'batch_size': 10}\n",
      "0.788265304298 (0.0432823764463) with: {'epochs': 50, 'batch_size': 10}\n",
      "0.772959185802 (0.0573177537304) with: {'epochs': 100, 'batch_size': 10}\n",
      "0.780612249764 (0.0300745493047) with: {'epochs': 10, 'batch_size': 20}\n",
      "0.78316327002 (0.0337785679152) with: {'epochs': 50, 'batch_size': 20}\n",
      "0.793367346483 (0.0317644706476) with: {'epochs': 100, 'batch_size': 20}\n",
      "0.729591829284 (0.0301926398124) with: {'epochs': 10, 'batch_size': 40}\n",
      "0.783163265458 (0.0531929367344) with: {'epochs': 50, 'batch_size': 40}\n",
      "0.778061223425 (0.0493287378951) with: {'epochs': 100, 'batch_size': 40}\n"
     ]
    }
   ],
   "source": [
    "# Do a grid search for the optimal batch size and number of epochs\n",
    "# import necessary packages\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "# Define a random seed\n",
    "seed = 6\n",
    "np.random.seed(seed)\n",
    "\n",
    "# Start defining the model\n",
    "def create_model():\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(4, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # compile the model\n",
    "    adam = Adam(lr = 0.01)\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# create the model\n",
    "model = KerasClassifier(build_fn = create_model, verbose = 1)\n",
    "\n",
    "# define the grid search parameters\n",
    "batch_size = [10, 20, 40]\n",
    "epochs = [10, 50, 100]\n",
    "\n",
    "# make a dictionary of the grid search parameters\n",
    "param_grid = dict(batch_size=batch_size, epochs=epochs)\n",
    "\n",
    "# build and fit the GridSearchCV\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, cv = KFold(random_state=seed), verbose = 10)\n",
    "grid_results = grid.fit(X_standardized, Y)\n",
    "\n",
    "# summarize the results\n",
    "print(\"Best: {0}, using {1}\".format(grid_results.best_score_, grid_results.best_params_))\n",
    "means = grid_results.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_results.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print('{0} ({1}) with: {2}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 9 candidates, totalling 27 fits\n",
      "[CV] learn_rate=0.001, dropout_rate=0.0 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.0, score=0.740458024367, total=   7.4s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.0 ..............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    7.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.001, dropout_rate=0.0, score=0.778625960113, total=   5.4s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.0 ..............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   12.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.001, dropout_rate=0.0, score=0.830769236271, total=   4.8s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.0 ...............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   17.8s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.01, dropout_rate=0.0, score=0.725190844245, total=   4.5s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.0 ...............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   22.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.01, dropout_rate=0.0, score=0.763358779991, total=   3.7s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.0 ...............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   26.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.01, dropout_rate=0.0, score=0.815384630974, total=   4.3s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.0 ................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   30.6s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.1, dropout_rate=0.0, score=0.740458014357, total=   5.6s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.0 ................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   36.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.1, dropout_rate=0.0, score=0.717557242353, total=   4.2s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.0 ................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   40.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.1, dropout_rate=0.0, score=0.815384612634, total=   4.4s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.1 ..............................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   45.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  learn_rate=0.001, dropout_rate=0.1, score=0.732824443861, total=   5.7s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.1 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.1, score=0.770992370507, total=   7.3s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.1 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.1, score=0.838461536628, total=   6.1s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.1 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.1, score=0.717557246448, total=   5.9s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.1 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.1, score=0.732824434761, total=   5.5s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.1 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.1, score=0.846153841569, total=   6.8s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.1 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.1, score=0.694656491507, total=   4.8s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.1 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.1, score=0.732824440676, total=   4.8s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.1 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.1, score=0.800000002751, total=   4.9s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.2 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.2, score=0.740458006167, total=   5.4s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.2 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.2, score=0.763358790001, total=   7.2s\n",
      "[CV] learn_rate=0.001, dropout_rate=0.2 ..............................\n",
      "[CV]  learn_rate=0.001, dropout_rate=0.2, score=0.799999998166, total=   5.6s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.2 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.2, score=0.725190835145, total=   5.3s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.2 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.2, score=0.78625954972, total=   5.5s\n",
      "[CV] learn_rate=0.01, dropout_rate=0.2 ...............................\n",
      "[CV]  learn_rate=0.01, dropout_rate=0.2, score=0.815384630974, total=   5.6s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.2 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.2, score=0.648854961832, total=   5.3s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.2 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.2, score=0.748091613973, total=   6.0s\n",
      "[CV] learn_rate=0.1, dropout_rate=0.2 ................................\n",
      "[CV]  learn_rate=0.1, dropout_rate=0.2, score=0.776923078757, total=   5.4s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:  2.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.783163272148, using {'learn_rate': 0.001, 'dropout_rate': 0.0}\n",
      "0.783163272148 (0.0494308487858) with: {'learn_rate': 0.001, 'dropout_rate': 0.0}\n",
      "0.767857150004 (0.0432823764463) with: {'learn_rate': 0.01, 'dropout_rate': 0.0}\n",
      "0.757653056815 (0.0573177537304) with: {'learn_rate': 0.1, 'dropout_rate': 0.0}\n",
      "0.780612251132 (0.0300745493047) with: {'learn_rate': 0.001, 'dropout_rate': 0.1}\n",
      "0.765306121537 (0.0337785679152) with: {'learn_rate': 0.01, 'dropout_rate': 0.1}\n",
      "0.742346945086 (0.0317644706476) with: {'learn_rate': 0.1, 'dropout_rate': 0.1}\n",
      "0.767857143009 (0.0301926398124) with: {'learn_rate': 0.001, 'dropout_rate': 0.2}\n",
      "0.775510210316 (0.0531929367344) with: {'learn_rate': 0.01, 'dropout_rate': 0.2}\n",
      "0.724489800176 (0.0493287378951) with: {'learn_rate': 0.1, 'dropout_rate': 0.2}\n"
     ]
    }
   ],
   "source": [
    "# Do a grid search for learning rate and dropout rate\n",
    "# import necessary packages\n",
    "from keras.layers import Dropout\n",
    "\n",
    "# Define a random seed\n",
    "seed = 6\n",
    "np.random.seed(seed)\n",
    "\n",
    "# Start defining the model\n",
    "def create_model(learn_rate, dropout_rate):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(4, input_dim = 8, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dropout(dropout_rate))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # compile the model\n",
    "    adam = Adam(lr = learn_rate)\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# create the model\n",
    "model = KerasClassifier(build_fn = create_model, epochs = 100, batch_size = 20, verbose = 0)\n",
    "\n",
    "# define the grid search parameters\n",
    "learn_rate = [0.001, 0.01, 0.1]\n",
    "dropout_rate = [0.0, 0.1, 0.2]\n",
    "\n",
    "# make a dictionary of the grid search parameters\n",
    "param_grid = dict(learn_rate=learn_rate, dropout_rate=dropout_rate)\n",
    "\n",
    "# build and fit the GridSearchCV\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, cv = KFold(random_state=seed), verbose = 10)\n",
    "grid_results = grid.fit(X_standardized, Y)\n",
    "\n",
    "# summarize the results\n",
    "print(\"Best: {0}, using {1}\".format(grid_results.best_score_, grid_results.best_params_))\n",
    "means = grid_results.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_results.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print('{0} ({1}) with: {2}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 12 candidates, totalling 36 fits\n",
      "[CV] activation=softmax, init=uniform ................................\n",
      "[CV]  activation=softmax, init=uniform, score=0.740458024367, total=  14.1s\n",
      "[CV] activation=softmax, init=uniform ................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:   14.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=uniform, score=0.755725200395, total=   4.1s\n",
      "[CV] activation=softmax, init=uniform ................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   18.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=uniform, score=0.815384621804, total=   4.4s\n",
      "[CV] activation=softmax, init=normal .................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   22.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=normal, score=0.610687024266, total=   4.2s\n",
      "[CV] activation=softmax, init=normal .................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   27.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=normal, score=0.755725200395, total=   8.0s\n",
      "[CV] activation=softmax, init=normal .................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   35.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=normal, score=0.823076926745, total=   5.0s\n",
      "[CV] activation=softmax, init=zero ...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   40.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=zero, score=0.610687024266, total=   4.3s\n",
      "[CV] activation=softmax, init=zero ...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   44.8s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=zero, score=0.69465649583, total=   3.9s\n",
      "[CV] activation=softmax, init=zero ...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   48.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=softmax, init=zero, score=0.699999988079, total=   4.1s\n",
      "[CV] activation=relu, init=uniform ...................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   53.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  activation=relu, init=uniform, score=0.725190854255, total=   4.6s\n",
      "[CV] activation=relu, init=uniform ...................................\n",
      "[CV]  activation=relu, init=uniform, score=0.770992369597, total=   4.1s\n",
      "[CV] activation=relu, init=uniform ...................................\n",
      "[CV]  activation=relu, init=uniform, score=0.85384615568, total=   4.4s\n",
      "[CV] activation=relu, init=normal ....................................\n",
      "[CV]  activation=relu, init=normal, score=0.732824434761, total=   4.1s\n",
      "[CV] activation=relu, init=normal ....................................\n",
      "[CV]  activation=relu, init=normal, score=0.770992369597, total=   4.5s\n",
      "[CV] activation=relu, init=normal ....................................\n",
      "[CV]  activation=relu, init=normal, score=0.807692307692, total=   4.5s\n",
      "[CV] activation=relu, init=zero ......................................\n",
      "[CV] . activation=relu, init=zero, score=0.610687024266, total=   3.9s\n",
      "[CV] activation=relu, init=zero ......................................\n",
      "[CV] .. activation=relu, init=zero, score=0.69465649583, total=   4.1s\n",
      "[CV] activation=relu, init=zero ......................................\n",
      "[CV] . activation=relu, init=zero, score=0.699999988079, total=   4.1s\n",
      "[CV] activation=tanh, init=uniform ...................................\n",
      "[CV]  activation=tanh, init=uniform, score=0.75572519448, total=   8.3s\n",
      "[CV] activation=tanh, init=uniform ...................................\n",
      "[CV]  activation=tanh, init=uniform, score=0.770992379607, total=   4.3s\n",
      "[CV] activation=tanh, init=uniform ...................................\n",
      "[CV]  activation=tanh, init=uniform, score=0.830769236271, total=   4.4s\n",
      "[CV] activation=tanh, init=normal ....................................\n",
      "[CV]  activation=tanh, init=normal, score=0.763358784086, total=   4.3s\n",
      "[CV] activation=tanh, init=normal ....................................\n",
      "[CV]  activation=tanh, init=normal, score=0.778625969213, total=   4.0s\n",
      "[CV] activation=tanh, init=normal ....................................\n",
      "[CV]  activation=tanh, init=normal, score=0.830769227101, total=   4.4s\n",
      "[CV] activation=tanh, init=zero ......................................\n",
      "[CV] . activation=tanh, init=zero, score=0.610687024266, total=   4.1s\n",
      "[CV] activation=tanh, init=zero ......................................\n",
      "[CV] .. activation=tanh, init=zero, score=0.69465649583, total=   4.1s\n",
      "[CV] activation=tanh, init=zero ......................................\n",
      "[CV] . activation=tanh, init=zero, score=0.699999988079, total=   3.9s\n",
      "[CV] activation=linear, init=uniform .................................\n",
      "[CV]  activation=linear, init=uniform, score=0.770992364592, total=   3.9s\n",
      "[CV] activation=linear, init=uniform .................................\n",
      "[CV]  activation=linear, init=uniform, score=0.763358790001, total=   4.0s\n",
      "[CV] activation=linear, init=uniform .................................\n",
      "[CV]  activation=linear, init=uniform, score=0.846153855324, total=   3.9s\n",
      "[CV] activation=linear, init=normal ..................................\n",
      "[CV]  activation=linear, init=normal, score=0.770992355492, total=   4.0s\n",
      "[CV] activation=linear, init=normal ..................................\n",
      "[CV]  activation=linear, init=normal, score=0.763358790001, total=   4.0s\n",
      "[CV] activation=linear, init=normal ..................................\n",
      "[CV]  activation=linear, init=normal, score=0.830769236271, total=   3.9s\n",
      "[CV] activation=linear, init=zero ....................................\n",
      "[CV]  activation=linear, init=zero, score=0.610687024266, total=   3.9s\n",
      "[CV] activation=linear, init=zero ....................................\n",
      "[CV]  activation=linear, init=zero, score=0.69465649583, total=   3.8s\n",
      "[CV] activation=linear, init=zero ....................................\n",
      "[CV]  activation=linear, init=zero, score=0.699999988079, total=   3.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  36 out of  36 | elapsed:  2.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.793367353173, using {'activation': 'linear', 'init': 'uniform'}\n",
      "0.770408171628 (0.0494308487858) with: {'activation': 'softmax', 'init': 'uniform'}\n",
      "0.7295918416 (0.0432823764463) with: {'activation': 'softmax', 'init': 'normal'}\n",
      "0.668367345874 (0.0573177537304) with: {'activation': 'softmax', 'init': 'zero'}\n",
      "0.783163271844 (0.0300745493047) with: {'activation': 'relu', 'init': 'uniform'}\n",
      "0.770408166763 (0.0337785679152) with: {'activation': 'relu', 'init': 'normal'}\n",
      "0.668367345874 (0.0317644706476) with: {'activation': 'relu', 'init': 'zero'}\n",
      "0.785714293165 (0.0301926398124) with: {'activation': 'tanh', 'init': 'uniform'}\n",
      "0.790816332157 (0.0531929367344) with: {'activation': 'tanh', 'init': 'normal'}\n",
      "0.668367345874 (0.0493287378951) with: {'activation': 'tanh', 'init': 'zero'}\n"
     ]
    }
   ],
   "source": [
    "# Do a grid search to optimize kernel initialization and activation functions\n",
    "# import necessary packages\n",
    "\n",
    "# Define a random seed\n",
    "seed = 6\n",
    "np.random.seed(seed)\n",
    "\n",
    "# Start defining the model\n",
    "def create_model(activation, init):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(8, input_dim = 8, kernel_initializer= init, activation= activation))\n",
    "    model.add(Dense(4, input_dim = 8, kernel_initializer= init, activation= activation))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # compile the model\n",
    "    adam = Adam(lr = 0.001)\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# create the model\n",
    "model = KerasClassifier(build_fn = create_model, epochs = 100, batch_size = 20, verbose = 0)\n",
    "\n",
    "# define the grid search parameters\n",
    "activation = ['softmax', 'relu', 'tanh', 'linear']\n",
    "init = ['uniform', 'normal', 'zero']\n",
    "\n",
    "# make a dictionary of the grid search parameters\n",
    "param_grid = dict(activation = activation, init = init)\n",
    "\n",
    "# build and fit the GridSearchCV\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, cv = KFold(random_state=seed), verbose = 10)\n",
    "grid_results = grid.fit(X_standardized, Y)\n",
    "\n",
    "# summarize the results\n",
    "print(\"Best: {0}, using {1}\".format(grid_results.best_score_, grid_results.best_params_))\n",
    "means = grid_results.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_results.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print('{0} ({1}) with: {2}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 9 candidates, totalling 27 fits\n",
      "[CV] neuron1=4, neuron2=2 ............................................\n",
      "[CV] ....... neuron1=4, neuron2=2, score=0.770992364592, total=   7.3s\n",
      "[CV] neuron1=4, neuron2=2 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    7.3s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=2, score=0.763358790001, total=   5.6s\n",
      "[CV] neuron1=4, neuron2=2 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:   13.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=2, score=0.830769236271, total=   5.4s\n",
      "[CV] neuron1=4, neuron2=4 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:   18.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=4, score=0.770992364592, total=   4.4s\n",
      "[CV] neuron1=4, neuron2=4 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   23.0s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=4, score=0.778625969213, total=   5.5s\n",
      "[CV] neuron1=4, neuron2=4 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   28.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=4, score=0.807692312277, total=   5.2s\n",
      "[CV] neuron1=4, neuron2=8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed:   33.8s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=8, score=0.763358774986, total=   5.7s\n",
      "[CV] neuron1=4, neuron2=8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed:   39.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=8, score=0.763358790001, total=   3.5s\n",
      "[CV] neuron1=4, neuron2=8 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed:   43.2s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=4, neuron2=8, score=0.830769236271, total=   4.3s\n",
      "[CV] neuron1=8, neuron2=2 ............................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed:   47.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ....... neuron1=8, neuron2=2, score=0.763358784086, total=   4.2s\n",
      "[CV] neuron1=8, neuron2=2 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=2, score=0.763358790001, total=   3.1s\n",
      "[CV] neuron1=8, neuron2=2 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=2, score=0.838461545797, total=   3.5s\n",
      "[CV] neuron1=8, neuron2=4 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=4, score=0.763358774986, total=   3.6s\n",
      "[CV] neuron1=8, neuron2=4 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=4, score=0.763358790001, total=   3.4s\n",
      "[CV] neuron1=8, neuron2=4 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=4, score=0.830769236271, total=   3.1s\n",
      "[CV] neuron1=8, neuron2=8 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=8, score=0.763358784086, total=   4.1s\n",
      "[CV] neuron1=8, neuron2=8 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=8, score=0.763358790001, total=   5.8s\n",
      "[CV] neuron1=8, neuron2=8 ............................................\n",
      "[CV] ....... neuron1=8, neuron2=8, score=0.830769236271, total=   4.1s\n",
      "[CV] neuron1=16, neuron2=2 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=2, score=0.763358784086, total=   3.8s\n",
      "[CV] neuron1=16, neuron2=2 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=2, score=0.763358790001, total=   3.8s\n",
      "[CV] neuron1=16, neuron2=2 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=2, score=0.846153855324, total=   3.3s\n",
      "[CV] neuron1=16, neuron2=4 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=4, score=0.770992364592, total=   4.1s\n",
      "[CV] neuron1=16, neuron2=4 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=4, score=0.763358790001, total=   5.7s\n",
      "[CV] neuron1=16, neuron2=4 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=4, score=0.838461541213, total=   4.0s\n",
      "[CV] neuron1=16, neuron2=8 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=8, score=0.763358784086, total=   3.9s\n",
      "[CV] neuron1=16, neuron2=8 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=8, score=0.763358790001, total=   5.1s\n",
      "[CV] neuron1=16, neuron2=8 ...........................................\n",
      "[CV] ...... neuron1=16, neuron2=8, score=0.830769236271, total=   4.3s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  27 out of  27 | elapsed:  2.0min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.790816335198, using {'neuron1': 16, 'neuron2': 2}\n",
      "0.78826531114 (0.0494308487858) with: {'neuron1': 4, 'neuron2': 2}\n",
      "0.785714291644 (0.0432823764463) with: {'neuron1': 4, 'neuron2': 4}\n",
      "0.785714290124 (0.0573177537304) with: {'neuron1': 4, 'neuron2': 8}\n",
      "0.788265314181 (0.0300745493047) with: {'neuron1': 8, 'neuron2': 2}\n",
      "0.785714290124 (0.0337785679152) with: {'neuron1': 8, 'neuron2': 4}\n",
      "0.785714293165 (0.0317644706476) with: {'neuron1': 8, 'neuron2': 8}\n",
      "0.790816335198 (0.0301926398124) with: {'neuron1': 16, 'neuron2': 2}\n",
      "0.790816330636 (0.0531929367344) with: {'neuron1': 16, 'neuron2': 4}\n",
      "0.785714293165 (0.0493287378951) with: {'neuron1': 16, 'neuron2': 8}\n"
     ]
    }
   ],
   "source": [
    "# Do a grid search to find the optimal number of neurons in each hidden layer\n",
    "# import necessary packages\n",
    "\n",
    "# Define a random seed\n",
    "seed = 6\n",
    "np.random.seed(seed)\n",
    "\n",
    "# Start defining the model\n",
    "def create_model(neuron1, neuron2):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(neuron1, input_dim = 8, kernel_initializer= 'uniform', activation= 'linear'))\n",
    "    model.add(Dense(neuron2, input_dim = neuron1, kernel_initializer= 'uniform', activation= 'linear'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    \n",
    "    # compile the model\n",
    "    adam = Adam(lr = 0.001)\n",
    "    model.compile(loss = 'binary_crossentropy', optimizer = adam, metrics = ['accuracy'])\n",
    "    return model\n",
    "\n",
    "# create the model\n",
    "model = KerasClassifier(build_fn = create_model, epochs = 100, batch_size = 20, verbose = 0)\n",
    "\n",
    "# define the grid search parameters\n",
    "neuron1 = [4, 8, 16]\n",
    "neuron2 = [2, 4, 8]\n",
    "\n",
    "# make a dictionary of the grid search parameters\n",
    "param_grid = dict(neuron1 = neuron1, neuron2 = neuron2)\n",
    "\n",
    "# build and fit the GridSearchCV\n",
    "grid = GridSearchCV(estimator = model, param_grid = param_grid, cv = KFold(random_state=seed), refit = True, verbose = 10)\n",
    "grid_results = grid.fit(X_standardized, Y)\n",
    "\n",
    "# summarize the results\n",
    "print(\"Best: {0}, using {1}\".format(grid_results.best_score_, grid_results.best_params_))\n",
    "means = grid_results.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_results.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print('{0} ({1}) with: {2}'.format(mean, stdev, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# generate predictions with optimal hyperparameters\n",
    "y_pred = grid.predict(X_standardized)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(392L, 1L)\n"
     ]
    }
   ],
   "source": [
    "print(y_pred.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0]\n",
      " [1]\n",
      " [0]\n",
      " [1]\n",
      " [1]]\n"
     ]
    }
   ],
   "source": [
    "print(y_pred[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7806122448979592\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.81      0.89      0.84       262\n",
      "          1       0.71      0.57      0.63       130\n",
      "\n",
      "avg / total       0.77      0.78      0.77       392\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Generate a classification report\n",
    "from sklearn.metrics import classification_report, accuracy_score\n",
    "\n",
    "print(accuracy_score(Y, y_pred))\n",
    "print(classification_report(Y, y_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}