prepare_nn_feat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "train_df = pd.read_csv(\"./input/train.csv\")\n",
    "test_df = pd.read_csv(\"./input/test.csv\")\n",
    "\n",
    "# trans text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "He shall find that I can feel my injuries; he shall learn to dread my revenge\" A few days after he arrived.\n",
      "he shall find that i can feel my injuries ; he shall learn to dread my revenge \" a few days after he arrived.\n"
     ]
    }
   ],
   "source": [
    "def trans_text(s):\n",
    "    s = s.replace(\"' \", \" ' \")\n",
    "    p = ['. ', '; ', '\" ', '! ', '? ', ', ',': ']\n",
    "    for tmp_p in p:\n",
    "        s=s.replace(tmp_p,' {}'.format(tmp_p))\n",
    "    s = s.lower()\n",
    "    return s\n",
    "\n",
    "print(train_df['text'][10])\n",
    "train_df['text'] = train_df['text'].apply(trans_text)\n",
    "test_df['text'] = test_df['text'].apply(trans_text)\n",
    "print(train_df['text'][10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import keras done\n"
     ]
    }
   ],
   "source": [
    "# add cnn feat\n",
    "from keras.layers import Embedding, CuDNNLSTM, Dense, Flatten, Dropout, LSTM\n",
    "from keras.models import Sequential, load_model\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Conv1D, GlobalMaxPooling1D, GlobalAveragePooling1D\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from sklearn import preprocessing\n",
    "from sklearn.metrics import log_loss\n",
    "from sklearn.model_selection import KFold\n",
    "import gc\n",
    "print('import keras done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def cnn done\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.94816, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0577 - acc: 0.4384 - val_loss: 0.9482 - val_acc: 0.5552\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.94816 to 0.69040, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7869 - acc: 0.6284 - val_loss: 0.6904 - val_acc: 0.6886\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.69040 to 0.54352, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.5316 - acc: 0.7812 - val_loss: 0.5435 - val_acc: 0.7728\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.54352 to 0.49323, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3550 - acc: 0.8710 - val_loss: 0.4932 - val_acc: 0.8130\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.49323 to 0.49017, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2503 - acc: 0.9183 - val_loss: 0.4902 - val_acc: 0.8194\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 1s - loss: 0.1858 - acc: 0.9384 - val_loss: 0.5109 - val_acc: 0.8232\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.1431 - acc: 0.9540 - val_loss: 0.5393 - val_acc: 0.8277\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 1s - loss: 0.1127 - acc: 0.9646 - val_loss: 0.5807 - val_acc: 0.8245\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 1s - loss: 0.0898 - acc: 0.9733 - val_loss: 0.6301 - val_acc: 0.8220\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.0801 - acc: 0.9749 - val_loss: 0.6642 - val_acc: 0.8200\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.94767, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0589 - acc: 0.4427 - val_loss: 0.9477 - val_acc: 0.5884\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.94767 to 0.59422, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7177 - acc: 0.7155 - val_loss: 0.5942 - val_acc: 0.7696\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.59422 to 0.47788, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4218 - acc: 0.8506 - val_loss: 0.4779 - val_acc: 0.8130\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.47788 to 0.45630, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2762 - acc: 0.9084 - val_loss: 0.4563 - val_acc: 0.8220\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 1s - loss: 0.1964 - acc: 0.9380 - val_loss: 0.4645 - val_acc: 0.8290\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 1s - loss: 0.1434 - acc: 0.9559 - val_loss: 0.5070 - val_acc: 0.8296\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.1093 - acc: 0.9662 - val_loss: 0.5481 - val_acc: 0.8258\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 1s - loss: 0.0851 - acc: 0.9742 - val_loss: 0.6091 - val_acc: 0.8226\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 1s - loss: 0.0683 - acc: 0.9811 - val_loss: 0.6627 - val_acc: 0.8239\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.0559 - acc: 0.9840 - val_loss: 0.7014 - val_acc: 0.8175\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.88449, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0323 - acc: 0.4579 - val_loss: 0.8845 - val_acc: 0.5903\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.88449 to 0.72885, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7416 - acc: 0.6276 - val_loss: 0.7289 - val_acc: 0.6362\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.72885 to 0.69843, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.5847 - acc: 0.7282 - val_loss: 0.6984 - val_acc: 0.6847\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.69843 to 0.69676, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4469 - acc: 0.8151 - val_loss: 0.6968 - val_acc: 0.7192\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 1s - loss: 0.3248 - acc: 0.8822 - val_loss: 0.7151 - val_acc: 0.7384\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 1s - loss: 0.2388 - acc: 0.9171 - val_loss: 0.7690 - val_acc: 0.7511\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.1788 - acc: 0.9420 - val_loss: 0.8440 - val_acc: 0.7588\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 1s - loss: 0.1396 - acc: 0.9596 - val_loss: 0.8707 - val_acc: 0.7671\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 1s - loss: 0.1133 - acc: 0.9674 - val_loss: 0.9472 - val_acc: 0.7703\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.0898 - acc: 0.9752 - val_loss: 1.0342 - val_acc: 0.7741\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.92193, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0568 - acc: 0.4252 - val_loss: 0.9219 - val_acc: 0.5986\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.92193 to 0.50587, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.6467 - acc: 0.7522 - val_loss: 0.5059 - val_acc: 0.8028\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.50587 to 0.42345, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3734 - acc: 0.8666 - val_loss: 0.4234 - val_acc: 0.8322\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.42345 to 0.41282, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2551 - acc: 0.9132 - val_loss: 0.4128 - val_acc: 0.8437\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss improved from 0.41282 to 0.41270, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1883 - acc: 0.9374 - val_loss: 0.4127 - val_acc: 0.8539\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 1s - loss: 0.1405 - acc: 0.9542 - val_loss: 0.4453 - val_acc: 0.8417\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.1071 - acc: 0.9675 - val_loss: 0.4635 - val_acc: 0.8488\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 1s - loss: 0.0857 - acc: 0.9735 - val_loss: 0.4953 - val_acc: 0.8437\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.0687 - acc: 0.9793 - val_loss: 0.5468 - val_acc: 0.8405\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 2s - loss: 0.0561 - acc: 0.9842 - val_loss: 0.5776 - val_acc: 0.8392\n",
      "------------------\n",
      "Train on 14097 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.87299, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 1.0334 - acc: 0.4620 - val_loss: 0.8730 - val_acc: 0.5935\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.87299 to 0.53802, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7006 - acc: 0.7102 - val_loss: 0.5380 - val_acc: 0.8003\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.53802 to 0.42464, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4082 - acc: 0.8530 - val_loss: 0.4246 - val_acc: 0.8392\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.42464 to 0.40846, saving model to /tmp/nn_model.h5\n",
      " - 2s - loss: 0.2773 - acc: 0.9025 - val_loss: 0.4085 - val_acc: 0.8468\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 2s - loss: 0.1939 - acc: 0.9357 - val_loss: 0.4303 - val_acc: 0.8449\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 2s - loss: 0.1455 - acc: 0.9557 - val_loss: 0.4713 - val_acc: 0.8366\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 1s - loss: 0.1098 - acc: 0.9669 - val_loss: 0.5121 - val_acc: 0.8347\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 1s - loss: 0.0890 - acc: 0.9737 - val_loss: 0.5453 - val_acc: 0.8264\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 2s - loss: 0.0732 - acc: 0.9794 - val_loss: 0.6294 - val_acc: 0.8207\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.0581 - acc: 0.9847 - val_loss: 0.6434 - val_acc: 0.8213\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "def get_cnn_feats(rnd=1):\n",
    "    # return train pred prob and test pred prob \n",
    "    train_pred, test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    best_val_train_pred, best_val_test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    FEAT_CNT = 5\n",
    "    NUM_WORDS = 30000\n",
    "    N = 10\n",
    "    MAX_LEN = 100\n",
    "    NUM_CLASSES = 3\n",
    "    MODEL_P = '/tmp/nn_model.h5'\n",
    "    \n",
    "    tmp_X = train_df['text']\n",
    "    tmp_Y = train_df['author']\n",
    "    tmp_X_test = test_df['text']\n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=NUM_WORDS)\n",
    "    tokenizer.fit_on_texts(tmp_X)\n",
    "\n",
    "    ttrain_x = tokenizer.texts_to_sequences(tmp_X)\n",
    "    ttrain_x = pad_sequences(ttrain_x, maxlen=MAX_LEN)\n",
    "    \n",
    "    ttest_x = tokenizer.texts_to_sequences(tmp_X_test)\n",
    "    ttest_x = pad_sequences(ttest_x, maxlen=MAX_LEN)\n",
    "\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(tmp_Y)\n",
    "\n",
    "    ttrain_y = lb.transform(tmp_Y)\n",
    "    \n",
    "    kf = KFold(n_splits=FEAT_CNT, shuffle=True, random_state=233*rnd)\n",
    "    for train_index, test_index in kf.split(tmp_X):\n",
    "        # prepare aug train, val\n",
    "        model = Sequential()\n",
    "        model.add(Embedding(NUM_WORDS, N, input_length=MAX_LEN))\n",
    "        model.add(Conv1D(16,\n",
    "                         3,\n",
    "                         padding='valid',\n",
    "                         activation='relu',\n",
    "                         strides=1))\n",
    "        model.add(GlobalAveragePooling1D())\n",
    "        model.add(Dense(16, activation='relu'))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(NUM_CLASSES, activation='softmax'))\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        #model.summary()\n",
    "\n",
    "        model_chk = ModelCheckpoint(filepath=MODEL_P, monitor='val_loss', save_best_only=True, verbose=1)\n",
    "        model.fit(ttrain_x[train_index], ttrain_y[train_index], \n",
    "                  validation_split=0.1,\n",
    "                  batch_size=32, epochs=10, \n",
    "                  verbose=2,\n",
    "                  callbacks=[model_chk],\n",
    "                  shuffle=False\n",
    "                 )\n",
    " \n",
    "        # save feat\n",
    "        train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        # best val model\n",
    "        model = load_model(MODEL_P)\n",
    "        best_val_train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        best_val_test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        # release\n",
    "        del model\n",
    "        gc.collect()\n",
    "        print('------------------')\n",
    "        \n",
    "    return train_pred,test_pred,best_val_train_pred,best_val_test_pred\n",
    "\n",
    "print('def cnn done')\n",
    "\n",
    "cnn_train1,cnn_test1,cnn_train2,cnn_test2 = get_cnn_feats(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def lstm done\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.84670, saving model to /tmp/nn_model.h5\n",
      " - 40s - loss: 1.0256 - acc: 0.4708 - val_loss: 0.8467 - val_acc: 0.5852\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.84670 to 0.52045, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.6458 - acc: 0.7157 - val_loss: 0.5204 - val_acc: 0.7837\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.52045 to 0.46248, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.3423 - acc: 0.8727 - val_loss: 0.4625 - val_acc: 0.8124\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 38s - loss: 0.2119 - acc: 0.9256 - val_loss: 0.4761 - val_acc: 0.8309\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 38s - loss: 0.1467 - acc: 0.9492 - val_loss: 0.5089 - val_acc: 0.8290\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 38s - loss: 0.1080 - acc: 0.9642 - val_loss: 0.5837 - val_acc: 0.8271\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 38s - loss: 0.0850 - acc: 0.9713 - val_loss: 0.6367 - val_acc: 0.8232\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 38s - loss: 0.0704 - acc: 0.9769 - val_loss: 0.6769 - val_acc: 0.8207\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 38s - loss: 0.0548 - acc: 0.9816 - val_loss: 0.7245 - val_acc: 0.8092\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 38s - loss: 0.0485 - acc: 0.9840 - val_loss: 0.8117 - val_acc: 0.8111\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.87811, saving model to /tmp/nn_model.h5\n",
      " - 39s - loss: 1.0394 - acc: 0.4561 - val_loss: 0.8781 - val_acc: 0.5514\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.87811 to 0.53548, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.6435 - acc: 0.7260 - val_loss: 0.5355 - val_acc: 0.7805\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.53548 to 0.47213, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.3360 - acc: 0.8762 - val_loss: 0.4721 - val_acc: 0.8226\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 37s - loss: 0.2127 - acc: 0.9256 - val_loss: 0.4956 - val_acc: 0.8296\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 37s - loss: 0.1444 - acc: 0.9502 - val_loss: 0.5695 - val_acc: 0.8232\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 38s - loss: 0.1052 - acc: 0.9641 - val_loss: 0.6272 - val_acc: 0.8175\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 38s - loss: 0.0820 - acc: 0.9713 - val_loss: 0.7093 - val_acc: 0.8111\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 38s - loss: 0.0681 - acc: 0.9769 - val_loss: 0.8117 - val_acc: 0.8060\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 38s - loss: 0.0576 - acc: 0.9804 - val_loss: 0.8211 - val_acc: 0.8073\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 37s - loss: 0.0439 - acc: 0.9850 - val_loss: 0.8747 - val_acc: 0.8047\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.83059, saving model to /tmp/nn_model.h5\n",
      " - 40s - loss: 1.0177 - acc: 0.4694 - val_loss: 0.8306 - val_acc: 0.6292\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.83059 to 0.49574, saving model to /tmp/nn_model.h5\n",
      " - 42s - loss: 0.6069 - acc: 0.7484 - val_loss: 0.4957 - val_acc: 0.7971\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.49574 to 0.45649, saving model to /tmp/nn_model.h5\n",
      " - 44s - loss: 0.3233 - acc: 0.8824 - val_loss: 0.4565 - val_acc: 0.8239\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 38s - loss: 0.2014 - acc: 0.9276 - val_loss: 0.4914 - val_acc: 0.8264\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 38s - loss: 0.1378 - acc: 0.9520 - val_loss: 0.5496 - val_acc: 0.8277\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 38s - loss: 0.1040 - acc: 0.9645 - val_loss: 0.6185 - val_acc: 0.8175\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 38s - loss: 0.0824 - acc: 0.9722 - val_loss: 0.6784 - val_acc: 0.8181\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 38s - loss: 0.0651 - acc: 0.9776 - val_loss: 0.7203 - val_acc: 0.8079\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 38s - loss: 0.0576 - acc: 0.9801 - val_loss: 0.8145 - val_acc: 0.8060\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 38s - loss: 0.0459 - acc: 0.9834 - val_loss: 0.8621 - val_acc: 0.8066\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.87999, saving model to /tmp/nn_model.h5\n",
      " - 39s - loss: 1.0484 - acc: 0.4446 - val_loss: 0.8800 - val_acc: 0.6420\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.87999 to 0.50032, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.6473 - acc: 0.7313 - val_loss: 0.5003 - val_acc: 0.7958\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.50032 to 0.45291, saving model to /tmp/nn_model.h5\n",
      " - 38s - loss: 0.3474 - acc: 0.8727 - val_loss: 0.4529 - val_acc: 0.8207\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss did not improve\n",
      " - 37s - loss: 0.2260 - acc: 0.9196 - val_loss: 0.4599 - val_acc: 0.8354\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 37s - loss: 0.1567 - acc: 0.9463 - val_loss: 0.4743 - val_acc: 0.8392\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 37s - loss: 0.1221 - acc: 0.9574 - val_loss: 0.5337 - val_acc: 0.8392\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 37s - loss: 0.0930 - acc: 0.9681 - val_loss: 0.5738 - val_acc: 0.8334\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 38s - loss: 0.0746 - acc: 0.9745 - val_loss: 0.6241 - val_acc: 0.8373\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 39s - loss: 0.0617 - acc: 0.9788 - val_loss: 0.6741 - val_acc: 0.8290\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 39s - loss: 0.0499 - acc: 0.9832 - val_loss: 0.7489 - val_acc: 0.8175\n",
      "------------------\n",
      "Train on 14097 samples, validate on 1567 samples\n",
      "Epoch 1/10\n",
      "Epoch 00001: val_loss improved from inf to 0.83864, saving model to /tmp/nn_model.h5\n",
      " - 41s - loss: 1.0212 - acc: 0.4601 - val_loss: 0.8386 - val_acc: 0.5852\n",
      "Epoch 2/10\n",
      "Epoch 00002: val_loss improved from 0.83864 to 0.70521, saving model to /tmp/nn_model.h5\n",
      " - 39s - loss: 0.7313 - acc: 0.6338 - val_loss: 0.7052 - val_acc: 0.6765\n",
      "Epoch 3/10\n",
      "Epoch 00003: val_loss improved from 0.70521 to 0.47606, saving model to /tmp/nn_model.h5\n",
      " - 39s - loss: 0.4650 - acc: 0.8081 - val_loss: 0.4761 - val_acc: 0.8271\n",
      "Epoch 4/10\n",
      "Epoch 00004: val_loss improved from 0.47606 to 0.47056, saving model to /tmp/nn_model.h5\n",
      " - 39s - loss: 0.2634 - acc: 0.9031 - val_loss: 0.4706 - val_acc: 0.8373\n",
      "Epoch 5/10\n",
      "Epoch 00005: val_loss did not improve\n",
      " - 40s - loss: 0.1741 - acc: 0.9384 - val_loss: 0.5400 - val_acc: 0.8347\n",
      "Epoch 6/10\n",
      "Epoch 00006: val_loss did not improve\n",
      " - 38s - loss: 0.1261 - acc: 0.9574 - val_loss: 0.5786 - val_acc: 0.8302\n",
      "Epoch 7/10\n",
      "Epoch 00007: val_loss did not improve\n",
      " - 38s - loss: 0.0988 - acc: 0.9667 - val_loss: 0.6404 - val_acc: 0.8271\n",
      "Epoch 8/10\n",
      "Epoch 00008: val_loss did not improve\n",
      " - 38s - loss: 0.0801 - acc: 0.9739 - val_loss: 0.7214 - val_acc: 0.8124\n",
      "Epoch 9/10\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 38s - loss: 0.0587 - acc: 0.9816 - val_loss: 0.8125 - val_acc: 0.8149\n",
      "Epoch 10/10\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 39s - loss: 0.0570 - acc: 0.9802 - val_loss: 0.7972 - val_acc: 0.8111\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "# add lstm feat\n",
    "def get_lstm_feats(rnd=1):\n",
    "    # return train pred prob and test pred prob \n",
    "    train_pred, test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    best_val_train_pred, best_val_test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    FEAT_CNT = 5\n",
    "    NUM_WORDS = 16000\n",
    "    N = 12\n",
    "    MAX_LEN = 300\n",
    "    NUM_CLASSES = 3\n",
    "    MODEL_P = '/tmp/nn_model.h5'\n",
    "    \n",
    "    tmp_X = train_df['text']\n",
    "    tmp_Y = train_df['author']\n",
    "    tmp_X_test = test_df['text']\n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=NUM_WORDS)\n",
    "    tokenizer.fit_on_texts(tmp_X)\n",
    "\n",
    "    ttrain_x = tokenizer.texts_to_sequences(tmp_X)\n",
    "    ttrain_x = pad_sequences(ttrain_x, maxlen=MAX_LEN)\n",
    "    \n",
    "    ttest_x = tokenizer.texts_to_sequences(tmp_X_test)\n",
    "    ttest_x = pad_sequences(ttest_x, maxlen=MAX_LEN)\n",
    "\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(tmp_Y)\n",
    "\n",
    "    ttrain_y = lb.transform(tmp_Y)\n",
    "    \n",
    "    kf = KFold(n_splits=FEAT_CNT, shuffle=True, random_state=233*rnd)\n",
    "    for train_index, test_index in kf.split(tmp_X):\n",
    "        # prepare aug train, val\n",
    "        model = Sequential()\n",
    "        model.add(Embedding(NUM_WORDS, N, input_length=MAX_LEN))\n",
    "        model.add(LSTM(N, dropout=0.2, recurrent_dropout=0.2, return_sequences=True))\n",
    "        model.add(Flatten())\n",
    "        model.add(Dense(64, activation='relu'))\n",
    "        model.add(Dropout(0.2))\n",
    "        model.add(Dense(NUM_CLASSES, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        #model.summary()\n",
    "\n",
    "        model_chk = ModelCheckpoint(filepath=MODEL_P, monitor='val_loss', save_best_only=True, verbose=1)\n",
    "        model.fit(ttrain_x[train_index], ttrain_y[train_index], \n",
    "                  validation_split=0.1,\n",
    "                  batch_size=128, epochs=10, \n",
    "                  verbose=2,\n",
    "                  callbacks=[model_chk],\n",
    "                  shuffle=False\n",
    "                 )\n",
    " \n",
    "        # save feat\n",
    "        train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        # best val model\n",
    "        model = load_model(MODEL_P)\n",
    "        best_val_train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        best_val_test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        del model\n",
    "        gc.collect()\n",
    "        print('------------------')\n",
    "        \n",
    "    return train_pred,test_pred,best_val_train_pred,best_val_test_pred\n",
    "\n",
    "print('def lstm done')\n",
    "lstm_train1,lstm_test1,lstm_train2,lstm_test2 = get_lstm_feats(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def cnn done\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/15\n",
      "Epoch 00001: val_loss improved from inf to 1.04838, saving model to /tmp/nn_model.h5\n",
      " - 3s - loss: 1.0796 - acc: 0.4047 - val_loss: 1.0484 - val_acc: 0.4601\n",
      "Epoch 2/15\n",
      "Epoch 00002: val_loss improved from 1.04838 to 0.84327, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9460 - acc: 0.5522 - val_loss: 0.8433 - val_acc: 0.6165\n",
      "Epoch 3/15\n",
      "Epoch 00003: val_loss improved from 0.84327 to 0.64991, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7044 - acc: 0.7246 - val_loss: 0.6499 - val_acc: 0.7549\n",
      "Epoch 4/15\n",
      "Epoch 00004: val_loss improved from 0.64991 to 0.52142, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.5012 - acc: 0.8296 - val_loss: 0.5214 - val_acc: 0.8060\n",
      "Epoch 5/15\n",
      "Epoch 00005: val_loss improved from 0.52142 to 0.45720, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3725 - acc: 0.8746 - val_loss: 0.4572 - val_acc: 0.8168\n",
      "Epoch 6/15\n",
      "Epoch 00006: val_loss improved from 0.45720 to 0.42354, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2889 - acc: 0.9047 - val_loss: 0.4235 - val_acc: 0.8271\n",
      "Epoch 7/15\n",
      "Epoch 00007: val_loss improved from 0.42354 to 0.40697, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2340 - acc: 0.9247 - val_loss: 0.4070 - val_acc: 0.8366\n",
      "Epoch 8/15\n",
      "Epoch 00008: val_loss improved from 0.40697 to 0.40081, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1926 - acc: 0.9403 - val_loss: 0.4008 - val_acc: 0.8417\n",
      "Epoch 9/15\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 1s - loss: 0.1602 - acc: 0.9501 - val_loss: 0.4021 - val_acc: 0.8449\n",
      "Epoch 10/15\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.1338 - acc: 0.9601 - val_loss: 0.4082 - val_acc: 0.8475\n",
      "Epoch 11/15\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 1s - loss: 0.1146 - acc: 0.9664 - val_loss: 0.4186 - val_acc: 0.8379\n",
      "Epoch 12/15\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 1s - loss: 0.0980 - acc: 0.9721 - val_loss: 0.4338 - val_acc: 0.8405\n",
      "Epoch 13/15\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 1s - loss: 0.0852 - acc: 0.9757 - val_loss: 0.4521 - val_acc: 0.8360\n",
      "Epoch 14/15\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 1s - loss: 0.0737 - acc: 0.9799 - val_loss: 0.4707 - val_acc: 0.8309\n",
      "Epoch 15/15\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 1s - loss: 0.0638 - acc: 0.9828 - val_loss: 0.4964 - val_acc: 0.8271\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/15\n",
      "Epoch 00001: val_loss improved from inf to 1.06446, saving model to /tmp/nn_model.h5\n",
      " - 4s - loss: 1.0821 - acc: 0.4027 - val_loss: 1.0645 - val_acc: 0.4091\n",
      "Epoch 2/15\n",
      "Epoch 00002: val_loss improved from 1.06446 to 0.87629, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9712 - acc: 0.5328 - val_loss: 0.8763 - val_acc: 0.5814\n",
      "Epoch 3/15\n",
      "Epoch 00003: val_loss improved from 0.87629 to 0.69105, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7453 - acc: 0.6895 - val_loss: 0.6910 - val_acc: 0.7262\n",
      "Epoch 4/15\n",
      "Epoch 00004: val_loss improved from 0.69105 to 0.55098, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.5457 - acc: 0.8064 - val_loss: 0.5510 - val_acc: 0.8009\n",
      "Epoch 5/15\n",
      "Epoch 00005: val_loss improved from 0.55098 to 0.47000, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4111 - acc: 0.8629 - val_loss: 0.4700 - val_acc: 0.8220\n",
      "Epoch 6/15\n",
      "Epoch 00006: val_loss improved from 0.47000 to 0.42403, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3216 - acc: 0.8941 - val_loss: 0.4240 - val_acc: 0.8322\n",
      "Epoch 7/15\n",
      "Epoch 00007: val_loss improved from 0.42403 to 0.39656, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2615 - acc: 0.9155 - val_loss: 0.3966 - val_acc: 0.8417\n",
      "Epoch 8/15\n",
      "Epoch 00008: val_loss improved from 0.39656 to 0.38515, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2163 - acc: 0.9306 - val_loss: 0.3852 - val_acc: 0.8424\n",
      "Epoch 9/15\n",
      "Epoch 00009: val_loss improved from 0.38515 to 0.37881, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1794 - acc: 0.9454 - val_loss: 0.3788 - val_acc: 0.8456\n",
      "Epoch 10/15\n",
      "Epoch 00010: val_loss improved from 0.37881 to 0.37819, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1526 - acc: 0.9552 - val_loss: 0.3782 - val_acc: 0.8468\n",
      "Epoch 11/15\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 1s - loss: 0.1282 - acc: 0.9626 - val_loss: 0.3903 - val_acc: 0.8449\n",
      "Epoch 12/15\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 1s - loss: 0.1112 - acc: 0.9690 - val_loss: 0.3984 - val_acc: 0.8456\n",
      "Epoch 13/15\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 1s - loss: 0.0968 - acc: 0.9729 - val_loss: 0.4020 - val_acc: 0.8481\n",
      "Epoch 14/15\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 1s - loss: 0.0845 - acc: 0.9762 - val_loss: 0.4152 - val_acc: 0.8449\n",
      "Epoch 15/15\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 1s - loss: 0.0736 - acc: 0.9803 - val_loss: 0.4216 - val_acc: 0.8449\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/15\n",
      "Epoch 00001: val_loss improved from inf to 1.06027, saving model to /tmp/nn_model.h5\n",
      " - 4s - loss: 1.0819 - acc: 0.4049 - val_loss: 1.0603 - val_acc: 0.4212\n",
      "Epoch 2/15\n",
      "Epoch 00002: val_loss improved from 1.06027 to 0.85466, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9692 - acc: 0.5270 - val_loss: 0.8547 - val_acc: 0.6343\n",
      "Epoch 3/15\n",
      "Epoch 00003: val_loss improved from 0.85466 to 0.62569, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.6987 - acc: 0.7392 - val_loss: 0.6257 - val_acc: 0.7703\n",
      "Epoch 4/15\n",
      "Epoch 00004: val_loss improved from 0.62569 to 0.50909, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4908 - acc: 0.8324 - val_loss: 0.5091 - val_acc: 0.8117\n",
      "Epoch 5/15\n",
      "Epoch 00005: val_loss improved from 0.50909 to 0.44902, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3709 - acc: 0.8758 - val_loss: 0.4490 - val_acc: 0.8264\n",
      "Epoch 6/15\n",
      "Epoch 00006: val_loss improved from 0.44902 to 0.41647, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2946 - acc: 0.9023 - val_loss: 0.4165 - val_acc: 0.8379\n",
      "Epoch 7/15\n",
      "Epoch 00007: val_loss improved from 0.41647 to 0.39884, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2392 - acc: 0.9227 - val_loss: 0.3988 - val_acc: 0.8411\n",
      "Epoch 8/15\n",
      "Epoch 00008: val_loss improved from 0.39884 to 0.38835, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1997 - acc: 0.9374 - val_loss: 0.3884 - val_acc: 0.8462\n",
      "Epoch 9/15\n",
      "Epoch 00009: val_loss improved from 0.38835 to 0.38684, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1654 - acc: 0.9508 - val_loss: 0.3868 - val_acc: 0.8462\n",
      "Epoch 10/15\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.1404 - acc: 0.9577 - val_loss: 0.3895 - val_acc: 0.8494\n",
      "Epoch 11/15\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 1s - loss: 0.1202 - acc: 0.9651 - val_loss: 0.3947 - val_acc: 0.8532\n",
      "Epoch 12/15\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 1s - loss: 0.1047 - acc: 0.9703 - val_loss: 0.4025 - val_acc: 0.8494\n",
      "Epoch 13/15\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 1s - loss: 0.0900 - acc: 0.9765 - val_loss: 0.4141 - val_acc: 0.8494\n",
      "Epoch 14/15\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 1s - loss: 0.0800 - acc: 0.9785 - val_loss: 0.4316 - val_acc: 0.8424\n",
      "Epoch 15/15\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 1s - loss: 0.0693 - acc: 0.9806 - val_loss: 0.4495 - val_acc: 0.8411\n",
      "------------------\n",
      "Train on 14096 samples, validate on 1567 samples\n",
      "Epoch 1/15\n",
      "Epoch 00001: val_loss improved from inf to 1.06730, saving model to /tmp/nn_model.h5\n",
      " - 4s - loss: 1.0826 - acc: 0.4039 - val_loss: 1.0673 - val_acc: 0.4065\n",
      "Epoch 2/15\n",
      "Epoch 00002: val_loss improved from 1.06730 to 0.85867, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9692 - acc: 0.5363 - val_loss: 0.8587 - val_acc: 0.6407\n",
      "Epoch 3/15\n",
      "Epoch 00003: val_loss improved from 0.85867 to 0.63466, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.6857 - acc: 0.7511 - val_loss: 0.6347 - val_acc: 0.7632\n",
      "Epoch 4/15\n",
      "Epoch 00004: val_loss improved from 0.63466 to 0.51697, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.4856 - acc: 0.8336 - val_loss: 0.5170 - val_acc: 0.8079\n",
      "Epoch 5/15\n",
      "Epoch 00005: val_loss improved from 0.51697 to 0.45678, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3674 - acc: 0.8776 - val_loss: 0.4568 - val_acc: 0.8239\n",
      "Epoch 6/15\n",
      "Epoch 00006: val_loss improved from 0.45678 to 0.42261, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2900 - acc: 0.9073 - val_loss: 0.4226 - val_acc: 0.8302\n",
      "Epoch 7/15\n",
      "Epoch 00007: val_loss improved from 0.42261 to 0.40737, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2338 - acc: 0.9247 - val_loss: 0.4074 - val_acc: 0.8366\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/15\n",
      "Epoch 00008: val_loss improved from 0.40737 to 0.39996, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1943 - acc: 0.9398 - val_loss: 0.4000 - val_acc: 0.8468\n",
      "Epoch 9/15\n",
      "Epoch 00009: val_loss did not improve\n",
      " - 1s - loss: 0.1614 - acc: 0.9499 - val_loss: 0.4004 - val_acc: 0.8481\n",
      "Epoch 10/15\n",
      "Epoch 00010: val_loss did not improve\n",
      " - 1s - loss: 0.1358 - acc: 0.9601 - val_loss: 0.4060 - val_acc: 0.8519\n",
      "Epoch 11/15\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 1s - loss: 0.1166 - acc: 0.9662 - val_loss: 0.4148 - val_acc: 0.8494\n",
      "Epoch 12/15\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 1s - loss: 0.0990 - acc: 0.9721 - val_loss: 0.4237 - val_acc: 0.8481\n",
      "Epoch 13/15\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 1s - loss: 0.0852 - acc: 0.9767 - val_loss: 0.4379 - val_acc: 0.8475\n",
      "Epoch 14/15\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 1s - loss: 0.0740 - acc: 0.9804 - val_loss: 0.4591 - val_acc: 0.8430\n",
      "Epoch 15/15\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 1s - loss: 0.0638 - acc: 0.9838 - val_loss: 0.4820 - val_acc: 0.8354\n",
      "------------------\n",
      "Train on 14097 samples, validate on 1567 samples\n",
      "Epoch 1/15\n",
      "Epoch 00001: val_loss improved from inf to 1.06072, saving model to /tmp/nn_model.h5\n",
      " - 4s - loss: 1.0819 - acc: 0.4068 - val_loss: 1.0607 - val_acc: 0.4231\n",
      "Epoch 2/15\n",
      "Epoch 00002: val_loss improved from 1.06072 to 0.87385, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.9749 - acc: 0.5276 - val_loss: 0.8739 - val_acc: 0.6165\n",
      "Epoch 3/15\n",
      "Epoch 00003: val_loss improved from 0.87385 to 0.66049, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.7193 - acc: 0.7242 - val_loss: 0.6605 - val_acc: 0.7696\n",
      "Epoch 4/15\n",
      "Epoch 00004: val_loss improved from 0.66049 to 0.53549, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.5088 - acc: 0.8250 - val_loss: 0.5355 - val_acc: 0.8003\n",
      "Epoch 5/15\n",
      "Epoch 00005: val_loss improved from 0.53549 to 0.46780, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3852 - acc: 0.8679 - val_loss: 0.4678 - val_acc: 0.8226\n",
      "Epoch 6/15\n",
      "Epoch 00006: val_loss improved from 0.46780 to 0.42802, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.3049 - acc: 0.8981 - val_loss: 0.4280 - val_acc: 0.8302\n",
      "Epoch 7/15\n",
      "Epoch 00007: val_loss improved from 0.42802 to 0.40488, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2481 - acc: 0.9188 - val_loss: 0.4049 - val_acc: 0.8366\n",
      "Epoch 8/15\n",
      "Epoch 00008: val_loss improved from 0.40488 to 0.39199, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.2046 - acc: 0.9354 - val_loss: 0.3920 - val_acc: 0.8456\n",
      "Epoch 9/15\n",
      "Epoch 00009: val_loss improved from 0.39199 to 0.38517, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1717 - acc: 0.9471 - val_loss: 0.3852 - val_acc: 0.8462\n",
      "Epoch 10/15\n",
      "Epoch 00010: val_loss improved from 0.38517 to 0.38464, saving model to /tmp/nn_model.h5\n",
      " - 1s - loss: 0.1446 - acc: 0.9562 - val_loss: 0.3846 - val_acc: 0.8488\n",
      "Epoch 11/15\n",
      "Epoch 00011: val_loss did not improve\n",
      " - 1s - loss: 0.1218 - acc: 0.9653 - val_loss: 0.3887 - val_acc: 0.8475\n",
      "Epoch 12/15\n",
      "Epoch 00012: val_loss did not improve\n",
      " - 1s - loss: 0.1055 - acc: 0.9691 - val_loss: 0.3962 - val_acc: 0.8449\n",
      "Epoch 13/15\n",
      "Epoch 00013: val_loss did not improve\n",
      " - 1s - loss: 0.0921 - acc: 0.9750 - val_loss: 0.4056 - val_acc: 0.8456\n",
      "Epoch 14/15\n",
      "Epoch 00014: val_loss did not improve\n",
      " - 1s - loss: 0.0798 - acc: 0.9791 - val_loss: 0.4172 - val_acc: 0.8405\n",
      "Epoch 15/15\n",
      "Epoch 00015: val_loss did not improve\n",
      " - 1s - loss: 0.0691 - acc: 0.9822 - val_loss: 0.4321 - val_acc: 0.8379\n",
      "------------------\n"
     ]
    }
   ],
   "source": [
    "def get_nn_feats(rnd=1):\n",
    "    # return train pred prob and test pred prob \n",
    "    train_pred, test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    best_val_train_pred, best_val_test_pred = np.zeros((19579,3)),np.zeros((8392,3))\n",
    "    FEAT_CNT = 5\n",
    "    NUM_WORDS = 30000\n",
    "    N = 10\n",
    "    MAX_LEN = 100\n",
    "    NUM_CLASSES = 3\n",
    "    MODEL_P = '/tmp/nn_model.h5'\n",
    "    \n",
    "    tmp_X = train_df['text']\n",
    "    tmp_Y = train_df['author']\n",
    "    tmp_X_test = test_df['text']\n",
    "    \n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    tmp_Y = lb.fit_transform(tmp_Y)\n",
    "    \n",
    "    tokenizer = Tokenizer(num_words=NUM_WORDS)\n",
    "    tokenizer.fit_on_texts(tmp_X)\n",
    "    \n",
    "    ttrain_x = tokenizer.texts_to_sequences(tmp_X)\n",
    "    ttrain_x = pad_sequences(ttrain_x, maxlen=MAX_LEN)\n",
    "    \n",
    "    ttest_x = tokenizer.texts_to_sequences(tmp_X_test)\n",
    "    ttest_x = pad_sequences(ttest_x, maxlen=MAX_LEN)\n",
    "\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(tmp_Y)\n",
    "\n",
    "    ttrain_y = lb.transform(tmp_Y)\n",
    "    \n",
    "    kf = KFold(n_splits=FEAT_CNT, shuffle=True, random_state=2331*rnd)\n",
    "    for train_index, test_index in kf.split(tmp_X):\n",
    "        # prepare aug train, val      \n",
    "        model = Sequential()\n",
    "        model.add(Embedding(NUM_WORDS, N, input_length=MAX_LEN))\n",
    "        model.add(GlobalAveragePooling1D())\n",
    "        model.add(Dense(50, activation='relu'))\n",
    "        model.add(Dropout(0.1))\n",
    "        model.add(Dense(NUM_CLASSES, activation='softmax'))\n",
    "\n",
    "        model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "        #model.summary()\n",
    "\n",
    "        model_chk = ModelCheckpoint(filepath=MODEL_P, monitor='val_loss', save_best_only=True, verbose=1)\n",
    "        model.fit(ttrain_x[train_index], ttrain_y[train_index], \n",
    "                  validation_split=0.1,\n",
    "                  batch_size=64, epochs=15, \n",
    "                  verbose=2,\n",
    "                  callbacks=[model_chk],\n",
    "                  shuffle=False\n",
    "                 )\n",
    " \n",
    "        # save feat\n",
    "        train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        # best val model\n",
    "        model = load_model(MODEL_P)\n",
    "        best_val_train_pred[test_index] = model.predict(ttrain_x[test_index])\n",
    "        best_val_test_pred += model.predict(ttest_x)/FEAT_CNT\n",
    "        \n",
    "        # release\n",
    "        del model\n",
    "        gc.collect()\n",
    "        print('------------------')\n",
    "        \n",
    "    return train_pred,test_pred,best_val_train_pred,best_val_test_pred\n",
    "\n",
    "print('def cnn done')\n",
    "\n",
    "nn_train1,nn_test1,nn_train2,nn_test2 = get_nn_feats(4)\n",
    "\n",
    "\n",
    "\n",
    "all_nn_train = np.hstack([lstm_train1, lstm_train2, \n",
    "                          cnn_train1, cnn_train2,\n",
    "                          nn_train1,nn_train2\n",
    "                         ])\n",
    "all_nn_test = np.hstack([lstm_test1, lstm_test2, \n",
    "                         cnn_test1, cnn_test2,\n",
    "                         nn_test1,nn_test2\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('nn_feat.pkl','wb') as fout:\n",
    "    pickle.dump([all_nn_train,all_nn_test],fout)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}