bagging_stacking_nn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import glob\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import log_loss\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import f1_score\n",
    "from keras.layers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "file path ../feature/bagging/rnn_model_bagging1.pkl\n",
      "(56590, 19) (56590, 19)\n",
      "file path ../feature/bagging/rnn_model_bagging2.pkl\n",
      "(56590, 19) (56590, 19)\n",
      "file path ../feature/bagging/rnn_model_bagging3.pkl\n",
      "(56590, 19) (56590, 19)\n",
      "(56590, 57)\n"
     ]
    }
   ],
   "source": [
    "train_x,test_x = [],[]\n",
    "for feat in sorted(glob.glob('../feature/bagging/*')):\n",
    "    print('file path',feat)\n",
    "    a,b,c = pickle.load(open(feat,'rb'))\n",
    "    print(a.shape,b.shape)\n",
    "    train_x.append(a)\n",
    "    test_x.append(c)\n",
    "# load y\n",
    "train = pd.read_csv(\"../input/new_data/train_set.csv\")\n",
    "train_x = np.nan_to_num(np.hstack(train_x))\n",
    "test_x = np.nan_to_num(np.hstack(test_x))\n",
    "print(train_x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y=b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def self_f1_score(data, y_hat):\n",
    "    y_true = data\n",
    "    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n",
    "    return f1_score(y_true, y_hat, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def self_f1_score(data, y_hat):\n",
    "    y_true = data\n",
    "    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n",
    "    return f1_score(y_true, y_hat, average='macro')\n",
    "def f1(y_true, y_pred):\n",
    "    def recall(y_true, y_pred):\n",
    "        \"\"\"Recall metric.\n",
    "\n",
    "        Only computes a batch-wise average of recall.\n",
    "\n",
    "        Computes the recall, a metric for multi-label classification of\n",
    "        how many relevant items are selected.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(y_true, y_pred):\n",
    "        \"\"\"Precision metric.\n",
    "\n",
    "        Only computes a batch-wise average of precision.\n",
    "\n",
    "        Computes the precision, a metric for multi-label classification of\n",
    "        how many selected items are relevant.\n",
    "        \"\"\"\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "    precision = precision(y_true, y_pred)\n",
    "    recall = recall(y_true, y_pred)\n",
    "    return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def model done\n"
     ]
    }
   ],
   "source": [
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.models import Model\n",
    "def get_nn():\n",
    "    cv1_input = Input(shape=(57,), dtype='float32')\n",
    "#     merged = concatenate([cv1_input])\n",
    "    merged = Dense(512,activation='relu')(cv1_input)\n",
    "    merged = Dropout(0.2)(merged)\n",
    "    x = Dense(19, activation=\"softmax\")(merged)\n",
    "    model = Model(inputs=cv1_input, outputs=x)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer='adam',\n",
    "                    metrics=[f1])\n",
    "    return model\n",
    "print('def model done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "def done\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import gc\n",
    "def kf_train(fold_cnt=3,rnd=1):\n",
    "    now_nfold=0\n",
    "    kf = KFold(n_splits=fold_cnt, shuffle=False, random_state=233*rnd)\n",
    "    test_pred, train_pred = np.zeros((102277,19)),np.zeros((102277,19))\n",
    "#     LRDecay = LearningRateScheduler(lr_decay)\n",
    "    for train_index, test_index in kf.split(train_x):\n",
    "        \n",
    "        now_nfold+=1\n",
    "        print (\"now is {} fold\".format(now_nfold))\n",
    "        curr_x,curr_y = train_x[train_index],train_y[train_index]\n",
    "        hold_out_x,hold_out_y = train_x[test_index],train_y[test_index]\n",
    "        d_test = test_x\n",
    "        batch_size = 64\n",
    "        epochs = 15\n",
    "\n",
    "        model = get_nn()\n",
    "\n",
    "        file_path=\"weights_base.best.h5\"\n",
    "        checkpoint = ModelCheckpoint(file_path, save_best_only=True,verbose=1, monitor='val_f1',  mode='max')\n",
    "        early = EarlyStopping( monitor='val_f1',  mode='max', patience=2,)\n",
    "        callbacks_list = [checkpoint,early]\n",
    "\n",
    "        model.fit(curr_x, curr_y, \n",
    "                  batch_size=batch_size, epochs=epochs, \n",
    "                    validation_data=(hold_out_x, hold_out_y), \n",
    "                  callbacks=callbacks_list)\n",
    "\n",
    "        \n",
    "        model.load_weights(file_path)\n",
    "       \n",
    "        train_pred[test_index] = model.predict(hold_out_x)\n",
    "        curr_test_pred = model.predict(d_test)\n",
    "        test_pred += curr_test_pred\n",
    "        \n",
    "        # clear\n",
    "        del model\n",
    "        gc.collect()\n",
    "        K.clear_session()\n",
    "    test_pred = test_pred / fold_cnt\n",
    "    print('-------------------------------')\n",
    "    return train_pred, test_pred\n",
    "\n",
    "\n",
    "print('def done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now is 1 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 12s 227us/step - loss: 0.8114 - f1: 0.7733 - val_loss: 0.7094 - val_f1: 0.8218\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.82178, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.7063 - f1: 0.8206 - val_loss: 0.7038 - val_f1: 0.8209\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.82178\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.6921 - f1: 0.8211 - val_loss: 0.6881 - val_f1: 0.8185\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.82178\n",
      "now is 2 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 72us/step - loss: 0.8129 - f1: 0.7725 - val_loss: 0.7068 - val_f1: 0.8207\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.82068, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.7077 - f1: 0.8206 - val_loss: 0.6927 - val_f1: 0.8191\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.82068\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.6930 - f1: 0.8207 - val_loss: 0.6823 - val_f1: 0.8193\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.82068\n",
      "now is 3 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 72us/step - loss: 0.8143 - f1: 0.7720 - val_loss: 0.6939 - val_f1: 0.8266\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.82658, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.7092 - f1: 0.8203 - val_loss: 0.6748 - val_f1: 0.8283\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.82658 to 0.82832, saving model to weights_base.best.h5\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 4s 69us/step - loss: 0.6935 - f1: 0.8197 - val_loss: 0.6634 - val_f1: 0.8270\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.82832\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.6782 - f1: 0.8199 - val_loss: 0.6539 - val_f1: 0.8269\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.82832\n",
      "now is 4 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 72us/step - loss: 0.8145 - f1: 0.7723 - val_loss: 0.6876 - val_f1: 0.8258\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.82577, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 4s 70us/step - loss: 0.7104 - f1: 0.8196 - val_loss: 0.6707 - val_f1: 0.8255\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.82577\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 4s 70us/step - loss: 0.6940 - f1: 0.8205 - val_loss: 0.6549 - val_f1: 0.8236\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.82577\n",
      "now is 5 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 69us/step - loss: 0.8082 - f1: 0.7751 - val_loss: 0.7378 - val_f1: 0.8115\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.81148, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 4s 71us/step - loss: 0.7041 - f1: 0.8212 - val_loss: 0.7275 - val_f1: 0.8102\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.81148\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 66us/step - loss: 0.6898 - f1: 0.8216 - val_loss: 0.7164 - val_f1: 0.8118\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.81148 to 0.81179, saving model to weights_base.best.h5\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.6749 - f1: 0.8221 - val_loss: 0.6992 - val_f1: 0.8112\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.81179\n",
      "Epoch 5/15\n",
      "50931/50931 [==============================] - 3s 67us/step - loss: 0.6623 - f1: 0.8212 - val_loss: 0.6942 - val_f1: 0.8107\n",
      "\n",
      "Epoch 00005: val_f1 did not improve from 0.81179\n",
      "now is 6 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.8112 - f1: 0.7738 - val_loss: 0.6968 - val_f1: 0.8246\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.82458, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 67us/step - loss: 0.7087 - f1: 0.8197 - val_loss: 0.6843 - val_f1: 0.8259\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.82458 to 0.82594, saving model to weights_base.best.h5\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 64us/step - loss: 0.6922 - f1: 0.8201 - val_loss: 0.6734 - val_f1: 0.8252\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.82594\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 64us/step - loss: 0.6777 - f1: 0.8202 - val_loss: 0.6603 - val_f1: 0.8253\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.82594\n",
      "now is 7 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 70us/step - loss: 0.8057 - f1: 0.7767 - val_loss: 0.7360 - val_f1: 0.8172\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.81724, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 4s 72us/step - loss: 0.7032 - f1: 0.8216 - val_loss: 0.7269 - val_f1: 0.8161\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.81724\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 4s 73us/step - loss: 0.6878 - f1: 0.8218 - val_loss: 0.7099 - val_f1: 0.8174\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.81724 to 0.81738, saving model to weights_base.best.h5\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.6744 - f1: 0.8214 - val_loss: 0.7000 - val_f1: 0.8132\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.81738\n",
      "Epoch 5/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.6611 - f1: 0.8211 - val_loss: 0.6963 - val_f1: 0.8136\n",
      "\n",
      "Epoch 00005: val_f1 did not improve from 0.81738\n",
      "now is 8 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 69us/step - loss: 0.8106 - f1: 0.7735 - val_loss: 0.7230 - val_f1: 0.8163\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.81625, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.7050 - f1: 0.8203 - val_loss: 0.7059 - val_f1: 0.8167\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.81625 to 0.81666, saving model to weights_base.best.h5\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 67us/step - loss: 0.6907 - f1: 0.8210 - val_loss: 0.6886 - val_f1: 0.8183\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.81666 to 0.81826, saving model to weights_base.best.h5\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 65us/step - loss: 0.6761 - f1: 0.8202 - val_loss: 0.6789 - val_f1: 0.8165\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.81826\n",
      "Epoch 5/15\n",
      "50931/50931 [==============================] - 4s 71us/step - loss: 0.6635 - f1: 0.8203 - val_loss: 0.6687 - val_f1: 0.8168\n",
      "\n",
      "Epoch 00005: val_f1 did not improve from 0.81826\n",
      "now is 9 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 75us/step - loss: 0.8130 - f1: 0.7721 - val_loss: 0.7201 - val_f1: 0.8198\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.81981, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 3s 66us/step - loss: 0.7059 - f1: 0.8206 - val_loss: 0.7066 - val_f1: 0.8193\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.81981\n",
      "Epoch 3/15\n",
      "50931/50931 [==============================] - 3s 66us/step - loss: 0.6899 - f1: 0.8204 - val_loss: 0.6908 - val_f1: 0.8196\n",
      "\n",
      "Epoch 00003: val_f1 did not improve from 0.81981\n",
      "now is 10 fold\n",
      "Train on 50931 samples, validate on 5659 samples\n",
      "Epoch 1/15\n",
      "50931/50931 [==============================] - 4s 74us/step - loss: 0.8095 - f1: 0.7734 - val_loss: 0.7168 - val_f1: 0.8197\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.81972, saving model to weights_base.best.h5\n",
      "Epoch 2/15\n",
      "50931/50931 [==============================] - 4s 70us/step - loss: 0.7044 - f1: 0.8204 - val_loss: 0.7098 - val_f1: 0.8167\n",
      "\n",
      "Epoch 00002: val_f1 did not improve from 0.81972\n",
      "Epoch 3/15\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50931/50931 [==============================] - 3s 67us/step - loss: 0.6904 - f1: 0.8207 - val_loss: 0.6963 - val_f1: 0.8204\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.81972 to 0.82037, saving model to weights_base.best.h5\n",
      "Epoch 4/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.6753 - f1: 0.8208 - val_loss: 0.6808 - val_f1: 0.8188\n",
      "\n",
      "Epoch 00004: val_f1 did not improve from 0.82037\n",
      "Epoch 5/15\n",
      "50931/50931 [==============================] - 3s 68us/step - loss: 0.6613 - f1: 0.8208 - val_loss: 0.6789 - val_f1: 0.8177\n",
      "\n",
      "Epoch 00005: val_f1 did not improve from 0.82037\n",
      "-------------------------------\n"
     ]
    }
   ],
   "source": [
    "train_pred, nn_res = kf_train(fold_cnt=10,rnd=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "name = [\"class_prob_%s\"%i for i in range(1,20)]\n",
    "df_lgb = pd.DataFrame(nn_res, columns=name)\n",
    "# df_lgb = df_lgb.drop('class_prob_20', axis=1)\n",
    "df_lgb.to_csv('../pro/stacking_nn_bagging.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}