part5_blending_and_stack.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xujin/AI/anaconda2/lib/python2.7/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "/home/xujin/AI/anaconda2/lib/python2.7/site-packages/sklearn/grid_search.py:43: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.\n",
      "  DeprecationWarning)\n",
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing, grid_search, metrics\n",
    "from sklearn.cross_validation import StratifiedKFold, KFold\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "import time\n",
    "import numpy as np\n",
    "import math\n",
    "import lightgbm as lgb\n",
    "from scipy import sparse\n",
    "from scipy.stats import skew, boxcox\n",
    "from sklearn.model_selection import train_test_split\n",
    "import gc\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.advanced_activations import PReLU,LeakyReLU,ELU,ParametricSoftplus,ThresholdedReLU,SReLU\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras import backend as K\n",
    "from keras.optimizers import SGD,Nadam\n",
    "from keras.regularizers import WeightRegularizer, ActivityRegularizer,l2, activity_l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def logregobj(labels, preds):\n",
    "    con = 2\n",
    "    x =preds-labels\n",
    "    grad =con*x / (np.abs(x)+con)\n",
    "    hess =con**2 / (np.abs(x)+con)**2\n",
    "    return grad, hess \n",
    "\n",
    "def log_mae(labels,preds,lift=200):\n",
    "    return mean_absolute_error(np.exp(labels)-lift, np.exp(preds)-lift)\n",
    "\n",
    "log_mae_scorer = metrics.make_scorer(log_mae, greater_is_better = False)\n",
    "\n",
    "def lgbm_eval_mae(yhat, dtrain, lift=200):\n",
    "    y = dtrain.get_label()\n",
    "    return 'mae', mean_absolute_error(np.exp(y)-lift, np.exp(yhat)-lift), False\n",
    "\n",
    "def xg_eval_mae(yhat, dtrain, lift=200):\n",
    "    y = dtrain.get_label()\n",
    "    return 'mae', mean_absolute_error(np.exp(y)-lift, np.exp(yhat)-lift)\n",
    "\n",
    "\n",
    "def search_model(train_x, train_y, est, param_grid, n_jobs, cv, refit=False):\n",
    "# Grid Search for the best model\n",
    "    model = grid_search.GridSearchCV(estimator  = est,\n",
    "                                     param_grid = param_grid,\n",
    "                                     scoring    = log_mae_scorer,\n",
    "                                     verbose    = 10,\n",
    "                                     n_jobs  = n_jobs,\n",
    "                                     iid        = True,\n",
    "                                     refit    = refit,\n",
    "                                     cv      = cv)\n",
    "    # Fit Grid Search Model\n",
    "    model.fit(train_x, train_y)\n",
    "    print(\"Best score: %0.3f\" % model.best_score_)\n",
    "    print(\"Best parameters set:\", model.best_params_)\n",
    "    print(\"Scores:\", model.grid_scores_)\n",
    "    return model\n",
    "\n",
    "# custom metric function for Keras\n",
    "def mae_log(y_true, y_pred): \n",
    "    return K.mean(K.abs((K.exp(y_pred)-200) - (K.exp(y_true)-200)))\n",
    "\n",
    "# Keras deosn't support sparse matrix. \n",
    "# The following functions are useful to split a large sparse matrix into\n",
    "# smaller batches so they can be loaded into mem.\n",
    "def batch_generator(X, y, batch_size, shuffle):\n",
    "    number_of_batches = np.ceil(X.shape[0]/batch_size)\n",
    "    counter = 0\n",
    "    sample_index = np.arange(X.shape[0])\n",
    "    if shuffle:\n",
    "        np.random.shuffle(sample_index)\n",
    "    while True:\n",
    "        batch_index = sample_index[batch_size*counter:batch_size*(counter+1)]\n",
    "        X_batch = X[batch_index,:].toarray()\n",
    "        y_batch = y[batch_index]\n",
    "        counter += 1\n",
    "        yield X_batch, y_batch\n",
    "        if (counter == number_of_batches):\n",
    "            if shuffle:\n",
    "                np.random.shuffle(sample_index)\n",
    "            counter = 0\n",
    "            \n",
    "def batch_generatorp(X, batch_size, shuffle):\n",
    "    number_of_batches = X.shape[0] / np.ceil(X.shape[0]/batch_size)\n",
    "    counter = 0\n",
    "    sample_index = np.arange(X.shape[0])\n",
    "    while True:\n",
    "        batch_index = sample_index[batch_size * counter:batch_size * (counter + 1)]\n",
    "        X_batch = X[batch_index, :].toarray()\n",
    "        counter += 1\n",
    "        yield X_batch\n",
    "        if (counter == number_of_batches):\n",
    "            counter = 0            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Blending function for XGBoost\n",
    "def xgb_blend(estimators, train_x, train_y, test_x, fold, early_stopping_rounds=0):\n",
    "    print (\"Blend %d estimators for %d folds\" % (len(estimators), fold))\n",
    "    skf = list(KFold(len(train_y), fold))\n",
    "    \n",
    "    train_blend_x = np.zeros((train_x.shape[0], len(estimators)))\n",
    "    test_blend_x = np.zeros((test_x.shape[0], len(estimators)))\n",
    "    scores = np.zeros ((len(skf),len(estimators)))\n",
    "    best_rounds = np.zeros ((len(skf),len(estimators)))\n",
    "    \n",
    "    for j, est in enumerate(estimators):\n",
    "        print (\"Model %d: %s\" %(j+1, est))\n",
    "        test_blend_x_j = np.zeros((test_x.shape[0], len(skf)))\n",
    "        for i, (train, val) in enumerate(skf):\n",
    "            print (\"Model %d fold %d\" %(j+1,i+1))\n",
    "            fold_start = time.time() \n",
    "            train_x_fold = train_x[train]\n",
    "            train_y_fold = train_y[train]\n",
    "            val_x_fold = train_x[val]\n",
    "            val_y_fold = train_y[val]\n",
    "            \n",
    "            est.set_params( n_estimators=10000)\n",
    "            est.fit(train_x_fold,train_y_fold,\n",
    "                    eval_set=[(val_x_fold, val_y_fold)],\n",
    "                    eval_metric=xg_eval_mae,\n",
    "                    early_stopping_rounds=early_stopping_rounds,\n",
    "                    verbose=False\n",
    "                   )\n",
    "            best_round=est.best_iteration\n",
    "            best_rounds[i,j]=best_round\n",
    "            print (\"best round %d\" % (best_round))\n",
    "            val_y_predict_fold = est.predict(val_x_fold,ntree_limit=best_round)\n",
    "            score = log_mae(val_y_fold, val_y_predict_fold,200)\n",
    "            print (\"Score: \", score)\n",
    "            scores[i,j]=score\n",
    "            train_blend_x[val, j] = val_y_predict_fold\n",
    "            test_blend_x_j[:,i] = est.predict(test_x,ntree_limit=best_round)\n",
    "            print (\"Model %d fold %d fitting finished in %0.3fs\" % (j+1,i+1, time.time() - fold_start))            \n",
    "   \n",
    "        test_blend_x[:,j] = test_blend_x_j.mean(1)\n",
    "        print (\"Score for model %d is %f\" % (j+1,np.mean(scores[:,j])))\n",
    "    print (\"Score for blended models is %f\" % (np.mean(scores)))\n",
    "    return (train_blend_x, test_blend_x, scores,best_rounds )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Blending function for LightGBM\n",
    "def lgbm_blend(estimators, train_x, train_y, test_x, fold, early_stopping_rounds=0):\n",
    "    print (\"Blend %d estimators for %d folds\" % (len(estimators), fold))\n",
    "    skf = list(KFold(len(train_y), fold))\n",
    "    \n",
    "    train_blend_x = np.zeros((train_x.shape[0], len(estimators)))\n",
    "    test_blend_x = np.zeros((test_x.shape[0], len(estimators)))\n",
    "    scores = np.zeros ((len(skf),len(estimators)))\n",
    "    best_rounds = np.zeros ((len(skf),len(estimators)))\n",
    "    \n",
    "    for j, est in enumerate(estimators):\n",
    "        print (\"Model %d: %s\" %(j+1, est))\n",
    "        test_blend_x_j = np.zeros((test_x.shape[0], len(skf)))\n",
    "        for i, (train, val) in enumerate(skf):\n",
    "            print (\"Model %d fold %d\" %(j+1,i+1))\n",
    "            fold_start = time.time() \n",
    "            train_x_fold = train_x[train]\n",
    "            train_y_fold = train_y[train]\n",
    "            val_x_fold = train_x[val]\n",
    "            val_y_fold = train_y[val]\n",
    "\n",
    "            est.set_params( n_estimators=100000)\n",
    "            est.fit(train_x_fold,\n",
    "                    train_y_fold,\n",
    "                    eval_set=[(val_x_fold, val_y_fold)],\n",
    "                    eval_metric=lgbm_eval_mae,\n",
    "                    early_stopping_rounds=early_stopping_rounds,\n",
    "                    verbose=False\n",
    "                   )\n",
    "            best_round=est.best_iteration\n",
    "            best_rounds[i,j]=best_round\n",
    "            print (\"best round %d\" % (best_round))\n",
    "            val_y_predict_fold = est.predict(val_x_fold,num_iteration=best_round)\n",
    "            score = log_mae(val_y_fold, val_y_predict_fold,200)\n",
    "            print (\"Score: \", score)\n",
    "            scores[i,j]=score\n",
    "            train_blend_x[val, j] = val_y_predict_fold\n",
    "            test_blend_x_j[:,i] = est.predict(test_x,num_iteration=best_round)\n",
    "            print (\"Model %d fold %d fitting finished in %0.3fs\" % (j+1,i+1, time.time() - fold_start))            \n",
    "   \n",
    "        test_blend_x[:,j] = test_blend_x_j.mean(1)\n",
    "        print (\"Score for model %d is %f\" % (j+1,np.mean(scores[:,j])))\n",
    "    print (\"Score for blended models is %f\" % (np.mean(scores)))\n",
    "    return (train_blend_x, test_blend_x, scores,best_rounds )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading train data finished in 2.644s\n",
      "Loading test data finished in 1.575s\n"
     ]
    }
   ],
   "source": [
    "start = time.time() \n",
    "train_data = pd.read_csv('../input/train.csv')\n",
    "train_size=train_data.shape[0]\n",
    "print (\"Loading train data finished in %0.3fs\" % (time.time() - start))\n",
    "\n",
    "start = time.time()\n",
    "test_data = pd.read_csv('../input/test.csv')\n",
    "print (\"Loading test data finished in %0.3fs\" % (time.time() - start)) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Merge train and test\n",
    "Save our time on duplicating logics for train and test and will also ensure the transformations applied on train and test are the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Full Data set created.\n"
     ]
    }
   ],
   "source": [
    "full_data=pd.concat([train_data,test_data])\n",
    "del( train_data, test_data)\n",
    "print (\"Full Data set created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data_types = full_data.dtypes  \n",
    "cat_cols = list(data_types[data_types=='object'].index)\n",
    "num_cols = list(data_types[data_types=='int64'].index) + list(data_types[data_types=='float64'].index)\n",
    "\n",
    "id_col = 'id'\n",
    "target_col = 'loss'\n",
    "num_cols.remove('id')\n",
    "num_cols.remove('loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Numeric features\n",
    "\n",
    "Two preprocessings on numeric features are applied:\n",
    "\n",
    "1. Apply box-cox transformations for skewed numeric features.\n",
    "\n",
    "2. Scale numeric features so they will fall in the range between 0 and 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "skewed_cols = full_data[num_cols].apply(lambda x: skew(x.dropna()))\n",
    "\n",
    "SSL = preprocessing.StandardScaler()\n",
    "skewed_cols = skewed_cols[skewed_cols > 0.25].index.values\n",
    "for skewed_col in skewed_cols:\n",
    "    full_data[skewed_col], lam = boxcox(full_data[skewed_col] + 1)\n",
    "for num_col in num_cols:\n",
    "    full_data[num_col] = SSL.fit_transform(full_data[num_col].values.reshape(-1,1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model LE Coding\n",
    "### Categorical features\n",
    "1. Label Encoding (Factorizing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label enconding finished in 34.448849 seconds\n"
     ]
    }
   ],
   "source": [
    "LBL = preprocessing.LabelEncoder()\n",
    "start=time.time()\n",
    "for cat_col in cat_cols:\n",
    "    full_data[cat_col] = LBL.fit_transform(full_data[cat_col])\n",
    "print ('Label enconding finished in %f seconds' % (time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lift = 200\n",
    "\n",
    "train_x = full_data[:train_size].drop(['loss','id'], axis=1).values\n",
    "test_x = full_data[train_size:].drop(['loss','id'], axis=1).values\n",
    "train_y = np.log(full_data[:train_size].loss.values + lift)\n",
    "ID = full_data.id[:train_size].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LE + LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimators = [lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=9255,\n",
    "                     num_leaves=81,\n",
    "                     min_child_samples=191,\n",
    "                     colsample_bytree=0.300000,\n",
    "                     subsample=1.000000,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1139.406737              \n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=9220,\n",
    "                     num_leaves=95,\n",
    "                     min_child_samples=220,\n",
    "                     colsample_bytree=0.261269,\n",
    "                     subsample=1.000000,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1139.631716\n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=9263,\n",
    "                     num_leaves=104,\n",
    "                     min_child_samples=190,\n",
    "                     colsample_bytree=0.300000,\n",
    "                     subsample=1.000000,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1139.849854 \n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                              \n",
    "                     max_bin=9248,\n",
    "                     num_leaves=149,\n",
    "                     min_child_samples=220,\n",
    "                     colsample_bytree=0.300000,\n",
    "                     subsample=1.000000,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False)\n",
    "#               score -1139.883523              \n",
    "              ]\n",
    "\n",
    "(train_blend_x_gbm_le,\n",
    " test_blend_x_gbm_le,\n",
    " blend_scores_gbm_le,\n",
    " best_rounds_gbm_le) = lgbm_blend(estimators, \n",
    "                                  train_x,train_y, \n",
    "                                  test_x,\n",
    "                                  10,\n",
    "                                  1000)\n",
    "\n",
    "print (np.mean(blend_scores_gbm_le,axis=0))\n",
    "print (np.mean(best_rounds_gbm_le,axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LE + XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimators = [xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=7,\n",
    "                              min_child_weight=88,\n",
    "                              colsample_bytree=0.615498,\n",
    "                              subsample=0.810715,\n",
    "                              gamma=1.562494,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1143.321167\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=6,\n",
    "                              min_child_weight=115,\n",
    "                              colsample_bytree=0.855791,\n",
    "                              subsample=0.916137,\n",
    "                              gamma=1.357693,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1144.113800\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=8,\n",
    "                              min_child_weight=102,\n",
    "                              colsample_bytree=0.600000,\n",
    "                              subsample=1.000000,\n",
    "                              gamma=0.950000,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1144.603485\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=8,\n",
    "                              min_child_weight=113,\n",
    "                              colsample_bytree=0.600000,\n",
    "                              subsample=1.000000,\n",
    "                              gamma=0.950000 ,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             )\n",
    "#               score -1145.356110\n",
    "              \n",
    "              ]\n",
    "\n",
    "(train_blend_x_xgb_le,\n",
    " test_blend_x_xgb_le,\n",
    " blend_scores_xgb_le,\n",
    " best_rounds_xgb_le) = xgb_blend(estimators,\n",
    "                                 train_x,train_y,\n",
    "                                 test_x,\n",
    "                                 4,\n",
    "                                 500)\n",
    "\n",
    "print (np.mean(blend_scores_xgb_le,axis=0))\n",
    "print (np.mean(best_rounds_xgb_le,axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Categorical features\n",
    "1. Label Encoding (Factorizing)\n",
    "2. One-hot-encoded categorical features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "One-hot-encoding finished in 9.933163 seconds\n",
      "(313864, 1176)\n"
     ]
    }
   ],
   "source": [
    "OHE = preprocessing.OneHotEncoder(sparse=True)\n",
    "start=time.time()\n",
    "full_data_sparse=OHE.fit_transform(full_data[cat_cols])\n",
    "print ('One-hot-encoding finished in %f seconds' % (time.time()-start))\n",
    "\n",
    "print (full_data_sparse.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(313864, 1190)\n"
     ]
    }
   ],
   "source": [
    "lift = 200\n",
    "\n",
    "full_data_sparse = sparse.hstack((full_data_sparse\n",
    "                                  ,full_data[num_cols])\n",
    "                                 , format='csr'\n",
    "                                 )\n",
    "print (full_data_sparse.shape)\n",
    "train_x = full_data_sparse[:train_size]\n",
    "test_x = full_data_sparse[train_size:]\n",
    "train_y = np.log(full_data[:train_size].loss.values + lift)\n",
    "ID = full_data.id[:train_size].values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OHE + LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimators = [lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=526,\n",
    "                     num_leaves=68,\n",
    "                     min_child_samples=127,\n",
    "                     colsample_bytree=0.218683,\n",
    "                     subsample=0.961961,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1139.877375            \n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=457,\n",
    "                     num_leaves=54,\n",
    "                     min_child_samples=125,\n",
    "                     colsample_bytree=0.383468,\n",
    "                     subsample=0.949582,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1140.332236\n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                             \n",
    "                     max_bin=514,\n",
    "                     num_leaves=40,\n",
    "                     min_child_samples=126,\n",
    "                     colsample_bytree=0.325435,\n",
    "                     subsample=0.923560,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False),\n",
    "#               score -1140.546101\n",
    "              lgb.LGBMRegressor(learning_rate=0.005,                              \n",
    "                     max_bin=514,\n",
    "                     num_leaves=40,\n",
    "                     min_child_samples=127,\n",
    "                     colsample_bytree=0.464765,\n",
    "                     subsample=0.968715,\n",
    "                     subsample_freq=1,\n",
    "                     silent=False)\n",
    "#               score -1140.593041             \n",
    "              ]\n",
    "\n",
    "(train_blend_x_gbm_ohe,\n",
    " test_blend_x_gbm_ohe,\n",
    " blend_scores_gbm_ohe,\n",
    " best_rounds_gbm_ohe) = lgbm_blend(estimators, \n",
    "                                   train_x, train_y, \n",
    "                                   test_x,\n",
    "                                   10,\n",
    "                                   500)\n",
    "\n",
    "print (np.mean(blend_scores_gbm_ohe,axis=0))\n",
    "print (np.mean(best_rounds_gbm_ohe,axis=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "estimators = [xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=11,\n",
    "                              min_child_weight=200,\n",
    "                              colsample_bytree=0.05,\n",
    "                              subsample=1.0,\n",
    "                              gamma=0.5,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1140.645843\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=15,\n",
    "                              min_child_weight=177,\n",
    "                              colsample_bytree=0.209870,\n",
    "                              subsample=0.916137,\n",
    "                              gamma=0.916137,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1140.646576\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=13,\n",
    "                              min_child_weight=127,\n",
    "                              colsample_bytree=0.087842,\n",
    "                              subsample=0.969313,\n",
    "                              gamma=0.573770,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             ),\n",
    "#               score -1141.266846\n",
    "              xgb.XGBRegressor(objective=logregobj,\n",
    "                              learning_rate=0.01, \n",
    "                              max_depth=16,\n",
    "                              min_child_weight=130,\n",
    "                              colsample_bytree=0.060157,\n",
    "                              subsample=0.971690,\n",
    "                              gamma=0.676203,\n",
    "                              nthread=-1,\n",
    "                              silent=True,\n",
    "                              seed=1234\n",
    "                             )\n",
    "#               score -1142.136109\n",
    "              ]\n",
    "\n",
    "(train_blend_x_xgb_ohe,\n",
    " test_blend_x_xgb_ohe,\n",
    " blend_scores_xgb_ohe,\n",
    " best_rounds_xgb_ohe) = xgb_blend(estimators, \n",
    "                                      train_x, \n",
    "                                      train_y, \n",
    "                                      test_x,\n",
    "                                      4,\n",
    "                                      1000)\n",
    "\n",
    "print (np.mean(blend_scores_xgb_ohe,axis=0))\n",
    "print (np.mean(best_rounds_xgb_ohe,axis=0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### OHE + Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# early_stop = EarlyStopping(monitor='val_mae_log', patience=5, verbose=0, mode='auto')\n",
    "# checkpointer = ModelCheckpoint(filepath=\"weights.hdf5\", monitor='val_mae_log', verbose=1, \n",
    "#                                save_best_only=True, mode='min')\n",
    "\n",
    "def nn_model(params):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(params['input_size'], input_dim = params['input_dim']))\n",
    "\n",
    "    model.add(PReLU())\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dropout(params['input_drop_out']))\n",
    "        \n",
    "    model.add(Dense(params['hidden_size0']))\n",
    "    model.add(PReLU())\n",
    "    model.add(BatchNormalization())    \n",
    "    model.add(Dropout(params['hidden_drop_out0']))\n",
    "\n",
    "    model.add(Dense(params['hidden_size1']))\n",
    "    model.add(PReLU())\n",
    "    model.add(BatchNormalization())    \n",
    "    model.add(Dropout(params['hidden_drop_out1']))    \n",
    "    \n",
    "    model.add(Dense(1))\n",
    "    model.compile(loss = 'mae', metrics=[mae_log], optimizer = 'adadelta')\n",
    "    return(model)\n",
    "\n",
    "\n",
    "def nn_blend_data(parameters, train_x, train_y, test_x, fold, early_stopping_rounds=0, batch_size=128):\n",
    "    print (\"Blend %d estimators for %d folds\" % (len(parameters), fold))\n",
    "    skf = list(KFold(len(train_y), fold))\n",
    "    \n",
    "    train_blend_x = np.zeros((train_x.shape[0], len(parameters)))\n",
    "    test_blend_x = np.zeros((test_x.shape[0], len(parameters)))\n",
    "    scores = np.zeros ((len(skf),len(parameters)))\n",
    "    best_rounds = np.zeros ((len(skf),len(parameters)))\n",
    " \n",
    "    for j, nn_params in enumerate(parameters):\n",
    "        print (\"Model %d: %s\" %(j+1, nn_params))\n",
    "        test_blend_x_j = np.zeros((test_x.shape[0], len(skf)))\n",
    "        for i, (train, val) in enumerate(skf):\n",
    "            print (\"Model %d fold %d\" %(j+1,i+1))\n",
    "            fold_start = time.time() \n",
    "            train_x_fold = train_x[train]\n",
    "            train_y_fold = train_y[train]\n",
    "            val_x_fold = train_x[val]\n",
    "            val_y_fold = train_y[val]\n",
    "\n",
    "            model = nn_model(nn_params)\n",
    "            print (model)\n",
    "            fit= model.fit_generator(generator=batch_generator(\n",
    "                    train_x_fold, train_y_fold, batch_size, True),\n",
    "                                     nb_epoch=60,\n",
    "                                     samples_per_epoch=train_x_fold.shape[0],\n",
    "                                     validation_data=(val_x_fold.todense(), val_y_fold),\n",
    "                                     verbose = 0,\n",
    "                                     callbacks=[ModelCheckpoint(filepath=\"weights.hdf5\", \n",
    "                                                                monitor='val_mae_log', \n",
    "                                                                verbose=0, save_best_only=True, \n",
    "                                                                mode='min')\n",
    "                                                ]\n",
    "                                     )\n",
    "            \n",
    "            model.load_weights(\"weights.hdf5\")\n",
    "            # Compile model (required to make predictions)\n",
    "            model.compile(loss = 'mae', metrics=[mae_log], optimizer = 'adadelta')\n",
    "\n",
    "            val_y_predict_fold = model.predict_generator(\n",
    "                generator=batch_generatorp(val_x_fold, batch_size, True),\n",
    "                val_samples=val_x_fold.shape[0]\n",
    "            )\n",
    "            \n",
    "            score = log_mae(val_y_fold, val_y_predict_fold,200)\n",
    "            print \"Score: \", score, mean_absolute_error(val_y_fold, val_y_predict_fold)\n",
    "            scores[i,j]=score\n",
    "            train_blend_x[val, j] = val_y_predict_fold.reshape(val_y_predict_fold.shape[0])\n",
    "            \n",
    "            model.load_weights(\"weights.hdf5\")\n",
    "            # Compile model (required to make predictions)\n",
    "            model.compile(loss = 'mae', metrics=[mae_log], optimizer = 'adadelta')            \n",
    "            test_blend_x_j[:,i] = model.predict_generator(generator=batch_generatorp(\n",
    "                    test_x, batch_size, True),\n",
    "                                                          val_samples=test_x.shape[0]\n",
    "                                                         ).reshape(test_x.shape[0])\n",
    "            print \"Model %d fold %d fitting finished in %0.3fs\" % (j+1,i+1, time.time() - fold_start)          \n",
    "   \n",
    "        test_blend_x[:,j] = test_blend_x_j.mean(1)\n",
    "        print \"Score for model %d is %f\" % (j+1,np.mean(scores[:,j]))\n",
    "    print \"Score for blended models is %f\" % (np.mean(scores))\n",
    "    return (train_blend_x, test_blend_x, scores,best_rounds )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bagging_num = 10\n",
    "nn_parameters = []\n",
    "\n",
    "nn_parameter =  { \n",
    "     'input_size' :400 ,\n",
    "     'input_dim' : train_x.shape[1],\n",
    "     'input_drop_out' : 0.4 ,\n",
    "     'hidden_size0' : 200 ,\n",
    "     'hidden_drop_out0' :0.2,\n",
    "     'hidden_size1' : 50 ,\n",
    "     'hidden_drop_out1' :0.2,    \n",
    "     'learning_rate': 0.1,\n",
    "     'optimizer': 'adadelta'}\n",
    "\n",
    "for i in range(bagging_num):\n",
    "    nn_parameters.append(nn_parameter)\n",
    "\n",
    "(train_blend_x_ohe_mlp,\n",
    " test_blend_x_ohe_mlp,\n",
    " blend_scores_ohe_mlp,\n",
    " best_round_ohe_mlp) = nn_blend_data(nn_parameters,\n",
    "                                     train_x,\n",
    "                                     train_y,\n",
    "                                     test_x,\n",
    "                                     4,\n",
    "                                     5)\n",
    "\n",
    "print np.mean(blend_scores_ohe_mlp,axis=0)\n",
    "print log_mae(np.mean(train_blend_x_ohe_mlp,axis=1).reshape(train_size,1),train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Blending\n",
    "1. Ridge Regression\n",
    "  * Ridge is focused on finding out weight of each feature which is exactly what we are interested in.\n",
    "2. XGB linear\n",
    "\n",
    "Specifically, we will simply average predictions from MLP models before using them for blending."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train_blend = np.hstack((\n",
    "        train_blend_x_gbm_le,\n",
    "        train_blend_x_xgb_le,\n",
    "        train_blend_x_gbm_ohe,\n",
    "        train_blend_x_xgb_ohe,\n",
    "        np.mean(train_blend_x_ohe_mlp,axis=1).reshape(train_size,1)\n",
    "        ))\n",
    "\n",
    "test_blend = np.hstack((\n",
    "        test_blend_x_gbm_le,\n",
    "        test_blend_x_xgb_le,\n",
    "        test_blend_x_gbm_ohe,\n",
    "        test_blend_x_xgb_ohe,\n",
    "        np.mean(test_blend_x_ohe_mlp,axis=1).reshape(test_x.shape[0],1)\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ridge\n",
    "from sklearn.linear_model import ElasticNet,Ridge,LinearRegression\n",
    "print  (\"Blending.\")\n",
    "\n",
    "param_grid = {\n",
    "    'alpha':[0,0.00001,0.00003,0.0001,0.0003,0.001,0.003,0.01,0.03,0.1,0.3,1,3,10,15,20,\n",
    "             25,30,35,40,45,50,55,60,70]\n",
    "}\n",
    "model = search_model(train_blend, \n",
    "                     train_y, \n",
    "                     Ridge(), \n",
    "                     param_grid, \n",
    "                     n_jobs=1, \n",
    "                     cv=4, \n",
    "                     refit=True\n",
    "                    )   \n",
    "print \"\\nbest subsample:\", model.best_params_\n",
    "\n",
    "print '\\nBest score: ',model.best_score_\n",
    "print '\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "start = time.time() \n",
    "# XGBoost gblinear\n",
    "params = {\n",
    "    'eta': 0.1,\n",
    "    'booster': 'gblinear',\n",
    "    'lambda': 0,\n",
    "    'alpha': 0, # you can try different values for alpha\n",
    "    'lambda_bias' : 0,\n",
    "    'silent': 0,\n",
    "    'verbose_eval': True,\n",
    "    'seed': 1234\n",
    "}\n",
    "\n",
    "\n",
    "cv_all = xgb.cv(params,xgb.DMatrix(train_blend, label=train_y,missing=np.nan),\n",
    "                num_boost_round=400000, nfold=4, feval=xg_eval_mae,seed=1234,\n",
    "                callbacks=[xgb.callback.early_stop(500)])\n",
    "\n",
    "print (\"\\nLoading train data finished in %0.3fs\" % (time.time() - start))    \n",
    "print cv_all[cv_all['test-mae-mean'] == cv_all['test-mae-mean'].min()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "xgtrain_blend = xgb.DMatrix(train_blend,label=train_y,missing=np.nan)\n",
    "\n",
    "xgb_model=xgb.train(params, xgtrain_blend,\n",
    "                    num_boost_round=<>, # best boost round based on previous step\n",
    "                    feval=xg_eval_mae)\n",
    "\n",
    "pred_y_gblinear = np.exp(xgb_model.predict(xgb.DMatrix(test_blend))) - lift\n",
    "\n",
    "results = pd.DataFrame()\n",
    "results['id'] = full_data[train_size:].id\n",
    "results['loss'] = pred_y_gblinear\n",
    "print (\"Submission created.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final submission \n",
    "  weights: [0.5,0.5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_y = pred_y_ridge*0.5 + pred_y_gblinear*0.5\n",
    "\n",
    "results = pd.DataFrame()\n",
    "results['id'] = full_data[train_size:].id\n",
    "results['loss'] = pred_y\n",
    "# results.to_csv(\"../output/sub_final.csv\", index=False)\n",
    "print (\"Submission created.\")"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}