Kaggle/[Kaggle] M5_Forecasting.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "pd.set_option('max_columns', None)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "plt.style.use('fivethirtyeight')\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, OneHotEncoder\n",
    "%matplotlib inline\n",
    "import copy\n",
    "import datetime\n",
    "from sklearn.utils import shuffle\n",
    "from scipy import stats\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "import optuna\n",
    "from optuna.visualization import plot_optimization_history\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
    "from sklearn import model_selection\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn import preprocessing, metrics\n",
    "from sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold, GroupKFold, GridSearchCV, train_test_split, TimeSeriesSplit\n",
    "\n",
    "import json\n",
    "import ast\n",
    "import time\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai import *\n",
    "from fastai.collab import *\n",
    "from fastai.tabular import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'C:\\\\Users\\\\bokhy\\\\Desktop\\\\Python\\\\Python-Projects\\\\Kaggle\\\\m5-forecasting-accuracy\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = [\"int16\", \"int32\", \"int64\", \"float16\", \"float32\", \"float64\"]\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == \"int\":\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if (\n",
    "                    c_min > np.finfo(np.float16).min\n",
    "                    and c_max < np.finfo(np.float16).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif (\n",
    "                    c_min > np.finfo(np.float32).min\n",
    "                    and c_max < np.finfo(np.float32).max\n",
    "                ):\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose:\n",
    "        print(\n",
    "            \"Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)\".format(\n",
    "                end_mem, 100 * (start_mem - end_mem) / start_mem\n",
    "            )\n",
    "        )\n",
    "    return df\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to read the data and merge it (ignoring some columns, this is a very fst model)\n",
    "def read_data():\n",
    "    print('Reading files...')\n",
    "    calendar = pd.read_csv(os.path.join(path, 'calendar.csv')).pipe(reduce_mem_usage)\n",
    "#    calendar = reduce_mem_usage(calendar)\n",
    "    print('Calendar has {} rows and {} columns'.format(calendar.shape[0], calendar.shape[1]))\n",
    "    sell_prices = pd.read_csv(os.path.join(path,'sell_prices.csv')).pipe(reduce_mem_usage)\n",
    "#    sell_prices = reduce_mem_usage(sell_prices)\n",
    "    print('Sell prices has {} rows and {} columns'.format(sell_prices.shape[0], sell_prices.shape[1]))\n",
    "    sales_train_val = pd.read_csv(os.path.join(path,'sales_train_validation.csv')).pipe(reduce_mem_usage)\n",
    "    print('Sales train validation has {} rows and {} columns'.format(sales_train_val.shape[0], sales_train_val.shape[1]))\n",
    "    submission = pd.read_csv(os.path.join(path,'sample_submission.csv')).pipe(reduce_mem_usage)\n",
    "    print('submission has {} rows and {} columns'.format(submission.shape[0], submission.shape[1]))\n",
    "\n",
    "    return calendar, sell_prices, sales_train_val, submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading files...\n",
      "Mem. usage decreased to  0.12 Mb (41.9% reduction)\n",
      "Calendar has 1969 rows and 14 columns\n",
      "Mem. usage decreased to 130.48 Mb (37.5% reduction)\n",
      "Sell prices has 6841121 rows and 4 columns\n",
      "Mem. usage decreased to 95.00 Mb (78.7% reduction)\n",
      "Sales train validation has 30490 rows and 1919 columns\n",
      "Mem. usage decreased to  2.09 Mb (84.5% reduction)\n",
      "submission has 60980 rows and 29 columns\n"
     ]
    }
   ],
   "source": [
    "calendar1, sell_prices1, sales_train_val1, submission1 = read_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "calendar = calendar1.copy()\n",
    "sell_prices = sell_prices1.copy()\n",
    "sales_train_val = sales_train_val1.copy()\n",
    "submission = submission1.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30490\n",
      "28\n"
     ]
    }
   ],
   "source": [
    "NUM_ITEMS = sales_train_val.shape[0]  # 30490\n",
    "DAYS_PRED = submission.shape[1] - 1  # 28\n",
    "print(NUM_ITEMS)\n",
    "print(DAYS_PRED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to  0.08 Mb (36.9% reduction)\n",
      "Mem. usage decreased to 94.01 Mb (0.4% reduction)\n",
      "Mem. usage decreased to 45.67 Mb (41.7% reduction)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def transform(df, cols):\n",
    "    for col in cols:\n",
    "        # Leave NaN as it is.\n",
    "        le = LabelEncoder()\n",
    "        not_null = df[col][df[col].notnull()]\n",
    "        df[col] = pd.Series(le.fit_transform(not_null), index=not_null.index)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "calendar = transform(\n",
    "    calendar, [\"event_name_1\", \"event_type_1\", \"event_name_2\", \"event_type_2\"]\n",
    ").pipe(reduce_mem_usage)\n",
    "\n",
    "sales_train_val = transform(\n",
    "    sales_train_val, [\"item_id\", \"dept_id\", \"cat_id\", \"store_id\", \"state_id\"],\n",
    ").pipe(reduce_mem_usage)\n",
    "\n",
    "sell_prices = transform(sell_prices, [\"item_id\", \"store_id\"]).pipe(\n",
    "    reduce_mem_usage\n",
    ")\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def melt(\n",
    "    sales_train_val, submission, nrows=55_000_000, verbose=True,\n",
    "):\n",
    "    # melt sales data, get it ready for training\n",
    "    id_columns = [\"id\", \"item_id\", \"dept_id\", \"cat_id\", \"store_id\", \"state_id\"]\n",
    "\n",
    "    # get product table.\n",
    "    product = sales_train_val[id_columns]\n",
    "\n",
    "    sales_train_val = sales_train_val.melt(\n",
    "        id_vars=id_columns, var_name=\"d\", value_name=\"demand\",\n",
    "    )\n",
    "\n",
    "    sales_train_val = reduce_mem_usage(sales_train_val, verbose=False)\n",
    "\n",
    "    if verbose:\n",
    "        print(\"melted\")\n",
    "        display(sales_train_val)\n",
    "\n",
    "    # separate test dataframes.\n",
    "    vals = submission[submission[\"id\"].str.endswith(\"validation\")]\n",
    "    evals = submission[submission[\"id\"].str.endswith(\"evaluation\")]\n",
    "\n",
    "    # change column names.\n",
    "    vals.columns = [\"id\"] + [f\"d_{d}\" for d in range(1914, 1914 + DAYS_PRED)]\n",
    "    evals.columns = [\"id\"] + [f\"d_{d}\" for d in range(1942, 1942 + DAYS_PRED)]\n",
    "\n",
    "    # merge with product table\n",
    "    evals[\"id\"] = evals[\"id\"].str.replace(\"_evaluation\", \"_validation\")\n",
    "    vals = vals.merge(product, how=\"left\", on=\"id\")\n",
    "    evals = evals.merge(product, how=\"left\", on=\"id\")\n",
    "    evals[\"id\"] = evals[\"id\"].str.replace(\"_validation\", \"_evaluation\")\n",
    "\n",
    "    if verbose:\n",
    "        print(\"validation\")\n",
    "        display(vals)\n",
    "\n",
    "        print(\"evaluation\")\n",
    "        display(evals)\n",
    "\n",
    "    vals = vals.melt(id_vars=id_columns, var_name=\"d\", value_name=\"demand\")\n",
    "    evals = evals.melt(id_vars=id_columns, var_name=\"d\", value_name=\"demand\")\n",
    "\n",
    "    sales_train_val[\"part\"] = \"train\"\n",
    "    vals[\"part\"] = \"validation\"\n",
    "    evals[\"part\"] = \"evaluation\"\n",
    "\n",
    "    data = pd.concat([sales_train_val, vals, evals], axis=0)\n",
    "\n",
    "    del sales_train_val, vals, evals\n",
    "\n",
    "    data = data.loc[nrows:]\n",
    "\n",
    "    # delete evaluation for now.\n",
    "    data = data[data[\"part\"] != \"evaluation\"]\n",
    "\n",
    "    gc.collect()\n",
    "\n",
    "    if verbose:\n",
    "        print(\"data\")\n",
    "        display(data)\n",
    "\n",
    "    return data\n",
    "\n",
    "\n",
    "def extract_d(df):\n",
    "    return df[\"d\"].str.extract(r\"d_(\\d+)\").astype(np.int16)\n",
    "\n",
    "\n",
    "def merge_calendar(data, calendar):\n",
    "    calendar = calendar.drop([\"weekday\", \"wday\", \"month\", \"year\"], axis=1)\n",
    "    return data.merge(calendar, how=\"left\", on=\"d\").assign(d=extract_d)\n",
    "\n",
    "\n",
    "def merge_sell_prices(data, sell_prices):\n",
    "    return data.merge(sell_prices, how=\"left\", on=[\"store_id\", \"item_id\", \"wm_yr_wk\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "melted\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>d</th>\n",
       "      <th>demand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_validation</td>\n",
       "      <td>1437</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>d_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_validation</td>\n",
       "      <td>1438</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>d_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_validation</td>\n",
       "      <td>1439</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>d_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_validation</td>\n",
       "      <td>1440</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>d_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_validation</td>\n",
       "      <td>1441</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>d_1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58327365</th>\n",
       "      <td>FOODS_3_823_WI_3_validation</td>\n",
       "      <td>1432</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1913</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58327366</th>\n",
       "      <td>FOODS_3_824_WI_3_validation</td>\n",
       "      <td>1433</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1913</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58327367</th>\n",
       "      <td>FOODS_3_825_WI_3_validation</td>\n",
       "      <td>1434</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1913</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58327368</th>\n",
       "      <td>FOODS_3_826_WI_3_validation</td>\n",
       "      <td>1435</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1913</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58327369</th>\n",
       "      <td>FOODS_3_827_WI_3_validation</td>\n",
       "      <td>1436</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1913</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>58327370 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     id  item_id  dept_id  cat_id  store_id  \\\n",
       "0         HOBBIES_1_001_CA_1_validation     1437        3       1         0   \n",
       "1         HOBBIES_1_002_CA_1_validation     1438        3       1         0   \n",
       "2         HOBBIES_1_003_CA_1_validation     1439        3       1         0   \n",
       "3         HOBBIES_1_004_CA_1_validation     1440        3       1         0   \n",
       "4         HOBBIES_1_005_CA_1_validation     1441        3       1         0   \n",
       "...                                 ...      ...      ...     ...       ...   \n",
       "58327365    FOODS_3_823_WI_3_validation     1432        2       0         9   \n",
       "58327366    FOODS_3_824_WI_3_validation     1433        2       0         9   \n",
       "58327367    FOODS_3_825_WI_3_validation     1434        2       0         9   \n",
       "58327368    FOODS_3_826_WI_3_validation     1435        2       0         9   \n",
       "58327369    FOODS_3_827_WI_3_validation     1436        2       0         9   \n",
       "\n",
       "          state_id       d  demand  \n",
       "0                0     d_1       0  \n",
       "1                0     d_1       0  \n",
       "2                0     d_1       0  \n",
       "3                0     d_1       0  \n",
       "4                0     d_1       0  \n",
       "...            ...     ...     ...  \n",
       "58327365         2  d_1913       1  \n",
       "58327366         2  d_1913       0  \n",
       "58327367         2  d_1913       0  \n",
       "58327368         2  d_1913       3  \n",
       "58327369         2  d_1913       0  \n",
       "\n",
       "[58327370 rows x 8 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "validation\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>d_1914</th>\n",
       "      <th>d_1915</th>\n",
       "      <th>d_1916</th>\n",
       "      <th>d_1917</th>\n",
       "      <th>d_1918</th>\n",
       "      <th>d_1919</th>\n",
       "      <th>d_1920</th>\n",
       "      <th>d_1921</th>\n",
       "      <th>d_1922</th>\n",
       "      <th>d_1923</th>\n",
       "      <th>d_1924</th>\n",
       "      <th>d_1925</th>\n",
       "      <th>d_1926</th>\n",
       "      <th>d_1927</th>\n",
       "      <th>d_1928</th>\n",
       "      <th>d_1929</th>\n",
       "      <th>d_1930</th>\n",
       "      <th>d_1931</th>\n",
       "      <th>d_1932</th>\n",
       "      <th>d_1933</th>\n",
       "      <th>d_1934</th>\n",
       "      <th>d_1935</th>\n",
       "      <th>d_1936</th>\n",
       "      <th>d_1937</th>\n",
       "      <th>d_1938</th>\n",
       "      <th>d_1939</th>\n",
       "      <th>d_1940</th>\n",
       "      <th>d_1941</th>\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1437</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1438</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1439</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1440</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1441</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30485</th>\n",
       "      <td>FOODS_3_823_WI_3_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1432</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30486</th>\n",
       "      <td>FOODS_3_824_WI_3_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1433</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30487</th>\n",
       "      <td>FOODS_3_825_WI_3_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1434</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30488</th>\n",
       "      <td>FOODS_3_826_WI_3_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1435</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30489</th>\n",
       "      <td>FOODS_3_827_WI_3_validation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1436</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30490 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  id  d_1914  d_1915  d_1916  d_1917  d_1918  \\\n",
       "0      HOBBIES_1_001_CA_1_validation       0       0       0       0       0   \n",
       "1      HOBBIES_1_002_CA_1_validation       0       0       0       0       0   \n",
       "2      HOBBIES_1_003_CA_1_validation       0       0       0       0       0   \n",
       "3      HOBBIES_1_004_CA_1_validation       0       0       0       0       0   \n",
       "4      HOBBIES_1_005_CA_1_validation       0       0       0       0       0   \n",
       "...                              ...     ...     ...     ...     ...     ...   \n",
       "30485    FOODS_3_823_WI_3_validation       0       0       0       0       0   \n",
       "30486    FOODS_3_824_WI_3_validation       0       0       0       0       0   \n",
       "30487    FOODS_3_825_WI_3_validation       0       0       0       0       0   \n",
       "30488    FOODS_3_826_WI_3_validation       0       0       0       0       0   \n",
       "30489    FOODS_3_827_WI_3_validation       0       0       0       0       0   \n",
       "\n",
       "       d_1919  d_1920  d_1921  d_1922  d_1923  d_1924  d_1925  d_1926  d_1927  \\\n",
       "0           0       0       0       0       0       0       0       0       0   \n",
       "1           0       0       0       0       0       0       0       0       0   \n",
       "2           0       0       0       0       0       0       0       0       0   \n",
       "3           0       0       0       0       0       0       0       0       0   \n",
       "4           0       0       0       0       0       0       0       0       0   \n",
       "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "30485       0       0       0       0       0       0       0       0       0   \n",
       "30486       0       0       0       0       0       0       0       0       0   \n",
       "30487       0       0       0       0       0       0       0       0       0   \n",
       "30488       0       0       0       0       0       0       0       0       0   \n",
       "30489       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "       d_1928  d_1929  d_1930  d_1931  d_1932  d_1933  d_1934  d_1935  d_1936  \\\n",
       "0           0       0       0       0       0       0       0       0       0   \n",
       "1           0       0       0       0       0       0       0       0       0   \n",
       "2           0       0       0       0       0       0       0       0       0   \n",
       "3           0       0       0       0       0       0       0       0       0   \n",
       "4           0       0       0       0       0       0       0       0       0   \n",
       "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "30485       0       0       0       0       0       0       0       0       0   \n",
       "30486       0       0       0       0       0       0       0       0       0   \n",
       "30487       0       0       0       0       0       0       0       0       0   \n",
       "30488       0       0       0       0       0       0       0       0       0   \n",
       "30489       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "       d_1937  d_1938  d_1939  d_1940  d_1941  item_id  dept_id  cat_id  \\\n",
       "0           0       0       0       0       0     1437        3       1   \n",
       "1           0       0       0       0       0     1438        3       1   \n",
       "2           0       0       0       0       0     1439        3       1   \n",
       "3           0       0       0       0       0     1440        3       1   \n",
       "4           0       0       0       0       0     1441        3       1   \n",
       "...       ...     ...     ...     ...     ...      ...      ...     ...   \n",
       "30485       0       0       0       0       0     1432        2       0   \n",
       "30486       0       0       0       0       0     1433        2       0   \n",
       "30487       0       0       0       0       0     1434        2       0   \n",
       "30488       0       0       0       0       0     1435        2       0   \n",
       "30489       0       0       0       0       0     1436        2       0   \n",
       "\n",
       "       store_id  state_id  \n",
       "0             0         0  \n",
       "1             0         0  \n",
       "2             0         0  \n",
       "3             0         0  \n",
       "4             0         0  \n",
       "...         ...       ...  \n",
       "30485         9         2  \n",
       "30486         9         2  \n",
       "30487         9         2  \n",
       "30488         9         2  \n",
       "30489         9         2  \n",
       "\n",
       "[30490 rows x 34 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluation\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>d_1942</th>\n",
       "      <th>d_1943</th>\n",
       "      <th>d_1944</th>\n",
       "      <th>d_1945</th>\n",
       "      <th>d_1946</th>\n",
       "      <th>d_1947</th>\n",
       "      <th>d_1948</th>\n",
       "      <th>d_1949</th>\n",
       "      <th>d_1950</th>\n",
       "      <th>d_1951</th>\n",
       "      <th>d_1952</th>\n",
       "      <th>d_1953</th>\n",
       "      <th>d_1954</th>\n",
       "      <th>d_1955</th>\n",
       "      <th>d_1956</th>\n",
       "      <th>d_1957</th>\n",
       "      <th>d_1958</th>\n",
       "      <th>d_1959</th>\n",
       "      <th>d_1960</th>\n",
       "      <th>d_1961</th>\n",
       "      <th>d_1962</th>\n",
       "      <th>d_1963</th>\n",
       "      <th>d_1964</th>\n",
       "      <th>d_1965</th>\n",
       "      <th>d_1966</th>\n",
       "      <th>d_1967</th>\n",
       "      <th>d_1968</th>\n",
       "      <th>d_1969</th>\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1437</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1438</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1439</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1440</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1441</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30485</th>\n",
       "      <td>FOODS_3_823_WI_3_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1432</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30486</th>\n",
       "      <td>FOODS_3_824_WI_3_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1433</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30487</th>\n",
       "      <td>FOODS_3_825_WI_3_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1434</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30488</th>\n",
       "      <td>FOODS_3_826_WI_3_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1435</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30489</th>\n",
       "      <td>FOODS_3_827_WI_3_evaluation</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1436</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>30490 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  id  d_1942  d_1943  d_1944  d_1945  d_1946  \\\n",
       "0      HOBBIES_1_001_CA_1_evaluation       0       0       0       0       0   \n",
       "1      HOBBIES_1_002_CA_1_evaluation       0       0       0       0       0   \n",
       "2      HOBBIES_1_003_CA_1_evaluation       0       0       0       0       0   \n",
       "3      HOBBIES_1_004_CA_1_evaluation       0       0       0       0       0   \n",
       "4      HOBBIES_1_005_CA_1_evaluation       0       0       0       0       0   \n",
       "...                              ...     ...     ...     ...     ...     ...   \n",
       "30485    FOODS_3_823_WI_3_evaluation       0       0       0       0       0   \n",
       "30486    FOODS_3_824_WI_3_evaluation       0       0       0       0       0   \n",
       "30487    FOODS_3_825_WI_3_evaluation       0       0       0       0       0   \n",
       "30488    FOODS_3_826_WI_3_evaluation       0       0       0       0       0   \n",
       "30489    FOODS_3_827_WI_3_evaluation       0       0       0       0       0   \n",
       "\n",
       "       d_1947  d_1948  d_1949  d_1950  d_1951  d_1952  d_1953  d_1954  d_1955  \\\n",
       "0           0       0       0       0       0       0       0       0       0   \n",
       "1           0       0       0       0       0       0       0       0       0   \n",
       "2           0       0       0       0       0       0       0       0       0   \n",
       "3           0       0       0       0       0       0       0       0       0   \n",
       "4           0       0       0       0       0       0       0       0       0   \n",
       "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "30485       0       0       0       0       0       0       0       0       0   \n",
       "30486       0       0       0       0       0       0       0       0       0   \n",
       "30487       0       0       0       0       0       0       0       0       0   \n",
       "30488       0       0       0       0       0       0       0       0       0   \n",
       "30489       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "       d_1956  d_1957  d_1958  d_1959  d_1960  d_1961  d_1962  d_1963  d_1964  \\\n",
       "0           0       0       0       0       0       0       0       0       0   \n",
       "1           0       0       0       0       0       0       0       0       0   \n",
       "2           0       0       0       0       0       0       0       0       0   \n",
       "3           0       0       0       0       0       0       0       0       0   \n",
       "4           0       0       0       0       0       0       0       0       0   \n",
       "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "30485       0       0       0       0       0       0       0       0       0   \n",
       "30486       0       0       0       0       0       0       0       0       0   \n",
       "30487       0       0       0       0       0       0       0       0       0   \n",
       "30488       0       0       0       0       0       0       0       0       0   \n",
       "30489       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "       d_1965  d_1966  d_1967  d_1968  d_1969  item_id  dept_id  cat_id  \\\n",
       "0           0       0       0       0       0     1437        3       1   \n",
       "1           0       0       0       0       0     1438        3       1   \n",
       "2           0       0       0       0       0     1439        3       1   \n",
       "3           0       0       0       0       0     1440        3       1   \n",
       "4           0       0       0       0       0     1441        3       1   \n",
       "...       ...     ...     ...     ...     ...      ...      ...     ...   \n",
       "30485       0       0       0       0       0     1432        2       0   \n",
       "30486       0       0       0       0       0     1433        2       0   \n",
       "30487       0       0       0       0       0     1434        2       0   \n",
       "30488       0       0       0       0       0     1435        2       0   \n",
       "30489       0       0       0       0       0     1436        2       0   \n",
       "\n",
       "       store_id  state_id  \n",
       "0             0         0  \n",
       "1             0         0  \n",
       "2             0         0  \n",
       "3             0         0  \n",
       "4             0         0  \n",
       "...         ...       ...  \n",
       "30485         9         2  \n",
       "30486         9         2  \n",
       "30487         9         2  \n",
       "30488         9         2  \n",
       "30489         9         2  \n",
       "\n",
       "[30490 rows x 34 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>d</th>\n",
       "      <th>demand</th>\n",
       "      <th>part</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27000000</th>\n",
       "      <td>HOUSEHOLD_2_009_TX_2_validation</td>\n",
       "      <td>2542</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>d_886</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27000001</th>\n",
       "      <td>HOUSEHOLD_2_010_TX_2_validation</td>\n",
       "      <td>2543</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>d_886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27000002</th>\n",
       "      <td>HOUSEHOLD_2_011_TX_2_validation</td>\n",
       "      <td>2544</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>d_886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27000003</th>\n",
       "      <td>HOUSEHOLD_2_012_TX_2_validation</td>\n",
       "      <td>2545</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>d_886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27000004</th>\n",
       "      <td>HOUSEHOLD_2_013_TX_2_validation</td>\n",
       "      <td>2546</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>d_886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853715</th>\n",
       "      <td>FOODS_3_823_WI_3_validation</td>\n",
       "      <td>1432</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1941</td>\n",
       "      <td>0</td>\n",
       "      <td>validation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853716</th>\n",
       "      <td>FOODS_3_824_WI_3_validation</td>\n",
       "      <td>1433</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1941</td>\n",
       "      <td>0</td>\n",
       "      <td>validation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853717</th>\n",
       "      <td>FOODS_3_825_WI_3_validation</td>\n",
       "      <td>1434</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1941</td>\n",
       "      <td>0</td>\n",
       "      <td>validation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853718</th>\n",
       "      <td>FOODS_3_826_WI_3_validation</td>\n",
       "      <td>1435</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1941</td>\n",
       "      <td>0</td>\n",
       "      <td>validation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>853719</th>\n",
       "      <td>FOODS_3_827_WI_3_validation</td>\n",
       "      <td>1436</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>d_1941</td>\n",
       "      <td>0</td>\n",
       "      <td>validation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>32181090 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       id  item_id  dept_id  cat_id  store_id  \\\n",
       "27000000  HOUSEHOLD_2_009_TX_2_validation     2542        6       2         5   \n",
       "27000001  HOUSEHOLD_2_010_TX_2_validation     2543        6       2         5   \n",
       "27000002  HOUSEHOLD_2_011_TX_2_validation     2544        6       2         5   \n",
       "27000003  HOUSEHOLD_2_012_TX_2_validation     2545        6       2         5   \n",
       "27000004  HOUSEHOLD_2_013_TX_2_validation     2546        6       2         5   \n",
       "...                                   ...      ...      ...     ...       ...   \n",
       "853715        FOODS_3_823_WI_3_validation     1432        2       0         9   \n",
       "853716        FOODS_3_824_WI_3_validation     1433        2       0         9   \n",
       "853717        FOODS_3_825_WI_3_validation     1434        2       0         9   \n",
       "853718        FOODS_3_826_WI_3_validation     1435        2       0         9   \n",
       "853719        FOODS_3_827_WI_3_validation     1436        2       0         9   \n",
       "\n",
       "          state_id       d  demand        part  \n",
       "27000000         1   d_886       1       train  \n",
       "27000001         1   d_886       0       train  \n",
       "27000002         1   d_886       0       train  \n",
       "27000003         1   d_886       0       train  \n",
       "27000004         1   d_886       0       train  \n",
       "...            ...     ...     ...         ...  \n",
       "853715           2  d_1941       0  validation  \n",
       "853716           2  d_1941       0  validation  \n",
       "853717           2  d_1941       0  validation  \n",
       "853718           2  d_1941       0  validation  \n",
       "853719           2  d_1941       0  validation  \n",
       "\n",
       "[32181090 rows x 9 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to 1749.35 Mb (0.0% reduction)\n"
     ]
    }
   ],
   "source": [
    "data = melt(sales_train_val, submission, nrows=27_000_000)\n",
    "del sales_train_val\n",
    "gc.collect()\n",
    "\n",
    "data = merge_calendar(data, calendar)\n",
    "del calendar\n",
    "gc.collect()\n",
    "\n",
    "data = merge_sell_prices(data, sell_prices)\n",
    "del sell_prices\n",
    "gc.collect()\n",
    "\n",
    "data = reduce_mem_usage(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start date: 2011-12-22\n",
      "end date: 2016-05-22\n",
      "data shape: (49181090, 19)\n"
     ]
    }
   ],
   "source": [
    "print(\"start date:\", data[\"date\"].min())\n",
    "print(\"end date:\", data[\"date\"].max())\n",
    "print(\"data shape:\", data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_engineering(data):\n",
    "    \n",
    "    # demand features\n",
    "    data['lag_t28'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28))\n",
    "    data['lag_t29'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(29))\n",
    "    data['lag_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(30))\n",
    "    # Mean\n",
    "    data['rolling_mean_t7'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(7).mean())\n",
    "    data['rolling_mean_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(30).mean())\n",
    "    data['rolling_mean_t60'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(60).mean())\n",
    "    data['rolling_mean_t90'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(90).mean())\n",
    "    data['rolling_mean_t180'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(180).mean())\n",
    "    # Std\n",
    "    data['rolling_std_t7'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(7).std())\n",
    "    data['rolling_std_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(30).std())\n",
    "    data['rolling_std_t60'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(60).std())\n",
    "    data['rolling_std_t90'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(90).std())\n",
    "    data['rolling_std_t180'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(DAYS_PRED).rolling(180).std())\n",
    "    # others\n",
    "    data[\"rolling_skew_t30\"] = data.groupby([\"id\"])[\"demand\"].transform(lambda x: x.shift(DAYS_PRED).rolling(30).skew())\n",
    "    data[\"rolling_kurt_t30\"] = data.groupby([\"id\"])[\"demand\"].transform(lambda x: x.shift(DAYS_PRED).rolling(30).kurt())\n",
    "    \n",
    "    # price features\n",
    "    data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))\n",
    "    data['price_change_t1'] = (data['lag_price_t1'] - data['sell_price']) / (data['lag_price_t1'])\n",
    "    data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1).rolling(365).max())\n",
    "    data['price_change_t365'] = (data['rolling_price_max_t365'] - data['sell_price']) / (data['rolling_price_max_t365'])\n",
    "    data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7).std())\n",
    "    data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30).std())\n",
    "    data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1)\n",
    "    \n",
    "    # time features\n",
    "    data['date'] = pd.to_datetime(data['date'])\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to 2915.58 Mb (48.6% reduction)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = feature_engineering(data).pipe(reduce_mem_usage)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_datepart(df, fldname, drop=True, time=False):\n",
    "    \"Helper function that adds columns relevant to a date.\"\n",
    "    fld = df[fldname]\n",
    "    fld_dtype = fld.dtype\n",
    "    if isinstance(fld_dtype, pd.core.dtypes.dtypes.DatetimeTZDtype):\n",
    "        fld_dtype = np.datetime64\n",
    "\n",
    "    if not np.issubdtype(fld_dtype, np.datetime64):\n",
    "        df[fldname] = fld = pd.to_datetime(fld, infer_datetime_format=True)\n",
    "    targ_pre = re.sub('[Dd]ate$', '', fldname)\n",
    "    attr = ['Year', 'Month', 'Quarter','Week', 'Day', 'Dayofweek', 'Dayofyear',\n",
    "            'Is_month_end', 'Is_month_start', 'Is_quarter_end', 'Is_quarter_start', 'Is_year_end', 'Is_year_start']\n",
    "    if time: attr = attr + ['Hour', 'Minute', 'Second']\n",
    "    for n in attr: df[targ_pre + n] = getattr(fld.dt, n.lower())\n",
    "    df[targ_pre + 'Elapsed'] = fld.astype(np.int64) // 10 ** 9\n",
    "    if drop: df.drop(fldname, axis=1, inplace=True)\n",
    "    df[\"is_weekend\"] = df[\"Dayofweek\"].isin([5, 6]).astype(np.int8)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mem. usage decreased to 3529.38 Mb (30.7% reduction)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = add_datepart(data, \"date\", drop=False).pipe(reduce_mem_usage)\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>d</th>\n",
       "      <th>demand</th>\n",
       "      <th>part</th>\n",
       "      <th>date</th>\n",
       "      <th>wm_yr_wk</th>\n",
       "      <th>event_name_1</th>\n",
       "      <th>event_type_1</th>\n",
       "      <th>event_name_2</th>\n",
       "      <th>event_type_2</th>\n",
       "      <th>snap_CA</th>\n",
       "      <th>snap_TX</th>\n",
       "      <th>snap_WI</th>\n",
       "      <th>sell_price</th>\n",
       "      <th>lag_t28</th>\n",
       "      <th>lag_t29</th>\n",
       "      <th>lag_t30</th>\n",
       "      <th>rolling_mean_t7</th>\n",
       "      <th>rolling_mean_t30</th>\n",
       "      <th>rolling_mean_t60</th>\n",
       "      <th>rolling_mean_t90</th>\n",
       "      <th>rolling_mean_t180</th>\n",
       "      <th>rolling_std_t7</th>\n",
       "      <th>rolling_std_t30</th>\n",
       "      <th>rolling_std_t60</th>\n",
       "      <th>rolling_std_t90</th>\n",
       "      <th>rolling_std_t180</th>\n",
       "      <th>rolling_skew_t30</th>\n",
       "      <th>rolling_kurt_t30</th>\n",
       "      <th>price_change_t1</th>\n",
       "      <th>price_change_t365</th>\n",
       "      <th>rolling_price_std_t7</th>\n",
       "      <th>rolling_price_std_t30</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Quarter</th>\n",
       "      <th>Week</th>\n",
       "      <th>Day</th>\n",
       "      <th>Dayofweek</th>\n",
       "      <th>Dayofyear</th>\n",
       "      <th>Is_month_end</th>\n",
       "      <th>Is_month_start</th>\n",
       "      <th>Is_quarter_end</th>\n",
       "      <th>Is_quarter_start</th>\n",
       "      <th>Is_year_end</th>\n",
       "      <th>Is_year_start</th>\n",
       "      <th>Elapsed</th>\n",
       "      <th>is_weekend</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOUSEHOLD_2_009_TX_2_validation</td>\n",
       "      <td>2542</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>886</td>\n",
       "      <td>1</td>\n",
       "      <td>train</td>\n",
       "      <td>2013-07-02</td>\n",
       "      <td>11323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.941406</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1372723200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOUSEHOLD_2_010_TX_2_validation</td>\n",
       "      <td>2543</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>2013-07-02</td>\n",
       "      <td>11323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6.968750</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1372723200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOUSEHOLD_2_011_TX_2_validation</td>\n",
       "      <td>2544</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>2013-07-02</td>\n",
       "      <td>11323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.621094</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1372723200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOUSEHOLD_2_012_TX_2_validation</td>\n",
       "      <td>2545</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>2013-07-02</td>\n",
       "      <td>11323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4.429688</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1372723200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOUSEHOLD_2_013_TX_2_validation</td>\n",
       "      <td>2546</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>886</td>\n",
       "      <td>0</td>\n",
       "      <td>train</td>\n",
       "      <td>2013-07-02</td>\n",
       "      <td>11323</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.941406</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2013</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>1372723200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                id  item_id  dept_id  cat_id  store_id  \\\n",
       "0  HOUSEHOLD_2_009_TX_2_validation     2542        6       2         5   \n",
       "1  HOUSEHOLD_2_010_TX_2_validation     2543        6       2         5   \n",
       "2  HOUSEHOLD_2_011_TX_2_validation     2544        6       2         5   \n",
       "3  HOUSEHOLD_2_012_TX_2_validation     2545        6       2         5   \n",
       "4  HOUSEHOLD_2_013_TX_2_validation     2546        6       2         5   \n",
       "\n",
       "   state_id    d  demand   part       date  wm_yr_wk  event_name_1  \\\n",
       "0         1  886       1  train 2013-07-02     11323           NaN   \n",
       "1         1  886       0  train 2013-07-02     11323           NaN   \n",
       "2         1  886       0  train 2013-07-02     11323           NaN   \n",
       "3         1  886       0  train 2013-07-02     11323           NaN   \n",
       "4         1  886       0  train 2013-07-02     11323           NaN   \n",
       "\n",
       "   event_type_1  event_name_2  event_type_2  snap_CA  snap_TX  snap_WI  \\\n",
       "0           NaN           NaN           NaN        1        0        1   \n",
       "1           NaN           NaN           NaN        1        0        1   \n",
       "2           NaN           NaN           NaN        1        0        1   \n",
       "3           NaN           NaN           NaN        1        0        1   \n",
       "4           NaN           NaN           NaN        1        0        1   \n",
       "\n",
       "   sell_price  lag_t28  lag_t29  lag_t30  rolling_mean_t7  rolling_mean_t30  \\\n",
       "0    5.941406      NaN      NaN      NaN              NaN               NaN   \n",
       "1    6.968750      NaN      NaN      NaN              NaN               NaN   \n",
       "2    5.621094      NaN      NaN      NaN              NaN               NaN   \n",
       "3    4.429688      NaN      NaN      NaN              NaN               NaN   \n",
       "4    5.941406      NaN      NaN      NaN              NaN               NaN   \n",
       "\n",
       "   rolling_mean_t60  rolling_mean_t90  rolling_mean_t180  rolling_std_t7  \\\n",
       "0               NaN               NaN                NaN             NaN   \n",
       "1               NaN               NaN                NaN             NaN   \n",
       "2               NaN               NaN                NaN             NaN   \n",
       "3               NaN               NaN                NaN             NaN   \n",
       "4               NaN               NaN                NaN             NaN   \n",
       "\n",
       "   rolling_std_t30  rolling_std_t60  rolling_std_t90  rolling_std_t180  \\\n",
       "0              NaN              NaN              NaN               NaN   \n",
       "1              NaN              NaN              NaN               NaN   \n",
       "2              NaN              NaN              NaN               NaN   \n",
       "3              NaN              NaN              NaN               NaN   \n",
       "4              NaN              NaN              NaN               NaN   \n",
       "\n",
       "   rolling_skew_t30  rolling_kurt_t30  price_change_t1  price_change_t365  \\\n",
       "0               NaN               NaN              NaN                NaN   \n",
       "1               NaN               NaN              NaN                NaN   \n",
       "2               NaN               NaN              NaN                NaN   \n",
       "3               NaN               NaN              NaN                NaN   \n",
       "4               NaN               NaN              NaN                NaN   \n",
       "\n",
       "   rolling_price_std_t7  rolling_price_std_t30  Year  Month  Quarter  Week  \\\n",
       "0                   NaN                    NaN  2013      7        3    27   \n",
       "1                   NaN                    NaN  2013      7        3    27   \n",
       "2                   NaN                    NaN  2013      7        3    27   \n",
       "3                   NaN                    NaN  2013      7        3    27   \n",
       "4                   NaN                    NaN  2013      7        3    27   \n",
       "\n",
       "   Day  Dayofweek  Dayofyear  Is_month_end  Is_month_start  Is_quarter_end  \\\n",
       "0    2          1        183         False           False           False   \n",
       "1    2          1        183         False           False           False   \n",
       "2    2          1        183         False           False           False   \n",
       "3    2          1        183         False           False           False   \n",
       "4    2          1        183         False           False           False   \n",
       "\n",
       "   Is_quarter_start  Is_year_end  Is_year_start     Elapsed  is_weekend  \n",
       "0             False        False          False  1372723200           0  \n",
       "1             False        False          False  1372723200           0  \n",
       "2             False        False          False  1372723200           0  \n",
       "3             False        False          False  1372723200           0  \n",
       "4             False        False          False  1372723200           0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "start date: 2013-07-02 00:00:00\n",
      "end date: 2016-05-22 00:00:00\n",
      "data shape: (32181090, 53)\n"
     ]
    }
   ],
   "source": [
    "data = data.sort_values(\"date\")\n",
    "\n",
    "print(\"start date:\", data[\"date\"].min())\n",
    "print(\"end date:\", data[\"date\"].max())\n",
    "print(\"data shape:\", data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomTimeSeriesSplitter:\n",
    "    def __init__(self, n_splits=5, train_days=80, test_days=20, day_col=\"d\"):\n",
    "        self.n_splits = n_splits\n",
    "        self.train_days = train_days\n",
    "        self.test_days = test_days\n",
    "        self.day_col = day_col\n",
    "\n",
    "    def split(self, X, y=None, groups=None):\n",
    "        SEC_IN_DAY = 3600 * 24\n",
    "        sec = (X[self.day_col] - X[self.day_col].iloc[0]) * SEC_IN_DAY\n",
    "        duration = sec.max()\n",
    "\n",
    "        train_sec = self.train_days * SEC_IN_DAY\n",
    "        test_sec = self.test_days * SEC_IN_DAY\n",
    "        total_sec = test_sec + train_sec\n",
    "\n",
    "        if self.n_splits == 1:\n",
    "            train_start = duration - total_sec\n",
    "            train_end = train_start + train_sec\n",
    "\n",
    "            train_mask = (sec >= train_start) & (sec < train_end)\n",
    "            test_mask = sec >= train_end\n",
    "\n",
    "            yield sec[train_mask].index.values, sec[test_mask].index.values\n",
    "\n",
    "        else:\n",
    "            # step = (duration - total_sec) / (self.n_splits - 1)\n",
    "            step = DAYS_PRED * SEC_IN_DAY\n",
    "\n",
    "            for idx in range(self.n_splits):\n",
    "                # train_start = idx * step\n",
    "                shift = (self.n_splits - (idx + 1)) * step\n",
    "                train_start = duration - total_sec - shift\n",
    "                train_end = train_start + train_sec\n",
    "                test_end = train_end + test_sec\n",
    "\n",
    "                train_mask = (sec > train_start) & (sec <= train_end)\n",
    "\n",
    "                if idx == self.n_splits - 1:\n",
    "                    test_mask = sec > train_end\n",
    "                else:\n",
    "                    test_mask = (sec > train_end) & (sec <= test_end)\n",
    "\n",
    "                yield sec[train_mask].index.values, sec[test_mask].index.values\n",
    "\n",
    "    def get_n_splits(self):\n",
    "        return self.n_splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "day_col = \"d\"\n",
    "cv_params = {\n",
    "    \"n_splits\": 2,\n",
    "    \"train_days\": 365 * 2,\n",
    "    \"test_days\": DAYS_PRED,\n",
    "    \"day_col\": day_col,\n",
    "}\n",
    "cv = CustomTimeSeriesSplitter(**cv_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_cv_days(cv, X, dt_col, day_col):\n",
    "    for ii, (tr, tt) in enumerate(cv.split(X)):\n",
    "        print(f\"\\n----- Fold: ({ii + 1} / {cv.n_splits}) -----\\n\")\n",
    "        tr_start = X.iloc[tr][dt_col].min()\n",
    "        tr_end = X.iloc[tr][dt_col].max()\n",
    "        tr_days = X.iloc[tr][day_col].max() - X.iloc[tr][day_col].min() + 1\n",
    "\n",
    "        tt_start = X.iloc[tt][dt_col].min()\n",
    "        tt_end = X.iloc[tt][dt_col].max()\n",
    "        tt_days = X.iloc[tt][day_col].max() - X.iloc[tt][day_col].min() + 1\n",
    "\n",
    "        df = pd.DataFrame(\n",
    "            {\n",
    "                \"start\": [tr_start, tt_start],\n",
    "                \"end\": [tr_end, tt_end],\n",
    "                \"days\": [tr_days, tt_days],\n",
    "            },\n",
    "            index=[\"train\", \"test\"],\n",
    "        )\n",
    "\n",
    "        display(df)\n",
    "\n",
    "\n",
    "def plot_cv_indices(cv, X, dt_col, lw=10):\n",
    "    n_splits = cv.get_n_splits()\n",
    "    _, ax = plt.subplots(figsize=(20, n_splits))\n",
    "\n",
    "    # Generate the training/testing visualizations for each CV split\n",
    "    for ii, (tr, tt) in enumerate(cv.split(X)):\n",
    "        # Fill in indices with the training/test groups\n",
    "        indices = np.array([np.nan] * len(X))\n",
    "        indices[tt] = 1\n",
    "        indices[tr] = 0\n",
    "\n",
    "        # Visualize the results\n",
    "        ax.scatter(\n",
    "            X[dt_col],\n",
    "            [ii + 0.5] * len(indices),\n",
    "            c=indices,\n",
    "            marker=\"_\",\n",
    "            lw=lw,\n",
    "            cmap=plt.cm.coolwarm,\n",
    "            vmin=-0.2,\n",
    "            vmax=1.2,\n",
    "        )\n",
    "\n",
    "    # Formatting\n",
    "    MIDDLE = 15\n",
    "    LARGE = 20\n",
    "    ax.set_xlabel(\"Datetime\", fontsize=LARGE)\n",
    "    ax.set_xlim([X[dt_col].min(), X[dt_col].max()])\n",
    "    ax.set_ylabel(\"CV iteration\", fontsize=LARGE)\n",
    "    ax.set_yticks(np.arange(n_splits) + 0.5)\n",
    "    ax.set_yticklabels(list(range(n_splits)))\n",
    "    ax.invert_yaxis()\n",
    "    ax.tick_params(axis=\"both\", which=\"major\", labelsize=MIDDLE)\n",
    "    ax.set_title(\"{}\".format(type(cv).__name__), fontsize=LARGE)\n",
    "    return ax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----- Fold: (1 / 2) -----\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>days</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>2014-03-29</td>\n",
       "      <td>2016-03-27</td>\n",
       "      <td>730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>2016-03-28</td>\n",
       "      <td>2016-04-24</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start        end  days\n",
       "train 2014-03-29 2016-03-27   730\n",
       "test  2016-03-28 2016-04-24    28"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "----- Fold: (2 / 2) -----\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>days</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train</th>\n",
       "      <td>2014-04-26</td>\n",
       "      <td>2016-04-24</td>\n",
       "      <td>730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test</th>\n",
       "      <td>2016-04-25</td>\n",
       "      <td>2016-05-22</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start        end  days\n",
       "train 2014-04-26 2016-04-24   730\n",
       "test  2016-04-25 2016-05-22    28"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABR0AAAC5CAYAAAClSZi1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XeYLGWZ9/HvjySGlSNiThhQzLpiYJXlKCLKa0RdEQxgWOMrht01LAqii6uua8A1rIrIGvAVE7KiKIioKyIKoiLKggdBBUWS5HS/fzw10jQ9M90zPTOn+3w/11VXTVU9VXV3zcOp5p4npKqQJEmSJEmSpHFZb6UDkCRJkiRJkjRdTDpKkiRJkiRJGiuTjpIkSZIkSZLGyqSjJEmSJEmSpLEy6ShJkiRJkiRprEw6SpIkSZIkSRork46SJElTIMmaJGtWOo610TQ+mySbJ6kkB/TtP6Dbv/mKBCZJktQx6ShJkpZVki2T7JfkZ0kuSHJFkt8l+e8kz0+y8TLFUUmOWo57DasnkTTKsnql455LklVJ9klyQpKLklye5LdJjknyriQPXOkYl0KSeyT5SJL/TXJpkouT/DrJ4UnelORWyxjLwARlz/HV3fG9lysmSZI0/TZY6QAkSdK6I8mbgL1of/g8BvgEcBFwK2A18FHgJcBWKxTiSjsfePOA/Xt160HH1nTr7ZYioMVIclvge8DmwGnAp4BzgdsBWwKvBC4Fjl/iUJb12SR5FPDfwMbA94GvAZfQnsMDgO2B/wHOXoLbvx74V+C3S3BtSZKkoZl0lCRJyyLJG2hJszOAp1fVDwaUeTzwmuWObW1RVecDe/fvT7JXd/x6x3rOPXXJAlu4fWiJtv2BF1RV9R5MchvgNksdxAo8mw/TEo67VdUn+g8muR9w3lLcuKp+D/x+Ka4tSZI0CrtXS5KkJdeNL7c3cCWw46CEI0BVHQo8tjtnzi6fg8bpS7JRklck+XGS85Jc0pX7cpJHd2V2SzKT/Nq2r6vy3n3X+7skR3fdwC9N8tMkr09yg9niSXKTJO9OckZ3zglJntyV2SDJG5KckuSyJKcmefmQj3FOszyP3brPtVuS7ZN8p+vi/MckH0+yqiv3wCSHds/soiSHzDYmYJJNk7wtyS+6z3dBkiOSPGZA8b/p1vv1JxyhJciq6scD7nGj7jmf0HVLvijJ95M8c0DZv9STJA/puumfm55xDQc9m57zn5nkW91nv6z7XHvO8jveJslXkpzZdRM/q+smvldPmVsCdwMuGJRw7D73iVV1Rt+1Z+rPJkne33VBvyzJSV2dzqBrDYjxgL7Pvjfw6+7wc/vq+25dl+tvdcf3yhxd90d8VpXkqCS3TvLR7vNcnWS3YT6HJEmafLZ0lCRJy2F3YEPgoKr62VwFq+ryRdznAOCZwM+AA2ldd28LPIKWzPwmcAKtxeVewOndOTOOmvkhyb60rqrnAJ+mdQN/HLAvsEOS7avqyr77bwh8A9gU+DKwURfP57uk3EuBhwKHAZcDTwf2S/LHqvrsIj73fJ4IPB44FPgQLRm4G3DnJK8DjgC+A3wMuC/wBOCuSe5bVdfMXCTJnWjPaPOu/NeAG3fX/lqSF1XVR3ru+6dufXfac59Xlwg9Engg8GNaK8n1gB2ATye5d1XtOeDUrWm/r+9252wGXDHPvT4GPA84E/gCrXv7w4C3ANt1v+OrurKPpXWZvhA4hNZ9eVPgnrTf60zX9wuAq4CbJLlN1/JwWBvR6ugq4KBu+6nAe4F7AC8b4VozjuqutwfwE+BLPcdOoH1mgOcC36bnvwGu7bo/0rPqsSltGIWLunOuYWm6lEuSpLVRVbm4uLi4uLi4LOlCS2oVrYvtsOes7s7Ze5bja4A1Pdub0JIaxwHrDyh/877tAo6a5dpbd8d/A9y6Z/8GwFe6Y28YEE91x2/Qs3+bbv+5wA+BVT3H7kJLjB0/z7Oo9rVtzjLXeR7dvt26c68Ctu3Zvx4tOToT1659532sO/akvv1Hdc945779q2gJrEuBW/Xsf3l3nQuBdwCP7v89DPgcB3Tn/FPf/o1pSc5rgAcMqCcFvGgBz+YLwA37ju3dHdujZ9/nu333H3D9zfq2D+7Kngr8Ay3RfKMhfn9FS5r21p9Nu+sU8Lc9+zfv9h0wy/PbfL6yI/y3NtKz6q2ztOT/BnN9dhcXFxcXF5fpXOxeLUmSlsPMuH1nLuE9CgitBeE11ztY9afrnTG753Xrt1bVWT3XuIo25uQ1wAtmOfeV1dNas6q+Q+veejPgtdXGbZw5dhptopX7Jll/hPhG9Zmq+nbPfa8B/qvb/FlVfaqv/IHd+gEzO5LcH9gW+HxVHdRbuPtMe9ESg0/tOfQfwNtoLUD/kZboPCdtFuePdNf8iyQ3B54FHFdV7+i7x2XAa2m/410GfMYTqurDs3z+QfagJWOfV1WX9h17C62V5q4DzusvS1Wd07frhbQE3Z2Bd9Ja+/05yU+SvDVzz1z9+r76c24XD7QWwythoc/qCuAf6votICVJ0jrA7tWSJGk5zIxHd71x/calqi5M8hVa1+ATknye1gX4B1V1yYiX++tufeSA+/wqyZm0rsmrepOIwPk1eNKS39ESUD8acOy3wPrArVm6GYePmyUm5ogJ4PY9+7bu1ptk8Dibt+jW95zZUVUFvCHJO2jdox9Ge7YPpSVtd0/ykrq2S/aDac9itrE8N+y/R49jB+wbKMmNgPvTus6/cpbhEi/vu8+ngJ2AHyT5LG0cxO9V1fUS6VV1HvDUblzFHWizsT8YuF+3vCTJY6vqh32nXkWb1brfUd36gUN8vLFa4LOasaaq/rCE4UmSpLWYSUdJkrQcfgdsyXWTWEvhGbTWcLtw7Rh7lyU5mNbiatjx5Dbp1rONx/d74I5dud6k4wWzlL8KoKoGHZ9pBbbhgGPjMtd9h43p5t16+26ZzU36d3SJ2c92C0luDLwO2JM2puUh3e9m5h4P7pah7wGcNWDfbG5GS4TfgtZCc15V9YVcO7v684AXAST5Ea114jcGnLOGNpP1h7uytwc+QEuMf4SelqSdc6rq6gG3n/lsmww4ttRGflY9RvmdSJKkKWP3akmStBy+2623G+GcmS7Ss/2R9HoJmKq6tKr2rqq705KCz+ru/SzaOHvDmknE3XqW47fpK7cumPmse1RV5ljm7QJcVRdX1Rvpxi8EHt53j3fPc49HDrrsAj7L8fPc5zrN+qrqv6vqUbRE3HbAu4F7A4cmudcQn/tMYGdat+P7d93Je202Szf7mXq4EvVtQc+qs2QtmyVJ0trPpKMkSVoOHweupHU5nTM5k+QG3Y/ndes7DChzN9rkJbOqqjO6sQp3AE4BHtGX5LmG1pV3kOO79epZ7n174Nd9Xaun3THdepsxXvPP3XomYXUs7fcyzntcT1VdBPwcuHeSTRdw/sVVdWRVvZo2m/lGtJnNh3E5s8+qvQFtZvF+q7v18QOODWOm9eRs9X3W44t9VpIkad1l0lGSJC25rpvp3rTkzH8n2WpQuSSPBQ7rNk+mzXr8pCS37ClzQ+B9A869RZKHDrjsjYG/onUZ7k32/IkBCc3O/t16zyQzYxXStUL7N9p3qI/Ncu5UqqrjaGNk7pTkeYPKJLlv3+/qH5Pce5ayjwAeSfu9fL+7xx9oYyduleSNSa7XyjXJXZPcedEfCP6dVh/3T3K9BHaSmyX5657t7bq6129mUphLunI37mKfbbKYV9K6h580y+RGb+tJvNMl+vbsNj8+34eaxXm0Vod3nOX4TByzHR/pWUmSJIFjOkqSpGVSVft2SaS9gB8m+R/aBCcX0RI3fwts0e2jqq5M8l7gjcDxSb5I++6yPW2MyN/13eJ2wDFJfgH8GDgDuCnweFr31PdV1Z97yh8B7NxNPvMjWvLr6Ko6uqr+p5v85J+An3VjQl5Ma812H1q34HeO7+lMjF1ok+t8LMkrgB/QxrS8PW2ClPvQJpyZmTxkV+AdSU6mtZT8PS0JfG/gUbQWjq+pqt7f5ctp9WAf4NlJvgucDdyWNlnJg4Fn0mYEX7Cq2j/Jg4CXAqcm+TrwG2BT2qQ/f0tL8r24O+VdwOZJjgLW0BLYD+o+x+nAzIzeG3ax75XkWOAEWtJvU1o38vvS6tLMdXv9ntbd/GdJDumu9TRad/4PVNXRC/ysFyX5AbBNkk8Bv6K1bjykqk4EfkmbPGjnJFd0z6GA/6qq0xfwrCRJkkw6SpKk5VNV+yT5HC158Uhgd2BjWkurE4C3A5/sOWUvWguyFwJ/T5uY4iBaq8mT+i6/piu/urv2ZsC5tITK67g2KTRjD1piZTtgR1rrxTcDR3exvjbJ8bQk2HNoCaBTaa3O3lVVs3WRnVpVdWaXfPq/wFNpScX1ab+Xk4D9gJ/2nLI78H9oibnVtORvaAmuzwAfrKrv9pSfmYV8W9rve5fuPhvTEo+nAK8CrjdpywI/z8uSHEZLlj2a1mX/XFpC7Z1cty7uCzyFNhP1o2ndwH/T7X9PN2M1tNa5jwMeQ0syPpk2CctltETpe7vyawaEdEV37X1pYz9uBpwG/Cvt2S7Gs2ljUD6WlrQNcCZwYlVdneQp3X3+jtYyOLTk+ukw8rOSJEkiVY7vLEmSJK2kJGsAqmrzlY1EkiRpPBzTUZIkSZIkSdJYmXSUJEmSJEmSNFYLGtMxyY1p47isP+h4Vf1mMUFJkiRJkiRJmlwjjemY5NnAa2kzB86mqsoJaiRJkiRJkqR11NDJwSS7AfsDVwPfAc4ArlpsAEnuRZuNb2vgfOCjwJur6urechdccIEz3kiSJEmSJEkrZJNNNsmwZUdpkfgPwHnAI6rqFyNHNUCSmwHfBE4CngTcFXgXbazJPcdxD0mSJEmSJEnLa5Sk492AT4wr4dh5MXBDYKequhD4RpKbAnsneUe3T5IkSZIkSdIEGSXpeC5w2Zjv/zjg633JxYOAtwPbAl8Z8/2m3imnnMIWW2yx0mFoglmH1i6v/dA1Kx2CJEmSJK1z/vn0F6x0CAPd9G37X2/f2vr/8euNUPZQYHWSoftuD2FL4OTeHd3M15d0xyRJkiRJkiRNmFGSjq8HbgB8KMlNxnT/m9Emj+l3XndMkiRJkiRJ0oQZpXv152gtEF8A7JLkFAYnDKuqthvhuoNmpc4s+4HWbFSz8/losaxDa5O7rnQAkiRJkqS1xGz/v76U/x+/0K7boyQdV/f8fGPgAbOUmzVZOMB5wKoB+zdhcEITWPiHXResrf34NTmsQ2uZIxzTUZIkSZLUDPr/9bX1/+OHTjpW1ShdsYd1Mn1jNya5Ay2pefLAMyRJkiRJkiSt1ZYikTiKw4AdkvxVz75nAJcC316ZkCRJkiRJkiQtxkonHT8EXA58Icmjk/w9sDfw71V14YpGJkmSJEmSJGlBRhnTEYAkO9Mmk3kgbezFC4EfAR+rqoNGuVZVnZdkO+D9wFdo4zi+m5Z4lKR13ttfvNJ/GxrO2jqGiCaL9UiLZR3SQlhvNC7WJS2G9WdttP9KBzDxhk46JglwILALbXbpq4E/ApsB2wGPSvLEqtpllACq6iTgUaOcI0mSJEmSJGntNUoTmhcBuwI/Bh4NbFxVtwE27rZ/BDwjyYvHHqUkSZIkSZKkiTFK0vF5wBrgb6vqyKq6GqCqrq6qI4Ftu+PPH3eQkiRJkiRJkibHKEnHewFfrKpLBx3s9n8JuOc4ApMkSZIkSZI0mUZJOhZtLMe5zHdckiRJkiRJ0pQbJen4C2CnJDccdLDb/2TgpHEEJkmSJEmSJGkyjZJ03B+4I3B0ku2SbACQZP0kjwS+BdwJ5xSXJEmSJEmS1mkbjFD2w8A2wDOBw4FrkpwLbEpLXgb4f1X1obFHKUmSJEmSJGliDN3SsZpdgV2BI4ELaAnHC7rtXatq5yWJUpIkSZIkSdLEGKWlIwBV9RngM0sQiyRJkiRJkqQpMMqYjpIkSZIkSZI0L5OOkiRJkiRJksZq1u7VSa4BrgHuVVW/6rZriGtWVY3cbVuSJEmSJEnSdJgrOXg0Lcl4Sd+2JEmSJEmSJM1q1qRjVa2ea1uSJEmSJEmSBnFMR0mSJEmSJEljNXTSMclpSV4xT5mXJTlt8WFJkiRJkiRJmlSjtHTcHFg1T5lVwJ0WHI0kSZIkSZKkiTfu7tU3Aa4Y8zUlSZIkSZIkTZC5Zq8myR37dq0asA9gfeCOwNMAu1dLkiRJkiRJ67A5k47AGqB6tvfoltkEePUiY5IkSZIkSZI0weZLOh5ISzoGeA5wInDCgHJXA38Cjqiqw8caoSRJkiRJkqSJMmfSsap2m/k5yXOAL1bVPksdlCRJkiRJkqTJNV9Lx7+oqnFPOiNJkiRJkiRpCplIlCRJkiRJkjRWQ7d0nJHkwcAOwO2AGwwoUlX1/MUGJkmSJEmSJGkyDZ10TBLgAOBZtIllZiaYmVE9+006SpIkSZIkSeuoUbpXvxx4NvBfwFa0BON7gL8B3gD8GTgIuMuYY5QkSZIkSZI0QUbpXv1c4JczM1q3ho+cX1XHAMck+TpwDPAN4ONjjlOSJEmSJEnShBilpeM9gCP79v0laVlVxwOHAi8dQ1ySJEmSJEmSJtQoSccAF/RsXwxs2lfmFGDLxQYlSZIkSZIkaXKNknT8LW3G6hmnAQ/qK7MFLRkpSZIkSZIkaR01StLxWK6bZDwMeEiSNya5d5KXAU+ijesoSZIkSZIkaR01StLx88D6Se7cbb8DOB14M3AisB9wPvC6sUYoSZIkSZIkaaIMPXt1VX0J+FLP9rlJHgi8ELgrsAY4sKp+P+4gJUmSJEmSJE2OoZOOg1TVBcC/jSkWSZIkSZIkSVNg6O7VSa5O8umlDEaSJEmSJEnS5BtlTMc/08ZwlCRJkiRJkqRZjZJ0PB6411IFIkmSJEmSJGk6jJJ0fDuwY5LtlyoYSZIkSZIkSZNvlIlkbgl8DTgsyZeAHwJnAdVfsKoOHE94kiRJkiRJkibNKEnHA2gJxgA7dQtcN+mYbtukoyRJkiRJkrSOGiXpuPuSRSFJkiRJkiRpagyddKyqTyxlIJIkSZIkSZKmwygTyUiSJEmSJEnSvEbpXg1AklsATwXuCdy4ql7Qs//OwE+r6tKxRilJkiRJkiRpYoyUdEzyfOB9wMZcO2nMC7rDtwK+D/w98LExxihJkiRJkiRpggzdvTrJ9sB/Ar8CngJ8sPd4Vf0M+Dnw5HEGKEmSJEmSJGmyjNLS8bXA74Ftq+rCJA8cUOZEYOuxRCZJkiRJkiRpIo0ykcxWwKFVdeEcZc4Ebr24kCRJkiRJkiRNslGSjhsBF89TZhVw9cLDkSRJkiRJkjTpRkk6rgEeNE+ZhwK/XHA0kiRJkiRJkibeKEnHLwPbJHn6oINJdgfuB3x+HIFJkiRJkiRJmkyjTCTzDmBn4DNJngZsApDk5cA2wE7AKcB+4w5SkiRJkiRJ0uQYOulYVecl2RY4EOht7fi+bv0dYJeqmm/cR0mSJEmSJElTbJSWjlTVb4DVSe4HbA3cHLgAOKaqfrQE8UmSJEmSJEmaMCMlHWdU1YnAiWOORZIkSZIkSdIUGHoimSSnJXnFPGVeluS0xYclSZIkSZIkaVKNMnv15sCqecqsAu604GgkSZIkSZIkTbxRko7DuAlwxZivKUmSJEmSJGmCzDmmY5I79u1aNWAfwPrAHYGnAXavliRJkiRJktZh800kswaonu09umU2AV69yJgkSZIkSZIkTbD5ko4H0pKOAZ5Dm7H6hAHlrgb+BBxRVYePNUJJkiRJkiRJE2XOpGNV7Tbzc5LnAF+sqn2WOihJkiRJkiRJk2u+lo5/UVXjnnRGkiRJkiRJ0hQykShJkiRJkiRprGZt6Zhkf9p4jm+oqrO77WFUVT1/LNFJkiRJkiRJmjhzda/ejZZ0fDtwdrc9jAJMOkqSJEmSJEnrqLmSjnfu1r/t25YkSZIkSZKkWc2adKyq0+faliRJkiRJkqRBnEhGkiRJkiRJ0liZdJQkSZIkSZI0ViYdJUmSJEmSJI2VSUdJkiRJkiRJY2XSUZIkSZIkSdJYmXSUJEmSJEmSNFZzJh2TPDGJiUlJkiRJkiRJQ5svofgl4PQk+yS503IEJEmSJEmSJGmyzZd0PAK4LbAncGqSw5I8Jcn6Sx+aJEmSJEmSpEk0Z9KxqrYH7gLsC/we2AE4GDgjyb8kucvShyhJkiRJkiRpksw7XmNVnV5VewJ3Ap4EHApsBrwe+FWSw5M8LckGSxuqJEmSJEmSpEkw9CQxVXVNVX2lqp4E3JHW5fp04NHAZ4HfJnl7ki2WJlRJkiRJkiRJk2BBM1NX1VlVtW9V3RXYHvgccFPgH4BfjDE+SZIkSZIkSRNmHF2ivw1sCmwOPGQM15MkSZIkSZI0wRacdExyD+AFwHNoYzwG+DXwsfGEJkmSJEmSJGkSjZR0TLIx8He0ZOPDaYnGK4EvAB+pqsPHHqEkSZIkSZKkiTJU0jHJA4AXArvQxm4McCrwUeDjVfWHJYtQkiRJkiRJ0kRJVc1+MHkRLdn4QFqi8QrgS8B/VtWRYwkguRvwj8DDgPsA36mq1f3lLrjggtkD1V+ccsopbLGFE4hr4axD6vfaD12z0iFIkiRJ0jrnn09/wUqHAMBN37b/X37eZJNNMux587V0/GC3/hXwEeATVXXOyNHN7d7AjsAxwEZjvrYkSZIkSZKkZTZf0vEztFaN317CGL5SVV8GSHIwbVIaSZIkSZIkSRNqzqRjVe261AFUlf32JEmSJEmSpCmy3lwHk9wgybFJjkiy4RzlNurKHDNXOUmSJEmSJEnTb77u1bsCDwKeUFVXzlaoqq5I8k7gq905B4wtwgFOOeWUpbz8xPP5aLGsQ7quu650AJIkSZKkCTNf0nEn4LSq+up8F6qqryU5BXg6S5x0dGbd2TnzsBbLOqTrOcJRMCRJkiRJo5mzezXwQOCoEa53NPCABUcjSZIkSZIkaeLNl3TcDDh7hOudDdx84eFIkiRJkiRJmnTzda++FLjJCNe7CXDZwsPRYtktVotlHVK/t794vr9PSZIkSZLGb/+VDmBR5ks6ngE8eITrbQX8ZpQAktwI2LHbvB1w0yRP67a/WlWXjHI9SZIkSZIkSStrvqTjUcBLk2xVVcfNVTDJg4C/AfYbMYZbAp/r2zezfWdgzYjXkyRJkiRJkrSCUlWzH0zuAfyc1uJxx6r6xSzltgS+CtwBuE9V/XIJYpUkSZIkSZI0AeZMOgIkeROwN3AFcDBwJHAmUMDtge2ApwI3AN5UVW9dwnglSZIkSZIkreXmnR2gqvYB9gQC7AJ8BDgM+Brw0W7fesA/m3Bskjw9ySFJfpvkoiQ/SvLMAeVemOSUJJd1ZbbrO36LJO9LcmySK5KsmeV+H0pycnev85IcneTRI8Q7ZxxdmUck+X5X5ndJ/iXJfN3ztUDLXYf6znllkkpy8Ajxjj0OLcwU1p3V3TX7l38d9h4a3bTVo66M77FltALfhY6a5d+KjYeM1/fYWmAK643vsBUwbfWoK+M7bBktdx3qyt4pyWeSnJvkkiQ/SfLYIeP1HbYWmcL6s6h32VBTklbVvsDdgbcA3wJOBn7Z/bwPcPeqetsw11pHvBq4CHgV8ETac/p0kv87UyDJzsCHgAOBx9G6sR+a5D4917kd8AzgLOCEOe53Q+D9wFOAZwHnAIcledh8gQ4TR5I7A98Azu7u8TZgD+Df5ru+Fmy569DMNW8JvAn447CBLkUcWpRpqzszdgW27ln+Y9j7aEGmqh75HlsRK1GHvsV1/53YGrh8vkB9j61Vpq3ezPAdtrymqh75DlsRy1qHktwB+D6wCti9u+d/0f4/f06+w9ZK01Z/ZizsXVZVLmNegM0G7Ps08Oue7V8C+/dsrwf8FPhk776en/8NWDPk/denzSL+viHKDhPHh4HTgA169r0CuBK4zUo/72lcVqoOAR+j/QN1FHDwkLEuWV12se4Aq2nDedxnpZ/turRMYT3yPTbldWiUOrPAOuR7zHqzkHrjO8x6NI565Dts+uvQQcB3esuPuQ75DrP+LKb+LOpdNlRLR42mqs4ZsPt42kzdJLkLreXo/+s55xrarN2P69u3kPtfDZwPbDRXuWHjAB4AHFVVV/XsO5w2+/ljFhKj5rYSdSjJg4G/A143wjlLWpc1ummrO1oZU1iPfI8ts5X+LjSstSUONdNWb7QyprAe+Q5bZstZh5JsAuwEfGDUOre21GVd17TVn8Uy6bh8/gY4qft5y259cl+ZXwCbJrnFqBdPs0GSmyd5FbAFsP88pw0bx8a0iYR6zXQXuOeosWrBlqwOJQmti/47quq3I5w69rqsJTENdefIJFcnWZNkzyTrjxKnxmKS65HvsbXDkn4XAh7TjWN0SZKvJ7nfEOf4Hlv7TUO98R228ia5HvkOWzssVR36a2BDoJJ8L8mVSc5M8vru+9FcfIdNjmmoPwt6l5l0XAZpA3E+iWv7vN+sW5/fV/S8vuOjeAatif05tLE3n1FVx85zzrBx/C/w4L4yD+nWm44eqka1DHVod+DWjD42zFLUZY3RFNSdC4B/BXYDdgC+BLwZ+PcR76dFmIJ65HtshS1DHfo2bYyzHYC/B+4IfCfJ5vOc53tsLTYF9cZ32FpgCuqR77AVtsR16Nbd+sO0LrKPoTUeeivwknnO9R02Aaag/izqXeaMV0use9l8GvhyVR3Qd7j6i8+yfxhfp72MNqMN8HlQkh2r6qgujt7fdXVdsIeN44PAN5K8sfv5brRKd3W3aAktdR3qmmTvC7yiqi6do9xi6pBWwDTUnao6ntYdYcY3k1wOvDrJW2bpvqAxmoZ6hO+xFbUc34Wqaq+eze8k+SbtL/ev7BbfYxNmGuqN77CVNw31CN9hK2oZ6tBMQ7DDqmpmiJlvJbk98HrgA10cvsMm0DTUn8W+y2zpuISSbAocRpvU5Vk9h2Yyx6v6TpnZ7s80z6uqzquq46rqa1X1bNrsRft0cWxOawU5s5w6ShxV9U1gT+CfaTOKHk0b7P9c2ixqWiLLVIfeAJwBHJ5kVZJVtD9IbNhtr7/YOqTlN+V15+DuPsN0XdIiTEs98j22cpbzu1BDtISYAAAJTElEQVSvqjoL+B6t29GivwtpeU15vfEdtkympR75Dls5y1SHzu3W3+rbfyRw+yQ39R02maa8/gz9LrOl4xJJciPgUNpkLv+nqi7uOTzTZ35L4PSe/VsC51bVH8cQwvHAzt3Pv+O6TfJnxgAZOo6q+pck7wXuDJxJmyH7LcAxY4hVAyxjHboHsBXX/qPT6zxgG+BYFlmHtHzWobrjX2+X0LTVI99jy28t+C4E1/47sejvQloe61C98R22hKatHvkOW37LWId+MVsI3foafIdNnHWo/sz7LjPpuAS6pqufo03m8vCq+kPv8ao6LcmvgKfTukWTZL1u+7Ax3D/A1sCvu/tdARzXX27UOKrqItr06STZi1Yxv7nYeHV9y1yH9gTe07fvPbSxG/YCfjquOqSlt47UnacCVwEnjhivhjSt9cj32PJZC74L3Qp4ON2ker7HJsM6Um98hy2xaa1HvsOWz3LWoapak+TnwHa0cflmbAec2v3ewXfYxFhH6s/Q7zKTjkvjA8COtAGFN03ysJ5jx1fV5cDewCeTrKE1v38urVLu0nuhJE/rfrw7cKOe7W9X1R+TbAO8BvgCrdnuzbtrPQx4whCxzhtHkrt128fS6szjgefRMvZXDXEPjW7Z6lBV/az/5knOB86ZGRN0HmOJY4j7aDhTVXeSfJDWleiHtJkbdwReDrynqv40xD20MNNWj3yPLb/l/C50P+BttC/4p9MmcXg97a/7/QntQcYSxxD30fymqt74Dlsx01aPfIctv2WrQ93PbwQ+n+SdwOHAauDZwHOGiHWccWg8pqr+LPpdVlUuY16ANbRmpoOWzXvKvZA2G9nlwI+B7QZca7brrO6Ob07rT39md50zac14tx4h3jnjoL08j6a1OrkYOArYZqWf8zQvy1mHZrn/UcDB46pDC43DxboDvIL2F7Q/d2V+ThuUfb2VftbTvExhPfI9NsV1CLgd8FXg97Qvw38CPg9sOa46tNC67LJu1xt8h1mPxlOPfIdNcR3qKfcsWlfZK7prvnhcdWiUOFysPwP+DVrUuyzdRSRJkiRJkiRpLJy9WpIkSZIkSdJYmXSUJEmSJEmSNFYmHSVJkiRJkiSNlUlHSZIkSZIkSWNl0lGSJEmSJEnSWJl0lCRJkiRJkjRWJh0lSZK0VkmyOkkl2XulY5EkSdLCmHSUJEmaQF1Srne5PMkfk/w4yUeTPC7J+mO6127dPXYb0/U27653wDiuJ0mSpLXPBisdgCRJkhblzd16fWAVcG/g2cDzgeOS7FpVv1qp4BboWOCewDkrHYgkSZIWxqSjJEnSBKuqvfv3JbkVsB/wdOCbSbaqqj8sd2wLVVWXACevdBySJElaOLtXS5IkTZmqOhvYGTgKuAPwht7jSR6U5L1JfpLk3CSXJTklybuS3Kyv7FHAx7vNj/d16d68p9wGSV6a5JgkFya5JMnxSV6eZL2ecnsDv+42n9t3vd26MgPHdExyVLd/wyRvSnJqF/vJSV7YU+7FSX6a5NIkZyZ5c28Mfdd8aJKDk5yV5IokZyT5cJLbDvm4JUmSNIAtHSVJkqZQVV2T5K3AauCZSV5VVdUdfiHwFODbwDdpXbP/Gng18LgkD62qP3dlDwDOB54EfBk4oec25wMk2RD4CrAD8Evg08BlwCNpLS4fSuvyDS0RugrYA/gJ8KWe6/Veey4Hddf8KnAl8DTgP5NcCdwPeC5wKHAE8ETgTcAlwNt7L5Jkd+AjwOXAIcAZwBbAC4AnJHlYVf1myJgkSZLUI9d+95QkSdKkSFIAVZU5ytwAuIj2h+a7VNWvu/13As6sqqv7yj8f+Cjwuqp6e8/+3WitHXevqgMG3GdvYC/g/cArZ67bTWTzn8DzgCdX1Ze7/ZvTWjt+oqp2G3C91cC3gDf3dh/vWl1uCxwHbF9VM0nPu9C6Y19MS4Q+oqp+2x1bBfwvUMBtquqqbv/dgZ8BvwG2nSnfHXsU8A3gkKp6yqBnK0mSpLnZvVqSJGlKVdXlwJ+6zVv07D+9P+HY2R+4kNZicShdt+WXA2cBr+q9bvfza2gJv11H/gCze91MwrG7z2nAd2ktKN/Sm0Dsyn0F2Ay4Xc81XgJsCOzRW74750hay8cnJPmrMcYtSZK0zrB7tSRJ0nSbaQn5l+4tXXfoF9HGfbwXsAnX/WN0b3JuPncHbg6cAuyZDGx4eSltNupxOW7Avt916x8NODaTVLw9cHr389bdetskDx5wzi1p3c7vPss1JUmSNAeTjpIkSVMqycbApt3mH3sOfZY2puNptHEaz6KNawjwSuAGI9zm5t16C1oX69ncZIRrzqmqLhiw+6puPdexDXv2zcT9j/PcbmxxS5IkrUtMOkqSJE2vR9C+751dVWsAkmxFSzh+E9ixqq6cKdx1lf6nEe8xk+T7YlXttOiIl89M3JtU1YUrGokkSdIUckxHSZKkKdQlEP+52/x0z6G7detDehOOnYcANxxwuZlxGtcfcOxk2uQtD+u6bQ9jrustl2O69TYrGIMkSdLUMukoSZI0ZZLcEjgIWE2bnXnfnsNruvXqAef8xyyXnJmM5o79B7rZoPcDbgO8L8n1kpZJbpPkXj27zqONMXm96y2j9wNXAu/uZrK+jiQbJTEhKUmStEB2r5YkSZpgSfbuflyPNnvzvWndqjcCjgV2rapzek75IfA9YKck/0Ob9flWwOOAX3LthCy9vg9cArwyyabA2d3+/brxFd8C3B94MW3G5yNpk7fckjbW48NprS5PAqiqi5L8ANgmyaeAX9FaPx5SVScu6oEMqapOTvI82ozdP0/ytS6ODWnJ0G1o42BuuRzxSJIkTRuTjpIkSZNtZvKWK4A/02ZnPhD4PHB4VV3TW7iqrk7yROCtwI7AK2gJwo92+07qv0FVnZfkqd29dgdu3B36JHBBVV2Z5MnAs4DdgMfTJmD5I/Br4I3Ap/ou+2zg3cBjgWfSZtk+E1iWpCNAVX0yyU+A1wCPBB4DXExLvB5Mm3BHkiRJC5CqWukYJEmSJEmSJE0Rx3SUJEmSJEmSNFYmHSVJkiRJkiSNlUlHSZIkSZIkSWNl0lGSJEmSJEnSWJl0lCRJkiRJkjRWJh0lSZIkSZIkjZVJR0mSJEmSJEljZdJRkiRJkiRJ0liZdJQkSZIkSZI0ViYdJUmSJEmSJI3V/we04UVl/vhs7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x144 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sample = data.iloc[::1000][[day_col, 'date']].reset_index(drop=True)\n",
    "show_cv_days(cv, sample, 'date', day_col)\n",
    "plot_cv_indices(cv, sample, 'date')\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = [\n",
    "    \"item_id\",\n",
    "    \"dept_id\",\n",
    "    \"cat_id\",\n",
    "    \"store_id\",\n",
    "    \"state_id\",\n",
    "    \"event_name_1\",\n",
    "    \"event_type_1\",\n",
    "    \"event_name_2\",\n",
    "    \"event_type_2\",\n",
    "    \"snap_CA\",\n",
    "    \"snap_TX\",\n",
    "    \"snap_WI\",\n",
    "    \"sell_price\",\n",
    "    # demand features.\n",
    "    \"lag_t28\",\n",
    "    \"lag_t29\",\n",
    "    \"lag_t30\",\n",
    "    \"rolling_std_t7\",\n",
    "    \"rolling_std_t30\",\n",
    "    \"rolling_std_t60\",\n",
    "    \"rolling_std_t90\",\n",
    "    \"rolling_std_t180\",\n",
    "    \n",
    "    \"rolling_mean_t7\",\n",
    "    \"rolling_mean_t30\",\n",
    "    \"rolling_mean_t60\",\n",
    "    \"rolling_mean_t90\",\n",
    "    \"rolling_mean_t180\",\n",
    "    \n",
    "    \"rolling_skew_t30\",\n",
    "    \"rolling_kurt_t30\",\n",
    "    # price features\n",
    "    \"price_change_t1\",\n",
    "    \"price_change_t365\",\n",
    "    \"rolling_price_std_t7\",\n",
    "    \"rolling_price_std_t30\",\n",
    "    # time features.\n",
    "    \"Year\",\n",
    "    \"Month\",\n",
    "    \"Quarter\",\n",
    "    \"Week\",\n",
    "    \"Day\",\n",
    "    \"Dayofweek\",\n",
    "    \"Dayofyear\",\n",
    "    'Is_month_end',\n",
    "    'Is_month_start',\n",
    "    'Is_quarter_end',\n",
    "    'Is_quarter_start',\n",
    "    'Is_year_end',\n",
    "    'Is_year_start',\n",
    "    'is_weekend'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train shape: (31327370, 47)\n",
      "X_test shape: (853720, 46)\n"
     ]
    }
   ],
   "source": [
    "# prepare training and test data.\n",
    "# 2011-01-29 ~ 2016-04-24 : d_1    ~ d_1913\n",
    "# 2016-04-25 ~ 2016-05-22 : d_1914 ~ d_1941 (public)\n",
    "# 2016-05-23 ~ 2016-06-19 : d_1942 ~ d_1969 (private)\n",
    "\n",
    "mask = data[\"date\"] <= \"2016-04-24\"\n",
    "\n",
    "# Attach \"d\" to X_train for cross validation.\n",
    "X_train = data[mask][[day_col] + features].reset_index(drop=True)\n",
    "y_train = data[mask][\"demand\"].reset_index(drop=True)\n",
    "X_test = data[~mask][features].reset_index(drop=True)\n",
    "\n",
    "# keep these two columns to use later.\n",
    "id_date = data[~mask][[\"id\", \"date\"]].reset_index(drop=True)\n",
    "\n",
    "gc.collect()\n",
    "\n",
    "print(\"X_train shape:\", X_train.shape)\n",
    "print(\"X_test shape:\", X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_lgb(bst_params, fit_params, X, y, cv, drop_when_train=None):\n",
    "    models = []\n",
    "\n",
    "    if drop_when_train is None:\n",
    "        drop_when_train = []\n",
    "\n",
    "    for idx_fold, (idx_trn, idx_val) in enumerate(cv.split(X, y)):\n",
    "        print(f\"\\n---------- Fold: ({idx_fold + 1} / {cv.get_n_splits()}) ----------\\n\")\n",
    "\n",
    "        X_trn, X_val = X.iloc[idx_trn], X.iloc[idx_val]\n",
    "        y_trn, y_val = y.iloc[idx_trn], y.iloc[idx_val]\n",
    "        train_set = lgb.Dataset(X_trn.drop(drop_when_train, axis=1), label=y_trn)\n",
    "        val_set = lgb.Dataset(X_val.drop(drop_when_train, axis=1), label=y_val)\n",
    "\n",
    "        model = lgb.train(\n",
    "            bst_params,\n",
    "            train_set,\n",
    "            valid_sets=[train_set, val_set],\n",
    "            valid_names=[\"train\", \"valid\"],\n",
    "            **fit_params\n",
    "        )\n",
    "        models.append(model)\n",
    "\n",
    "        del idx_trn, idx_val, X_trn, X_val, y_trn, y_val\n",
    "        gc.collect()\n",
    "\n",
    "    return models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_xgb(bst_params, X, y, cv, drop_when_train=None):\n",
    "    models2 = []\n",
    "\n",
    "    if drop_when_train is None:\n",
    "        drop_when_train = []\n",
    "        \n",
    "    for idx_fold, (idx_trn, idx_val) in enumerate(cv.split(X, y)):\n",
    "        print(f\"\\n---------- Fold: ({idx_fold + 1} / {cv.get_n_splits()}) ----------\\n\")\n",
    "\n",
    "        X_trn, X_val = X.iloc[idx_trn], X.iloc[idx_val]\n",
    "        y_trn, y_val = y.iloc[idx_trn], y.iloc[idx_val]\n",
    "        \n",
    "        train_set = xgb.DMatrix(X_trn.drop(drop_when_train, axis=1), label=y_trn)\n",
    "        val_set = xgb.DMatrix(X_val.drop(drop_when_train, axis=1), label=y_val)        \n",
    " \n",
    "        model2 = xgb.train(\n",
    "            params_xgb,\n",
    "            train_set,\n",
    "            evals=[(train_set, 'train'), (val_set, 'valid')],\n",
    "            num_boost_round = 100_000,\n",
    "            early_stopping_rounds = 50,\n",
    "            verbose_eval = 100\n",
    "        )\n",
    "        models2.append(model2)\n",
    "\n",
    "        del idx_trn, idx_val, X_trn, X_val, y_trn, y_val\n",
    "        gc.collect()\n",
    "        \n",
    "    return models2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'num_leaves': 275,\n",
    "          'min_child_weight': 0.034,\n",
    "          'feature_fraction': 0.379,\n",
    "          'bagging_fraction': 0.418,\n",
    "          'min_data_in_leaf': 106,\n",
    "          'objective': 'regression',\n",
    "          'max_depth': -1,\n",
    "          'learning_rate': 0.007,\n",
    "          \"boosting_type\": \"gbdt\",\n",
    "          \"bagging_seed\": 11,\n",
    "          \"metric\": 'rmse',\n",
    "          \"verbosity\": -1,\n",
    "          'reg_alpha': 0.3899,\n",
    "          'reg_lambda': 0.648,\n",
    "          'random_state': 623,\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_xgb = {'colsample_bytree': 0.8,                 \n",
    "              'learning_rate': 0.01,\n",
    "              'max_depth': 30,\n",
    "              'subsample': 0.8,\n",
    "              'objective':'reg:squarederror',\n",
    "              'eval_metric':'rmse',\n",
    "              'min_child_weight':2,\n",
    "              'gamma':0.25,\n",
    "              'n_estimators':5000\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "fit_params = {\n",
    "    \"num_boost_round\": 100_000,\n",
    "    \"early_stopping_rounds\": 50,\n",
    "    \"verbose_eval\": 100,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------- Fold: (1 / 2) ----------\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttrain's rmse: 2.70363\tvalid's rmse: 2.75311\n",
      "[200]\ttrain's rmse: 2.38684\tvalid's rmse: 2.43498\n",
      "[300]\ttrain's rmse: 2.27532\tvalid's rmse: 2.33134\n",
      "[400]\ttrain's rmse: 2.22735\tvalid's rmse: 2.29182\n",
      "[500]\ttrain's rmse: 2.19843\tvalid's rmse: 2.27376\n",
      "[600]\ttrain's rmse: 2.17818\tvalid's rmse: 2.26265\n",
      "[700]\ttrain's rmse: 2.16154\tvalid's rmse: 2.25429\n",
      "[800]\ttrain's rmse: 2.14602\tvalid's rmse: 2.24793\n",
      "[900]\ttrain's rmse: 2.1322\tvalid's rmse: 2.2427\n",
      "[1000]\ttrain's rmse: 2.12075\tvalid's rmse: 2.23875\n",
      "[1100]\ttrain's rmse: 2.11037\tvalid's rmse: 2.23544\n",
      "[1200]\ttrain's rmse: 2.10001\tvalid's rmse: 2.23271\n",
      "[1300]\ttrain's rmse: 2.09094\tvalid's rmse: 2.23021\n",
      "[1400]\ttrain's rmse: 2.08279\tvalid's rmse: 2.22862\n",
      "[1500]\ttrain's rmse: 2.07566\tvalid's rmse: 2.22744\n",
      "[1600]\ttrain's rmse: 2.06898\tvalid's rmse: 2.22616\n",
      "[1700]\ttrain's rmse: 2.06263\tvalid's rmse: 2.22497\n",
      "[1800]\ttrain's rmse: 2.05682\tvalid's rmse: 2.22398\n",
      "[1900]\ttrain's rmse: 2.051\tvalid's rmse: 2.22311\n",
      "[2000]\ttrain's rmse: 2.04586\tvalid's rmse: 2.22225\n",
      "[2100]\ttrain's rmse: 2.04069\tvalid's rmse: 2.22129\n",
      "[2200]\ttrain's rmse: 2.03593\tvalid's rmse: 2.22043\n",
      "[2300]\ttrain's rmse: 2.03094\tvalid's rmse: 2.21997\n",
      "[2400]\ttrain's rmse: 2.02634\tvalid's rmse: 2.21883\n",
      "[2500]\ttrain's rmse: 2.02198\tvalid's rmse: 2.21826\n",
      "[2600]\ttrain's rmse: 2.01788\tvalid's rmse: 2.21777\n",
      "[2700]\ttrain's rmse: 2.01384\tvalid's rmse: 2.21719\n",
      "[2800]\ttrain's rmse: 2.01001\tvalid's rmse: 2.2167\n",
      "[2900]\ttrain's rmse: 2.00616\tvalid's rmse: 2.21644\n",
      "[3000]\ttrain's rmse: 2.00235\tvalid's rmse: 2.2161\n",
      "[3100]\ttrain's rmse: 1.9986\tvalid's rmse: 2.21558\n",
      "[3200]\ttrain's rmse: 1.99476\tvalid's rmse: 2.21495\n",
      "[3300]\ttrain's rmse: 1.99138\tvalid's rmse: 2.2146\n",
      "[3400]\ttrain's rmse: 1.98784\tvalid's rmse: 2.21436\n",
      "[3500]\ttrain's rmse: 1.98446\tvalid's rmse: 2.21382\n",
      "[3600]\ttrain's rmse: 1.98121\tvalid's rmse: 2.21339\n",
      "[3700]\ttrain's rmse: 1.97806\tvalid's rmse: 2.21303\n",
      "[3800]\ttrain's rmse: 1.975\tvalid's rmse: 2.21276\n",
      "[3900]\ttrain's rmse: 1.97212\tvalid's rmse: 2.21239\n",
      "[4000]\ttrain's rmse: 1.96925\tvalid's rmse: 2.21192\n",
      "Early stopping, best iteration is:\n",
      "[4042]\ttrain's rmse: 1.96799\tvalid's rmse: 2.21177\n",
      "\n",
      "---------- Fold: (2 / 2) ----------\n",
      "\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttrain's rmse: 2.70098\tvalid's rmse: 2.63181\n",
      "[200]\ttrain's rmse: 2.38298\tvalid's rmse: 2.29384\n",
      "[300]\ttrain's rmse: 2.27151\tvalid's rmse: 2.18532\n",
      "[400]\ttrain's rmse: 2.22389\tvalid's rmse: 2.14881\n",
      "[500]\ttrain's rmse: 2.1951\tvalid's rmse: 2.13358\n",
      "[600]\ttrain's rmse: 2.1747\tvalid's rmse: 2.126\n",
      "[700]\ttrain's rmse: 2.15792\tvalid's rmse: 2.12192\n",
      "[800]\ttrain's rmse: 2.14225\tvalid's rmse: 2.11959\n",
      "[900]\ttrain's rmse: 2.12845\tvalid's rmse: 2.11932\n",
      "[1000]\ttrain's rmse: 2.117\tvalid's rmse: 2.11852\n",
      "[1100]\ttrain's rmse: 2.10651\tvalid's rmse: 2.11875\n",
      "Early stopping, best iteration is:\n",
      "[1054]\ttrain's rmse: 2.11136\tvalid's rmse: 2.11832\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "models = train_lgb(\n",
    "    params, fit_params, X_train, y_train, cv, drop_when_train=[day_col]\n",
    ")\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "---------- Fold: (1 / 2) ----------\n",
      "\n",
      "[14:15:18] WARNING: C:\\Jenkins\\workspace\\xgboost-win64_release_0.90\\src\\learner.cc:686: Tree method is automatically selected to be 'approx' for faster speed. To use old behavior (exact greedy algorithm on single machine), set tree_method to 'exact'.\n",
      "[0]\ttrain-rmse:3.65813\tvalid-rmse:3.73164\n",
      "Multiple eval metrics have been passed: 'valid-rmse' will be used for early stopping.\n",
      "\n",
      "Will train until valid-rmse hasn't improved in 50 rounds.\n"
     ]
    }
   ],
   "source": [
    "models2 = train_xgb(\n",
    "    params_xgb, X_train, y_train, cv, drop_when_train=[day_col]\n",
    ")\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rmse(y_true, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LGB\n",
    "imp_type = \"gain\"\n",
    "importances = np.zeros(X_test.shape[1])\n",
    "preds = np.zeros(X_test.shape[0])\n",
    "\n",
    "for model in models:\n",
    "    preds += model.predict(X_test)\n",
    "    importances += model.feature_importance(imp_type)\n",
    "\n",
    "preds = preds / cv.get_n_splits()\n",
    "importances = importances / cv.get_n_splits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGB\n",
    "imp_type = \"gain\"\n",
    "importances2 = np.zeros(X_test.shape[1])\n",
    "preds2 = np.zeros(X_test.shape[0])\n",
    "\n",
    "for model in models2:\n",
    "    preds2 += model.predict(X_test)\n",
    "    importances2 += model.feature_importance(imp_type)\n",
    "\n",
    "preds2 = preds2 / cv.get_n_splits()\n",
    "importances2 = importances2 / cv.get_n_splits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LGB\n",
    "def make_submission(test, submission):\n",
    "    preds = test[[\"id\", \"date\", \"demand\"]]\n",
    "    preds = preds.pivot(index=\"id\", columns=\"date\", values=\"demand\").reset_index()\n",
    "    preds.columns = [\"id\"] + [\"F\" + str(d + 1) for d in range(DAYS_PRED)]\n",
    "\n",
    "    evals = submission[submission[\"id\"].str.endswith(\"evaluation\")]\n",
    "    vals = submission[[\"id\"]].merge(preds, how=\"inner\", on=\"id\")\n",
    "    final = pd.concat([vals, evals])\n",
    "\n",
    "    assert final.drop(\"id\", axis=1).isnull().sum().sum() == 0\n",
    "    assert final[\"id\"].equals(submission[\"id\"])\n",
    "\n",
    "    final.to_csv(\"M5_Forecasting.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission(id_date.assign(demand=preds), submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_submission2(test, submission):\n",
    "    preds2 = test[[\"id\", \"date\", \"demand\"]]\n",
    "    preds2 = preds2.pivot(index=\"id\", columns=\"date\", values=\"demand\").reset_index()\n",
    "    preds2.columns = [\"id\"] + [\"F\" + str(d + 1) for d in range(DAYS_PRED)]\n",
    "\n",
    "    evals = submission[submission[\"id\"].str.endswith(\"evaluation\")]\n",
    "    vals = submission[[\"id\"]].merge(preds2, how=\"inner\", on=\"id\")\n",
    "    final2 = pd.concat([vals, evals])\n",
    "\n",
    "    assert final2.drop(\"id\", axis=1).isnull().sum().sum() == 0\n",
    "    assert final2[\"id\"].equals(submission[\"id\"])\n",
    "\n",
    "    final2.to_csv(\"M5_Forecasting_xgb.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_submission2(id_date.assign(demand=preds2), submission)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "M5_lgb = pd.read_csv(os.path.join(path, 'M5_Forecasting.csv'))\n",
    "M5_xgb = pd.read_csv(os.path.join(path, 'M5_Forecasting_xgb.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv(os.path.join(path, 'WSampleSubmissionStage1_2020.csv'))\n",
    "# \n",
    "submission = 0.85*preds + 0.15*preds2\n",
    "submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = data[data['date'] <= '2016-04-24']\n",
    "y = x.sort_values('date')['demand']\n",
    "test = data[(data['date'] > '2016-04-24')]\n",
    "x = x.sort_values('date')\n",
    "test = test.sort_values('date')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_fold = 5 #5 for timely purpose of the kernel\n",
    "folds = TimeSeriesSplit(n_splits=n_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.70546\tvalid_1's rmse: 3.63974\n",
      "[200]\ttraining's rmse: 3.57824\tvalid_1's rmse: 3.48565\n",
      "[300]\ttraining's rmse: 3.47161\tvalid_1's rmse: 3.35357\n",
      "[400]\ttraining's rmse: 3.37402\tvalid_1's rmse: 3.23889\n",
      "[500]\ttraining's rmse: 3.28827\tvalid_1's rmse: 3.13854\n",
      "[600]\ttraining's rmse: 3.21325\tvalid_1's rmse: 3.05215\n",
      "[700]\ttraining's rmse: 3.14758\tvalid_1's rmse: 2.97932\n",
      "[800]\ttraining's rmse: 3.09042\tvalid_1's rmse: 2.91678\n",
      "[900]\ttraining's rmse: 3.03823\tvalid_1's rmse: 2.86132\n",
      "[1000]\ttraining's rmse: 2.99145\tvalid_1's rmse: 2.81476\n",
      "[1100]\ttraining's rmse: 2.95289\tvalid_1's rmse: 2.77446\n",
      "[1200]\ttraining's rmse: 2.91696\tvalid_1's rmse: 2.7402\n",
      "[1300]\ttraining's rmse: 2.88552\tvalid_1's rmse: 2.71058\n",
      "[1400]\ttraining's rmse: 2.8561\tvalid_1's rmse: 2.68557\n",
      "[1500]\ttraining's rmse: 2.83148\tvalid_1's rmse: 2.66464\n",
      "[1600]\ttraining's rmse: 2.80789\tvalid_1's rmse: 2.64667\n",
      "[1700]\ttraining's rmse: 2.78702\tvalid_1's rmse: 2.63086\n",
      "[1800]\ttraining's rmse: 2.76695\tvalid_1's rmse: 2.61723\n",
      "[1900]\ttraining's rmse: 2.74876\tvalid_1's rmse: 2.60495\n",
      "[2000]\ttraining's rmse: 2.7328\tvalid_1's rmse: 2.59479\n",
      "[2100]\ttraining's rmse: 2.71847\tvalid_1's rmse: 2.5859\n",
      "[2200]\ttraining's rmse: 2.70497\tvalid_1's rmse: 2.57867\n",
      "[2300]\ttraining's rmse: 2.69195\tvalid_1's rmse: 2.57209\n",
      "[2400]\ttraining's rmse: 2.67995\tvalid_1's rmse: 2.56711\n",
      "[2500]\ttraining's rmse: 2.66792\tvalid_1's rmse: 2.56243\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\ttraining's rmse: 2.66792\tvalid_1's rmse: 2.56243\n",
      "val rmse score is 2.562434159186203\n",
      "Fold: 2\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.66082\tvalid_1's rmse: 3.33163\n",
      "[200]\ttraining's rmse: 3.51151\tvalid_1's rmse: 3.16515\n",
      "[300]\ttraining's rmse: 3.38318\tvalid_1's rmse: 3.02322\n",
      "[400]\ttraining's rmse: 3.26796\tvalid_1's rmse: 2.89885\n",
      "[500]\ttraining's rmse: 3.16925\tvalid_1's rmse: 2.79373\n",
      "[600]\ttraining's rmse: 3.08253\tvalid_1's rmse: 2.70481\n",
      "[700]\ttraining's rmse: 3.00672\tvalid_1's rmse: 2.6299\n",
      "[800]\ttraining's rmse: 2.94051\tvalid_1's rmse: 2.56739\n",
      "[900]\ttraining's rmse: 2.88407\tvalid_1's rmse: 2.51622\n",
      "[1000]\ttraining's rmse: 2.83365\tvalid_1's rmse: 2.47352\n",
      "[1100]\ttraining's rmse: 2.79081\tvalid_1's rmse: 2.43891\n",
      "[1200]\ttraining's rmse: 2.75287\tvalid_1's rmse: 2.41055\n",
      "[1300]\ttraining's rmse: 2.71996\tvalid_1's rmse: 2.38792\n",
      "[1400]\ttraining's rmse: 2.69105\tvalid_1's rmse: 2.36942\n",
      "[1500]\ttraining's rmse: 2.66566\tvalid_1's rmse: 2.35477\n",
      "[1600]\ttraining's rmse: 2.64293\tvalid_1's rmse: 2.34342\n",
      "[1700]\ttraining's rmse: 2.62251\tvalid_1's rmse: 2.33448\n",
      "[1800]\ttraining's rmse: 2.6043\tvalid_1's rmse: 2.32755\n",
      "[1900]\ttraining's rmse: 2.58814\tvalid_1's rmse: 2.32261\n",
      "[2000]\ttraining's rmse: 2.5731\tvalid_1's rmse: 2.31886\n",
      "[2100]\ttraining's rmse: 2.55898\tvalid_1's rmse: 2.31636\n",
      "[2200]\ttraining's rmse: 2.54722\tvalid_1's rmse: 2.31447\n",
      "[2300]\ttraining's rmse: 2.53633\tvalid_1's rmse: 2.31348\n",
      "[2400]\ttraining's rmse: 2.52579\tvalid_1's rmse: 2.31279\n",
      "Early stopping, best iteration is:\n",
      "[2432]\ttraining's rmse: 2.52273\tvalid_1's rmse: 2.31251\n",
      "val rmse score is 2.312509513018695\n",
      "Fold: 3\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.55862\tvalid_1's rmse: 3.28525\n",
      "[200]\ttraining's rmse: 3.40847\tvalid_1's rmse: 3.14595\n",
      "[300]\ttraining's rmse: 3.27858\tvalid_1's rmse: 3.02674\n",
      "[400]\ttraining's rmse: 3.1655\tvalid_1's rmse: 2.92371\n",
      "[500]\ttraining's rmse: 3.06629\tvalid_1's rmse: 2.8339\n",
      "[600]\ttraining's rmse: 2.98038\tvalid_1's rmse: 2.75655\n",
      "[700]\ttraining's rmse: 2.90573\tvalid_1's rmse: 2.69063\n",
      "[800]\ttraining's rmse: 2.84081\tvalid_1's rmse: 2.63412\n",
      "[900]\ttraining's rmse: 2.7834\tvalid_1's rmse: 2.58531\n",
      "[1000]\ttraining's rmse: 2.73411\tvalid_1's rmse: 2.54382\n",
      "[1100]\ttraining's rmse: 2.69123\tvalid_1's rmse: 2.50866\n",
      "[1200]\ttraining's rmse: 2.65395\tvalid_1's rmse: 2.47869\n",
      "[1300]\ttraining's rmse: 2.62116\tvalid_1's rmse: 2.45269\n",
      "[1400]\ttraining's rmse: 2.59265\tvalid_1's rmse: 2.43105\n",
      "[1500]\ttraining's rmse: 2.56807\tvalid_1's rmse: 2.41252\n",
      "[1600]\ttraining's rmse: 2.54614\tvalid_1's rmse: 2.39659\n",
      "[1700]\ttraining's rmse: 2.52656\tvalid_1's rmse: 2.38275\n",
      "[1800]\ttraining's rmse: 2.50833\tvalid_1's rmse: 2.37041\n",
      "[1900]\ttraining's rmse: 2.4929\tvalid_1's rmse: 2.36004\n",
      "[2000]\ttraining's rmse: 2.47875\tvalid_1's rmse: 2.35104\n",
      "[2100]\ttraining's rmse: 2.4663\tvalid_1's rmse: 2.3431\n",
      "[2200]\ttraining's rmse: 2.45475\tvalid_1's rmse: 2.33613\n",
      "[2300]\ttraining's rmse: 2.4442\tvalid_1's rmse: 2.33007\n",
      "[2400]\ttraining's rmse: 2.43441\tvalid_1's rmse: 2.32477\n",
      "[2500]\ttraining's rmse: 2.42546\tvalid_1's rmse: 2.31993\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\ttraining's rmse: 2.42546\tvalid_1's rmse: 2.31993\n",
      "val rmse score is 2.319926115740396\n",
      "Fold: 4\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.49087\tvalid_1's rmse: 3.515\n",
      "[200]\ttraining's rmse: 3.34205\tvalid_1's rmse: 3.34375\n",
      "[300]\ttraining's rmse: 3.21355\tvalid_1's rmse: 3.19718\n",
      "[400]\ttraining's rmse: 3.10152\tvalid_1's rmse: 3.06839\n",
      "[500]\ttraining's rmse: 3.00348\tvalid_1's rmse: 2.95578\n",
      "[600]\ttraining's rmse: 2.91841\tvalid_1's rmse: 2.8582\n",
      "[700]\ttraining's rmse: 2.84463\tvalid_1's rmse: 2.77384\n",
      "[800]\ttraining's rmse: 2.78066\tvalid_1's rmse: 2.70133\n",
      "[900]\ttraining's rmse: 2.72429\tvalid_1's rmse: 2.63835\n",
      "[1000]\ttraining's rmse: 2.67596\tvalid_1's rmse: 2.58473\n",
      "[1100]\ttraining's rmse: 2.63411\tvalid_1's rmse: 2.53888\n",
      "[1200]\ttraining's rmse: 2.59781\tvalid_1's rmse: 2.49958\n",
      "[1300]\ttraining's rmse: 2.56597\tvalid_1's rmse: 2.46571\n",
      "[1400]\ttraining's rmse: 2.53839\tvalid_1's rmse: 2.43681\n",
      "[1500]\ttraining's rmse: 2.51452\tvalid_1's rmse: 2.41219\n",
      "[1600]\ttraining's rmse: 2.49335\tvalid_1's rmse: 2.39089\n",
      "[1700]\ttraining's rmse: 2.47462\tvalid_1's rmse: 2.37259\n",
      "[1800]\ttraining's rmse: 2.45733\tvalid_1's rmse: 2.35635\n",
      "[1900]\ttraining's rmse: 2.44266\tvalid_1's rmse: 2.34266\n",
      "[2000]\ttraining's rmse: 2.4294\tvalid_1's rmse: 2.33081\n",
      "[2100]\ttraining's rmse: 2.4176\tvalid_1's rmse: 2.32055\n",
      "[2200]\ttraining's rmse: 2.40672\tvalid_1's rmse: 2.3114\n",
      "[2300]\ttraining's rmse: 2.39689\tvalid_1's rmse: 2.30355\n",
      "[2400]\ttraining's rmse: 2.38782\tvalid_1's rmse: 2.2966\n",
      "[2500]\ttraining's rmse: 2.37957\tvalid_1's rmse: 2.29054\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\ttraining's rmse: 2.37957\tvalid_1's rmse: 2.29054\n",
      "val rmse score is 2.2905405526662017\n",
      "Fold: 5\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.49421\tvalid_1's rmse: 3.37779\n",
      "[200]\ttraining's rmse: 3.33976\tvalid_1's rmse: 3.2265\n",
      "[300]\ttraining's rmse: 3.20672\tvalid_1's rmse: 3.09723\n",
      "[400]\ttraining's rmse: 3.09072\tvalid_1's rmse: 2.98472\n",
      "[500]\ttraining's rmse: 2.98917\tvalid_1's rmse: 2.88698\n",
      "[600]\ttraining's rmse: 2.90096\tvalid_1's rmse: 2.80223\n",
      "[700]\ttraining's rmse: 2.82465\tvalid_1's rmse: 2.72926\n",
      "[800]\ttraining's rmse: 2.75861\tvalid_1's rmse: 2.66672\n",
      "[900]\ttraining's rmse: 2.70055\tvalid_1's rmse: 2.61246\n",
      "[1000]\ttraining's rmse: 2.65089\tvalid_1's rmse: 2.56616\n",
      "[1100]\ttraining's rmse: 2.6081\tvalid_1's rmse: 2.5271\n",
      "[1200]\ttraining's rmse: 2.57107\tvalid_1's rmse: 2.49369\n",
      "[1300]\ttraining's rmse: 2.53874\tvalid_1's rmse: 2.4647\n",
      "[1400]\ttraining's rmse: 2.51078\tvalid_1's rmse: 2.44004\n",
      "[1500]\ttraining's rmse: 2.48671\tvalid_1's rmse: 2.41898\n",
      "[1600]\ttraining's rmse: 2.46553\tvalid_1's rmse: 2.40099\n",
      "[1700]\ttraining's rmse: 2.44688\tvalid_1's rmse: 2.38542\n",
      "[1800]\ttraining's rmse: 2.42971\tvalid_1's rmse: 2.37164\n",
      "[1900]\ttraining's rmse: 2.41517\tvalid_1's rmse: 2.36013\n",
      "[2000]\ttraining's rmse: 2.40213\tvalid_1's rmse: 2.35017\n",
      "[2100]\ttraining's rmse: 2.39057\tvalid_1's rmse: 2.34137\n",
      "[2200]\ttraining's rmse: 2.37994\tvalid_1's rmse: 2.33372\n",
      "[2300]\ttraining's rmse: 2.37042\tvalid_1's rmse: 2.32709\n",
      "[2400]\ttraining's rmse: 2.36166\tvalid_1's rmse: 2.32119\n",
      "[2500]\ttraining's rmse: 2.35375\tvalid_1's rmse: 2.31606\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\ttraining's rmse: 2.35375\tvalid_1's rmse: 2.31606\n",
      "val rmse score is 2.316060039994052\n",
      "mean rmse score over folds is 2.3602940761211095\n"
     ]
    }
   ],
   "source": [
    "columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'Year', 'Month', 'Week', 'Day', 'Dayofweek', 'Dayofyear', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2', \n",
    "            'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90', \n",
    "            'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30','Is_month_end','Is_month_start','Is_quarter_end','Is_quarter_start','Is_year_end','Is_year_start','is_weekend']\n",
    "splits = folds.split(x, y)\n",
    "y_preds = np.zeros(test.shape[0])\n",
    "y_oof = np.zeros(x.shape[0])\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = columns\n",
    "mean_score = []\n",
    "for fold_n, (train_index, valid_index) in enumerate(splits):\n",
    "    print('Fold:',fold_n+1)\n",
    "    X_train, X_valid = x[columns].iloc[train_index], x[columns].iloc[valid_index]\n",
    "    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n",
    "    dtrain = lgb.Dataset(X_train, label=y_train)\n",
    "    dvalid = lgb.Dataset(X_valid, label=y_valid)\n",
    "    clf = lgb.train(params, dtrain, 2500, valid_sets = [dtrain, dvalid],early_stopping_rounds = 50, verbose_eval=100)\n",
    "    feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()\n",
    "    y_pred_valid = clf.predict(X_valid,num_iteration=clf.best_iteration)\n",
    "    y_oof[valid_index] = y_pred_valid\n",
    "    val_score = np.sqrt(metrics.mean_squared_error(y_pred_valid, y_valid))\n",
    "    print(f'val rmse score is {val_score}')\n",
    "    mean_score.append(val_score)\n",
    "    y_preds += clf.predict(test[columns], num_iteration=clf.best_iteration)/n_fold\n",
    "    del X_train, X_valid, y_train, y_valid\n",
    "    gc.collect()\n",
    "print('mean rmse score over folds is',np.mean(mean_score))\n",
    "test['demand'] = y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>F4</th>\n",
       "      <th>F5</th>\n",
       "      <th>F6</th>\n",
       "      <th>F7</th>\n",
       "      <th>F8</th>\n",
       "      <th>F9</th>\n",
       "      <th>F10</th>\n",
       "      <th>F11</th>\n",
       "      <th>F12</th>\n",
       "      <th>F13</th>\n",
       "      <th>F14</th>\n",
       "      <th>F15</th>\n",
       "      <th>F16</th>\n",
       "      <th>F17</th>\n",
       "      <th>F18</th>\n",
       "      <th>F19</th>\n",
       "      <th>F20</th>\n",
       "      <th>F21</th>\n",
       "      <th>F22</th>\n",
       "      <th>F23</th>\n",
       "      <th>F24</th>\n",
       "      <th>F25</th>\n",
       "      <th>F26</th>\n",
       "      <th>F27</th>\n",
       "      <th>F28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_validation</td>\n",
       "      <td>0.865531</td>\n",
       "      <td>0.807942</td>\n",
       "      <td>0.782789</td>\n",
       "      <td>0.768486</td>\n",
       "      <td>0.775392</td>\n",
       "      <td>0.850230</td>\n",
       "      <td>0.867979</td>\n",
       "      <td>0.750338</td>\n",
       "      <td>0.929047</td>\n",
       "      <td>0.984478</td>\n",
       "      <td>1.084697</td>\n",
       "      <td>1.025586</td>\n",
       "      <td>1.185424</td>\n",
       "      <td>1.196782</td>\n",
       "      <td>0.989096</td>\n",
       "      <td>0.863416</td>\n",
       "      <td>0.819320</td>\n",
       "      <td>0.804049</td>\n",
       "      <td>0.851128</td>\n",
       "      <td>1.134578</td>\n",
       "      <td>1.021611</td>\n",
       "      <td>0.883556</td>\n",
       "      <td>0.860246</td>\n",
       "      <td>0.881730</td>\n",
       "      <td>0.968237</td>\n",
       "      <td>0.923052</td>\n",
       "      <td>1.047521</td>\n",
       "      <td>1.044283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_validation</td>\n",
       "      <td>0.428786</td>\n",
       "      <td>0.408093</td>\n",
       "      <td>0.397258</td>\n",
       "      <td>0.380068</td>\n",
       "      <td>0.372334</td>\n",
       "      <td>0.369134</td>\n",
       "      <td>0.356890</td>\n",
       "      <td>0.295321</td>\n",
       "      <td>0.286300</td>\n",
       "      <td>0.276204</td>\n",
       "      <td>0.275311</td>\n",
       "      <td>0.276456</td>\n",
       "      <td>0.297520</td>\n",
       "      <td>0.296572</td>\n",
       "      <td>0.254984</td>\n",
       "      <td>0.254566</td>\n",
       "      <td>0.254775</td>\n",
       "      <td>0.253560</td>\n",
       "      <td>0.256841</td>\n",
       "      <td>0.280211</td>\n",
       "      <td>0.278067</td>\n",
       "      <td>0.251165</td>\n",
       "      <td>0.250718</td>\n",
       "      <td>0.310173</td>\n",
       "      <td>0.300669</td>\n",
       "      <td>0.303097</td>\n",
       "      <td>0.306454</td>\n",
       "      <td>0.298681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_validation</td>\n",
       "      <td>0.523659</td>\n",
       "      <td>0.512263</td>\n",
       "      <td>0.510883</td>\n",
       "      <td>0.507747</td>\n",
       "      <td>0.509175</td>\n",
       "      <td>0.588526</td>\n",
       "      <td>0.613127</td>\n",
       "      <td>0.402131</td>\n",
       "      <td>0.391712</td>\n",
       "      <td>0.374083</td>\n",
       "      <td>0.386098</td>\n",
       "      <td>0.387282</td>\n",
       "      <td>0.422815</td>\n",
       "      <td>0.398209</td>\n",
       "      <td>0.357443</td>\n",
       "      <td>0.346912</td>\n",
       "      <td>0.385322</td>\n",
       "      <td>0.459259</td>\n",
       "      <td>0.588733</td>\n",
       "      <td>0.703190</td>\n",
       "      <td>0.806529</td>\n",
       "      <td>0.662586</td>\n",
       "      <td>0.687901</td>\n",
       "      <td>0.679124</td>\n",
       "      <td>0.607307</td>\n",
       "      <td>0.620870</td>\n",
       "      <td>0.717829</td>\n",
       "      <td>0.673876</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_validation</td>\n",
       "      <td>1.960018</td>\n",
       "      <td>1.750663</td>\n",
       "      <td>1.615915</td>\n",
       "      <td>1.516198</td>\n",
       "      <td>1.746814</td>\n",
       "      <td>1.868266</td>\n",
       "      <td>1.849573</td>\n",
       "      <td>1.559118</td>\n",
       "      <td>1.672507</td>\n",
       "      <td>1.585481</td>\n",
       "      <td>1.505082</td>\n",
       "      <td>1.611071</td>\n",
       "      <td>2.169794</td>\n",
       "      <td>2.302821</td>\n",
       "      <td>1.956956</td>\n",
       "      <td>1.748004</td>\n",
       "      <td>1.827239</td>\n",
       "      <td>1.742337</td>\n",
       "      <td>1.802949</td>\n",
       "      <td>1.889524</td>\n",
       "      <td>2.105226</td>\n",
       "      <td>1.922538</td>\n",
       "      <td>1.822523</td>\n",
       "      <td>1.633256</td>\n",
       "      <td>1.613948</td>\n",
       "      <td>1.781871</td>\n",
       "      <td>2.448790</td>\n",
       "      <td>2.345165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_validation</td>\n",
       "      <td>0.865271</td>\n",
       "      <td>0.787341</td>\n",
       "      <td>0.983298</td>\n",
       "      <td>1.166901</td>\n",
       "      <td>1.123261</td>\n",
       "      <td>1.268960</td>\n",
       "      <td>1.490975</td>\n",
       "      <td>1.196775</td>\n",
       "      <td>1.221046</td>\n",
       "      <td>1.044642</td>\n",
       "      <td>1.011785</td>\n",
       "      <td>1.019694</td>\n",
       "      <td>1.221476</td>\n",
       "      <td>1.103937</td>\n",
       "      <td>1.012103</td>\n",
       "      <td>0.949233</td>\n",
       "      <td>0.961080</td>\n",
       "      <td>0.956120</td>\n",
       "      <td>1.052778</td>\n",
       "      <td>1.215995</td>\n",
       "      <td>1.192255</td>\n",
       "      <td>0.909365</td>\n",
       "      <td>0.924303</td>\n",
       "      <td>0.921379</td>\n",
       "      <td>0.960074</td>\n",
       "      <td>1.018071</td>\n",
       "      <td>1.221927</td>\n",
       "      <td>1.458850</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              id        F1        F2        F3        F4  \\\n",
       "0  HOBBIES_1_001_CA_1_validation  0.865531  0.807942  0.782789  0.768486   \n",
       "1  HOBBIES_1_002_CA_1_validation  0.428786  0.408093  0.397258  0.380068   \n",
       "2  HOBBIES_1_003_CA_1_validation  0.523659  0.512263  0.510883  0.507747   \n",
       "3  HOBBIES_1_004_CA_1_validation  1.960018  1.750663  1.615915  1.516198   \n",
       "4  HOBBIES_1_005_CA_1_validation  0.865271  0.787341  0.983298  1.166901   \n",
       "\n",
       "         F5        F6        F7        F8        F9       F10       F11  \\\n",
       "0  0.775392  0.850230  0.867979  0.750338  0.929047  0.984478  1.084697   \n",
       "1  0.372334  0.369134  0.356890  0.295321  0.286300  0.276204  0.275311   \n",
       "2  0.509175  0.588526  0.613127  0.402131  0.391712  0.374083  0.386098   \n",
       "3  1.746814  1.868266  1.849573  1.559118  1.672507  1.585481  1.505082   \n",
       "4  1.123261  1.268960  1.490975  1.196775  1.221046  1.044642  1.011785   \n",
       "\n",
       "        F12       F13       F14       F15       F16       F17       F18  \\\n",
       "0  1.025586  1.185424  1.196782  0.989096  0.863416  0.819320  0.804049   \n",
       "1  0.276456  0.297520  0.296572  0.254984  0.254566  0.254775  0.253560   \n",
       "2  0.387282  0.422815  0.398209  0.357443  0.346912  0.385322  0.459259   \n",
       "3  1.611071  2.169794  2.302821  1.956956  1.748004  1.827239  1.742337   \n",
       "4  1.019694  1.221476  1.103937  1.012103  0.949233  0.961080  0.956120   \n",
       "\n",
       "        F19       F20       F21       F22       F23       F24       F25  \\\n",
       "0  0.851128  1.134578  1.021611  0.883556  0.860246  0.881730  0.968237   \n",
       "1  0.256841  0.280211  0.278067  0.251165  0.250718  0.310173  0.300669   \n",
       "2  0.588733  0.703190  0.806529  0.662586  0.687901  0.679124  0.607307   \n",
       "3  1.802949  1.889524  2.105226  1.922538  1.822523  1.633256  1.613948   \n",
       "4  1.052778  1.215995  1.192255  0.909365  0.924303  0.921379  0.960074   \n",
       "\n",
       "        F26       F27       F28  \n",
       "0  0.923052  1.047521  1.044283  \n",
       "1  0.303097  0.306454  0.298681  \n",
       "2  0.620870  0.717829  0.673876  \n",
       "3  1.781871  2.448790  2.345165  \n",
       "4  1.018071  1.221927  1.458850  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def predict(test, submission):\n",
    "    predictions = test[['id', 'date', 'demand']]\n",
    "    predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand').reset_index()\n",
    "    predictions.columns = ['id'] + ['F' + str(i + 1) for i in range(28)]\n",
    "    evaluation_rows = [row for row in submission['id'] if 'evaluation' in row] \n",
    "    evaluation = submission[submission['id'].isin(evaluation_rows)]\n",
    "    validation = submission[['id']].merge(predictions, on = 'id')\n",
    "    final = pd.concat([validation, evaluation])\n",
    "    #final.to_csv('submission.csv', index = False)\n",
    "    return final\n",
    "\n",
    "subs = predict(test, submission)\n",
    "subs.to_csv('M5_Forecasting.csv',index = False)\n",
    "subs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = x.copy()\n",
    "test_df = test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = train_df.sort_values('date')['demand']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      17816316\n",
       "1       4562185\n",
       "2       2232229\n",
       "3       1177400\n",
       "4        706316\n",
       "         ...   \n",
       "273           1\n",
       "275           1\n",
       "276           1\n",
       "278           1\n",
       "607           1\n",
       "Name: demand, Length: 287, dtype: int64"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df['demand'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df = test_df.drop(['id', 'part','id_encode'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df.drop(['id', 'part','id_encode'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "procs=[FillMissing, Categorify, Normalize]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = ['item_id','dept_id','cat_id','store_id','state_id','event_name_1','event_type_1','event_name_2','event_type_2','snap_CA','snap_TX','snap_WI',\n",
    "           'Year','Month','Week','Day','Dayofweek','Dayofyear','Is_month_end','Is_month_start','Is_quarter_end','Is_quarter_start','Is_year_end','Is_year_start']\n",
    "\n",
    "cont_vars = ['wm_yr_wk','sell_price','lag_t28','lag_t29','lag_t30','rolling_mean_t7','rolling_std_t7','rolling_mean_t30','rolling_mean_t90','rolling_mean_t180','rolling_std_t30',\n",
    "            'price_change_t1','price_change_t365','rolling_price_std_t7','rolling_price_std_t30',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "dep_var = 'demand'\n",
    "df = train_df[cat_vars + cont_vars + [dep_var,'date']].copy()\n",
    "# or\n",
    "# train_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('2016-04-25 00:00:00'), Timestamp('2016-05-22 00:00:00'))"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For valid_inx in Time Series,\n",
    "# we want to have the valid set's time is almost at the end of the train data\n",
    "test_df['date'].min(), test_df['date'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "range(0, 855879)\n"
     ]
    }
   ],
   "source": [
    "cut = train_df['date'][(train_df['date'] == train_df['date'][len(test_df)])].index.max()\n",
    "cut\n",
    "valid_idx = range(cut)\n",
    "print(valid_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create DataBunch\n",
    "# anytime you're trying to predict something like a population or a dollar amount of sales, always use log = True (RMSPE becomes root mean squared error)\n",
    "data = (TabularList.from_df(df, path=path, cat_names=cat_vars, cont_names=cont_vars, procs=procs)\n",
    "                   .split_by_idx(valid_idx)\n",
    "                   .label_from_df(cols=dep_var, label_cls=FloatList, log = True) \n",
    "                   .databunch())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>item_id</th>\n",
       "      <th>dept_id</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>store_id</th>\n",
       "      <th>state_id</th>\n",
       "      <th>event_name_1</th>\n",
       "      <th>event_type_1</th>\n",
       "      <th>event_name_2</th>\n",
       "      <th>event_type_2</th>\n",
       "      <th>snap_CA</th>\n",
       "      <th>snap_TX</th>\n",
       "      <th>snap_WI</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Week</th>\n",
       "      <th>Day</th>\n",
       "      <th>Dayofweek</th>\n",
       "      <th>Dayofyear</th>\n",
       "      <th>Is_month_end</th>\n",
       "      <th>Is_month_start</th>\n",
       "      <th>Is_quarter_end</th>\n",
       "      <th>Is_quarter_start</th>\n",
       "      <th>Is_year_end</th>\n",
       "      <th>Is_year_start</th>\n",
       "      <th>sell_price_na</th>\n",
       "      <th>lag_t28_na</th>\n",
       "      <th>lag_t29_na</th>\n",
       "      <th>lag_t30_na</th>\n",
       "      <th>rolling_mean_t7_na</th>\n",
       "      <th>rolling_std_t7_na</th>\n",
       "      <th>rolling_mean_t30_na</th>\n",
       "      <th>rolling_mean_t90_na</th>\n",
       "      <th>rolling_mean_t180_na</th>\n",
       "      <th>rolling_std_t30_na</th>\n",
       "      <th>price_change_t1_na</th>\n",
       "      <th>price_change_t365_na</th>\n",
       "      <th>rolling_price_std_t7_na</th>\n",
       "      <th>rolling_price_std_t30_na</th>\n",
       "      <th>wm_yr_wk</th>\n",
       "      <th>sell_price</th>\n",
       "      <th>lag_t28</th>\n",
       "      <th>lag_t29</th>\n",
       "      <th>lag_t30</th>\n",
       "      <th>rolling_mean_t7</th>\n",
       "      <th>rolling_std_t7</th>\n",
       "      <th>rolling_mean_t30</th>\n",
       "      <th>rolling_mean_t90</th>\n",
       "      <th>rolling_mean_t180</th>\n",
       "      <th>rolling_std_t30</th>\n",
       "      <th>price_change_t1</th>\n",
       "      <th>price_change_t365</th>\n",
       "      <th>rolling_price_std_t7</th>\n",
       "      <th>rolling_price_std_t30</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>963</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>8</td>\n",
       "      <td>34</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>229</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.7130</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>-19824.0000</td>\n",
       "      <td>nan</td>\n",
       "      <td>-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>171</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2014</td>\n",
       "      <td>7</td>\n",
       "      <td>27</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>183</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>-0.7386</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>-19824.0000</td>\n",
       "      <td>nan</td>\n",
       "      <td>-inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1252</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>9</td>\n",
       "      <td>39</td>\n",
       "      <td>25</td>\n",
       "      <td>4</td>\n",
       "      <td>268</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.7808</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>-19824.0000</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>532</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>119</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.4959</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>-19824.0000</td>\n",
       "      <td>nan</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2336</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>30</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2015</td>\n",
       "      <td>10</td>\n",
       "      <td>44</td>\n",
       "      <td>26</td>\n",
       "      <td>0</td>\n",
       "      <td>299</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>0.8486</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>nan</td>\n",
       "      <td>-19824.0000</td>\n",
       "      <td>nan</td>\n",
       "      <td>-inf</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0000, 6.5909], device='cuda:0')"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a Learner\n",
    "\n",
    "# y values are going to be taken the log first, we need to make sure that the y_range we want is also the log\n",
    "# multiply it by a little bit (1.2) to give the opportunity to give exact data (like last time)\n",
    "# Then our y_range will be from zero to a bit more than the maximum --> Make sure to convert to tensor!\n",
    "\n",
    "max_log_y = np.log(np.max(train_df['demand'])*1.2) \n",
    "# max_y = np.max(train_df['demand'])\n",
    "y_range = torch.tensor([0, max_log_y], device=defaults.device)\n",
    "y_range"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PS is drop-out\n",
    "# Weight matrix is going to have to go from a 1000 activation input to a 500 activation output, \n",
    "# which means it's going to have to be 500,000 elements in that weight matrix. \n",
    "# That's an awful lot for a data set with only a few hundred thousand rows. \n",
    "# So this is going to overfit, and we need to make sure it doesn't. so we have to regularize using dropout\n",
    "\n",
    "learn = tabular_learner(data, layers=[1000,500], ps=[0.001,0.01], emb_drop=0.04, \n",
    "                        y_range=y_range, metrics= exp_rmspe)\n",
    "#learn = tabular_learner(data, layers=[200,100],\n",
    " #                       y_range=y_range, metrics= exp_rmspe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "193"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>exp_rmspe</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='progress-bar-interrupted' max='429242', style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      Interrupted\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    },
    {
     "ename": "OverflowError",
     "evalue": "cannot serialize a bytes object larger than 4 GiB",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOverflowError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-60-d81c6bd29d71>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr_find\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastai\\train.py\u001b[0m in \u001b[0;36mlr_find\u001b[1;34m(learn, start_lr, end_lr, num_it, stop_div, wd)\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[0mcb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLRFinder\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mend_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnum_it\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstop_div\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m     \u001b[0mepochs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mceil\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnum_it\u001b[0m\u001b[1;33m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m     \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstart_lr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m def to_fp16(learn:Learner, loss_scale:float=None, max_noskip:int=1000, dynamic:bool=True, clip:float=None,\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, epochs, lr, wd, callbacks)\u001b[0m\n\u001b[0;32m    198\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mwd\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[0mcallbacks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mcb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallback_fns\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdefaults\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextra_callback_fns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlistify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 200\u001b[1;33m         \u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mcallbacks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    201\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcreate_opt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mwd\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mFloats\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m->\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastai\\basic_train.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(epochs, learn, callbacks, metrics)\u001b[0m\n\u001b[0;32m     97\u001b[0m             \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_dl\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m             \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_epoch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m             \u001b[1;32mfor\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0myb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mprogress_bar\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain_dl\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparent\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpbar\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m                 \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m                 \u001b[0mloss\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mloss_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0myb\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_func\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mopt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcb_handler\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastprogress\\fastprogress.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     45\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mon_interrupt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     48\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mval\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastprogress\\fastprogress.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtotal\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m             \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mo\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[1;33m>=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtotal\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m                 \u001b[1;32myield\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\fastai\\basic_data.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     73\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m         \u001b[1;34m\"Process and returns items from `DataLoader`.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 75\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdl\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;32myield\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mproc_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     76\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    276\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0m_SingleProcessDataLoaderIter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 278\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0m_MultiProcessingDataLoaderIter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    279\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    280\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\torch\\utils\\data\\dataloader.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, loader)\u001b[0m\n\u001b[0;32m    680\u001b[0m             \u001b[1;31m#     before it starts, and __del__ tries to join but will get:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    681\u001b[0m             \u001b[1;31m#     AssertionError: can only join a started process.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 682\u001b[1;33m             \u001b[0mw\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstart\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    683\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_index_queues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex_queue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    684\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\multiprocessing\\process.py\u001b[0m in \u001b[0;36mstart\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    110\u001b[0m                \u001b[1;34m'daemonic processes are not allowed to have children'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[0m_cleanup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 112\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_popen\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    113\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_sentinel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_popen\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msentinel\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    114\u001b[0m         \u001b[1;31m# Avoid a refcycle if the target function holds an indirect\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\multiprocessing\\context.py\u001b[0m in \u001b[0;36m_Popen\u001b[1;34m(process_obj)\u001b[0m\n\u001b[0;32m    221\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mstaticmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    222\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 223\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_default_context\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mProcess\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    224\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    225\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0mDefaultContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\multiprocessing\\context.py\u001b[0m in \u001b[0;36m_Popen\u001b[1;34m(process_obj)\u001b[0m\n\u001b[0;32m    320\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0m_Popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    321\u001b[0m             \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mpopen_spawn_win32\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mPopen\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 322\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mPopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    323\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    324\u001b[0m     \u001b[1;32mclass\u001b[0m \u001b[0mSpawnContext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mBaseContext\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\multiprocessing\\popen_spawn_win32.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, process_obj)\u001b[0m\n\u001b[0;32m     87\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     88\u001b[0m                 \u001b[0mreduction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprep_data\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_child\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 89\u001b[1;33m                 \u001b[0mreduction\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprocess_obj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mto_child\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     90\u001b[0m             \u001b[1;32mfinally\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     91\u001b[0m                 \u001b[0mset_spawning_popen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\multiprocessing\\reduction.py\u001b[0m in \u001b[0;36mdump\u001b[1;34m(obj, file, protocol)\u001b[0m\n\u001b[0;32m     58\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m     \u001b[1;34m'''Replacement for pickle.dump() using ForkingPickler.'''\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 60\u001b[1;33m     \u001b[0mForkingPickler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprotocol\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdump\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     61\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m \u001b[1;31m#\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOverflowError\u001b[0m: cannot serialize a bytes object larger than 4 GiB"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to compute the gradients, there might not be enough points.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAEGCAYAAAD4yOuIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAEshJREFUeJzt3X+w5XVdx/HnS1ZwFL38cEBgURi5Y7NZaSVIZTEiuEzhYuGENbk6UNnkTP6oxJoAUUwdCTO1MWVr0zEJpnSbIEKJMY0UUhtbi84C6l53U2OXq2RCC+/+ON/Fs5d7797LPefecz/3+Zg5c77fz/fz/X7f3/PH9zWf7/d7zklVIUlSix6z0gVIkjQqhpwkqVmGnCSpWYacJKlZhpwkqVnrVrqA5TA9Pe0jpJLUuImJicxscyQnSWqWISdJapYhtwi9Xm+lS5Ckpoz6vGrISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpo1NiGXZGOSO5LsSHLxLMsPS3JNt/wzSU6asfypSe5L8pvLVbMkabyNRcglOQR4D3AOsAF4aZINM7pdCOytqlOAq4C3zVh+FXDDqGuVJK0eYxFywKnAjqq6q6oeAD4CbJrRZxOwtZu+DjgzSQCSnAfcBWxfpnolSavAuITcCcDOgfmprm3WPlW1D5gGjk7yBOD1wBuXoU5J0iqybqUL6GSWtlpgnzcCV1XVfd3Abl69Xm/x1Q1xfUnSgZZyXp2cnJx3+biE3BRw4sD8emDXHH2mkqwDJoA9wGnA+UneDhwBPJTku1X17tl2dLAPZD69Xm9J60uSDjTq8+q4hNxtwGSSk4GvARcAvzCjzzZgM3ArcD5wc1UV8Lz9HZJcBtw3V8BJktaWsQi5qtqX5FXAjcAhwJaq2p7kcuD2qtoGXA18MMkO+iO4C1auYknSajAWIQdQVdcD189ou2Rg+rvASw6yjctGUpwkaVUal6crJUkaOkNOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1KyxCbkkG5PckWRHkotnWX5Ykmu65Z9JclLXflaSf0nyxe79+ctduyRpPI1FyCU5BHgPcA6wAXhpkg0zul0I7K2qU4CrgLd17f8NnFtVPwBsBj64PFVLksbdWIQccCqwo6ruqqoHgI8Am2b02QRs7aavA85Mkqr6fFXt6tq3A49LctiyVC1JGmvjEnInADsH5qe6tln7VNU+YBo4ekafnwM+X1X3j6hOSdIqsm6lC+hklrZaTJ8k30//EubZ8+2o1+sturhhri9JOtBSzquTk5PzLh+XkJsCThyYXw/smqPPVJJ1wASwByDJeuCvgZdV1Z3z7ehgH8h8er3ektaXJB1o1OfVcblceRswmeTkJIcCFwDbZvTZRv/BEoDzgZurqpIcAfwt8Iaq+vSyVSxJGntjEXLdPbZXATcC/w78ZVVtT3J5khd13a4Gjk6yA3gtsP9rBq8CTgF+L8kXutcxy3wIkqQxlKqZt77aMz09PZSD9HKlJA3XMM+rExMTj3h2YyxGcpIkjYIhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJapYhJ0lqliEnSWqWISdJataCQy7Ja5M8q5t+bpKvJrkryemjK0+SpEdvMSO51wB3d9O/D/wBcAXwzmEXJUnSMKxbRN+JqppO8kTgh4AXVNWDSa4cUW2SJC3JYkJuZ5IfA74f+GQXcE8CHhxNaZIkLc1iQu63gOuAB4Cf69p+BvjssIuSJGkYFhxyVXU9cPyM5mu7lyRJY2cxT1duSHJsN314kjcCbwAeO4xCkmxMckeSHUkunmX5YUmu6ZZ/JslJA8ve0LXfkeSFw6hHkrT6Lebpyg8DR3TT7wB+EjgdeN9Si0hyCPAe4BxgA/DSJBtmdLsQ2FtVpwBXAW/r1t0AXED/XuFG4L3d9iRJa9xiQu6kqrojSYAXAy8BzgeGMXI6FdhRVXdV1QPAR4BNM/psArZ209cBZ3a1bAI+UlX3V9XdwI5ue5KkNW4xIXd/9/WBU4GdVfXfwP3A44ZQxwnAzoH5qa5t1j5VtQ+YBo5e4LqSpDVoMU9Xfhi4GXgi8O6u7Yf53hfElyKztNUC+yxk3Yf1er1FlDX89SVJB1rKeXVycnLe5Yt5uvI1Sc4G/q+q/qFrfoj+L6Es1RRw4sD8emDXHH2mkqwDJoA9C1z3YQf7QObT6/WWtL4k6UCjPq8u6geaq+rvgTuTnJ7kqVV1e1XdPIQ6bgMmk5yc5FD6D5Jsm9FnG7C5mz4fuLmqqmu/oHv68mRgEr+7J0liESO5JMfRfyDkufRHUEcnuRV4aVXNOXJaiKral+RVwI3AIcCWqtqe5HLg9qraBlwNfDDJjm7/F3Trbk/yl8CXgH3Ar1eVv8IiSSL9wdACOiYfBb4KvKGq/ifJE4C3ACdX1YtGWOOSTU9PL+wgD8LLlZI0XMM8r05MTDziGY3FPHjyE8BxVfV/AF3Q/TbwtaFUJ0nSkC3mntxe+l/UHvQM4N7hlSNJ0vAsZiT3duDjSa4GvgI8DXgF8HujKEySpKVa8Eiuqt4P/DzwZODc7v2X6D+yL0nS2FnMSI7u6wIPf2UgyWHADcAlQ65LkqQlW9T35OYw2y+OSJK04oYRckN5PF+SpGE76OXKJM+fZ/GhQ6xFkqShWsg9uasPsvyrwyhEkqRhO2jIVdXJy1GIJEnDNox7cpIkjSVDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLULENOktQsQ06S1CxDTpLUrBUPuSRHJbkpSa97P3KOfpu7Pr0km7u2xyf52yT/kWR7krcub/WSpHG24iEHXAx8oqomgU908wdIchRwKXAacCpw6UAYvqOqvg94NvDjSc5ZnrIlSeNuHEJuE7C1m94KnDdLnxcCN1XVnqraC9wEbKyq71TVPwBU1QPA54D1y1CzJGkVGIeQO7aqdgN078fM0ucEYOfA/FTX9rAkRwDn0h8NSpLEuuXYSZKPA0+ZZdHvLnQTs7TVwPbXAX8BvKuq7ppvQ71eb4G7HM36kqQDLeW8Ojk5Oe/yZQm5qnrBXMuSfD3JcVW1O8lxwDdm6TYFnDEwvx64ZWD+T4BeVb3zYLUc7AOZT6/XW9L6kqQDjfq8Og6XK7cBm7vpzcDHZulzI3B2kiO7B07O7tpI8mZgAnj1MtQqSVpFxiHk3gqclaQHnNXNk+RHk3wAoKr2AG8Cbutel1fVniTr6V/y3AB8LskXkly0EgchSRo/y3K5cj5VdQ9w5izttwMXDcxvAbbM6DPF7PfrJEkai5GcJEkjYchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKaZchJkpplyEmSmmXISZKateIhl+SoJDcl6XXvR87Rb3PXp5dk8yzLtyX5t9FXLElaLVY85ICLgU9U1STwiW7+AEmOAi4FTgNOBS4dDMMkPwvctzzlSpJWi3EIuU3A1m56K3DeLH1eCNxUVXuqai9wE7ARIMnhwGuBNy9DrZKkVWQcQu7YqtoN0L0fM0ufE4CdA/NTXRvAm4Arge+MskhJ0uqzbjl2kuTjwFNmWfS7C93ELG2V5FnAKVX1miQnLWRDvV5vgbsczfqSpAMt5bw6OTk57/JlCbmqesFcy5J8PclxVbU7yXHAN2bpNgWcMTC/HrgFOB34kSRfpn8sxyS5parOYA4H+0Dm0+v1lrS+JOlAoz6vjsPlym3A/qclNwMfm6XPjcDZSY7sHjg5G7ixqv64qo6vqpOAnwD+c76AkyStLeMQcm8FzkrSA87q5knyo0k+AFBVe+jfe7ute13etUmSNKdluVw5n6q6BzhzlvbbgYsG5rcAW+bZzpeBZ46gREnSKjUOIzlJkkbCkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNcuQkyQ1y5CTJDXLkJMkNStVtdI1jNz09HT7BylJa9zExERmtjmSkyQ1y5CTJDVrTVyulCStTY7kJEnNWnMhl2RLkm8k+bchbW9zkl732jzQ/iNJvphkR5J3JXnEDVFJWu2W8Zx6RZKdSe5bzPbWXMgBfwZsXOxKSW5JctKMtqOAS4HTgFOBS5Mc2S3+Y+BXgMnuteh9StIq8Gcszzn1b7q2RVlzIVdVnwT2DLYleXqSv0vyL0n+Mcn3LXBzLwRuqqo9VbUXuAnYmOQ44ElVdWv1b3r+OXDeMI9DksbBcpxTu/38c1XtXmx96xa7QqP+BHhlVfWSnAa8F3j+AtY7Adg5MD/VtZ3QTc9sl6S1YNjn1EdtzYdcksOBHwOuHbhtdli37BXAb3RtpwDXJ3kAuLuqXgzMdp+t5mmXpKaN6Jz6qK35kKN/yfbeqnrWzAVV9afAn0L/+jHw8qr68kCXKeCMgfn1wC1d+/oZ7buGWLMkjatRnFOXVMyaVlXfAu5O8hKA9P3QAle/ETg7yZHdzdGzgRu768bfTvLc7qnKlwEfG0X9kjRORnFOXUo9ay7kkvwFcCvwjCRTSS4EfhG4MMm/AtuBTQvZVlXtAd4E3Na9Lu/aAH4N+ACwA7gTuGGoByJJY2C5zqlJ3p5kCnh8t5/LFlSfv3giSWrVmhvJSZLWDkNOktQsQ06S1CxDTpLULENOktQsQ05apZLcMPgr7ZIeyZCTFinJl5O8YKXrqKpzqmrrsLeb5IwkDyW5L8m3k9zR/RzTQte/LMmHhl2X9GgYctIYSrLSP7m3q6oOB54EvAZ4f5JnrHBN0qIZctIQJfmZJF9Icm+Sf0rygwPLLk5yZzc6+lKSFw8se3mSTye5Kske4LKu7VNJ3pFkb5K7k5wzsM4tSS4aWH++vicn+WS3748nec9CRlvVdz39v1IZPJY/7P7A8lvd36k8r2vfCPwO8PPdSPBfu/aJJFcn2Z3ka0nenOSQJXzU0oIYctKQJPlhYAvwq8DRwPuAbUkO67rcCTwPmADeCHyo++/B/U4D7gKOAa4YaLsDeDLwduDqZM5/mZ+v74eBz3Z1XQb80gKP6TFJXtRtc8fAotuAZwFHddu+NsnjqurvgLcA11TV4VW1/zcLtwL76P/y/LPp/ybhRQupQVoKQ04anl8G3ldVn6mqB7v7ZfcDzwWoqmuraldVPVRV1wA9Dvyn411V9UdVta+q/rdr+0pVvb+qHqQfFMcBx86x/1n7Jnkq8Bzgkqp6oKo+BWw7yLEcn+Re4H+BvwZeW1Wf37+wqj5UVfd0tV5J/69UZr2cmeRY4Bzg1VX1P1X1DeAq4IKD1CAtmSEnDc/TgNd1lyrv7ULiROB4gCQvG7iUeS/wTPojpP12PnKT/Nf+iar6Tjd5+Bz7n6vv8cCegba59jVoV1UdQf+e3LuY8YeXSV6X5N+TTHfHMjHjWAY9DXgssHvg2N9Hf8QqjdRK39yWWrITuKKqrpi5IMnTgPcDZwK3VtWDSb7AgX8SOapfS98NHJXk8QNBd+JCVqyq+5O8HrgjyXlV9dHu/tvr6R/L9qp6KMlevncsM49jJ/0R7ZOrat+Sj0ZaBEdy0qPz2CSPG3itox9ir0xyWvcfWk9I8tNJngg8gf7J/5vw8D8kP3M5Cq2qrwC303+Y5dAkpwPnLmL9B4ArgUu6pifSv7/2TWBdkkvoj/j2+zpwUpLHdOvvBv4euDLJk7r7fE9P8lNLPTbpYAw56dG5nv79qv2vy6rqdvr35d4N7KX/oMbLAarqS/SD4lb6IfADwKeXsd5fBE4H7gHeDFxDf3S1UFuApyY5l/6fWN4A/CfwFeC7HHj589ru/Z4kn+umXwYcCnyJ/mdzHf17htJI+X9y0hqU5BrgP6rq0pWuRRolR3LSGpDkOd0lwsd032XbBHx0peuSRs0HT6S14SnAX9H/ntwU8GuDXwmQWuXlSklSs7xcKUlqliEnSWqWISdJapYhJ0lqliEnSWqWISdJatb/A4hJK4jt7MAIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.recorder.plot(suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.fit_one_cycle(4, 3.31e-1, wd=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.unfreeze()\n",
    "learn.fit_one_cycle(5, max_lr=slice(1e-6,1e-3), pct_start=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# And we can predict on a row of dataframe that has the right cat_names and cont_names.\n",
    "learn.predict(test_df.iloc[300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = data[data['date'] <= '2016-04-24']\n",
    "y = x.sort_values('date')['demand']\n",
    "test = data[(data['date'] > '2016-04-24')]\n",
    "x = x.sort_values('date')\n",
    "test = test.sort_values('date')\n",
    "del data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "x.to_csv('M5_Forecasting_Train.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test.to_csv('M5_Forecasting_Test.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_fold = 5 #5 for timely purpose of the kernel\n",
    "folds = TimeSeriesSplit(n_splits=n_fold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = x.drop('demand', axis=1)\n",
    "#y = data.demand\n",
    "testdf = test.drop('demand', axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = X.columns.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    train_x, test_x, train_y, test_y = train_test_split(X[features], y, test_size=0.25)\n",
    "    dtrain = xgb.DMatrix(train_x, label=train_y)\n",
    "    dtest = xgb.DMatrix(test_x, label=test_y)\n",
    "\n",
    "    param = {\n",
    "        'silent': 1,\n",
    "        'objective': 'reg:squarederror',\n",
    "        'booster': trial.suggest_categorical('booster', ['gbtree', 'gblinear', 'dart']),\n",
    "        'lambda': trial.suggest_loguniform('lambda', 1e-8, 1.0),\n",
    "        'alpha': trial.suggest_loguniform('alpha', 1e-8, 1.0)\n",
    "    }\n",
    "\n",
    "    if param['booster'] == 'gbtree' or param['booster'] == 'dart':\n",
    "        param['max_depth'] = trial.suggest_int('max_depth', 1, 100)\n",
    "        param['n_estimators'] = trial.suggest_int('n_estimators', 0, 10000)\n",
    "        param['min_child_weight'] = trial.suggest_int('min_child_weight', 1, 20)\n",
    "        param['scale_pos_weight'] = trial.suggest_int('scale_pos_weight', 1, 100)\n",
    "        param['subsample'] = trial.suggest_discrete_uniform('subsample', 0.5, 0.9, 0.1)\n",
    "        param['colsample_bytree'] = trial.suggest_discrete_uniform('colsample_bytree', 0.5, 0.9, 0.1)\n",
    "        param['eta'] = trial.suggest_loguniform('eta', 1e-8, 1.0)\n",
    "        param['gamma'] = trial.suggest_loguniform('gamma', 1e-8, 1.0)\n",
    "        param['grow_policy'] = trial.suggest_categorical('grow_policy', ['depthwise', 'lossguide'])\n",
    "    if param['booster'] == 'dart':\n",
    "        param['sample_type'] = trial.suggest_categorical('sample_type', ['uniform', 'weighted'])\n",
    "        param['normalize_type'] = trial.suggest_categorical('normalize_type', ['tree', 'forest'])\n",
    "        param['rate_drop'] = trial.suggest_loguniform('rate_drop', 1e-8, 1.0)\n",
    "        param['skip_drop'] = trial.suggest_loguniform('skip_drop', 1e-8, 1.0)\n",
    "\n",
    "    bst = xgb.train(param, dtrain)\n",
    "    preds = bst.predict(dtest)\n",
    "    pred_labels = np.rint(preds)\n",
    "    mse = sklearn.metrics.mean_squared_error(test_y, pred_labels)\n",
    "    return mse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, accuracy_score, roc_auc_score, log_loss, classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W 2020-03-05 20:09:08,614] Setting status of trial#0 as TrialState.FAIL because of the following error: OSError(22, 'Windows Error 0xe06d7363', None, -529697949, None)\n",
      "Traceback (most recent call last):\n",
      "  File \"c:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\optuna\\study.py\", line 569, in _run_trial\n",
      "    result = func(trial)\n",
      "  File \"<ipython-input-30-2b1d79aaf94f>\", line 30, in objective\n",
      "    bst = xgb.train(param, dtrain)\n",
      "  File \"c:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\training.py\", line 216, in train\n",
      "    xgb_model=xgb_model, callbacks=callbacks)\n",
      "  File \"c:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\training.py\", line 74, in _train_internal\n",
      "    bst.update(dtrain, i, obj)\n",
      "  File \"c:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\core.py\", line 1109, in update\n",
      "    dtrain.handle))\n",
      "OSError: [WinError -529697949] Windows Error 0xe06d7363\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "[WinError -529697949] Windows Error 0xe06d7363",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-33-f8e5fe73f20d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mstudy\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0moptuna\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcreate_study\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mstudy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobjective\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_trials\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstudy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\optuna\\study.py\u001b[0m in \u001b[0;36moptimize\u001b[1;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[0;32m    300\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mn_jobs\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    301\u001b[0m                 self._optimize_sequential(func, n_trials, timeout, catch, callbacks,\n\u001b[1;32m--> 302\u001b[1;33m                                           gc_after_trial, None)\n\u001b[0m\u001b[0;32m    303\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    304\u001b[0m                 \u001b[0mtime_start\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnow\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\optuna\\study.py\u001b[0m in \u001b[0;36m_optimize_sequential\u001b[1;34m(self, func, n_trials, timeout, catch, callbacks, gc_after_trial, time_start)\u001b[0m\n\u001b[0;32m    536\u001b[0m                     \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    537\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 538\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_trial_and_callbacks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    539\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mremove_session\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    540\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\optuna\\study.py\u001b[0m in \u001b[0;36m_run_trial_and_callbacks\u001b[1;34m(self, func, catch, callbacks, gc_after_trial)\u001b[0m\n\u001b[0;32m    548\u001b[0m         \u001b[1;31m# type: (...) -> None\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    549\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 550\u001b[1;33m         \u001b[0mtrial\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_run_trial\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgc_after_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    551\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallbacks\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m             \u001b[0mfrozen_trial\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_storage\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_trial\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_trial_id\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\optuna\\study.py\u001b[0m in \u001b[0;36m_run_trial\u001b[1;34m(self, func, catch, gc_after_trial)\u001b[0m\n\u001b[0;32m    567\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    568\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 569\u001b[1;33m             \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    570\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mexceptions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTrialPruned\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    571\u001b[0m             message = 'Setting status of trial#{} as {}. {}'.format(trial_number,\n",
      "\u001b[1;32m<ipython-input-30-2b1d79aaf94f>\u001b[0m in \u001b[0;36mobjective\u001b[1;34m(trial)\u001b[0m\n\u001b[0;32m     28\u001b[0m         \u001b[0mparam\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'skip_drop'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrial\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msuggest_loguniform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'skip_drop'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1e-8\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1.0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 30\u001b[1;33m     \u001b[0mbst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mxgb\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     31\u001b[0m     \u001b[0mpreds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtest\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m     \u001b[0mpred_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpreds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36mtrain\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, maximize, early_stopping_rounds, evals_result, verbose_eval, xgb_model, callbacks, learning_rates)\u001b[0m\n\u001b[0;32m    214\u001b[0m                            \u001b[0mevals\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mevals\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    215\u001b[0m                            \u001b[0mobj\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeval\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfeval\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 216\u001b[1;33m                            xgb_model=xgb_model, callbacks=callbacks)\n\u001b[0m\u001b[0;32m    217\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    218\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\training.py\u001b[0m in \u001b[0;36m_train_internal\u001b[1;34m(params, dtrain, num_boost_round, evals, obj, feval, xgb_model, callbacks)\u001b[0m\n\u001b[0;32m     72\u001b[0m         \u001b[1;31m# Skip the first update if it is a recovery step.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mversion\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;36m2\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mbst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_rabit_checkpoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mversion\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\bokhy\\appdata\\local\\programs\\python\\python37\\lib\\site-packages\\xgboost\\core.py\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, dtrain, iteration, fobj)\u001b[0m\n\u001b[0;32m   1107\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mfobj\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1108\u001b[0m             _check_call(_LIB.XGBoosterUpdateOneIter(self.handle, ctypes.c_int(iteration),\n\u001b[1;32m-> 1109\u001b[1;33m                                                     dtrain.handle))\n\u001b[0m\u001b[0;32m   1110\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1111\u001b[0m             \u001b[0mpred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdtrain\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOSError\u001b[0m: [WinError -529697949] Windows Error 0xe06d7363"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study()\n",
    "study.optimize(objective, n_trials=50)\n",
    "print(study.best_trial)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trial = study.best_trial\n",
    "params = trial.params\n",
    "params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params_xgb = {'colsample_bytree': 0.8,                 \n",
    "              'learning_rate': 0.001,\n",
    "              'max_depth': 30,\n",
    "              'subsample': 1,\n",
    "              'objective':'reg:squarederror',\n",
    "              'eval_metric':'rmse',\n",
    "              'min_child_weight':2,\n",
    "              'gamma':0.25,\n",
    "              'n_estimators':5000\n",
    "              }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2', \n",
    "            'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90', \n",
    "            'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30']\n",
    "splits = folds.split(X, y)\n",
    "y_preds_xgb = np.zeros(testdf.shape[0])\n",
    "y_oof = np.zeros(x.shape[0])\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = columns\n",
    "mean_score = []\n",
    "for fold_n, (train_index, valid_index) in enumerate(splits):\n",
    "    print('Fold:',fold_n+1)\n",
    "    X_train, X_valid = X[columns].iloc[train_index], X[columns].iloc[valid_index]\n",
    "    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n",
    "    dtrain = lgb.Dataset(X_train, label=y_train)\n",
    "    dvalid = lgb.Dataset(X_valid, label=y_valid)\n",
    "    clf = lgb.train(params, dtrain, 2500, valid_sets = [dtrain, dvalid],early_stopping_rounds = 50, verbose_eval=100)\n",
    "    feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()\n",
    "    y_pred_valid = clf.predict(X_valid,num_iteration=clf.best_iteration)\n",
    "    y_oof[valid_index] = y_pred_valid\n",
    "    val_score = np.sqrt(metrics.mean_squared_error(y_pred_valid, y_valid))\n",
    "    print(f'val rmse score is {val_score}')\n",
    "    mean_score.append(val_score)\n",
    "    y_preds_xgb += clf.predict(testdf[columns], num_iteration=clf.best_iteration)/n_fold\n",
    "    del X_train, X_valid, y_train, y_valid\n",
    "    gc.collect()\n",
    "print('mean rmse score over folds is',np.mean(mean_score))\n",
    "testdf['demand'] = y_preds_xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(testdf, submission):\n",
    "    predictions = test[['id', 'date', 'demand']]\n",
    "    predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand').reset_index()\n",
    "    predictions.columns = ['id'] + ['F' + str(i + 1) for i in range(28)]\n",
    "    evaluation_rows = [row for row in submission['id'] if 'evaluation' in row] \n",
    "    evaluation = submission[submission['id'].isin(evaluation_rows)]\n",
    "    validation = submission[['id']].merge(predictions, on = 'id')\n",
    "    final = pd.concat([validation, evaluation])\n",
    "    #final.to_csv('submission.csv', index = False)\n",
    "    return final\n",
    "\n",
    "subs = predict(testdf, submission)\n",
    "subs.to_csv('M5_Forecasting.csv',index = False)\n",
    "subs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = {'num_leaves': 275,\n",
    "          'min_child_weight': 0.034,\n",
    "          'feature_fraction': 0.379,\n",
    "          'bagging_fraction': 0.418,\n",
    "          'min_data_in_leaf': 106,\n",
    "          'objective': 'regression',\n",
    "          'max_depth': -1,\n",
    "          'learning_rate': 0.007,\n",
    "          \"boosting_type\": \"gbdt\",\n",
    "          \"bagging_seed\": 11,\n",
    "          \"metric\": 'rmse',\n",
    "          \"verbosity\": -1,\n",
    "          'reg_alpha': 0.3899,\n",
    "          'reg_lambda': 0.648,\n",
    "          'random_state': 623,\n",
    "         }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fold: 1\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.1552\tvalid_1's rmse: 2.98899\n",
      "[200]\ttraining's rmse: 2.87094\tvalid_1's rmse: 2.69436\n",
      "[300]\ttraining's rmse: 2.74535\tvalid_1's rmse: 2.59525\n",
      "[400]\ttraining's rmse: 2.67405\tvalid_1's rmse: 2.5638\n",
      "[500]\ttraining's rmse: 2.62141\tvalid_1's rmse: 2.55228\n",
      "[600]\ttraining's rmse: 2.58111\tvalid_1's rmse: 2.55157\n",
      "Early stopping, best iteration is:\n",
      "[615]\ttraining's rmse: 2.57499\tvalid_1's rmse: 2.55094\n",
      "val rmse score is 2.5509445868657474\n",
      "Fold: 2\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 3.01106\tvalid_1's rmse: 2.62845\n",
      "[200]\ttraining's rmse: 2.71467\tvalid_1's rmse: 2.37049\n",
      "[300]\ttraining's rmse: 2.59085\tvalid_1's rmse: 2.31785\n",
      "[400]\ttraining's rmse: 2.52654\tvalid_1's rmse: 2.31449\n",
      "Early stopping, best iteration is:\n",
      "[363]\ttraining's rmse: 2.5468\tvalid_1's rmse: 2.31336\n",
      "val rmse score is 2.3133613909046784\n",
      "Fold: 3\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 2.90803\tvalid_1's rmse: 2.68665\n",
      "[200]\ttraining's rmse: 2.60297\tvalid_1's rmse: 2.43097\n",
      "[300]\ttraining's rmse: 2.48854\tvalid_1's rmse: 2.34634\n",
      "[400]\ttraining's rmse: 2.43123\tvalid_1's rmse: 2.31193\n",
      "[500]\ttraining's rmse: 2.39358\tvalid_1's rmse: 2.29587\n",
      "[600]\ttraining's rmse: 2.36278\tvalid_1's rmse: 2.28569\n",
      "[700]\ttraining's rmse: 2.33932\tvalid_1's rmse: 2.27912\n",
      "[800]\ttraining's rmse: 2.31976\tvalid_1's rmse: 2.27486\n",
      "[900]\ttraining's rmse: 2.30263\tvalid_1's rmse: 2.27209\n",
      "[1000]\ttraining's rmse: 2.28688\tvalid_1's rmse: 2.26975\n",
      "[1100]\ttraining's rmse: 2.27374\tvalid_1's rmse: 2.26808\n",
      "[1200]\ttraining's rmse: 2.26165\tvalid_1's rmse: 2.26666\n",
      "[1300]\ttraining's rmse: 2.24869\tvalid_1's rmse: 2.26528\n",
      "[1400]\ttraining's rmse: 2.23844\tvalid_1's rmse: 2.26412\n",
      "[1500]\ttraining's rmse: 2.22869\tvalid_1's rmse: 2.26333\n",
      "[1600]\ttraining's rmse: 2.21956\tvalid_1's rmse: 2.26256\n",
      "[1700]\ttraining's rmse: 2.21182\tvalid_1's rmse: 2.26213\n",
      "[1800]\ttraining's rmse: 2.20377\tvalid_1's rmse: 2.26134\n",
      "[1900]\ttraining's rmse: 2.19615\tvalid_1's rmse: 2.26068\n",
      "[2000]\ttraining's rmse: 2.18878\tvalid_1's rmse: 2.26029\n",
      "[2100]\ttraining's rmse: 2.18215\tvalid_1's rmse: 2.2601\n",
      "[2200]\ttraining's rmse: 2.17544\tvalid_1's rmse: 2.25979\n",
      "[2300]\ttraining's rmse: 2.16947\tvalid_1's rmse: 2.2596\n",
      "Early stopping, best iteration is:\n",
      "[2313]\ttraining's rmse: 2.16854\tvalid_1's rmse: 2.2595\n",
      "val rmse score is 2.2595045487040153\n",
      "Fold: 4\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 2.84653\tvalid_1's rmse: 2.77196\n",
      "[200]\ttraining's rmse: 2.54884\tvalid_1's rmse: 2.43889\n",
      "[300]\ttraining's rmse: 2.43879\tvalid_1's rmse: 2.32607\n",
      "[400]\ttraining's rmse: 2.3853\tvalid_1's rmse: 2.28332\n",
      "[500]\ttraining's rmse: 2.35171\tvalid_1's rmse: 2.26356\n",
      "[600]\ttraining's rmse: 2.32516\tvalid_1's rmse: 2.25199\n",
      "[700]\ttraining's rmse: 2.30465\tvalid_1's rmse: 2.24507\n",
      "[800]\ttraining's rmse: 2.2876\tvalid_1's rmse: 2.23987\n",
      "[900]\ttraining's rmse: 2.27243\tvalid_1's rmse: 2.2355\n",
      "[1000]\ttraining's rmse: 2.25862\tvalid_1's rmse: 2.23234\n",
      "[1100]\ttraining's rmse: 2.24711\tvalid_1's rmse: 2.22994\n",
      "[1200]\ttraining's rmse: 2.23608\tvalid_1's rmse: 2.22799\n",
      "[1300]\ttraining's rmse: 2.22458\tvalid_1's rmse: 2.22645\n",
      "[1400]\ttraining's rmse: 2.21521\tvalid_1's rmse: 2.2254\n",
      "[1500]\ttraining's rmse: 2.20635\tvalid_1's rmse: 2.2244\n",
      "[1600]\ttraining's rmse: 2.19841\tvalid_1's rmse: 2.22362\n",
      "[1700]\ttraining's rmse: 2.19132\tvalid_1's rmse: 2.22292\n",
      "[1800]\ttraining's rmse: 2.18444\tvalid_1's rmse: 2.22219\n",
      "[1900]\ttraining's rmse: 2.17791\tvalid_1's rmse: 2.22168\n",
      "[2000]\ttraining's rmse: 2.17165\tvalid_1's rmse: 2.22132\n",
      "[2100]\ttraining's rmse: 2.16588\tvalid_1's rmse: 2.22097\n",
      "[2200]\ttraining's rmse: 2.15997\tvalid_1's rmse: 2.22059\n",
      "[2300]\ttraining's rmse: 2.15437\tvalid_1's rmse: 2.21991\n",
      "[2400]\ttraining's rmse: 2.14918\tvalid_1's rmse: 2.21958\n",
      "[2500]\ttraining's rmse: 2.14403\tvalid_1's rmse: 2.21934\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[2500]\ttraining's rmse: 2.14403\tvalid_1's rmse: 2.21934\n",
      "val rmse score is 2.2193373708764783\n",
      "Fold: 5\n",
      "Training until validation scores don't improve for 50 rounds\n",
      "[100]\ttraining's rmse: 2.82596\tvalid_1's rmse: 2.72176\n",
      "[200]\ttraining's rmse: 2.52048\tvalid_1's rmse: 2.43807\n",
      "[300]\ttraining's rmse: 2.41035\tvalid_1's rmse: 2.3423\n",
      "[400]\ttraining's rmse: 2.35934\tvalid_1's rmse: 2.30699\n",
      "[500]\ttraining's rmse: 2.32875\tvalid_1's rmse: 2.2902\n",
      "[600]\ttraining's rmse: 2.3046\tvalid_1's rmse: 2.2811\n",
      "[700]\ttraining's rmse: 2.28564\tvalid_1's rmse: 2.27482\n",
      "[800]\ttraining's rmse: 2.26986\tvalid_1's rmse: 2.27046\n",
      "[900]\ttraining's rmse: 2.25578\tvalid_1's rmse: 2.2667\n",
      "[1000]\ttraining's rmse: 2.24306\tvalid_1's rmse: 2.26423\n",
      "[1100]\ttraining's rmse: 2.23199\tvalid_1's rmse: 2.2625\n",
      "[1200]\ttraining's rmse: 2.22175\tvalid_1's rmse: 2.26104\n",
      "[1300]\ttraining's rmse: 2.21159\tvalid_1's rmse: 2.25979\n",
      "[1400]\ttraining's rmse: 2.20314\tvalid_1's rmse: 2.2591\n",
      "[1500]\ttraining's rmse: 2.19486\tvalid_1's rmse: 2.25808\n",
      "[1600]\ttraining's rmse: 2.18763\tvalid_1's rmse: 2.25775\n",
      "[1700]\ttraining's rmse: 2.18084\tvalid_1's rmse: 2.25747\n",
      "[1800]\ttraining's rmse: 2.17446\tvalid_1's rmse: 2.25725\n",
      "[1900]\ttraining's rmse: 2.16844\tvalid_1's rmse: 2.25682\n",
      "[2000]\ttraining's rmse: 2.16264\tvalid_1's rmse: 2.25656\n",
      "[2100]\ttraining's rmse: 2.15727\tvalid_1's rmse: 2.25629\n",
      "[2200]\ttraining's rmse: 2.1519\tvalid_1's rmse: 2.25619\n",
      "[2300]\ttraining's rmse: 2.14654\tvalid_1's rmse: 2.25591\n",
      "[2400]\ttraining's rmse: 2.14182\tvalid_1's rmse: 2.25582\n",
      "Early stopping, best iteration is:\n",
      "[2444]\ttraining's rmse: 2.1396\tvalid_1's rmse: 2.25571\n",
      "val rmse score is 2.255714843558955\n",
      "mean rmse score over folds is 2.3197725481819753\n"
     ]
    }
   ],
   "source": [
    "columns = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2', \n",
    "            'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90', \n",
    "            'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30']\n",
    "splits = folds.split(x, y)\n",
    "y_preds = np.zeros(test.shape[0])\n",
    "y_oof = np.zeros(x.shape[0])\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = columns\n",
    "mean_score = []\n",
    "for fold_n, (train_index, valid_index) in enumerate(splits):\n",
    "    print('Fold:',fold_n+1)\n",
    "    X_train, X_valid = x[columns].iloc[train_index], x[columns].iloc[valid_index]\n",
    "    y_train, y_valid = y.iloc[train_index], y.iloc[valid_index]\n",
    "    dtrain = lgb.Dataset(X_train, label=y_train)\n",
    "    dvalid = lgb.Dataset(X_valid, label=y_valid)\n",
    "    clf = lgb.train(params, dtrain, 2500, valid_sets = [dtrain, dvalid],early_stopping_rounds = 50, verbose_eval=100)\n",
    "    feature_importances[f'fold_{fold_n + 1}'] = clf.feature_importance()\n",
    "    y_pred_valid = clf.predict(X_valid,num_iteration=clf.best_iteration)\n",
    "    y_oof[valid_index] = y_pred_valid\n",
    "    val_score = np.sqrt(metrics.mean_squared_error(y_pred_valid, y_valid))\n",
    "    print(f'val rmse score is {val_score}')\n",
    "    mean_score.append(val_score)\n",
    "    y_preds += clf.predict(test[columns], num_iteration=clf.best_iteration)/n_fold\n",
    "    del X_train, X_valid, y_train, y_valid\n",
    "    gc.collect()\n",
    "print('mean rmse score over folds is',np.mean(mean_score))\n",
    "test['demand'] = y_preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>F4</th>\n",
       "      <th>F5</th>\n",
       "      <th>F6</th>\n",
       "      <th>F7</th>\n",
       "      <th>F8</th>\n",
       "      <th>F9</th>\n",
       "      <th>F10</th>\n",
       "      <th>F11</th>\n",
       "      <th>F12</th>\n",
       "      <th>F13</th>\n",
       "      <th>F14</th>\n",
       "      <th>F15</th>\n",
       "      <th>F16</th>\n",
       "      <th>F17</th>\n",
       "      <th>F18</th>\n",
       "      <th>F19</th>\n",
       "      <th>F20</th>\n",
       "      <th>F21</th>\n",
       "      <th>F22</th>\n",
       "      <th>F23</th>\n",
       "      <th>F24</th>\n",
       "      <th>F25</th>\n",
       "      <th>F26</th>\n",
       "      <th>F27</th>\n",
       "      <th>F28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_validation</td>\n",
       "      <td>0.840346</td>\n",
       "      <td>0.757049</td>\n",
       "      <td>0.730179</td>\n",
       "      <td>0.710340</td>\n",
       "      <td>0.756308</td>\n",
       "      <td>0.881373</td>\n",
       "      <td>0.930050</td>\n",
       "      <td>0.736683</td>\n",
       "      <td>0.911305</td>\n",
       "      <td>0.941631</td>\n",
       "      <td>0.986345</td>\n",
       "      <td>1.017312</td>\n",
       "      <td>1.238508</td>\n",
       "      <td>1.181409</td>\n",
       "      <td>0.942887</td>\n",
       "      <td>0.802565</td>\n",
       "      <td>0.749132</td>\n",
       "      <td>0.759155</td>\n",
       "      <td>0.847691</td>\n",
       "      <td>1.202421</td>\n",
       "      <td>1.063079</td>\n",
       "      <td>0.841947</td>\n",
       "      <td>0.801952</td>\n",
       "      <td>0.824973</td>\n",
       "      <td>0.915340</td>\n",
       "      <td>0.898192</td>\n",
       "      <td>1.082797</td>\n",
       "      <td>1.094313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_validation</td>\n",
       "      <td>0.385132</td>\n",
       "      <td>0.335629</td>\n",
       "      <td>0.321714</td>\n",
       "      <td>0.295971</td>\n",
       "      <td>0.312593</td>\n",
       "      <td>0.335121</td>\n",
       "      <td>0.330239</td>\n",
       "      <td>0.230958</td>\n",
       "      <td>0.208576</td>\n",
       "      <td>0.185881</td>\n",
       "      <td>0.164029</td>\n",
       "      <td>0.206332</td>\n",
       "      <td>0.257377</td>\n",
       "      <td>0.242941</td>\n",
       "      <td>0.181729</td>\n",
       "      <td>0.166552</td>\n",
       "      <td>0.164873</td>\n",
       "      <td>0.164409</td>\n",
       "      <td>0.188273</td>\n",
       "      <td>0.240910</td>\n",
       "      <td>0.240607</td>\n",
       "      <td>0.178932</td>\n",
       "      <td>0.162536</td>\n",
       "      <td>0.241204</td>\n",
       "      <td>0.221869</td>\n",
       "      <td>0.242488</td>\n",
       "      <td>0.276836</td>\n",
       "      <td>0.270644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_validation</td>\n",
       "      <td>0.482840</td>\n",
       "      <td>0.453671</td>\n",
       "      <td>0.452803</td>\n",
       "      <td>0.447799</td>\n",
       "      <td>0.474878</td>\n",
       "      <td>0.586725</td>\n",
       "      <td>0.644362</td>\n",
       "      <td>0.359141</td>\n",
       "      <td>0.332036</td>\n",
       "      <td>0.301958</td>\n",
       "      <td>0.298563</td>\n",
       "      <td>0.345099</td>\n",
       "      <td>0.409679</td>\n",
       "      <td>0.360242</td>\n",
       "      <td>0.299273</td>\n",
       "      <td>0.273750</td>\n",
       "      <td>0.323895</td>\n",
       "      <td>0.404617</td>\n",
       "      <td>0.575065</td>\n",
       "      <td>0.731944</td>\n",
       "      <td>0.844626</td>\n",
       "      <td>0.629296</td>\n",
       "      <td>0.626875</td>\n",
       "      <td>0.618692</td>\n",
       "      <td>0.542670</td>\n",
       "      <td>0.591095</td>\n",
       "      <td>0.741278</td>\n",
       "      <td>0.705748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_validation</td>\n",
       "      <td>1.944721</td>\n",
       "      <td>1.642529</td>\n",
       "      <td>1.498953</td>\n",
       "      <td>1.402535</td>\n",
       "      <td>1.743661</td>\n",
       "      <td>1.986201</td>\n",
       "      <td>2.063033</td>\n",
       "      <td>1.641121</td>\n",
       "      <td>1.713807</td>\n",
       "      <td>1.585660</td>\n",
       "      <td>1.412963</td>\n",
       "      <td>1.685572</td>\n",
       "      <td>2.425858</td>\n",
       "      <td>2.429269</td>\n",
       "      <td>2.017233</td>\n",
       "      <td>1.723854</td>\n",
       "      <td>1.811848</td>\n",
       "      <td>1.721299</td>\n",
       "      <td>1.849481</td>\n",
       "      <td>2.059332</td>\n",
       "      <td>2.317281</td>\n",
       "      <td>1.987846</td>\n",
       "      <td>1.769339</td>\n",
       "      <td>1.568850</td>\n",
       "      <td>1.570194</td>\n",
       "      <td>1.833747</td>\n",
       "      <td>2.697069</td>\n",
       "      <td>2.546463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_validation</td>\n",
       "      <td>0.824027</td>\n",
       "      <td>0.723736</td>\n",
       "      <td>0.912673</td>\n",
       "      <td>1.092498</td>\n",
       "      <td>1.081765</td>\n",
       "      <td>1.306467</td>\n",
       "      <td>1.599981</td>\n",
       "      <td>1.165997</td>\n",
       "      <td>1.178859</td>\n",
       "      <td>0.966913</td>\n",
       "      <td>0.886214</td>\n",
       "      <td>0.990038</td>\n",
       "      <td>1.288195</td>\n",
       "      <td>1.135575</td>\n",
       "      <td>1.012675</td>\n",
       "      <td>0.919669</td>\n",
       "      <td>0.923333</td>\n",
       "      <td>0.921345</td>\n",
       "      <td>1.066479</td>\n",
       "      <td>1.321764</td>\n",
       "      <td>1.288149</td>\n",
       "      <td>0.882401</td>\n",
       "      <td>0.877133</td>\n",
       "      <td>0.875785</td>\n",
       "      <td>0.917395</td>\n",
       "      <td>1.034123</td>\n",
       "      <td>1.327560</td>\n",
       "      <td>1.569137</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              id        F1        F2        F3        F4  \\\n",
       "0  HOBBIES_1_001_CA_1_validation  0.840346  0.757049  0.730179  0.710340   \n",
       "1  HOBBIES_1_002_CA_1_validation  0.385132  0.335629  0.321714  0.295971   \n",
       "2  HOBBIES_1_003_CA_1_validation  0.482840  0.453671  0.452803  0.447799   \n",
       "3  HOBBIES_1_004_CA_1_validation  1.944721  1.642529  1.498953  1.402535   \n",
       "4  HOBBIES_1_005_CA_1_validation  0.824027  0.723736  0.912673  1.092498   \n",
       "\n",
       "         F5        F6        F7        F8        F9       F10       F11  \\\n",
       "0  0.756308  0.881373  0.930050  0.736683  0.911305  0.941631  0.986345   \n",
       "1  0.312593  0.335121  0.330239  0.230958  0.208576  0.185881  0.164029   \n",
       "2  0.474878  0.586725  0.644362  0.359141  0.332036  0.301958  0.298563   \n",
       "3  1.743661  1.986201  2.063033  1.641121  1.713807  1.585660  1.412963   \n",
       "4  1.081765  1.306467  1.599981  1.165997  1.178859  0.966913  0.886214   \n",
       "\n",
       "        F12       F13       F14       F15       F16       F17       F18  \\\n",
       "0  1.017312  1.238508  1.181409  0.942887  0.802565  0.749132  0.759155   \n",
       "1  0.206332  0.257377  0.242941  0.181729  0.166552  0.164873  0.164409   \n",
       "2  0.345099  0.409679  0.360242  0.299273  0.273750  0.323895  0.404617   \n",
       "3  1.685572  2.425858  2.429269  2.017233  1.723854  1.811848  1.721299   \n",
       "4  0.990038  1.288195  1.135575  1.012675  0.919669  0.923333  0.921345   \n",
       "\n",
       "        F19       F20       F21       F22       F23       F24       F25  \\\n",
       "0  0.847691  1.202421  1.063079  0.841947  0.801952  0.824973  0.915340   \n",
       "1  0.188273  0.240910  0.240607  0.178932  0.162536  0.241204  0.221869   \n",
       "2  0.575065  0.731944  0.844626  0.629296  0.626875  0.618692  0.542670   \n",
       "3  1.849481  2.059332  2.317281  1.987846  1.769339  1.568850  1.570194   \n",
       "4  1.066479  1.321764  1.288149  0.882401  0.877133  0.875785  0.917395   \n",
       "\n",
       "        F26       F27       F28  \n",
       "0  0.898192  1.082797  1.094313  \n",
       "1  0.242488  0.276836  0.270644  \n",
       "2  0.591095  0.741278  0.705748  \n",
       "3  1.833747  2.697069  2.546463  \n",
       "4  1.034123  1.327560  1.569137  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def predict(test, submission):\n",
    "    predictions = test[['id', 'date', 'demand']]\n",
    "    predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand').reset_index()\n",
    "    predictions.columns = ['id'] + ['F' + str(i + 1) for i in range(28)]\n",
    "    evaluation_rows = [row for row in submission['id'] if 'evaluation' in row] \n",
    "    evaluation = submission[submission['id'].isin(evaluation_rows)]\n",
    "    validation = submission[['id']].merge(predictions, on = 'id')\n",
    "    final = pd.concat([validation, evaluation])\n",
    "    #final.to_csv('submission.csv', index = False)\n",
    "    return final\n",
    "\n",
    "subs = predict(test, submission)\n",
    "subs.to_csv('M5_Forecasting.csv',index = False)\n",
    "subs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'C:\\\\Users\\\\bokhy\\\\Desktop\\\\'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_a = pd.read_csv(os.path.join(path,'submission_a.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_col = submission_a[['id']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_b = pd.read_csv(os.path.join(path,'submission_b.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_c = pd.read_csv(os.path.join(path,'submission_c.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_d = pd.read_csv(os.path.join(path,'submission_d.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_a.drop(['id'], axis=1, inplace=True)\n",
    "submission_b.drop(['id'], axis=1, inplace=True)\n",
    "submission_c.drop(['id'], axis=1, inplace=True)\n",
    "submission_d.drop(['id'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "w1 = 0.10\n",
    "w2 = 0.10\n",
    "w3 = 0.50\n",
    "w4 = 0.30"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "ensemble = w1*submission_a + w2*submission_b + w3*submission_c + w4*submission_d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pd.concat([id_col, ensemble], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>F4</th>\n",
       "      <th>F5</th>\n",
       "      <th>F6</th>\n",
       "      <th>F7</th>\n",
       "      <th>F8</th>\n",
       "      <th>F9</th>\n",
       "      <th>F10</th>\n",
       "      <th>F11</th>\n",
       "      <th>F12</th>\n",
       "      <th>F13</th>\n",
       "      <th>F14</th>\n",
       "      <th>F15</th>\n",
       "      <th>F16</th>\n",
       "      <th>F17</th>\n",
       "      <th>F18</th>\n",
       "      <th>F19</th>\n",
       "      <th>F20</th>\n",
       "      <th>F21</th>\n",
       "      <th>F22</th>\n",
       "      <th>F23</th>\n",
       "      <th>F24</th>\n",
       "      <th>F25</th>\n",
       "      <th>F26</th>\n",
       "      <th>F27</th>\n",
       "      <th>F28</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HOBBIES_1_001_CA_1_validation</td>\n",
       "      <td>0.855554</td>\n",
       "      <td>0.783716</td>\n",
       "      <td>0.752670</td>\n",
       "      <td>0.731978</td>\n",
       "      <td>0.760978</td>\n",
       "      <td>0.870851</td>\n",
       "      <td>0.904322</td>\n",
       "      <td>0.749212</td>\n",
       "      <td>0.900048</td>\n",
       "      <td>0.910087</td>\n",
       "      <td>0.971497</td>\n",
       "      <td>1.011839</td>\n",
       "      <td>1.233625</td>\n",
       "      <td>1.223187</td>\n",
       "      <td>0.989130</td>\n",
       "      <td>0.848318</td>\n",
       "      <td>0.785516</td>\n",
       "      <td>0.781597</td>\n",
       "      <td>0.859779</td>\n",
       "      <td>1.205632</td>\n",
       "      <td>1.049711</td>\n",
       "      <td>0.866122</td>\n",
       "      <td>0.833176</td>\n",
       "      <td>0.864190</td>\n",
       "      <td>0.949474</td>\n",
       "      <td>0.933339</td>\n",
       "      <td>1.086479</td>\n",
       "      <td>1.111789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HOBBIES_1_002_CA_1_validation</td>\n",
       "      <td>0.364804</td>\n",
       "      <td>0.320856</td>\n",
       "      <td>0.311469</td>\n",
       "      <td>0.295248</td>\n",
       "      <td>0.304716</td>\n",
       "      <td>0.339879</td>\n",
       "      <td>0.342275</td>\n",
       "      <td>0.229858</td>\n",
       "      <td>0.206530</td>\n",
       "      <td>0.182979</td>\n",
       "      <td>0.165634</td>\n",
       "      <td>0.210959</td>\n",
       "      <td>0.264840</td>\n",
       "      <td>0.248136</td>\n",
       "      <td>0.193720</td>\n",
       "      <td>0.170606</td>\n",
       "      <td>0.172387</td>\n",
       "      <td>0.172155</td>\n",
       "      <td>0.198182</td>\n",
       "      <td>0.255883</td>\n",
       "      <td>0.257907</td>\n",
       "      <td>0.189714</td>\n",
       "      <td>0.167463</td>\n",
       "      <td>0.225593</td>\n",
       "      <td>0.210779</td>\n",
       "      <td>0.232802</td>\n",
       "      <td>0.285695</td>\n",
       "      <td>0.281309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HOBBIES_1_003_CA_1_validation</td>\n",
       "      <td>0.481296</td>\n",
       "      <td>0.453292</td>\n",
       "      <td>0.451491</td>\n",
       "      <td>0.442689</td>\n",
       "      <td>0.464222</td>\n",
       "      <td>0.563045</td>\n",
       "      <td>0.627168</td>\n",
       "      <td>0.398160</td>\n",
       "      <td>0.364867</td>\n",
       "      <td>0.340281</td>\n",
       "      <td>0.339347</td>\n",
       "      <td>0.383211</td>\n",
       "      <td>0.458588</td>\n",
       "      <td>0.416606</td>\n",
       "      <td>0.347422</td>\n",
       "      <td>0.318691</td>\n",
       "      <td>0.358401</td>\n",
       "      <td>0.431497</td>\n",
       "      <td>0.572877</td>\n",
       "      <td>0.722756</td>\n",
       "      <td>0.827787</td>\n",
       "      <td>0.659365</td>\n",
       "      <td>0.659563</td>\n",
       "      <td>0.655541</td>\n",
       "      <td>0.570861</td>\n",
       "      <td>0.636648</td>\n",
       "      <td>0.776139</td>\n",
       "      <td>0.733348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HOBBIES_1_004_CA_1_validation</td>\n",
       "      <td>1.996432</td>\n",
       "      <td>1.721847</td>\n",
       "      <td>1.593628</td>\n",
       "      <td>1.506835</td>\n",
       "      <td>1.793652</td>\n",
       "      <td>2.024145</td>\n",
       "      <td>2.047356</td>\n",
       "      <td>1.677166</td>\n",
       "      <td>1.713577</td>\n",
       "      <td>1.639180</td>\n",
       "      <td>1.504542</td>\n",
       "      <td>1.727294</td>\n",
       "      <td>2.383253</td>\n",
       "      <td>2.398187</td>\n",
       "      <td>2.002172</td>\n",
       "      <td>1.764882</td>\n",
       "      <td>1.833272</td>\n",
       "      <td>1.775409</td>\n",
       "      <td>1.879175</td>\n",
       "      <td>2.070773</td>\n",
       "      <td>2.280710</td>\n",
       "      <td>1.971223</td>\n",
       "      <td>1.785484</td>\n",
       "      <td>1.636987</td>\n",
       "      <td>1.614011</td>\n",
       "      <td>1.865098</td>\n",
       "      <td>2.638427</td>\n",
       "      <td>2.490398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HOBBIES_1_005_CA_1_validation</td>\n",
       "      <td>0.900292</td>\n",
       "      <td>0.787578</td>\n",
       "      <td>0.976314</td>\n",
       "      <td>1.143517</td>\n",
       "      <td>1.123724</td>\n",
       "      <td>1.346601</td>\n",
       "      <td>1.639200</td>\n",
       "      <td>1.221375</td>\n",
       "      <td>1.234414</td>\n",
       "      <td>1.044392</td>\n",
       "      <td>0.952305</td>\n",
       "      <td>1.051562</td>\n",
       "      <td>1.299507</td>\n",
       "      <td>1.164933</td>\n",
       "      <td>1.063349</td>\n",
       "      <td>0.974244</td>\n",
       "      <td>0.995090</td>\n",
       "      <td>0.993842</td>\n",
       "      <td>1.124986</td>\n",
       "      <td>1.340301</td>\n",
       "      <td>1.321160</td>\n",
       "      <td>0.956594</td>\n",
       "      <td>0.933047</td>\n",
       "      <td>0.932122</td>\n",
       "      <td>0.974477</td>\n",
       "      <td>1.060844</td>\n",
       "      <td>1.337230</td>\n",
       "      <td>1.571955</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              id        F1        F2        F3        F4  \\\n",
       "0  HOBBIES_1_001_CA_1_validation  0.855554  0.783716  0.752670  0.731978   \n",
       "1  HOBBIES_1_002_CA_1_validation  0.364804  0.320856  0.311469  0.295248   \n",
       "2  HOBBIES_1_003_CA_1_validation  0.481296  0.453292  0.451491  0.442689   \n",
       "3  HOBBIES_1_004_CA_1_validation  1.996432  1.721847  1.593628  1.506835   \n",
       "4  HOBBIES_1_005_CA_1_validation  0.900292  0.787578  0.976314  1.143517   \n",
       "\n",
       "         F5        F6        F7        F8        F9       F10       F11  \\\n",
       "0  0.760978  0.870851  0.904322  0.749212  0.900048  0.910087  0.971497   \n",
       "1  0.304716  0.339879  0.342275  0.229858  0.206530  0.182979  0.165634   \n",
       "2  0.464222  0.563045  0.627168  0.398160  0.364867  0.340281  0.339347   \n",
       "3  1.793652  2.024145  2.047356  1.677166  1.713577  1.639180  1.504542   \n",
       "4  1.123724  1.346601  1.639200  1.221375  1.234414  1.044392  0.952305   \n",
       "\n",
       "        F12       F13       F14       F15       F16       F17       F18  \\\n",
       "0  1.011839  1.233625  1.223187  0.989130  0.848318  0.785516  0.781597   \n",
       "1  0.210959  0.264840  0.248136  0.193720  0.170606  0.172387  0.172155   \n",
       "2  0.383211  0.458588  0.416606  0.347422  0.318691  0.358401  0.431497   \n",
       "3  1.727294  2.383253  2.398187  2.002172  1.764882  1.833272  1.775409   \n",
       "4  1.051562  1.299507  1.164933  1.063349  0.974244  0.995090  0.993842   \n",
       "\n",
       "        F19       F20       F21       F22       F23       F24       F25  \\\n",
       "0  0.859779  1.205632  1.049711  0.866122  0.833176  0.864190  0.949474   \n",
       "1  0.198182  0.255883  0.257907  0.189714  0.167463  0.225593  0.210779   \n",
       "2  0.572877  0.722756  0.827787  0.659365  0.659563  0.655541  0.570861   \n",
       "3  1.879175  2.070773  2.280710  1.971223  1.785484  1.636987  1.614011   \n",
       "4  1.124986  1.340301  1.321160  0.956594  0.933047  0.932122  0.974477   \n",
       "\n",
       "        F26       F27       F28  \n",
       "0  0.933339  1.086479  1.111789  \n",
       "1  0.232802  0.285695  0.281309  \n",
       "2  0.636648  0.776139  0.733348  \n",
       "3  1.865098  2.638427  2.490398  \n",
       "4  1.060844  1.337230  1.571955  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions.to_csv('M5_Ensemble.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60980, 29)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}