notebooks/1_entsoe_forecast_only.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Category 1: Using the ENTSO-E forecast only\n",
    "The first model category will just use the current available ENTSO-E forecast and try to create a better forecast in terms of mean absolute error."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model category specific configuration\n",
    "These parameters are model category specific\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Model category name used throughout the subsequent analysis\n",
    "model_cat_id = \"01\"\n",
    "\n",
    "# Which features from the dataset should be loaded:\n",
    "# ['all', 'actual', 'entsoe', 'weather_t', 'weather_i', 'holiday', 'weekday', 'hour', 'month']\n",
    "features = ['actual', 'entsoe']\n",
    "\n",
    "# LSTM Layer configuration\n",
    "# ========================\n",
    "# Stateful True or false\n",
    "layer_conf = [ True, True, True ]\n",
    "# Number of neurons per layer\n",
    "cells = [[ 5, 10, 20, 30, 50, 75, 100, 125, 150 ], [0, 10, 20, 50], [0, 10, 15, 20]]\n",
    "# Regularization per layer\n",
    "dropout = [0, 0.1, 0.2]\n",
    "# Size of how many samples are used for one forward/backward pass\n",
    "batch_size = [8]\n",
    "# In a sense this is the output neuron dimension, or how many timesteps the neuron should output. Currently not implemented, defaults to 1.\n",
    "timesteps = [1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Module imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import math\n",
    "import itertools\n",
    "import datetime as dt\n",
    "import pytz\n",
    "import time as t\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from pandas import read_csv\n",
    "from pandas import datetime\n",
    "from numpy import newaxis\n",
    "\n",
    "import matplotlib as mpl\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats as stats\n",
    "from statsmodels.tsa import stattools\n",
    "from tabulate import tabulate\n",
    "\n",
    "import math\n",
    "import keras as keras\n",
    "from keras import backend as K\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, Dropout, LSTM\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.utils import np_utils\n",
    "from keras.models import load_model\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "\n",
    "from IPython.display import HTML\n",
    "from IPython.display import display\n",
    "%matplotlib notebook\n",
    "mpl.rcParams['figure.figsize'] = (9,5)\n",
    "\n",
    "# Import custom module functions\n",
    "module_path = os.path.abspath(os.path.join('../'))\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "\n",
    "from lstm_load_forecasting import data, lstm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overall configuration\n",
    "These parameters are later used, but shouldn't have to change between different model categories (model 1-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Directory with dataset\n",
    "path = os.path.join(os.path.abspath(''), '../data/fulldataset.csv')\n",
    "\n",
    "# Splitdate for train and test data. As the TBATS and ARIMA benchmark needs 2 full cycle of all seasonality, needs to be after jan 01. \n",
    "loc_tz = pytz.timezone('Europe/Zurich')\n",
    "split_date = loc_tz.localize(dt.datetime(2017,2,1,0,0,0,0))\n",
    "\n",
    "# Validation split percentage\n",
    "validation_split = 0.2\n",
    "# How many epochs in total\n",
    "epochs = 30\n",
    "# Set verbosity level. 0 for only per model, 1 for progress bar...\n",
    "verbose = 0\n",
    "\n",
    "# Dataframe containing the relevant data from training of all models\n",
    "results = pd.DataFrame(columns=['model_name', 'config', 'dropout',\n",
    "                                'train_loss', 'train_rmse', 'train_mae', 'train_mape', \n",
    "                                'valid_loss', 'valid_rmse', 'valid_mae', 'valid_mape', \n",
    "                                'test_rmse', 'test_mae', 'test_mape',\n",
    "                                'epochs', 'batch_train', 'input_shape',\n",
    "                                'total_time', 'time_step', 'splits'\n",
    "                               ])\n",
    "# Early stopping parameters\n",
    "early_stopping = True\n",
    "min_delta = 0.006\n",
    "patience = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation and model generation\n",
    "Necessary preliminary steps and then the generation of all possible models based on the settings at the top of this notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\n",
      "| Number of model configs generated | 432 |\n"
     ]
    }
   ],
   "source": [
    "# Generate output folders and files\n",
    "res_dir = '../results/notebook_' + model_cat_id + '/'\n",
    "plot_dir = '../plots/notebook_' + model_cat_id + '/'\n",
    "model_dir = '../models/notebook_' + model_cat_id + '/'\n",
    "os.makedirs(res_dir, exist_ok=True)\n",
    "os.makedirs(model_dir, exist_ok=True)\n",
    "output_table = res_dir + model_cat_id + '_results_' + t.strftime(\"%Y%m%d\") + '.csv'\n",
    "test_output_table = res_dir + model_cat_id + '_test_results' + t.strftime(\"%Y%m%d\") + '.csv'\n",
    "\n",
    "# Generate model combinations\n",
    "models = []\n",
    "models = lstm.generate_combinations(\n",
    "    model_name=model_cat_id + '_', layer_conf=layer_conf, cells=cells, dropout=dropout, \n",
    "    batch_size=batch_size, timesteps=[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the data:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Load data and prepare for standardization\n",
    "df = data.load_dataset(path=path, modules=features)\n",
    "df_scaled = df.copy()\n",
    "df_scaled = df_scaled.dropna()\n",
    "\n",
    "# Get all float type columns and standardize them\n",
    "floats = [key for key in dict(df_scaled.dtypes) if dict(df_scaled.dtypes)[key] in ['float64']]\n",
    "scaler = StandardScaler()\n",
    "scaled_columns = scaler.fit_transform(df_scaled[floats])\n",
    "df_scaled[floats] = scaled_columns\n",
    "\n",
    "# Split in train and test dataset\n",
    "df_train = df_scaled.loc[(df_scaled.index < split_date )].copy()\n",
    "df_test = df_scaled.loc[df_scaled.index >= split_date].copy()\n",
    "\n",
    "# Split in features and label data\n",
    "y_train = df_train['actual'].copy()\n",
    "X_train = df_train.drop('actual', 1).copy()\n",
    "y_test = df_test['actual'].copy()\n",
    "X_test = df_test.drop('actual', 1).copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running through all generated models\n",
    "Note: Depending on the above settings, this can take very long!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "========================= Model 1/8 =========================\n",
      "| Starting with model | 01_1_l-5                   |\n",
      "| Starting time       | 2017-06-16 18:03:45.264842 |\n",
      "========================= Model 2/8 =========================\n",
      "| Starting with model | 01_2_l-5_l-10              |\n",
      "| Starting time       | 2017-06-16 18:04:24.647617 |\n",
      "========================= Model 3/8 =========================\n",
      "| Starting with model | 01_3_l-5_l-10              |\n",
      "| Starting time       | 2017-06-16 18:05:21.940684 |\n",
      "========================= Model 4/8 =========================\n",
      "| Starting with model | 01_4_l-5_l-10_l-10         |\n",
      "| Starting time       | 2017-06-16 18:06:06.565460 |\n",
      "========================= Model 5/8 =========================\n",
      "| Starting with model | 01_5_l-10                  |\n",
      "| Starting time       | 2017-06-16 18:08:52.052885 |\n",
      "========================= Model 6/8 =========================\n",
      "| Starting with model | 01_6_l-10_l-10             |\n",
      "| Starting time       | 2017-06-16 18:09:22.277358 |\n",
      "========================= Model 7/8 =========================\n",
      "| Starting with model | 01_7_l-10_l-10             |\n",
      "| Starting time       | 2017-06-16 18:10:06.038210 |\n",
      "========================= Model 8/8 =========================\n",
      "| Starting with model | 01_8_l-10_l-10_l-10        |\n",
      "| Starting time       | 2017-06-16 18:10:56.996150 |\n"
     ]
    }
   ],
   "source": [
    "start_time = t.time()\n",
    "for idx, m in enumerate(models):\n",
    "    stopper = t.time()\n",
    "    print('========================= Model {}/{} ========================='.format(idx+1, len(models)))\n",
    "    print(tabulate([['Starting with model', m['name']], ['Starting time', datetime.fromtimestamp(stopper)]],\n",
    "                   tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "    try:\n",
    "        # Creating the Keras Model\n",
    "        model = lstm.create_model(layers=m['layers'], sample_size=X_train.shape[0], batch_size=m['batch_size'], \n",
    "                          timesteps=m['timesteps'], features=X_train.shape[1])\n",
    "        # Training...\n",
    "        history = lstm.train_model(model=model, mode='fit', y=y_train, X=X_train, \n",
    "                                   batch_size=m['batch_size'], timesteps=m['timesteps'], epochs=epochs, \n",
    "                                   rearrange=False, validation_split=validation_split, verbose=verbose, \n",
    "                                   early_stopping=early_stopping, min_delta=min_delta, patience=patience)\n",
    "\n",
    "        # Write results\n",
    "        min_loss = np.min(history.history['val_loss'])\n",
    "        min_idx = np.argmin(history.history['val_loss'])\n",
    "        min_epoch = min_idx + 1\n",
    "        \n",
    "        if verbose > 0:\n",
    "            print('______________________________________________________________________')\n",
    "            print(tabulate([['Minimum validation loss at epoch', min_epoch, 'Time: {}'.format(t.time()-stopper)],\n",
    "                        ['Training loss & MAE', history.history['loss'][min_idx], history.history['mean_absolute_error'][min_idx]  ], \n",
    "                        ['Validation loss & mae', history.history['val_loss'][min_idx], history.history['val_mean_absolute_error'][min_idx] ],\n",
    "                       ], tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "            print('______________________________________________________________________')\n",
    "        \n",
    "        \n",
    "        result = [{'model_name': m['name'], 'config': m, 'train_loss': history.history['loss'][min_idx], 'train_rmse': 0,\n",
    "                   'train_mae': history.history['mean_absolute_error'][min_idx], 'train_mape': 0,\n",
    "                   'valid_loss': history.history['val_loss'][min_idx], 'valid_rmse': 0, \n",
    "                   'valid_mae': history.history['val_mean_absolute_error'][min_idx],'valid_mape': 0, \n",
    "                   'test_rmse': 0, 'test_mae': 0, 'test_mape': 0, 'epochs': '{}/{}'.format(min_epoch, epochs), 'batch_train':m['batch_size'],\n",
    "                   'input_shape':(X_train.shape[0], timesteps, X_train.shape[1]), 'total_time':t.time()-stopper, \n",
    "                   'time_step':0, 'splits':str(split_date), 'dropout': m['layers'][0]['dropout']\n",
    "                  }]\n",
    "        results = results.append(result, ignore_index=True)\n",
    "        \n",
    "        # Saving the model and weights\n",
    "        model.save(model_dir + m['name'] + '.h5')\n",
    "        \n",
    "        # Write results to csv\n",
    "        results.to_csv(output_table, sep=';')\n",
    "        \n",
    "        #if not os.path.isfile(output_table):\n",
    "            #results.to_csv(output_table, sep=';')\n",
    "        #else: # else it exists so append without writing the header\n",
    "        #    results.to_csv(output_table,mode = 'a',header=False, sep=';')\n",
    "        \n",
    "        K.clear_session()\n",
    "        import tensorflow as tf\n",
    "        tf.reset_default_graph()\n",
    "        \n",
    "    # Shouldn't catch all errors, but for now...\n",
    "    except BaseException as e:\n",
    "        print('=============== ERROR {}/{} ============='.format(idx+1, len(models)))\n",
    "        print(tabulate([['Model:', m['name']], ['Config:', m]], tablefmt=\"jira\", numalign=\"right\", floatfmt=\".3f\"))\n",
    "        print('Error: {}'.format(e))\n",
    "        result = [{'model_name': m['name'], 'config': m, 'train_loss': str(e)}]\n",
    "        results = results.append(result, ignore_index=True)\n",
    "        results.to_csv(output_table,sep=';')\n",
    "        continue\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model selection based on the validation MAE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "Select the top 5 models based on the Mean Absolute Error in the validation data:\n",
    "http://scikit-learn.org/stable/modules/model_evaluation.html#mean-absolute-error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of the selected top models \n",
    "selection = 5\n",
    "# If run in the same instance not necessary. If run on the same day, then just use output_table\n",
    "results_fn = res_dir + model_cat_id + '_results_' + '20170616' + '.csv'\n",
    "\n",
    "results_csv = pd.read_csv(results_fn, delimiter=';')\n",
    "top_models = results_csv.nsmallest(selection, 'valid_mae')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate top 5 models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Init test results table\n",
    "test_results = pd.DataFrame(columns=['Model name', 'Mean absolute error', 'Mean squared error'])\n",
    "\n",
    "# Init empty predictions\n",
    "predictions = {}\n",
    "\n",
    "# Loop through models\n",
    "for index, row in top_models.iterrows():\n",
    "    filename = model_dir + row['model_name'] + '.h5'\n",
    "    model = load_model(filename)\n",
    "    batch_size = int(row['batch_train'])\n",
    "    \n",
    "    # Calculate scores\n",
    "    loss, mae = lstm.evaluate_model(model=model, X=X_test, y=y_test, batch_size=batch_size, timesteps=1, verbose=verbose)\n",
    "    \n",
    "    # Store results\n",
    "    result = [{'Model name': row['model_name'], \n",
    "               'Mean squared error': loss, 'Mean absolute error': mae\n",
    "              }]\n",
    "    test_results = test_results.append(result, ignore_index=True)\n",
    "    \n",
    "    # Generate predictions\n",
    "    model.reset_states()\n",
    "    model_predictions = lstm.get_predictions(model=model, X=X_test, batch_size=batch_size, timesteps=timesteps[0], verbose=verbose)\n",
    "    \n",
    "    # Save predictions\n",
    "    predictions[row['model_name']] = model_predictions\n",
    "    \n",
    "    K.clear_session()\n",
    "    import tensorflow as tf\n",
    "    tf.reset_default_graph()\n",
    "    \n",
    "\n",
    "test_results = test_results.sort_values('Mean absolute error', ascending=True)\n",
    "test_results = test_results.set_index(['Model name'])\n",
    "\n",
    "if not os.path.isfile(test_output_table):\n",
    "    test_results.to_csv(test_output_table, sep=';')\n",
    "else: # else it exists so append without writing the header\n",
    "    test_results.to_csv(test_output_table,mode = 'a',header=False, sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test dataset performance of the best 5 (out of 432 tested models):\n",
      "+--------------------+-----------------------+----------------------+\n",
      "| Model name         |   Mean absolute error |   Mean squared error |\n",
      "+====================+=======================+======================+\n",
      "| 01_386_l-150_d-0.1 |                 0.293 |                0.137 |\n",
      "+--------------------+-----------------------+----------------------+\n",
      "| 01_339_l-125_d-0.2 |                 0.296 |                0.141 |\n",
      "+--------------------+-----------------------+----------------------+\n",
      "| 01_385_l-150       |                 0.296 |                0.140 |\n",
      "+--------------------+-----------------------+----------------------+\n",
      "| 01_338_l-125_d-0.1 |                 0.300 |                0.141 |\n",
      "+--------------------+-----------------------+----------------------+\n",
      "| 01_291_l-100_d-0.2 |                 0.300 |                0.142 |\n",
      "+--------------------+-----------------------+----------------------+\n"
     ]
    }
   ],
   "source": [
    "print('Test dataset performance of the best {} (out of {} tested models):'.format(min(selection, len(models)), len(models)))\n",
    "print(tabulate(test_results, headers='keys', tablefmt=\"grid\", numalign=\"right\", floatfmt=\".3f\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}