6.1-MultinomialLogitAndProbitModels/03-logit-validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How to use cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas  as pd\n",
    "import numpy as np\n",
    "import biogeme.database as db\n",
    "import biogeme.biogeme as bio\n",
    "import biogeme.models as models\n",
    "import biogeme.optimization as opt\n",
    "from biogeme.expressions import Beta, DefineVariable\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Import Swissmetro data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pandas = pd.read_csv(\"../Data/swissmetro.dat\",sep='\\t')\n",
    "database = db.Database(\"data/swissmetro\", pandas)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Use collumn names as variables**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "globals().update(database.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Exclude some unwanted entries**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude = (( PURPOSE != 1 ) * ( PURPOSE != 3 ) + ( CHOICE == 0 )) > 0\n",
    "\n",
    "database.remove(exclude)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define some dummy variables**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "SM_COST = SM_CO * ( GA == 0 )\n",
    "TRAIN_COST = TRAIN_CO * ( GA == 0 )\n",
    "\n",
    "CAR_AV_SP = DefineVariable ('CAR_AV_SP', CAR_AV * ( SP !=0 ), database)\n",
    "TRAIN_AV_SP = DefineVariable ('TRAIN_AV_SP', TRAIN_AV * ( SP != 0 ), database)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Rescale some data**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_TT_SCALED   = DefineVariable('TRAIN_TT_SCALED',   TRAIN_TT / 100.0, database)\n",
    "TRAIN_COST_SCALED = DefineVariable('TRAIN_COST_SCALED', TRAIN_COST / 100, database)\n",
    "SM_TT_SCALED      = DefineVariable('SM_TT_SCALED',      SM_TT / 100.0   , database)\n",
    "SM_COST_SCALED    = DefineVariable('SM_COST_SCALED',    SM_COST / 100   , database)\n",
    "CAR_TT_SCALED     = DefineVariable('CAR_TT_SCALED',     CAR_TT / 100    , database)\n",
    "CAR_CO_SCALED     = DefineVariable('CAR_CO_SCALED',     CAR_CO / 100    , database)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Create parameters to be estimated**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "ASC_CAR = Beta('ASC_CAR',0,None ,None ,0)\n",
    "ASC_TRAIN = Beta('ASC_TRAIN',0,None ,None ,0)\n",
    "ASC_SM = Beta('ASC_SM',0,None ,None ,1)\n",
    "B_TIME = Beta('B_TIME',0,None ,None ,0)\n",
    "B_COST = Beta('B_COST',0,None ,None ,0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define the utility functions**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "V1 = ASC_TRAIN + \\\n",
    "     B_TIME * TRAIN_TT_SCALED + \\\n",
    "     B_COST * TRAIN_COST_SCALED\n",
    "V2 = ASC_SM + \\\n",
    "     B_TIME * SM_TT_SCALED + \\\n",
    "     B_COST * SM_COST_SCALED\n",
    "V3 = ASC_CAR + \\\n",
    "     B_TIME * CAR_TT_SCALED + \\\n",
    "     B_COST * CAR_CO_SCALED"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Associate utility functions with alternatives and associate availability of alternatives**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "V = {1: V1,\n",
    "     2: V2,\n",
    "     3: V3}\n",
    "\n",
    "av = {1: TRAIN_AV_SP,\n",
    "      2: SM_AV,\n",
    "      3: CAR_AV_SP}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define the model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "logprob = models.loglogit(V, av, CHOICE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Define the Biogeme object**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "biogeme  = bio.BIOGEME(database, logprob)\n",
    "\n",
    "biogeme.modelName = \"swissmetro_logit_validation\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Estimate the model**\n",
    "\n",
    "1. A `.html` can be generated with a report of the results and can be opened with a browser\n",
    "2. A `.pickle` file can also be generaetd with a snapshot with the results. This file can then be used in other scripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "HTML file:    swissmetro_logit_validation.html\n",
      "Pickle file:  None\n"
     ]
    }
   ],
   "source": [
    "biogeme.generateHtml = True\n",
    "biogeme.generatePickle = False\n",
    "\n",
    "results = biogeme.estimate()\n",
    "\n",
    "print(f\"HTML file:    {results.data.htmlFileName}\")\n",
    "print(f\"Pickle file:  {results.data.pickleFileName }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of estimated parameters               = 4\n",
      "Sample size                                  = 6768\n",
      "Excluded observations                        = 3960\n",
      "Init log likelihood                          = -6964.662979192295\n",
      "Final log likelihood                         = -5331.252007298093\n",
      "Likelihood ratio test for the init. model    = 3266.8219437884036\n",
      "Rho-square for the init. model               = 0.2345283579082288\n",
      "Rho-square-bar for the init. model           = 0.23395403004599769\n",
      "Akaike Information Criterion                 = 10670.504014596187\n",
      "Bayesian Information Criterion               = 10697.78385820133\n",
      "Final gradient norm                          = 0.015306327718063055\n",
      "Nbr of threads                               = 8\n"
     ]
    }
   ],
   "source": [
    "gs = results.getGeneralStatistics()\n",
    "\n",
    "for k,v in gs.items():\n",
    "    print(\"{}= {}\".format(k.ljust(45),v[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Log likelihood for 1354 validation data: -1070.4355819994128\n",
      "Log likelihood for 1354 validation data: -1085.69142450774\n",
      "Log likelihood for 1354 validation data: -1068.2759300553225\n",
      "Log likelihood for 1353 validation data: -1073.7030903233847\n",
      "Log likelihood for 1353 validation data: -1033.1459807649794\n"
     ]
    }
   ],
   "source": [
    "validation_results = biogeme.validate(results)\n",
    "\n",
    "for slide in validation_results:\n",
    "    print(f'Log likelihood for {slide.shape[0]} validation data: {slide[\"Loglikelihood\"].sum()}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}