FairKDTree_Git.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing libraries and dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "dfc = pd.read_csv(r\"C:\\Users\\Sina\\Desktop\\2023\\2023\\Project_1_Fairness_Reweighting\\Codes\\SchoolDataset_Coordinates_CA.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate_unemployment</th>\n",
       "      <th>percent_college</th>\n",
       "      <th>percent_married</th>\n",
       "      <th>median_income</th>\n",
       "      <th>average_act</th>\n",
       "      <th>percent_lunch</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>geocode</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.122475</td>\n",
       "      <td>0.605338</td>\n",
       "      <td>0.691299</td>\n",
       "      <td>80156.0</td>\n",
       "      <td>21.568987</td>\n",
       "      <td>0.208716</td>\n",
       "      <td>93510</td>\n",
       "      <td>Los Angeles County, California, 93510, United ...</td>\n",
       "      <td>34.465826</td>\n",
       "      <td>-118.198287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.078767</td>\n",
       "      <td>0.764101</td>\n",
       "      <td>0.659389</td>\n",
       "      <td>70074.0</td>\n",
       "      <td>23.693529</td>\n",
       "      <td>0.094731</td>\n",
       "      <td>95765</td>\n",
       "      <td>Rocklin, Placer County, California, 95765, Uni...</td>\n",
       "      <td>38.820964</td>\n",
       "      <td>-121.290739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.080710</td>\n",
       "      <td>0.855984</td>\n",
       "      <td>0.953930</td>\n",
       "      <td>113229.0</td>\n",
       "      <td>22.350427</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>95765</td>\n",
       "      <td>Rocklin, Placer County, California, 95765, Uni...</td>\n",
       "      <td>38.820964</td>\n",
       "      <td>-121.290739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.193734</td>\n",
       "      <td>0.489950</td>\n",
       "      <td>0.454356</td>\n",
       "      <td>28975.0</td>\n",
       "      <td>19.737485</td>\n",
       "      <td>0.525169</td>\n",
       "      <td>92345</td>\n",
       "      <td>Hesperia, San Bernardino County, California, 9...</td>\n",
       "      <td>34.425008</td>\n",
       "      <td>-117.323456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.212134</td>\n",
       "      <td>0.408170</td>\n",
       "      <td>0.658747</td>\n",
       "      <td>55332.0</td>\n",
       "      <td>18.663004</td>\n",
       "      <td>0.621418</td>\n",
       "      <td>92345</td>\n",
       "      <td>Hesperia, San Bernardino County, California, 9...</td>\n",
       "      <td>34.425008</td>\n",
       "      <td>-117.323456</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rate_unemployment  percent_college  percent_married  median_income  \\\n",
       "0           0.122475         0.605338         0.691299        80156.0   \n",
       "1           0.078767         0.764101         0.659389        70074.0   \n",
       "2           0.080710         0.855984         0.953930       113229.0   \n",
       "3           0.193734         0.489950         0.454356        28975.0   \n",
       "4           0.212134         0.408170         0.658747        55332.0   \n",
       "\n",
       "   average_act  percent_lunch  zip_code  \\\n",
       "0    21.568987       0.208716     93510   \n",
       "1    23.693529       0.094731     95765   \n",
       "2    22.350427       0.133333     95765   \n",
       "3    19.737485       0.525169     92345   \n",
       "4    18.663004       0.621418     92345   \n",
       "\n",
       "                                             geocode   latitude   longitude  \n",
       "0  Los Angeles County, California, 93510, United ...  34.465826 -118.198287  \n",
       "1  Rocklin, Placer County, California, 95765, Uni...  38.820964 -121.290739  \n",
       "2  Rocklin, Placer County, California, 95765, Uni...  38.820964 -121.290739  \n",
       "3  Hesperia, San Bernardino County, California, 9...  34.425008 -117.323456  \n",
       "4  Hesperia, San Bernardino County, California, 9...  34.425008 -117.323456  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate_unemployment</th>\n",
       "      <th>percent_college</th>\n",
       "      <th>percent_married</th>\n",
       "      <th>median_income</th>\n",
       "      <th>average_act</th>\n",
       "      <th>percent_lunch</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "      <td>1148.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.118608</td>\n",
       "      <td>0.581791</td>\n",
       "      <td>0.663165</td>\n",
       "      <td>61601.338850</td>\n",
       "      <td>20.479764</td>\n",
       "      <td>0.458978</td>\n",
       "      <td>93114.068815</td>\n",
       "      <td>36.629632</td>\n",
       "      <td>-109.427834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.053631</td>\n",
       "      <td>0.211320</td>\n",
       "      <td>0.157860</td>\n",
       "      <td>29156.750582</td>\n",
       "      <td>2.838364</td>\n",
       "      <td>0.241423</td>\n",
       "      <td>1863.498217</td>\n",
       "      <td>4.390662</td>\n",
       "      <td>34.736251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.001536</td>\n",
       "      <td>0.102423</td>\n",
       "      <td>0.162238</td>\n",
       "      <td>11600.000000</td>\n",
       "      <td>12.973138</td>\n",
       "      <td>-0.015385</td>\n",
       "      <td>90001.000000</td>\n",
       "      <td>-5.190223</td>\n",
       "      <td>-124.281180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.077556</td>\n",
       "      <td>0.419355</td>\n",
       "      <td>0.553291</td>\n",
       "      <td>40329.750000</td>\n",
       "      <td>18.510379</td>\n",
       "      <td>0.257288</td>\n",
       "      <td>91762.000000</td>\n",
       "      <td>33.987032</td>\n",
       "      <td>-121.387112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.109442</td>\n",
       "      <td>0.597131</td>\n",
       "      <td>0.668279</td>\n",
       "      <td>55592.000000</td>\n",
       "      <td>20.494505</td>\n",
       "      <td>0.467614</td>\n",
       "      <td>93031.500000</td>\n",
       "      <td>35.074912</td>\n",
       "      <td>-118.371124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.152350</td>\n",
       "      <td>0.750037</td>\n",
       "      <td>0.783062</td>\n",
       "      <td>76637.500000</td>\n",
       "      <td>22.380952</td>\n",
       "      <td>0.661031</td>\n",
       "      <td>94901.500000</td>\n",
       "      <td>37.920084</td>\n",
       "      <td>-117.701282</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.335133</td>\n",
       "      <td>0.979269</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>224531.000000</td>\n",
       "      <td>29.212454</td>\n",
       "      <td>0.977505</td>\n",
       "      <td>96161.000000</td>\n",
       "      <td>65.775638</td>\n",
       "      <td>120.243772</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       rate_unemployment  percent_college  percent_married  median_income  \\\n",
       "count        1148.000000      1148.000000      1148.000000    1148.000000   \n",
       "mean            0.118608         0.581791         0.663165   61601.338850   \n",
       "std             0.053631         0.211320         0.157860   29156.750582   \n",
       "min             0.001536         0.102423         0.162238   11600.000000   \n",
       "25%             0.077556         0.419355         0.553291   40329.750000   \n",
       "50%             0.109442         0.597131         0.668279   55592.000000   \n",
       "75%             0.152350         0.750037         0.783062   76637.500000   \n",
       "max             0.335133         0.979269         1.000000  224531.000000   \n",
       "\n",
       "       average_act  percent_lunch      zip_code     latitude    longitude  \n",
       "count  1148.000000    1148.000000   1148.000000  1148.000000  1148.000000  \n",
       "mean     20.479764       0.458978  93114.068815    36.629632  -109.427834  \n",
       "std       2.838364       0.241423   1863.498217     4.390662    34.736251  \n",
       "min      12.973138      -0.015385  90001.000000    -5.190223  -124.281180  \n",
       "25%      18.510379       0.257288  91762.000000    33.987032  -121.387112  \n",
       "50%      20.494505       0.467614  93031.500000    35.074912  -118.371124  \n",
       "75%      22.380952       0.661031  94901.500000    37.920084  -117.701282  \n",
       "max      29.212454       0.977505  96161.000000    65.775638   120.243772  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfc.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's apply a grid on top of the area"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calMisCalibration(labels, preds):\n",
    "    _preds = sum(preds)\n",
    "    _labels = sum(labels)\n",
    "    \n",
    "    if _labels==0:\n",
    "        print(\"ERROR: the label sum is zero\")\n",
    "    \n",
    "    \n",
    "    return _preds/_labels\n",
    "\n",
    "\n",
    "def calMisCalibrationAbsDiff(labels, preds):\n",
    "    _preds = sum(preds)\n",
    "    _labels = sum(labels)\n",
    "    if len(preds)==0:\n",
    "        #print(\"ERROR: There is no entry\")\n",
    "        return 0\n",
    "    \n",
    "    else:\n",
    "        return abs(_preds-_labels)/len(preds)\n",
    "        #return abs(_preds-_labels)\n",
    "    \n",
    "def calMisCalibrationForGivenIndexesInDF(labels, preds, indexes):    \n",
    "    mem_label = []\n",
    "    mem_preds = []\n",
    "    for i in indexes:\n",
    "        if i<len(preds):\n",
    "            mem_label.append(labels[i])\n",
    "            mem_preds.append(preds[i])\n",
    "    return calMisCalibrationAbsDiff(mem_label, mem_preds)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "def calECE(labels,preds,num_bins):\n",
    "    bins = np.linspace(0,1,num_bins+1)\n",
    "    ece = 0\n",
    "    for i in range(len(bins)-1):\n",
    "        \n",
    "        preds_in_bin = []\n",
    "        labels_in_bin = []\n",
    "        for idx,pred in enumerate(preds):\n",
    "            if i ==0:\n",
    "                if bins[i]<= pred<=bins[i+1]:\n",
    "                    preds_in_bin.append(preds[idx])\n",
    "                    labels_in_bin.append(labels[idx])  \n",
    "\n",
    "            else:\n",
    "                if bins[i]< pred<=bins[i+1]:\n",
    "                    preds_in_bin.append(preds[idx])\n",
    "                    labels_in_bin.append(labels[idx])\n",
    "        \n",
    "        #print(\"bin between {} to {} preds {} labels {}\".format(bins[i],bins[i+1],preds_in_bin,labels_in_bin))\n",
    "        \n",
    "        #ece+=len(preds_in_bin)*(abs(sum(labels_in_bin)- sum(preds_in_bin)))\n",
    "        ece+= (abs(sum(labels_in_bin)- sum(preds_in_bin)))\n",
    "    return ece/len(preds)\n",
    "    \n",
    "    \n",
    "def calMCE(labels,preds,num_bins):\n",
    "    bins = np.linspace(0,1,num_bins+1)\n",
    "    ece = 0\n",
    "    _max = []\n",
    "    for i in range(len(bins)-1):\n",
    "        \n",
    "        preds_in_bin = []\n",
    "        labels_in_bin = []\n",
    "        \n",
    "        for idx,pred in enumerate(preds):\n",
    "            if i ==0:\n",
    "                if bins[i]<= pred<=bins[i+1]:\n",
    "                    preds_in_bin.append(preds[idx])\n",
    "                    labels_in_bin.append(labels[idx])  \n",
    "\n",
    "            else:\n",
    "                if bins[i]< pred<=bins[i+1]:\n",
    "                    preds_in_bin.append(preds[idx])\n",
    "                    labels_in_bin.append(labels[idx])\n",
    "        \n",
    "        #print(\"bin between {} to {} preds {} labels {}\".format(bins[i],bins[i+1],preds_in_bin,labels_in_bin))\n",
    "        _max.append(        abs(sum(labels_in_bin)- sum(preds_in_bin))/len(labels_in_bin)        )\n",
    "    return max(_max)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1148\n",
      "range_X 1024 range_Y 1024 resolution [0.01953125, 0.01953125] range_coordinates [24.052235000000003, 44.052235, -128.243683, -108.243683]\n",
      "we are in \n",
      "1148\n",
      "we are out\n",
      "1057\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1596: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1745: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(ilocs[0], value)\n"
     ]
    }
   ],
   "source": [
    "\n",
    "def filter_area(df,bj_coordinates):\n",
    "    print(\"we are in \")\n",
    "    print(len(df))\n",
    "    df=df[(df['latitude']>bj_coordinates[0])&(df['latitude']<bj_coordinates[1])&(df['longitude']>bj_coordinates[2])&(df['longitude']<bj_coordinates[3])]\n",
    "    print(\"we are out\")\n",
    "    print(len(df))\n",
    "    return df\n",
    "\n",
    "\n",
    "      \n",
    "def generate_point_X(lat, lon, range_coordinates, grid_size):\n",
    "    return int(np.floor((lon-range_coordinates[2])/resolution[1]))\n",
    "\n",
    "\n",
    "def generate_point_Y(lat, lon, range_coordinates, grid_size):\n",
    "    return int(np.floor((lat-range_coordinates[0])/resolution[0]))\n",
    "\n",
    "\n",
    "\n",
    "def generate_point_ID(lat, lon, grid_size):\n",
    "    return int(lat*range_Y+lon)\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    \n",
    "    \"\"\"\n",
    "    Read Data \n",
    "    \"\"\"\n",
    "    df = dfc\n",
    "    \n",
    "    print(len(df))\n",
    "    \n",
    "    \"\"\"\n",
    "    Map variables\n",
    "    \"\"\"\n",
    "    #here you need to specify the the coordinates you would like to consider.\n",
    "    ### California ###\n",
    "    City_Lat = 34.052235\n",
    "    City_Lon = -118.243683   \n",
    "\n",
    "    \n",
    "    #thresh = 5\n",
    "    thresh = 10\n",
    "    range_coordinates = [City_Lat-thresh, City_Lat+thresh, City_Lon-thresh, City_Lon+thresh]   \n",
    "    grid_size =1024\n",
    "\n",
    "    range_Y = grid_size\n",
    "    range_X = grid_size\n",
    "    \n",
    "\n",
    "    resolution = [0.0, 0.0]\n",
    "    resolution[0] = (range_coordinates[1]-range_coordinates[0])/range_Y#103\n",
    "    resolution[1]  = (range_coordinates[3]-range_coordinates[2])/range_X#101\n",
    "    \n",
    "\n",
    "   \n",
    "    max_gridID = range_Y*range_Y+range_X\n",
    "\n",
    "    print(\"range_X {} range_Y {} resolution {} range_coordinates {}\".format(range_X,range_Y,resolution, range_coordinates))\n",
    "\n",
    "    \"\"\"\n",
    "    Data Process\n",
    "    \"\"\"\n",
    "    df = filter_area(df,range_coordinates)\n",
    "\n",
    "    df.loc[:, 'gridX'] = df.apply(lambda x: generate_point_X(x['latitude'], x['longitude'], range_coordinates, resolution), axis=1)\n",
    "    df.loc[:, 'gridY'] = df.apply(lambda x: generate_point_Y(x['latitude'], x['longitude'], range_coordinates, resolution), axis=1)\n",
    "    df.loc[:, 'gridID'] = df.apply(lambda x: generate_point_ID(x['gridY'], x['gridX'], grid_size), axis=1)\n",
    "    df = df.reset_index(drop = True)  \n",
    "\n",
    "\n",
    "    df_kdTree =  df[['rate_unemployment','percent_college','percent_married','median_income','average_act', 'percent_lunch','gridID','gridY','gridX']]\n",
    "    df = df[['rate_unemployment','percent_college','percent_married','median_income','average_act', 'percent_lunch','gridID']]\n",
    "    df.head()\n",
    "\n",
    "    from collections import defaultdict\n",
    "    d_regions = defaultdict(list)\n",
    "\n",
    "    for i in range(len(df)):\n",
    "        d_regions[df.loc[i,'gridID']].append(i)\n",
    "\n",
    "    df.gridID.value_counts()\n",
    "    #print(df.describe())\n",
    "    #df.loc[:,'average_act'] = df.apply(lambda x: 1 if  x['average_act']>=24 else 0 , axis=1)\n",
    "    df.loc[:,'average_act'] = df.apply(lambda x: 1 if  x['average_act']>=22 else 0 , axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['rate_unemployment', 'percent_college', 'percent_married',\n",
       "       'median_income', 'average_act', 'percent_lunch', 'gridID'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Initial Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(X_train) 845, len(X_test) 212, len(y_train) 845, len(y_test) 212\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.92      0.84       147\n",
      "           1       0.67      0.37      0.48        65\n",
      "\n",
      "    accuracy                           0.75       212\n",
      "   macro avg       0.72      0.64      0.66       212\n",
      "weighted avg       0.74      0.75      0.73       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.75\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-5.87969350e-11  2.28529807e-10  3.03618743e-11  2.28878069e-05\n",
      "  -5.26087999e-10 -3.75110432e-06]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.0901797394560016\n",
      " Test Calibration 1.0914144899091278\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.025680323156344793\n",
      " Test Calibration 0.028028027566477865\n",
      "\n",
      "Cross-Validation Accuracy Scores [0.75471698 0.71698113 0.76415094 0.77358491 0.73584906 0.81132075\n",
      " 0.82075472 0.7047619  0.66666667 0.78095238]\n",
      "Cross-Validation Accuracy Scores 0.752973944294699\n",
      "                   rate_unemployment  percent_college  percent_married  \\\n",
      "rate_unemployment           1.000000        -0.469815        -0.369085   \n",
      "percent_college            -0.469815         1.000000         0.508429   \n",
      "percent_married            -0.369085         0.508429         1.000000   \n",
      "median_income              -0.512166         0.719920         0.659265   \n",
      "average_act                -0.301128         0.513276         0.290254   \n",
      "percent_lunch               0.389700        -0.687281        -0.444991   \n",
      "gridID                      0.067025         0.130524        -0.022700   \n",
      "\n",
      "                   median_income  average_act  percent_lunch    gridID  \n",
      "rate_unemployment      -0.512166    -0.301128       0.389700  0.067025  \n",
      "percent_college         0.719920     0.513276      -0.687281  0.130524  \n",
      "percent_married         0.659265     0.290254      -0.444991 -0.022700  \n",
      "median_income           1.000000     0.423122      -0.581230 -0.013814  \n",
      "average_act             0.423122     1.000000      -0.659055  0.133347  \n",
      "percent_lunch          -0.581230    -0.659055       1.000000 -0.207776  \n",
      "gridID                 -0.013814     0.133347      -0.207776  1.000000  \n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(df.drop('average_act',axis=1), \n",
    "                                                    df['average_act'], test_size=0.20, \n",
    "                                                    random_state=101)\n",
    "\n",
    "\n",
    "\n",
    "print(\"len(X_train) {}, len(X_test) {}, len(y_train) {}, len(y_test) {}\".format(len(X_train),len(X_test),len(y_train),len(y_test)))\n",
    "X_train = df.drop(['average_act'],axis=1)\n",
    "y_train = df['average_act']\n",
    "\n",
    "#logistic regression model\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "logmodel = LogisticRegression()\n",
    "logmodel.fit(X_train,y_train)\n",
    "\n",
    "\n",
    "\n",
    "X_test_pred_labels = logmodel.predict(X_test)\n",
    "X_test_pred = logmodel.predict_proba(X_test)\n",
    "X_train_pred = logmodel.predict_proba(X_train)\n",
    "X_test_pred = X_test_pred[:,1]\n",
    "X_train_pred = X_train_pred[:,1]\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test,X_test_pred_labels))\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(\"--- Accuracy of the whole model ---\\n Accuracy {}\\n\".format(accuracy_score(y_test,X_test_pred_labels)))\n",
    "print(\"--- Coefficient significance---\\n  {}\\n\".format(logmodel.coef_))\n",
    "print(\"--- Miscalibration for the whole model---\\n Training Calibration {}\\n Test Calibration {}\\n\".format(format(calMisCalibration(y_train.to_numpy(), X_train_pred)) ,calMisCalibration(y_test.to_numpy(), X_test_pred) ))\n",
    "print(\"--- Miscalibration difference for the whole model---\\n Training Calibration {}\\n Test Calibration {}\\n\".format(format(calMisCalibrationAbsDiff(y_train.to_numpy(), X_train_pred)) ,calMisCalibrationAbsDiff(y_test.to_numpy(), X_test_pred) ))\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "scores = cross_val_score(logmodel, X_train, y_train, cv=10)\n",
    "print('Cross-Validation Accuracy Scores', scores)\n",
    "print('Cross-Validation Accuracy Scores', np.mean(scores))\n",
    "\n",
    "print(df[df.columns[:]].corr())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Select Data for KD Tree and also figure out the calibration for original grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ERROR: the label sum is zero\n",
      "--- region_ID --- 546306\n",
      " MisCalibration inf \n",
      " Micalibration difference 0.44654117388851433\n",
      "\n",
      "--- region_ID --- 774499\n",
      " MisCalibration 0.3180763328992209 \n",
      " Micalibration difference 0.6819236671007791\n",
      "\n",
      "ERROR: the label sum is zero\n",
      "--- region_ID --- 544303\n",
      " MisCalibration inf \n",
      " Micalibration difference 0.25798479785040773\n",
      "\n",
      "ERROR: the label sum is zero\n",
      "--- region_ID --- 542250\n",
      " MisCalibration inf \n",
      " Micalibration difference 0.4157161081401519\n",
      "\n",
      "ERROR: the label sum is zero\n",
      "--- region_ID --- 526877\n",
      " MisCalibration inf \n",
      " Micalibration difference 0.2570405477995814\n",
      "\n",
      "--- region_ID --- 548402\n",
      " MisCalibration 0.5051695790115666 \n",
      " Micalibration difference 0.24741521049421672\n",
      "\n",
      "ERROR: the label sum is zero\n",
      "--- region_ID --- 547380\n",
      " MisCalibration inf \n",
      " Micalibration difference 0.2432652365020705\n",
      "\n",
      "--- region_ID --- 716102\n",
      " MisCalibration 0.3540044357523965 \n",
      " Micalibration difference 0.6459955642476035\n",
      "\n",
      "--- region_ID --- 715077\n",
      " MisCalibration 0.5944339895396455 \n",
      " Micalibration difference 0.40556601046035445\n",
      "\n",
      "--- region_ID --- 714053\n",
      " MisCalibration 0.516114175438779 \n",
      " Micalibration difference 0.483885824561221\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-6-83d80194f2c3>:9: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  return _preds/_labels\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "counter = 0\n",
    "\n",
    "for region_id,idxs in d_regions.items():\n",
    "    counter+=1\n",
    "    a_label = y_train.to_numpy()\n",
    "    a_preds = X_train_pred\n",
    "    \n",
    "    \n",
    "    mem_label = []\n",
    "    mem_preds = []\n",
    "    for i in idxs:\n",
    "        if i<len(a_preds):\n",
    "            mem_label.append(a_label[i])\n",
    "            mem_preds.append(a_preds[i])\n",
    "    \n",
    "    print(\"--- region_ID --- {}\\n MisCalibration {} \\n Micalibration difference {}\\n\".format(region_id, calMisCalibration(mem_label, mem_preds), calMisCalibrationAbsDiff(mem_label, mem_preds)))\n",
    "    if counter==10:\n",
    "        break\n",
    "    #1/0\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fair KD-tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# The following applies KDTree for various heights and displays results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The tree height is 10\n",
      "------------------ KDtree is Done| Neighborhoods Generated ---------------\n",
      "res_oldModel   [(0.06040224017417853, 4), (0.005013179867988393, 5), (0.5223734625211411, 1), (0.15941815497818057, 2), (0.3639263375326226, 1), (0.43605962126648445, 1), (0.15853712537255657, 2), (0.5988139913416679, 1), (0.0026294821757152365, 2), (0.07756747080964227, 4), (0.11205484236307316, 2), (0.05814187162362817, 6), (0.14293106067719047, 2), (0.06885179283626626, 3), (0.14273319838061566, 3), (0.4659360626805882, 1), (0.10767019770103131, 2), (0.07756925384236812, 3), (0.035329767793964885, 6), (0.00630747603148276, 38), (0.18469855641527705, 2), (0.2180236652247499, 1), (0.1255773465553185, 2), (0.2275690596674695, 1), (0.3434297144553668, 1), (0.3807218900904067, 1), (0.2036984541580783, 2), (0.15051868307239444, 2), (0.08803828709656729, 3), (0.14646723577498913, 2), (0.1464012932628788, 2), (0.05252159939865203, 4), (0.5038915109074716, 1), (0.2853958306839012, 1), (0.4026786456541923, 1), (0.6239563341872005, 1), (0.26075728737149473, 2), (0.036547855292628086, 3), (0.12271023895184917, 3), (0.6137403310380994, 1), (0.7260659897929098, 1), (0.06407854444368088, 4), (0.06918760929126147, 3), (0.0388201403313115, 7), (0.08806085767589161, 3), (0.3350627222354173, 1), (0.3767908821694192, 1), (0.09136106549988775, 3), (0.2964624074962298, 1), (0.3059344767855995, 1), (0.18335332109968128, 2), (0.32009612255293995, 1), (0.46740825219424853, 1), (0.5867555272629892, 1), (0.24260629331294148, 1), (0.06509600313531538, 3), (0.2704404044725468, 2), (0.44199009194457484, 1), (0.10717550379421355, 3), (0.5361161961571969, 1), (0.32105794407158494, 1), (0.012002388278458379, 3), (0.40214513617183234, 1), (0.03123704155755452, 3), (0.3811791202270225, 1), (0.7714324069068252, 1), (0.3397482155692912, 1), (0.6801480798373853, 1), (0.1805814197156017, 2), (0.1597088559388226, 2), (0.14495772579314214, 2), (0.09030814859097819, 3), (0.6279828611015106, 1), (0.276414200310246, 2), (0.11844799580080605, 2), (0.20377199329465806, 1), (0.5770165704819405, 1), (0.26337210257077076, 1), (0.22398014554272155, 2), (0.02475898092848129, 8), (0.1149643685810241, 2), (0.34582930810261425, 1), (0.042069959480894106, 5), (0.4423580771437864, 1), (0.12138554201168311, 2), (0.054910253747642884, 4), (0.05753025650179357, 4), (0.4091975842026012, 1), (0.04158965019010741, 5), (0.3111215480741855, 1), (0.057048911661109716, 4), (0.12897590322817654, 2), (0.13365688282145488, 2), (0.1724481652262727, 2), (0.6383669997695967, 1), (0.10518334857988021, 2), (0.6142517100934892, 1), (0.7221004468178059, 1), (0.8034639893364053, 1), (0.6890229771834773, 1), (0.20128339455663335, 2), (0.23338960869634434, 2), (0.2920724769750757, 1), (0.1824337743280355, 2), (0.15041472215501653, 3), (0.4076301181715295, 1), (0.08710525849500424, 6), (0.052254770330257015, 6), (0.4520050461553594, 1), (0.38101048060710674, 1), (0.4583458735852247, 1), (0.43344918035195934, 1), (0.5067330195353643, 1), (0.42011753834061993, 1), (0.6656440608730279, 1), (0.0585620467722075, 2), (0.06932783171053344, 2), (0.46777968351561494, 1), (0.051797777451647264, 4), (0.010990691359419852, 4), (0.45172842874911334, 1), (0.24679557777681066, 2), (0.09684991842017948, 3), (0.48991077084574897, 1), (0.39802374615723396, 1), (0.01656914951747247, 6), (0.37665690458085244, 1), (0.25610635428447137, 1), (0.18394682652609684, 2), (0.02304952777352355, 3), (0.13846381932663213, 3), (0.37419266756616987, 1), (0.40369363339759035, 1), (0.1750603895325039, 2), (0.47504227363815954, 1), (0.6947058202185656, 1), (0.2046094331616323, 2), (0.22163613222712075, 2), (0.22994973744029862, 2), (0.6185207640139916, 1), (0.4416706959982618, 1), (0.1953724524852244, 1), (0.3755936959561067, 1), (0.27895368453640085, 1), (0.2331332730599488, 2), (0.09801328774363581, 4), (0.14106046762597974, 2), (0.19358308573467164, 2), (0.3817303188728993, 1), (0.3056592363087181, 1), (0.20029888462630457, 2), (0.4906254434788188, 1), (0.15314766161168927, 2), (0.21543494274857414, 2), (0.35263728557714247, 1), (0.19916293466509372, 3), (0.4665276999264156, 1), (0.31154968438803665, 1), (0.36828235186379676, 1), (0.05736608709409574, 2), (0.2628464375002465, 2), (0.0573223094776554, 2), (0.5108068694458069, 1), (0.1477674156996323, 2), (0.26765280022851135, 1), (0.40010821067694313, 1), (0.14856881180771264, 2), (0.6243783238147915, 1), (0.20656410903479727, 1), (0.5264705189836824, 1), (0.17840133960274662, 1), (0.31489527047363797, 2), (0.03291514573587373, 2), (0.4113687677223597, 1), (0.4988986097661501, 1), (0.1157153304717927, 2), (0.05670708672394109, 2), (0.6386196473107142, 1), (0.2963441527969144, 2), (0.5204759446411747, 1), (0.33705217641512175, 1), (0.23551348744687145, 1), (0.2767142985536363, 1), (0.07315047375199235, 2), (0.11919189403927201, 3), (0.12127034601519732, 2), (0.20182857049222794, 1), (0.1029423551827246, 2), (0.08845973356384544, 3), (0.30132208158329515, 1), (0.28092658854326796, 1), (0.37143845789972235, 1), (0.29635950746268697, 1), (0.4388078560000638, 1), (0.3698255624651075, 1), (0.09813472111416088, 2), (0.2883637516216559, 1), (0.3040186312761361, 1), (0.39939583186795335, 1), (0.49655736033949904, 1), (0.0042857669192098435, 4), (0.04318759382764741, 3), (0.23484151755868765, 1), (0.16412677149923866, 3), (0.6129279000698464, 1), (0.3344678365495093, 1), (0.21253794703895645, 2), (0.25646194018561974, 1), (0.40757068321437917, 1), (0.023695563032040123, 2), (0.11947716248241978, 3), (0.2763490228753997, 1), (0.5489962685371026, 1), (0.6429109686007408, 1), (0.4033475714144299, 1), (0.1440307089722701, 2), (0.3555049232065167, 1), (0.27125539333305243, 1), (0.29774286957192453, 1), (0.15376534767033753, 2), (0.4002074707670801, 1), (0.04398818628236495, 2), (0.1798775042106303, 2), (0.4163164146486959, 1), (0.1360300650028647, 3), (0.09916965518338301, 3), (0.48046686977131436, 1), (0.1927437397267947, 2), (0.22999455810286157, 1), (0.23799133358549365, 2), (0.273508699018577, 2), (0.14992790410019854, 2), (0.6339212961412797, 1), (0.1483049234894759, 2), (0.36576528840173816, 1), (0.15312580724474445, 2), (0.28588447443478227, 1), (0.19945305040692224, 2), (0.46416749331444396, 1), (0.1613944873601363, 2), (0.5003043789126639, 1), (0.14533656861666364, 2), (0.251889193261179, 1), (0.2666470993482322, 1), (0.13822622834694331, 2), (0.2956513024111405, 1), (0.17250654684110722, 2), (0.015018149518747607, 3), (0.273524567701318, 1), (0.46438418059305503, 1), (0.35386726218474773, 1), (0.1750830649452307, 2), (0.4257157994645586, 1), (0.15169566339867718, 2), (0.21159403782089514, 2), (0.25916845244552855, 1), (0.2641617783059001, 2), (0.0592411024449524, 2), (0.26367863304918926, 2), (0.21646246003562603, 2), (0.09444611513131267, 3), (0.23432330271164725, 1), (0.019486717459014487, 3), (0.13755198408457653, 2), (0.21468484854107617, 1), (0.1044049145069621, 2), (0.3730479148758827, 1), (0.7577483482051677, 1), (0.27154804832958357, 1), (0.6381926190341141, 1), (0.2967419927465345, 2), (0.1141472797047611, 2), (0.14900244794345324, 2), (0.2500706039060996, 2), (0.6766385897492354, 1), (0.06942482790764844, 3), (0.05746032206544177, 2), (0.18297929821625142, 1), (0.5955266925215815, 1), (0.19115131034194846, 1), (0.03000000708135872, 2), (0.31960809070161056, 1), (0.8317993874880458, 1), (0.5186676256605232, 1), (0.15316597155448308, 1), (0.4850718363755291, 1), (0.03927695957515587, 2), (0.45579695222853356, 1), (0.2625192382422681, 1), (0.04697634359972844, 3), (0.12449892537442334, 2), (0.08106750888637689, 3), (0.3240548831032957, 1), (0.14364967284012925, 2), (0.07484955275185838, 4), (0.0340561608044934, 5), (0.07093308601253888, 2), (0.02565580715458565, 8), (0.05501871288109902, 3), (0.42291466065952243, 1), (0.17072984438439037, 1), (0.20046652807892035, 2), (0.6951783670256052, 1), (0.1431495855308566, 2), (0.7954266846348772, 1), (0.5378940397795151, 1), (0.7093780757616068, 1), (0.2336983108363105, 1), (0.7669275333390047, 1), (0.6280114805969178, 1), (0.1308016579154671, 2), (0.29225830217286974, 2), (0.18922321965065136, 1), (0.521570851694414, 1), (0.3700315365252782, 2), (0.26205572039221636, 2), (0.2050063542575373, 1), (0.23251127587445797, 1), (0.20465750777080507, 1), (0.23170631896970464, 1), (0.2314816803539012, 1), (0.1540939601624443, 2), (0.08376828523945348, 2), (0.03494880203654677, 2), (0.003928117299986088, 3), (0.23182693530441595, 1), (0.32923330483544855, 1), (0.28183621821797034, 1), (0.40275201131858646, 1), (0.2896749952946968, 2), (0.0753793324989547, 2), (0.0389322940245625, 2), (0.2919258978815457, 1), (0.1994143814209429, 1), (0.5995797656873098, 1), (0.26844688335558453, 1), (0.44604950707702773, 1), (0.6234642079218572, 1), (0.20043758781782373, 2), (0.5044239835442648, 1), (0.12259551118587753, 2), (0.06275110786609785, 4), (0.04011545844174845, 4), (0.16242197789827456, 3), (0.6459955642476035, 1), (0.22821019956581434, 3), (0.26884774526431393, 1), (0.7439676822179686, 1), (0.6523730384877264, 1), (0.7659468318170575, 1), (0.15346492073571857, 2), (0.815142458172933, 1), (0.07960684733211698, 2), (0.1517906904271529, 2), (0.2411500803875466, 1), (0.7202421214990679, 1), (0.7873817567515928, 1), (0.8298745002188954, 1), (0.08245332656798043, 2), (0.09532161456697417, 2), (0.27659465888967205, 1), (0.8835428746858718, 1), (0.4113287619368589, 1), (0.21209700948999388, 1), (0.31545331295857243, 1), (0.16853497419814523, 1), (0.09507228256033909, 2), (0.11256116963937446, 1), (0.2415817021373872, 1), (0.11996302098199663, 1), (0.11718096146710368, 1), (0.9193084043859657, 1), (0.09539817708704819, 1), (0.1329138024802101, 1), (0.795374319587798, 1), (0.8686265781072021, 1), (0.8902517439796038, 1), (0.09784843556650172, 1), (0.06816376177775348, 1), (0.05177480909934343, 1), (0.9080883062176571, 1), (0.09430201106210605, 1), (0.4665578162611817, 2), (0.9293538446392333, 1), (0.8394052036506666, 1), (0.17018952506585797, 1), (0.13261369501140735, 1), (0.08177612402780922, 1), (0.15950928617897892, 1), (0.19099624545331428, 1), (0.12227107262167555, 1), (0.9133709964071285, 1), (0.4597794955038046, 2), (0.0753225353022684, 1), (0.21244776837849028, 2), (0.9251994661479952, 1), (0.8359848459674581, 1), (0.06880635474876261, 1), (0.9131612712103693, 1), (0.176926975052901, 3), (0.3332232774991486, 2), (0.6065152091888744, 1), (0.4145172122769162, 1), (0.4487722032277052, 1), (0.11587721826776304, 2), (0.35079791250784514, 1), (0.7891635777278608, 1), (0.25850478250961095, 1), (0.3801579001552231, 1), (0.0839182378692883, 3), (0.7909288731756524, 1), (0.3220861996023244, 1), (0.16873271220638375, 1), (0.14437430017928424, 1), (0.11394364602909607, 2), (0.1106234855028249, 2), (0.15519369460568028, 2), (0.17170485664317425, 1), (0.41280053043888476, 2), (0.2871977135751873, 1), (0.27618689491300147, 1), (0.20017850475184792, 1), (0.14332645748489897, 1), (0.1673800550061343, 1), (0.10554395698446398, 1), (0.07258282193484536, 2), (0.14565407297207925, 1), (0.9268783196686764, 1), (0.1849957665713816, 1), (0.0992019729405946, 1), (0.10282049585933706, 1), (0.10477152265491295, 1), (0.0987847830758934, 1), (0.15905576855592274, 4), (0.22738236274414894, 1), (0.19723282995504018, 1), (0.08877531934416111, 1), (0.12204444166346781, 1), (0.09354119887244422, 2), (0.11428372248545757, 2), (0.13055600979029275, 1), (0.09797040304545405, 1), (0.16774680462948857, 1), (0.10891394674929386, 1), (0.04906700677373732, 2), (0.9046775815479149, 1), (0.21022107462219475, 2), (0.08986867909580767, 2), (0.8447246267607216, 1), (0.3084794191452276, 1), (0.06424611611178938, 2), (0.13836512595904674, 2), (0.2237838133269216, 1), (0.1629056628674436, 1), (0.1833733860192352, 1), (0.22795930184993643, 1), (0.7348300157272463, 1), (0.6474189243351436, 1), (0.3241518953853359, 1), (0.31378696968434394, 2), (0.7902796149645954, 1), (0.2511909955458849, 1), (0.16585309358053357, 1), (0.10258274535025025, 2), (0.059402574248790106, 3), (0.07976381510147447, 3), (0.19892924843811893, 1), (0.18992510599205573, 1), (0.22014348732653846, 1), (0.12292694212440583, 1), (0.213299202027001, 1), (0.1474993581742395, 1), (0.154383220337567, 1), (0.13851989378292406, 1), (0.14564770499547589, 1), (0.08383881211832889, 2), (0.25926695769761965, 1), (0.15839064978174636, 1), (0.06999747249328758, 2), (0.12835506699683025, 2), (0.8549115335248306, 1), (0.13625185902932963, 1), (0.0710268711628897, 2), (0.1439246096437462, 1), (0.6430034776657609, 1), (0.3031152374345616, 1), (0.572899206808053, 1), (0.15053221902198277, 1), (0.150171698418871, 1), (0.15790903521269517, 1), (0.1531558119461568, 2), (0.19655512652378482, 1), (0.14465881987206475, 1), (0.02580382433881898, 6), (0.0019085369185597445, 2), (0.1344209024055656, 2), (0.453055593478699, 1), (0.44654117388851433, 1), (0.49798352160363213, 1), (0.37644123054319306, 1), (0.1545531075270542, 2), (0.20103280650911437, 1), (0.22364397151096546, 1), (0.2489123283841831, 1), (0.12714355222993134, 2), (0.3983261983195794, 1), (0.06449619946260193, 4), (0.7059658504132703, 1), (0.1357243647240905, 2), (0.2432652365020705, 1), (0.12079013284699983, 2), (0.29191478567553925, 1), (0.13663557319710162, 2), (0.12370760524710836, 2), (0.18222302878038715, 1), (0.22482552708064915, 1), (0.49110518414155807, 1), (0.033611067192768074, 2), (0.24024797965321168, 1), (0.016876552660128885, 2), (0.3625329815693786, 1), (0.21047893797809059, 1), (0.0514058259745296, 4), (0.2511646975399776, 1), (0.16099871164301058, 1), (0.15490406218530167, 1), (0.09806343975468562, 2), (0.2229282971921388, 1), (0.1166254893193806, 2), (0.1863751240711543, 1), (0.16889549030873, 2), (0.08207580375977401, 3), (0.30974814454351834, 1), (0.040065360442245325, 5), (0.7152034029994802, 1), (0.0020121698027949914, 4), (0.1503395135416302, 1), (0.09026710803677912, 1), (0.08382815833163032, 1), (0.29531905967999883, 1), (0.06580583156369621, 6), (0.06760290061784202, 2), (0.20183759809748608, 1), (0.04962331904346168, 4), (0.20119150596070567, 2), (0.14080043975417278, 1), (0.09023630215485928, 4), (0.8099456785718869, 1), (0.16513272772074233, 1), (0.09805634249309547, 2), (0.03935581144080088, 3), (0.07346731684307523, 2), (0.23460102654411455, 1), (0.10029727391037188, 1), (0.6348710734325325, 1), (0.045385453589277584, 3), (0.17556616020988175, 1), (0.414602183099784, 2), (0.027685338061883877, 3), (0.16403592473222742, 2), (0.07991415143947592, 2), (0.15174403942427842, 2), (0.19122653520494723, 1), (0.33025301267893614, 1), (0.1312910789281143, 3), (0.08733022795719017, 2), (0.46636138645399017, 1), (0.7952351414606451, 1), (0.15868709256741836, 1), (0.06105395377563402, 3), (0.16988970846422158, 1), (0.07763741580733571, 2), (0.19974068623633992, 1), (0.17469389914784186, 1), (0.04534863824938763, 4), (0.11175176283484661, 2), (0.2099123544168528, 1), (0.125919913295358, 1), (0.17616189745117658, 1), (0.051220547224474444, 3), (0.04300220195726631, 3), (0.11508316210598962, 2), (0.7586305318160667, 1), (0.16522986951478458, 1), (0.8361368173853359, 1), (0.21696280026599396, 1), (0.17041014704195123, 2), (0.1893901853353686, 1), (0.14059796980167105, 1), (0.8006316197510105, 1), (0.21907434846278084, 1), (0.06871631526172448, 3), (0.11652413499229711, 2), (0.16163124892276065, 1), (0.1828404728356184, 1), (0.8520676583719153, 1), (0.8995700769038004, 1), (0.40665403827289315, 1), (0.8463217298384105, 1), (0.8876995416108465, 1), (0.7621540338971785, 1), (0.8772022831834985, 1), (0.1650037141165804, 1), (0.8264469016559901, 1), (0.43166941088921945, 2), (0.14831114546464966, 3), (0.8019295776051133, 1), (0.16059420986339135, 1), (0.826174996262672, 1), (0.16782653524240973, 1), (0.06882515856279242, 1), (0.06644534236394203, 1), (0.8003913145654165, 1), (0.8855944004547873, 1), (0.8987048633197202, 1), (0.8298219681288355, 1)]\n",
      "------------------ results for the updated neighborhoods ---------------\n",
      "Partitioning ,  Average Calib 0.2826829799495195, Weight Calib Average 0.2038387745792144\n",
      "------------------ Run New Model ---------------\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.70      0.88      0.78       147\n",
      "           1       0.37      0.15      0.22        65\n",
      "\n",
      "    accuracy                           0.66       212\n",
      "   macro avg       0.54      0.52      0.50       212\n",
      "weighted avg       0.60      0.66      0.61       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.660377358490566\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-7.08365426e-07 -2.74282817e-07 -1.55487593e-06  3.08331667e-06\n",
      "  -4.20177137e-06 -1.53598034e-03]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.279988245286233\n",
      " Test Calibration 1.1836134606953292\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.0797317519689273\n",
      " Test Calibration 0.05629657993017169\n",
      "\n",
      "res_newModel   [(0.09791940935819829, 4), (0.029285508108064812, 5), (0.43870298392551643, 1), (0.26929779583410673, 2), (0.54069502191105, 1), (0.4294966575560767, 1), (0.26801643388011653, 2), (0.45157350938430807, 1), (0.030281653770508843, 2), (0.13313561071005983, 4), (0.25937949398760995, 2), (0.08944093333381954, 6), (0.2638724848318008, 2), (0.17096554897107905, 3), (0.18243741980532416, 3), (0.4395296353112401, 1), (0.25643136890019136, 2), (0.17199755948337533, 3), (0.0852694460154214, 6), (0.012945521047371707, 38), (0.26043090249952244, 2), (0.496774052442824, 1), (0.25128654382338717, 2), (0.4972166496678676, 1), (0.516160110575584, 1), (0.5217262944747391, 1), (0.26198857174021906, 2), (0.25465506716729186, 2), (0.16745663522155094, 3), (0.25326348267490373, 2), (0.25312771050263244, 2), (0.06945793795080107, 4), (0.5338598728307594, 1), (0.5020915353109191, 1), (0.45512636823410024, 1), (0.48521010972571244, 1), (0.23560752572964583, 2), (0.04258206933319316, 3), (0.17069206795702282, 3), (0.545025135489524, 1), (0.5615573826397188, 1), (0.12325525937476932, 4), (0.16113775136917682, 3), (0.07059136064144474, 7), (0.1627154630316209, 3), (0.49815481258852656, 1), (0.5039065577205757, 1), (0.1623840170390772, 3), (0.48880782902232844, 1), (0.4898135126408534, 1), (0.24967230638891974, 2), (0.4919072563021758, 1), (0.47884914123300604, 1), (0.49568224690552565, 1), (0.4462556136567094, 1), (0.051669961931109426, 3), (0.2604015480182843, 2), (0.4774452161462974, 1), (0.05790497313660323, 3), (0.5180525481931868, 1), (0.48745752896407124, 1), (0.04964330676257086, 3), (0.4760471360622939, 1), (0.05595308746803543, 3), (0.47462169311521907, 1), (0.5314530176293597, 1), (0.48671516830910116, 1), (0.5344463045892617, 1), (0.24328673069610623, 2), (0.2408318197675039, 2), (0.23828663684081963, 2), (0.058240370503053716, 3), (0.5117379046362678, 1), (0.25120105644014495, 2), (0.23238141511967736, 2), (0.4575395974638455, 1), (0.5132803539818477, 1), (0.4678503988847784, 1), (0.2453838045576494, 2), (0.04373827903822133, 8), (0.01780091462695088, 2), (0.47306343164606546, 1), (0.08964043163163797, 5), (0.48413607120422275, 1), (0.2266458707797172, 2), (0.11211488953977997, 4), (0.11229139949879731, 4), (0.4747203661648519, 1), (0.08831663154911784, 5), (0.4593617172763744, 1), (0.01201339276650154, 4), (0.22459661764299121, 2), (0.22491222367088354, 2), (0.23027426830115022, 2), (0.5373747365155624, 1), (0.024630066573614773, 2), (0.5357009498180235, 1), (0.5530794614937378, 1), (0.568239656517219, 1), (0.5484001709645889, 1), (0.23187069210947603, 2), (0.25928323462535374, 2), (0.44620754002565416, 1), (0.22805035452162825, 2), (0.1556889600331664, 3), (0.5155671357850803, 1), (0.07911382720388534, 6), (0.04833537552786482, 6), (0.5343768927747511, 1), (0.44235473682492393, 1), (0.5370150206578865, 1), (0.45063785452202193, 1), (0.5425972784958099, 1), (0.5288362883608644, 1), (0.5630866986851487, 1), (0.029701108549691713, 2), (0.03094296574146227, 2), (0.4533734183102384, 1), (0.049410749464136275, 4), (0.043084477511550465, 4), (0.4444678854543762, 1), (0.22497037376547402, 2), (0.04578943122200527, 3), (0.5492470513520507, 1), (0.4364305587617401, 1), (0.03399004645855438, 6), (0.5314062094944125, 1), (0.5137875227695219, 1), (0.26536653461587945, 2), (0.03466106444726935, 3), (0.146239289450642, 3), (0.43187863056443765, 1), (0.5383966659865185, 1), (0.26573637744352474, 2), (0.5508938409725328, 1), (0.5816438875024976, 1), (0.21636075580121883, 2), (0.21827887290852566, 2), (0.2753136220154815, 2), (0.5726232096599935, 1), (0.5494475463760033, 1), (0.5102142680836012, 1), (0.4250520938540593, 1), (0.4105937014324675, 1), (0.21849991290456766, 2), (0.1047879752897864, 4), (0.20165576814459024, 2), (0.20894360197350265, 2), (0.41611428716756316, 1), (0.40496462978159126, 1), (0.20858404493359706, 2), (0.43062379327087846, 1), (0.20198483993139926, 2), (0.2106051797139604, 2), (0.4095475868728908, 1), (0.14722043229495588, 3), (0.42342600776210876, 1), (0.5469831145743062, 1), (0.4090388700451586, 1), (0.02787079047825569, 2), (0.21506567306564217, 2), (0.044521913264236934, 2), (0.4262718239328885, 1), (0.1979136604483187, 2), (0.39160529418519974, 1), (0.4105785850858752, 1), (0.2717604148029136, 2), (0.5927755957588633, 1), (0.5316333981619448, 1), (0.580378011110497, 1), (0.5265658734456926, 1), (0.2977581420642627, 2), (0.04290922582636736, 2), (0.41264386241452833, 1), (0.5753342723146094, 1), (0.05569660486693795, 2), (0.031086671988675124, 2), (0.5996827731784327, 1), (0.2968136435946984, 2), (0.421816385111696, 1), (0.3947830816112236, 1), (0.38048495486419265, 1), (0.3860436350892281, 1), (0.05005848085248188, 2), (0.1332331717012157, 3), (0.19019641359395348, 2), (0.3728246080280793, 1), (0.05543225319339992, 2), (0.12801404302173955, 3), (0.3889273645124164, 1), (0.38540323403673865, 1), (0.39807725390333265, 1), (0.3873249473209452, 1), (0.576875582547552, 1), (0.3977415511121046, 1), (0.056242272974447644, 2), (0.3845126405051102, 1), (0.38654591019042944, 1), (0.39980957155942065, 1), (0.5867289125613737, 1), (0.032234517628429105, 4), (0.08378066333814214, 3), (0.3712569231016281, 1), (0.19650180578965978, 3), (0.6062628646982172, 1), (0.38531663622260637, 1), (0.2904835148058007, 2), (0.5577303943602866, 1), (0.58115985772177, 1), (0.043266238011770874, 2), (0.12936539055670151, 3), (0.3758865147785468, 1), (0.5998561475580693, 1), (0.6136627085536182, 1), (0.39239229832021233, 1), (0.18787644391973976, 2), (0.38527737401377793, 1), (0.37244758982311643, 1), (0.37559255394219077, 1), (0.1881239280097994, 2), (0.5853737155893024, 1), (0.05522229420250663, 2), (0.1907698441294573, 2), (0.389846669225919, 1), (0.1292326279481948, 3), (0.12388444050104891, 3), (0.5993214080714461, 1), (0.19129509217230076, 2), (0.35643556690829425, 1), (0.19538182079321406, 2), (0.20042211671069238, 2), (0.18352867582597365, 2), (0.4110229088539532, 1), (0.1822599725237256, 2), (0.3750960331767521, 1), (0.18318403404517097, 2), (0.36328991634508845, 1), (0.18862745450283758, 2), (0.38584730414996105, 1), (0.18336256471749413, 2), (0.38790870121800197, 1), (0.17970416002425837, 2), (0.35344642417135846, 1), (0.35522765393444056, 1), (0.17795957628387718, 2), (0.35908681884696864, 1), (0.1828389182361621, 2), (0.012275977068215113, 3), (0.3532664116307356, 1), (0.3801949158552493, 1), (0.3648879431107225, 1), (0.18230682244021043, 2), (0.6067753444914494, 1), (0.17842742113940147, 2), (0.186394493418513, 2), (0.34903809335592123, 1), (0.19306728269291717, 2), (0.051347218007663675, 2), (0.19203113589263116, 2), (0.18592661114747583, 2), (0.11652438432006747, 3), (0.34160992355443587, 1), (0.0043978411774620646, 3), (0.17321216947961438, 2), (0.3359119458657252, 1), (0.16722763700962517, 2), (0.35884843302630953, 1), (0.6605078561778621, 1), (0.34356913087345564, 1), (0.6231879466881439, 1), (0.30916502686492364, 2), (0.06834155944605469, 2), (0.18326710126882273, 2), (0.30328465543962135, 2), (0.6313238277474772, 1), (0.11683330730761338, 3), (0.06169597273363647, 2), (0.34386512411720604, 1), (0.40301173875825796, 1), (0.34503996686048555, 1), (0.06000324925004377, 2), (0.5879768451567617, 1), (0.6626880549474277, 1), (0.6161204459884235, 1), (0.5585455516598795, 1), (0.6123122508544774, 1), (0.06248900505270866, 2), (0.6087859664950099, 1), (0.3537408985756502, 1), (0.10990243503747564, 3), (0.17495752822759353, 2), (0.11615214759936549, 3), (0.3606011750203457, 1), (0.1774518757129897, 2), (0.041380666582273115, 4), (0.06601361296011551, 5), (0.16276317510929134, 2), (0.005854743404381531, 8), (0.10895216520370637, 3), (0.36613049726492264, 1), (0.3268541422423385, 1), (0.30001786897115956, 2), (0.6578202477513422, 1), (0.08627521091937013, 2), (0.6741440806042869, 1), (0.6377306662780917, 1), (0.6637991044452245, 1), (0.3273886477012688, 1), (0.6707721010276368, 1), (0.6532020747965075, 1), (0.30003036985000825, 2), (0.32228991766900456, 2), (0.5877354755483267, 1), (0.6375692832637655, 1), (0.33438968450184914, 2), (0.3191991278652943, 2), (0.3185308086813982, 1), (0.32326445385610425, 1), (0.318700612206877, 1), (0.3223476381500497, 1), (0.3220879865300433, 1), (0.1660594171795356, 2), (0.1553499487925118, 2), (0.07464331842331576, 2), (0.0005978214714250147, 3), (0.32036897119859703, 1), (0.6189155581756924, 1), (0.6122698309386443, 1), (0.3468694779292481, 1), (0.32555133899238126, 2), (0.08046382654297018, 2), (0.07717625281263268, 2), (0.3305399559874046, 1), (0.315140126278465, 1), (0.6552698075472569, 1), (0.3262915951870427, 1), (0.6365056046432943, 1), (0.6588931589779696, 1), (0.31553422184072877, 2), (0.35330141803434384, 1), (0.15932566253844355, 2), (0.07957415038800908, 4), (0.043158945175270635, 4), (0.215816091514061, 3), (0.6669705634632987, 1), (0.22406141014428196, 3), (0.3218955688619092, 1), (0.6803808336457484, 1), (0.669727891537824, 1), (0.6861898906503425, 1), (0.09764611276217613, 2), (0.6932281971662417, 1), (0.15068649455496785, 2), (0.0959034987792825, 2), (0.3148597674964523, 1), (0.6800628495077716, 1), (0.689956480559589, 1), (0.6980544016240438, 1), (0.14875962569834478, 2), (0.15087277779097574, 2), (0.31490712862760695, 1), (0.7155064937694696, 1), (0.3318956047745231, 1), (0.3041551945236516, 1), (0.319362455303707, 1), (0.29710474812669607, 1), (0.1504778900688255, 2), (0.28220773879022165, 1), (0.3077859325803188, 1), (0.2876672340280913, 1), (0.2870437472926132, 1), (0.7179767452968531, 1), (0.28242332410229737, 1), (0.2889105557398259, 1), (0.6985116026222691, 1), (0.7100725304972475, 1), (0.7166511520307811, 1), (0.2866838312566351, 1), (0.2767885371910311, 1), (0.2748142502790619, 1), (0.7138345411830166, 1), (0.2873725461620641, 1), (0.3617835686803309, 2), (0.7193353072405946, 1), (0.7100156254695913, 1), (0.2927771951364196, 1), (0.2854967298837229, 1), (0.27017019768591033, 1), (0.2886050358424914, 1), (0.2939331275946386, 1), (0.2891977086201493, 1), (0.7226766582471458, 1), (0.36149516634415313, 2), (0.27519745697662057, 1), (0.1103842088149844, 2), (0.726437458754918, 1), (0.7025026188466703, 1), (0.2707576058750946, 1), (0.7190384734617208, 1), (0.22504146536057723, 3), (0.3464986594942825, 2), (0.6847407982552093, 1), (0.6622653214192966, 1), (0.32111607291221356, 1), (0.14618867068101382, 2), (0.3088769687648306, 1), (0.7116367123887354, 1), (0.2961020607694431, 1), (0.31254238508697585, 1), (0.09748554830183596, 3), (0.7119099908863677, 1), (0.3017228081341545, 1), (0.2780453505380872, 1), (0.2727717309651081, 1), (0.10339205811777363, 2), (0.14203330526037644, 2), (0.10903781607441174, 2), (0.2789529399748913, 1), (0.36102175208639653, 2), (0.29581241940193526, 1), (0.29461315802491583, 1), (0.28231987758383337, 1), (0.27392125931780326, 1), (0.2767955256219565, 1), (0.2666910020450809, 1), (0.1371111754520604, 2), (0.27115888742437, 1), (0.7443573808343331, 1), (0.28146238684472363, 1), (0.26419066687631226, 1), (0.26483673268220537, 1), (0.2660959870966741, 1), (0.2645812409954114, 1), (0.12132580492944872, 4), (0.2823944810650028, 1), (0.2783273691445267, 1), (0.254789220911614, 1), (0.26338823211300727, 1), (0.13719837147532116, 2), (0.1067273775605393, 2), (0.2644175700669365, 1), (0.25602100896306057, 1), (0.27302963008478615, 1), (0.25936722794056394, 1), (0.12749780246458842, 2), (0.7433610691646713, 1), (0.12499517112506764, 2), (0.11591810060898872, 2), (0.7529100662103982, 1), (0.2685516106359315, 1), (0.1124852933907371, 2), (0.1312432143758957, 2), (0.2622442105449388, 1), (0.2512571155066659, 1), (0.25608390675923814, 1), (0.26247520650431844, 1), (0.7379968921116447, 1), (0.7268237853659287, 1), (0.2698942597655298, 1), (0.3623689287599834, 2), (0.7517444344273886, 1), (0.2575476818161522, 1), (0.2495032510914641, 1), (0.12681101777506396, 2), (0.08351263394731355, 3), (0.08671966658012359, 3), (0.25409341225549714, 1), (0.25272950968795305, 1), (0.2569929373037773, 1), (0.23827355963559316, 1), (0.2544891218371164, 1), (0.24394471615881666, 1), (0.24499802525548775, 1), (0.24250956705767274, 1), (0.24301076517172027, 1), (0.12351442597906258, 2), (0.26178615519863957, 1), (0.24548097255029244, 1), (0.12074451795984727, 2), (0.13015548441247704, 2), (0.7577457050183906, 1), (0.23997931169459763, 1), (0.12074416851425909, 2), (0.2407093153754362, 1), (0.728865603781403, 1), (0.2645877418521366, 1), (0.721357736005295, 1), (0.2395611752319786, 1), (0.23940111491052007, 1), (0.24076850453271093, 1), (0.12650063999540456, 2), (0.24632940112148224, 1), (0.23675222937392684, 1), (0.01585382210632789, 6), (0.11590548459563899, 2), (0.3517902074178574, 2), (0.26222509953045003, 1), (0.26144434603661926, 1), (0.2669576895881136, 1), (0.25378143564122063, 1), (0.1232534839488085, 2), (0.2334067382899456, 1), (0.23566039294090677, 1), (0.2374877844529656, 1), (0.11903593477727967, 2), (0.2553779105918699, 1), (0.059107270130832903, 4), (0.7590544047988144, 1), (0.11860870734928411, 2), (0.23334022736542465, 1), (0.11625918899194462, 2), (0.23873497142450906, 1), (0.11848223409707287, 2), (0.13350396735648493, 2), (0.22455496483061374, 1), (0.23026212772697585, 1), (0.2625877220433279, 1), (0.12194300911454162, 2), (0.23463505980199476, 1), (0.11638449522548014, 2), (0.2473062912414586, 1), (0.22880612411863255, 1), (0.057318935363078326, 4), (0.2335825387512254, 1), (0.2231578953208342, 1), (0.22227071824879796, 1), (0.11441006600622255, 2), (0.22774699852570926, 1), (0.1143831708105349, 2), (0.22201731700313962, 1), (0.14104928418247106, 2), (0.07656905390493975, 3), (0.2361055466049098, 1), (0.04510663170025864, 5), (0.7671245940501346, 1), (0.0035289751298941685, 4), (0.2224824816296149, 1), (0.20972887841230348, 1), (0.20775725044247292, 1), (0.24351929990415822, 1), (0.07092016570361144, 6), (0.11032763885385963, 2), (0.23104205565189917, 1), (0.056696797844928384, 4), (0.1459139179683376, 2), (0.21746983263588449, 1), (0.0711380981957003, 4), (0.7758763551786656, 1), (0.21943941677183945, 1), (0.11250444748447164, 2), (0.03607895497451127, 3), (0.10879662879753683, 2), (0.22973607957271952, 1), (0.20668840067549743, 1), (0.7550081116262952, 1), (0.07129635460606364, 3), (0.22199042264250402, 1), (0.3907320926730721, 2), (0.03456931853901487, 3), (0.14106833411467262, 2), (0.10789774185557838, 2), (0.13944218816691628, 2), (0.21999406031981733, 1), (0.2370839947576085, 1), (0.14566522993910067, 3), (0.1082916564339751, 2), (0.7432621717480515, 1), (0.7798762788886462, 1), (0.21976863309058126, 1), (0.04280364457106816, 3), (0.20728616463413152, 1), (0.10169085839399045, 2), (0.21129741716743639, 1), (0.20745378378444257, 1), (0.05200215522357377, 4), (0.10632052584572077, 2), (0.21142531150150784, 1), (0.19833606359633918, 1), (0.20642579551042356, 1), (0.06698250848775382, 3), (0.06553592460030504, 3), (0.10528276478410203, 2), (0.7886390118871791, 1), (0.20148077746146387, 1), (0.7985627550015842, 1), (0.20930394351525855, 1), (0.1493630680951071, 2), (0.20418684111161323, 1), (0.1965908588701995, 1), (0.7944902087058021, 1), (0.20857730174414513, 1), (0.06855013288053304, 3), (0.10532364735162206, 2), (0.19978170968792747, 1), (0.2026795611058627, 1), (0.7995674095114713, 1), (0.8081276699967452, 1), (0.7501443881706649, 1), (0.7970874797002696, 1), (0.8049271608481526, 1), (0.7878080177133564, 1), (0.8048106629839266, 1), (0.20409908903920793, 1), (0.7956599633561698, 1), (0.40244217699744594, 2), (0.15282595070029295, 3), (0.7983000597486951, 1), (0.19695282793922048, 1), (0.8017521497263103, 1), (0.20081190541172736, 1), (0.1852400191735272, 1), (0.18847982744043137, 1), (0.7947155330234017, 1), (0.8100687010625415, 1), (0.8094712796240539, 1), (0.7989738401882016, 1)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 616\n",
      "Updated Model,  Average Calib 0.316839227691729, Weight Calib Average 0.23137616668590494\n",
      "Seconds between date 1 and date 2 is  102.612200 seconds\n",
      "--------------------End -----------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "#all you need to figure out is the indexes for the area\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def runModel(dfc, d_new_regions, X_train, X_test,y_train, y_test):\n",
    "    dfc = dfc[['rate_unemployment','percent_college','percent_married','median_income','average_act', 'percent_lunch','gridID']]\n",
    "    for region_id,idxs in d_new_regions.items():\n",
    "        for i in idxs:\n",
    "\n",
    "            dfc.loc[i,'gridID']=region_id\n",
    "\n",
    "\n",
    "    X_test.loc[:,'gridID'] = dfc.loc[X_test.index,'gridID']\n",
    "    X_train.loc[:,'gridID'] = dfc.loc[X_train.index,'gridID']\n",
    "\n",
    "    \n",
    "    logmodel = LogisticRegression()\n",
    "    logmodel.fit(X_train,y_train)\n",
    "    X_test_pred_labels = logmodel.predict(X_test)\n",
    "    X_test_pred = logmodel.predict_proba(X_test)\n",
    "    X_train_pred = logmodel.predict_proba(X_train)\n",
    "    X_test_pred = X_test_pred[:,1]\n",
    "    X_train_pred = X_train_pred[:,1]\n",
    "\n",
    "    from sklearn.metrics import classification_report\n",
    "    print(classification_report(y_test,X_test_pred_labels))\n",
    "\n",
    "    from sklearn.metrics import accuracy_score\n",
    "    print(\"--- Accuracy of the whole model ---\\n Accuracy {}\\n\".format(accuracy_score(y_test,X_test_pred_labels)))\n",
    "    \n",
    "    mem_accuracy.append(accuracy_score(y_test,X_test_pred_labels))\n",
    "    \n",
    "    print(\"--- Coefficient significance---\\n  {}\\n\".format(logmodel.coef_))\n",
    "    mem_coeff.append(logmodel.coef_)\n",
    "\n",
    "    print(\"--- Miscalibration for the whole model---\\n Training Calibration {}\\n Test Calibration {}\\n\".format(format(calMisCalibration(y_train.to_numpy(), X_train_pred)) ,calMisCalibration(y_test.to_numpy(), X_test_pred) ))\n",
    "    mem_train_calib.append(calMisCalibration(y_train.to_numpy(), X_train_pred))\n",
    "    mem_test_calib.append(calMisCalibration(y_test.to_numpy(), X_test_pred))\n",
    "    \n",
    "    print(\"--- Miscalibration difference for the whole model---\\n Training Calibration {}\\n Test Calibration {}\\n\".format(format(calMisCalibrationAbsDiff(y_train.to_numpy(), X_train_pred)) ,calMisCalibrationAbsDiff(y_test.to_numpy(), X_test_pred) ))\n",
    "    mem_train_calib_diff.append(calMisCalibrationAbsDiff(y_train.to_numpy(), X_train_pred))\n",
    "    mem_test_calib_diff.append(calMisCalibrationAbsDiff(y_test.to_numpy(), X_test_pred) )\n",
    "    return X_train_pred\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "Return calibration for a given neighborhoods\n",
    "\"\"\"\n",
    "\n",
    "def returnCalibForNeighborhoods(dfc, neighborhoods, X_train_pred, y_train):\n",
    "\n",
    "    \"\"\"\n",
    "    Now that we have new neighborhoods, we come up with the d_new_regions dictionary \n",
    "    \"\"\"\n",
    "    #print(neighborhoods)\n",
    "    #1/0\n",
    "    d_new_regions = defaultdict(list)\n",
    "    counter = 0\n",
    "    for x_start,x_end, y_start,y_end in neighborhoods:\n",
    "\n",
    "        df_mem = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "        #print(\" tricky part len(df ) {}\\n \".format(len(df_mem)))\n",
    "        if len(df_mem)>0:\n",
    "            \n",
    "\n",
    "            d_new_regions[counter] = df_mem.index.to_numpy()\n",
    "        #d_new_regions[counter] = df_mem.index.to_numpy()\n",
    "        counter+=1\n",
    "    d_new_regions.keys()\n",
    "\n",
    "    \"\"\"\n",
    "    Now lets calculate calibration for each neighboorhood\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    counter = 0\n",
    "    res_holder = {}\n",
    "    mem = []\n",
    "    for region_id,idxs in d_new_regions.items():\n",
    "        counter+=1\n",
    "        a_label = y_train.to_numpy()\n",
    "        a_preds = X_train_pred\n",
    "\n",
    "\n",
    "        mem_label = []\n",
    "        mem_preds = []\n",
    "        for i in idxs:\n",
    "            if i<len(a_preds):\n",
    "                mem_label.append(a_label[i])\n",
    "                mem_preds.append(a_preds[i])\n",
    "\n",
    "        ################################################ you changed here \n",
    "        \n",
    "        mem.append( (calMisCalibrationAbsDiff(mem_label, mem_preds)/len(mem_label), len(mem_label)) )\n",
    "\n",
    "        #print(\"--- region_ID --- {}\\n MisCalibration {}\".format(region_id, calMisCalibrationAbsDiff(mem_label, mem_preds)))\n",
    "        #if counter==10:\n",
    "        #    break\n",
    "    #res_holder[height] = mem\n",
    "\n",
    "    return mem, d_new_regions\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def KDTree(dfc, x_start,x_end, y_start,y_end, num_iters, counter, division_type, num_samples):\n",
    "    \n",
    "    \n",
    "    \"\"\"\n",
    "    Here you have your stop conditions\n",
    "    \n",
    "    division_type: \n",
    "        (I)   medians\n",
    "        (II)  Exhaustive Search\n",
    "        (III) Sampling\n",
    "        (IV)  Fairness Objective Functions\n",
    "    \"\"\"\n",
    "    #print(\"counter {}\".format(counter))\n",
    "    if counter>=num_iters:\n",
    "\n",
    "        temp = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "        #print(\"attached temp length {} neighborhood {}\".format(len(temp),[x_start,x_end, y_start,y_end]))\n",
    "\n",
    "        neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "        \n",
    "        return \n",
    "    \n",
    "\n",
    "    \n",
    "    counter+=1\n",
    "    #print(\"counter {}\".format(counter))\n",
    "    \n",
    "    \n",
    "    if counter%2==0: # division is on x-axis\n",
    "\n",
    "        if division_type==\"medians\":\n",
    "            \n",
    "            \n",
    "            tot = len(dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<x_end)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)])\n",
    "            tot = tot//2\n",
    "            for i in range(x_start,x_end+1):\n",
    "                mem_left = dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<i)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)] \n",
    "                if len(mem_left)>= tot:           \n",
    "                    division_indexes = [i]          \n",
    "                    break\n",
    "                           \n",
    "                           \n",
    "                           \n",
    "            \n",
    "        elif division_type==\"Exhaustive Search\":\n",
    "            \n",
    "            \n",
    "            mem = []\n",
    "            mem_indexes = []\n",
    "            #print(\"hi\")\n",
    "            for idx in range(x_start,x_end):  \n",
    "                mem_left = dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<idx)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end) ]\n",
    "                mem_left = mem_left.index.to_numpy()\n",
    "                left_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_left) \n",
    "            \n",
    "                mem_right = dfc[(dfc['gridX']>=idx)&(dfc['gridX']<x_end)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end) ]\n",
    "                mem_right = mem_right.index.to_numpy()       \n",
    "                right_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_right) \n",
    "                #mem.append(right_miscalib+left_miscalib )\n",
    "\n",
    "                #if len(mem_right)>20 and len(mem_left)>20:\n",
    "                if True:\n",
    "                    #mem.append(right_miscalib+left_miscalib)\n",
    "                    mem.append(np.abs(right_miscalib-left_miscalib))\n",
    "                    mem_indexes.append(idx)\n",
    "            \n",
    "            if len(mem_indexes)==0:\n",
    "                temp = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "                #print(\"attached temp length {} neighborhood {}\".format(len(temp),[x_start,x_end, y_start,y_end]))\n",
    "                neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "                return                     \n",
    "\n",
    "            \n",
    "            _min = min(mem)\n",
    "            mem2 = []\n",
    "            for index,i in enumerate(mem):\n",
    "                if i==_min:\n",
    "                    mem2.append(mem_indexes[index])\n",
    "                \n",
    "            f = mem2[len(mem2)//2]\n",
    "            #print(\" Division on X axis - x_start  {} , x_end {} , y_start {} , y_end {} Index {} \\n\".format(x_start,x_end, y_start,y_end,f))\n",
    "            division_indexes = [f]\n",
    "            \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "        \n",
    "        elif division_type==\"Sampling\":\n",
    "            pass\n",
    "        elif division_type==\"Fairness Objective Functions\":\n",
    "            pass \n",
    "        \n",
    "        \n",
    "        for idx in division_indexes:\n",
    "            \n",
    "            mem_left = dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<idx)]\n",
    "            mem_left = mem_left.index.to_numpy()\n",
    "            left_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_left) \n",
    "            \n",
    "            mem_right = dfc[(dfc['gridX']>=idx)&(dfc['gridX']<x_end)]\n",
    "            mem_right = mem_right.index.to_numpy()       \n",
    "            right_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_right) \n",
    "            \n",
    "            #print(\"left calibration {} right calibration {}\".format(left_miscalib, right_miscalib))\n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        This needs to be modified\n",
    "        \"\"\"\n",
    "        #Now you should select the best index and then \n",
    "        best_index = division_indexes[0]\n",
    "        KDTree(dfc, x_start,best_index, y_start,y_end, num_iters, counter, division_type, num_samples)\n",
    "        KDTree(dfc, best_index,x_end, y_start,y_end, num_iters, counter, division_type, num_samples)\n",
    "        \n",
    "########################################################################################### \n",
    "    else:    # division is on y-axis\n",
    "        #print(\"yess\")\n",
    "        if division_type==\"medians\":\n",
    "           # division_indexes = [y_start + (y_end - y_start)//2]\n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "            tot = len(dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<x_end)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)])\n",
    "            tot = tot//2\n",
    "            for i in range(y_start,y_end+1):\n",
    "                mem_left = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start)&(dfc['gridY']<i)]\n",
    "                if len(mem_left)>= tot:\n",
    "                    division_indexes = [i]          \n",
    "                    break\n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "                               \n",
    "        elif division_type==\"Exhaustive Search\":\n",
    "            \n",
    "            mem = []\n",
    "            mem_indexes =  []\n",
    "            for idx in range(y_start,y_end):  \n",
    "                mem_left = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start)&(dfc['gridY']<idx)]\n",
    "                mem_left = mem_left.index.to_numpy()\n",
    "                left_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_left) \n",
    "            \n",
    "                mem_right = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=idx)&(dfc['gridY']<y_end)]\n",
    "                mem_right = mem_right.index.to_numpy()       \n",
    "                right_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_right) \n",
    "                \n",
    "                #df_mem = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "                \n",
    "\n",
    "                #if len(mem_right)>20 and len(mem_left)>20:\n",
    "                if True:\n",
    "                    \n",
    "                    #mem.append(right_miscalib+left_miscalib)\n",
    "                    mem.append(np.abs(right_miscalib-left_miscalib))\n",
    "                    mem_indexes.append(idx)\n",
    "\n",
    "            #print(\"len(mem_indexes)\")        \n",
    "            if len(mem_indexes)==0:\n",
    "                temp = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "                #print(\"attached temp length {} neighborhood {}\".format(len(temp),[x_start,x_end, y_start,y_end]))\n",
    "                \n",
    "                neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "                return                     \n",
    "                    \n",
    "                \n",
    "            _min = min(mem)\n",
    "            mem2 = []\n",
    "            for index,i in enumerate(mem):\n",
    "                if i==_min:\n",
    "                    mem2.append(mem_indexes[index])\n",
    "                \n",
    "            f = mem2[len(mem2)//2]\n",
    "            #print(\" Division on Y axis - x_start  {} , x_end {} , y_start {} , y_end {} Index {} \\n\".format(x_start,x_end, y_start,y_end,f))\n",
    "            division_indexes = [f]\n",
    "            \n",
    "            \n",
    "        elif division_type==\"Sampling\":\n",
    "            pass\n",
    "        elif division_type==\"Fairness Objective Functions\":\n",
    "            pass \n",
    "        \n",
    "        \n",
    "        \n",
    "        for idx in division_indexes:\n",
    "            \n",
    "            mem_left = dfc[(dfc['gridY']>=y_start)&(dfc['gridY']<idx)]\n",
    "            mem_left = mem_left.index.to_numpy()\n",
    "            left_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_left) \n",
    "            \n",
    "            mem_right = dfc[(dfc['gridY']>=idx)&(dfc['gridY']<y_end)]\n",
    "            mem_right = mem_right.index.to_numpy()       \n",
    "            right_miscalib = calMisCalibrationForGivenIndexesInDF(y_train.to_numpy(), X_train_pred, mem_right) \n",
    "            \n",
    "            #print(\"left calibration {} right calibration {}\".format(left_miscalib, right_miscalib))\n",
    "        \n",
    "        \n",
    "        \"\"\"\n",
    "        This needs to be modified\n",
    "        \"\"\"\n",
    "        #Now you should select the best index and then \n",
    "        best_index = division_indexes[0]\n",
    "        KDTree(dfc, x_start,x_end, y_start,best_index, num_iters, counter, division_type, num_samples)\n",
    "        KDTree(dfc, x_start,x_end, best_index ,y_end, num_iters, counter, division_type, num_samples)\n",
    "        \n",
    "        \n",
    "        \n",
    "    \n",
    "  \n",
    "\n",
    "\"\"\"\n",
    "let us define our memories\n",
    "\"\"\"\n",
    "mem_accuracy, mem_train_calib, mem_test_calib, mem_train_calib_diff, mem_test_calib_diff = [],[],[],[],[]\n",
    "mem_num_part,mem_ave_calib,mem_ave_weighted_calib,mem_ave_weighted_calib_firstModel = [],[],[],[]\n",
    "mem_coeff = []\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import time\n",
    "\n",
    "time1 = time.time()\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "Let us start by defining \n",
    "\"\"\"\n",
    "\n",
    "res_holder = {}\n",
    "\n",
    "#for height in range(1,3):\n",
    "for height in [10]:\n",
    "#for height in range(1,11):    \n",
    "    print(\"The tree height is {}\".format(height))\n",
    "\n",
    "    \"\"\"\n",
    "    APPLY KDTREE\n",
    "\n",
    "    When we set num_iters=2k it means k division in x-axis and k division in y-axis\n",
    "    division_type: \n",
    "        (I)   medians\n",
    "        (II)  Exhaustive Search\n",
    "        (III) Sampling\n",
    "        (IV)  Fairness Objective Functions\n",
    "    \n",
    "    \n",
    "    The output of KDTree is Neighborhoods\n",
    "    \"\"\"\n",
    "    neighborhoods = []\n",
    "    KDTree(df_kdTree, \n",
    "           x_start = 0,\n",
    "           x_end = grid_size, \n",
    "           y_start = 0, \n",
    "           y_end = grid_size, \n",
    "           num_iters = height, \n",
    "           counter = 0, \n",
    "           #division_type = \"Exhaustive Search\", \n",
    "           division_type = \"medians\", \n",
    "           num_samples = 5)\n",
    "    \n",
    "    \n",
    "    print(\"------------------ KDtree is Done| Neighborhoods Generated ---------------\")\n",
    "\n",
    "    res_oldModel,d_new_regions = returnCalibForNeighborhoods(df_kdTree, neighborhoods, X_train_pred, y_train)\n",
    "\n",
    "    print(\"res_oldModel   {}\".format(res_oldModel))\n",
    "    #1/0\n",
    "    print(\"------------------ results for the updated neighborhoods ---------------\")\n",
    "\n",
    "    \n",
    "\n",
    "    _sum = 0\n",
    "    _sum_weighted = 0\n",
    "    counter=0\n",
    "    counter_weighted =0 \n",
    "    for value in res_oldModel:\n",
    "        #print(\"Height: {},  Values {} \\n \\n\".format(height,value))\n",
    "\n",
    "\n",
    "\n",
    "        #for i in range(len(value)):\n",
    "        if value[0]>0:\n",
    "            _sum+= value[0]\n",
    "            _sum_weighted += value[0]*value[1]\n",
    "\n",
    "            counter_weighted += value[1]\n",
    "            counter+=1\n",
    "\n",
    "    print(\"Partitioning ,  Average Calib {}, Weight Calib Average {}\".format(_sum/counter, _sum_weighted/counter_weighted))\n",
    "\n",
    "\n",
    "    mem_ave_weighted_calib_firstModel.append( _sum_weighted/counter_weighted)\n",
    "        \n",
    "        \n",
    "    #1/0\n",
    "        \n",
    "        \n",
    "        \n",
    "    print(\"------------------ Run New Model ---------------\")\n",
    "    \n",
    "    X_train_pred_updated = runModel(df_kdTree, d_new_regions, X_train, X_test, y_train, y_test)\n",
    "    res_newModel,d_new_regions = returnCalibForNeighborhoods(df_kdTree, neighborhoods, X_train_pred_updated, y_train)\n",
    "    print(\"res_newModel   {}\".format(res_newModel))\n",
    "    print(\"------------------ results for the updated neighborhoods NewModel---------------\")\n",
    " \n",
    "    print(\" Number of Partition {}\".format(len(res_newModel)))\n",
    "    mem_num_part.append(len(res_newModel))\n",
    "\n",
    "    _sum = 0\n",
    "    _sum_weighted = 0\n",
    "    counter=0\n",
    "    counter_weighted =0 \n",
    "\n",
    "    \n",
    "    \n",
    "    for value in res_newModel:\n",
    "        #print(\"Height: {},  Values {} \\n \\n\".format(height,value))\n",
    "\n",
    "\n",
    "\n",
    "        #for i in range(len(value)):\n",
    "        if value[0]>0:\n",
    "            _sum+= value[0]\n",
    "            _sum_weighted += value[0]*value[1]\n",
    "\n",
    "            counter_weighted += value[1]\n",
    "            counter+=1\n",
    "\n",
    "\n",
    "    print(\"Updated Model,  Average Calib {}, Weight Calib Average {}\".format(_sum/counter, (_sum_weighted/counter_weighted)))\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    mem_ave_calib.append(_sum/counter)\n",
    "    mem_ave_weighted_calib.append( _sum_weighted/counter_weighted)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    time2 = time.time()\n",
    "    # Seconds between Date 1 and date 2\n",
    "    seconds = time2 - time1\n",
    "    print(\"Seconds between date 1 and date 2 is % f seconds\" % seconds)\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "   \n",
    "    \n",
    "    \n",
    "    print(\"--------------------End -----------------------------------------------------\")\n",
    "    #print(neighborhoods)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "res_FairKDtree = [mem_accuracy, \n",
    "                  mem_train_calib, \n",
    "                  mem_test_calib, \n",
    "                  mem_train_calib_diff, \n",
    "                  mem_test_calib_diff,\n",
    "                  mem_num_part,\n",
    "                  mem_ave_calib,\n",
    "                  mem_ave_weighted_calib,\n",
    "                  mem_ave_weighted_calib_firstModel,\n",
    "                  mem_coeff]\n",
    "import pickle \n",
    "\n",
    "with open('res_FairKDtree_CA', 'wb') as f: \n",
    "#with open('res_FairKDtree_TX', 'wb') as f: \n",
    "    pickle.dump(res_FairKDtree, f) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Second Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate_unemployment</th>\n",
       "      <th>percent_college</th>\n",
       "      <th>percent_married</th>\n",
       "      <th>median_income</th>\n",
       "      <th>average_act</th>\n",
       "      <th>percent_lunch</th>\n",
       "      <th>gridID</th>\n",
       "      <th>gridY</th>\n",
       "      <th>gridX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.122475</td>\n",
       "      <td>0.605338</td>\n",
       "      <td>0.691299</td>\n",
       "      <td>80156.0</td>\n",
       "      <td>21.568987</td>\n",
       "      <td>0.208716</td>\n",
       "      <td>546306</td>\n",
       "      <td>533</td>\n",
       "      <td>514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.078767</td>\n",
       "      <td>0.764101</td>\n",
       "      <td>0.659389</td>\n",
       "      <td>70074.0</td>\n",
       "      <td>23.693529</td>\n",
       "      <td>0.094731</td>\n",
       "      <td>774499</td>\n",
       "      <td>756</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.080710</td>\n",
       "      <td>0.855984</td>\n",
       "      <td>0.953930</td>\n",
       "      <td>113229.0</td>\n",
       "      <td>22.350427</td>\n",
       "      <td>0.133333</td>\n",
       "      <td>774499</td>\n",
       "      <td>756</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.193734</td>\n",
       "      <td>0.489950</td>\n",
       "      <td>0.454356</td>\n",
       "      <td>28975.0</td>\n",
       "      <td>19.737485</td>\n",
       "      <td>0.525169</td>\n",
       "      <td>544303</td>\n",
       "      <td>531</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.212134</td>\n",
       "      <td>0.408170</td>\n",
       "      <td>0.658747</td>\n",
       "      <td>55332.0</td>\n",
       "      <td>18.663004</td>\n",
       "      <td>0.621418</td>\n",
       "      <td>544303</td>\n",
       "      <td>531</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   rate_unemployment  percent_college  percent_married  median_income  \\\n",
       "0           0.122475         0.605338         0.691299        80156.0   \n",
       "1           0.078767         0.764101         0.659389        70074.0   \n",
       "2           0.080710         0.855984         0.953930       113229.0   \n",
       "3           0.193734         0.489950         0.454356        28975.0   \n",
       "4           0.212134         0.408170         0.658747        55332.0   \n",
       "\n",
       "   average_act  percent_lunch  gridID  gridY  gridX  \n",
       "0    21.568987       0.208716  546306    533    514  \n",
       "1    23.693529       0.094731  774499    756    355  \n",
       "2    22.350427       0.133333  774499    756    355  \n",
       "3    19.737485       0.525169  544303    531    559  \n",
       "4    18.663004       0.621418  544303    531    559  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kdTree.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are in the maing body and set the height  10 \n",
      "START\n",
      " The height is 1 \n",
      " Neighbors are [[0, 1024, 0, 640], [0, 1024, 640, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      1.00      0.82       147\n",
      "           1       0.00      0.00      0.00        65\n",
      "\n",
      "    accuracy                           0.69       212\n",
      "   macro avg       0.35      0.50      0.41       212\n",
      "weighted avg       0.48      0.69      0.57       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.6933962264150944\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-5.30527908e-10 -2.64655672e-10 -1.20942511e-09 -6.95785145e-06\n",
      "  -3.12002923e-09 -2.96549925e-10]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.3930799107220089\n",
      " Test Calibration 1.270483389866294\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.11193666331818793\n",
      " Test Calibration 0.08293122802504298\n",
      "\n",
      "temp_neighborhoods  [[0, 1024, 0, 640], [0, 1024, 640, 1024]]\n",
      "res_newModel   [(0.00020593353153880835, 633), (0.0003080010637821362, 424)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 2\n",
      "Updated Model,  Average Calib 0.0002569672976604723, Weight Calib Average 0.00024687642053707797\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 2 \n",
      " Neighbors are [[0, 486, 0, 640], [486, 1024, 0, 640], [0, 305, 640, 1024], [305, 1024, 640, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      1.00      0.82       147\n",
      "           1       0.00      0.00      0.00        65\n",
      "\n",
      "    accuracy                           0.69       212\n",
      "   macro avg       0.35      0.50      0.41       212\n",
      "weighted avg       0.48      0.69      0.57       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.6933962264150944\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-5.32514398e-10 -2.65433580e-10 -1.21378693e-09 -6.95796769e-06\n",
      "  -3.13179212e-09 -3.07823103e-09]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.3930741454124405\n",
      " Test Calibration 1.2704776956169161\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.11193502154129101\n",
      " Test Calibration 0.082929482146696\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 640], [486, 1024, 0, 640], [0, 305, 640, 1024], [305, 1024, 640, 1024]]\n",
      "res_newModel   [(0.0019233673086030169, 88), (0.00030953023281574154, 545), (0.000543269561976606, 80), (0.00012642986482606445, 344)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 4\n",
      "Updated Model,  Average Calib 0.0007256492420553573, Weight Calib Average 0.00040199029186370786\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 3 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 486, 549, 640], [486, 1024, 0, 514], [486, 1024, 514, 640], [0, 305, 640, 758], [0, 305, 758, 1024], [305, 1024, 640, 691], [305, 1024, 691, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.97      0.85       147\n",
      "           1       0.80      0.31      0.44        65\n",
      "\n",
      "    accuracy                           0.76       212\n",
      "   macro avg       0.78      0.64      0.65       212\n",
      "weighted avg       0.77      0.76      0.73       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.7641509433962265\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-1.75091683e-02 -6.84808680e-03 -3.84390310e-02  6.59551060e-06\n",
      "  -1.03682815e-01 -2.32226528e-01]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.2558983866798734\n",
      " Test Calibration 1.1703346772629557\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.07287172600817589\n",
      " Test Calibration 0.052225254821189254\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 486, 549, 640], [486, 1024, 0, 514], [486, 1024, 514, 640], [0, 305, 640, 758], [0, 305, 758, 1024], [305, 1024, 640, 691], [305, 1024, 691, 1024]]\n",
      "res_newModel   [(0.00517474951929196, 22), (0.003994261801678042, 66), (0.00040925394916363077, 412), (0.0012702204958739746, 133), (0.0007848888706259907, 55), (0.0017644341980971546, 25), (0.0003633401724544521, 120), (0.00019388793754257985, 224)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 8\n",
      "Updated Model,  Average Calib 0.0017443796180909733, Weight Calib Average 0.000841371033947738\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 4 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 640], [467, 486, 549, 640], [486, 559, 0, 514], [559, 1024, 0, 514], [486, 531, 514, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 434, 640, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 1024, 691, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      1.00      0.82       147\n",
      "           1       0.00      0.00      0.00        65\n",
      "\n",
      "    accuracy                           0.69       212\n",
      "   macro avg       0.35      0.50      0.41       212\n",
      "weighted avg       0.48      0.69      0.57       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.6933962264150944\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-1.90005135e-10 -1.31474403e-10 -4.61854652e-10 -6.95810388e-06\n",
      "  -1.10357862e-09 -4.72496103e-09]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.3930673776618447\n",
      " Test Calibration 1.2704710116562659\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.11193309430105515\n",
      " Test Calibration 0.08292743281913813\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 640], [467, 486, 549, 640], [486, 559, 0, 514], [559, 1024, 0, 514], [486, 531, 514, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 434, 640, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 1024, 691, 1024]]\n",
      "res_newModel   [(0.0031701672215162864, 22), (0.007460772845007079, 42), (0.0148195116965391, 24), (0.0007414951806185848, 282), (0.0015931529819335633, 130), (0.001545669183265693, 92), (0.005040466233411847, 41), (0.004507247224826656, 34), (0.003922645778988894, 21), (0.00902129482828291, 25), (0.0009811772064320721, 97), (0.0031053680134321927, 23), (0.00517815672392871, 29), (0.0007706518956948164, 195)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 14\n",
      "Updated Model,  Average Calib 0.004418412643848458, Weight Calib Average 0.0023008820092657962\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 5 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 497], [486, 559, 497, 514], [559, 1024, 0, 459], [559, 1024, 459, 514], [486, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 1024, 691, 755], [308, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.97      0.84       147\n",
      "           1       0.76      0.20      0.32        65\n",
      "\n",
      "    accuracy                           0.74       212\n",
      "   macro avg       0.75      0.59      0.58       212\n",
      "weighted avg       0.74      0.74      0.68       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.7358490566037735\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-2.21339867e-03 -8.65685747e-04 -4.85922016e-03  3.64421863e-06\n",
      "  -1.31069407e-02 -8.00877280e-02]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.2871277585685121\n",
      " Test Calibration 1.190361346516253\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.08176485840030477\n",
      " Test Calibration 0.05836550718658702\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 497], [486, 559, 497, 514], [559, 1024, 0, 459], [559, 1024, 459, 514], [486, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 1024, 691, 755], [308, 1024, 755, 1024]]\n",
      "res_newModel   [(0.005174846245660113, 22), (0.0057157455699148535, 21), (0.017979761662922405, 21), (0.012064547355707346, 24), (0.004590414301384849, 45), (0.0010419723148248593, 237), (0.0023967339847871844, 66), (0.002391723562546422, 64), (0.001961233456664512, 71), (0.006623082753879979, 21), (0.005747359302126377, 41), (0.003408882831318226, 34), (0.0035515926051285997, 21), (0.001764484089539377, 25), (0.0007042344705386442, 76), (0.0027516645780128767, 21), (0.004494653947662318, 23), (0.00224390414184565, 29), (0.00026630399028565026, 150), (0.0009131468934776613, 45)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 20\n",
      "Updated Model,  Average Calib 0.004289314402911396, Weight Calib Average 0.002625485423289469\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 6 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 497], [486, 534, 497, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n",
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.69      1.00      0.82       147\n",
      "           1       0.00      0.00      0.00        65\n",
      "\n",
      "    accuracy                           0.69       212\n",
      "   macro avg       0.35      0.50      0.41       212\n",
      "weighted avg       0.48      0.69      0.57       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.6933962264150944\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-1.01090812e-10 -9.66975653e-11 -2.66654270e-10 -6.95796330e-06\n",
      "  -5.77061362e-10 -5.27798148e-09]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.3930743163205592\n",
      " Test Calibration 1.2704778655733637\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.1119350702104904\n",
      " Test Calibration 0.08292953425598419\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 497], [486, 534, 497, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]]\n",
      "res_newModel   [(0.002529568362183723, 22), (0.01138125607326887, 21), (0.021254274208145456, 21), (0.014792683777745006, 24), (0.0020697907916787405, 45), (0.0013683633687813558, 206), (0.009582254841837622, 31), (0.009571242946970808, 22), (0.004750643080016445, 44), (0.0035201151570223754, 41), (0.007865188936632895, 23), (0.0030767243810172432, 42), (0.005369576811527714, 29), (0.006925716130944604, 21), (0.004363492193000975, 41), (0.004630441255161735, 34), (0.0024481207988739325, 21), (0.007849858554855739, 25), (0.005857788638987846, 43), (0.0074145633806175365, 33), (0.007004656489833698, 21), (0.0023969676805642368, 23), (0.003798996714705383, 29), (0.0009969560897316633, 118), (0.0037077998748051696, 32), (0.00034294567214797805, 24), (0.017399281261368235, 21)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 27\n",
      "Updated Model,  Average Calib 0.006380343239719518, Weight Calib Average 0.004679432298011334\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 7 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 534, 497, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 359, 702, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.97      0.83       147\n",
      "           1       0.75      0.18      0.30        65\n",
      "\n",
      "    accuracy                           0.73       212\n",
      "   macro avg       0.74      0.58      0.57       212\n",
      "weighted avg       0.74      0.73      0.67       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.7311320754716981\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-8.88062210e-04 -3.47330912e-04 -1.94962159e-03  2.79757498e-06\n",
      "  -5.25878198e-03 -4.88060095e-02]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.2968816153907523\n",
      " Test Calibration 1.1914587495786264\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.08454244676690297\n",
      " Test Calibration 0.058701975106654335\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 534, 497, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 359, 702, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]]\n",
      "res_newModel   [(0.005174746355210626, 22), (0.0057158333947643115, 21), (0.01797983225833548, 21), (0.012064616638635124, 24), (0.020768989665573008, 23), (0.0034944545658293523, 22), (0.0014283039088086433, 173), (0.008211186722638257, 33), (0.007114752222233689, 31), (0.007089304103797081, 22), (0.015523575593467903, 23), (0.0027276133992861127, 21), (0.0030534007132146813, 41), (0.008816358722716675, 23), (0.003224673640621045, 42), (0.004992151617590044, 29), (0.006623193316997494, 21), (0.005747403260606568, 41), (0.0034088172131105507, 34), (0.003551716675404872, 21), (0.0017644333816944225, 25), (0.004723961322756012, 43), (0.01175613341516421, 33), (0.002751549973544634, 21), (0.004494731298223403, 23), (0.0022439624664244277, 29), (0.0016619614607939542, 24), (0.00038036140782030326, 94), (0.0016347656932459917, 32), (0.009073010613856897, 24), (0.007657280241471233, 21)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 31\n",
      "Updated Model,  Average Calib 0.006285583073026999, Weight Calib Average 0.0050196860649726885\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 8 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 512], [512, 534, 497, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 755], [347, 359, 702, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.73      0.97      0.83       147\n",
      "           1       0.75      0.18      0.30        65\n",
      "\n",
      "    accuracy                           0.73       212\n",
      "   macro avg       0.74      0.58      0.57       212\n",
      "weighted avg       0.74      0.73      0.67       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.7311320754716981\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-7.98907149e-04 -3.12461914e-04 -1.75389355e-03  3.02483438e-06\n",
      "  -4.73083691e-03 -4.71844691e-02]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 1.294267787226636\n",
      " Test Calibration 1.1901648271670382\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.08379811159434003\n",
      " Test Calibration 0.058305253612535295\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 512], [512, 534, 497, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 755], [347, 359, 702, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]]\n",
      "res_newModel   [(0.002029576704921344, 22), (0.011352523856545683, 21), (0.0217194396081904, 21), (0.015465551671414696, 24), (0.013622598698526634, 23), (0.007355447429773749, 22), (0.0037233263460448507, 86), (0.0032230434798540034, 87), (0.008869532886086266, 33), (0.009292484739518679, 31), (0.00952080607365586, 22), (0.016258438825028726, 23), (0.0004840023881434375, 21), (0.0031334769563355186, 41), (0.006831138781099477, 23), (0.0026257008750598925, 42), (0.00427233603552617, 29), (0.005705889162158674, 21), (0.004017639982395119, 41), (0.004917511255603899, 34), (0.002182967444393321, 21), (0.007475640730538053, 25), (0.0056977575299111925, 43), (0.007538370658714362, 33), (0.00676943882972539, 21), (0.001769848641222432, 23), (0.0031218240775825233, 29), (0.0032073095652327874, 24), (0.0022446685460390213, 50), (0.0023587321877624914, 44), (0.003519995941669618, 32), (0.0002193281643200885, 24), (0.017210236297350773, 21)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 33\n",
      "Updated Model,  Average Calib 0.006598078314252883, Weight Calib Average 0.005774961105626444\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 9 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 509], [486, 512, 509, 512], [512, 534, 497, 502], [512, 534, 502, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 727], [308, 347, 727, 755], [347, 359, 702, 741], [347, 359, 741, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.94      0.87       147\n",
      "           1       0.78      0.49      0.60        65\n",
      "\n",
      "    accuracy                           0.80       212\n",
      "   macro avg       0.79      0.72      0.74       212\n",
      "weighted avg       0.80      0.80      0.79       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.8018867924528302\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-4.28956338e-01  4.88027809e-02 -9.43194352e-01  3.70477020e-05\n",
      "  -2.98298990e+00  3.36741630e-02]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 0.9487124479072047\n",
      " Test Calibration 0.9771166809232972\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.014605064503246357\n",
      " Test Calibration 0.0070161119810645545\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 509], [486, 512, 509, 512], [512, 534, 497, 502], [512, 534, 502, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 727], [308, 347, 727, 755], [347, 359, 702, 741], [347, 359, 741, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]]\n",
      "res_newModel   [(0.0022096718582925914, 22), (0.011530338493856212, 21), (0.021886015641816758, 21), (0.015643647088081736, 24), (0.013320533581902086, 23), (0.007616342562622832, 22), (0.006236737575890657, 39), (0.00838532971322708, 47), (0.012765232221080669, 21), (0.00419094172377398, 66), (0.008685988408042928, 33), (0.00915402020451156, 31), (0.009342536274867187, 22), (0.01608152720364871, 23), (0.0006106803724825008, 21), (0.0030596392490420967, 41), (0.00668249345890985, 23), (0.0025717124813560057, 42), (0.004219886838872925, 29), (0.005665377622790498, 21), (0.0039844948219022935, 41), (0.004928390888498476, 34), (0.0022212673025118934, 21), (0.007521144147731656, 25), (0.005658064028703815, 43), (0.007540346238557669, 33), (0.006683574687752411, 21), (0.0017272778774978907, 23), (0.003077215946996345, 29), (0.003102620163041553, 24), (0.006172588424812206, 28), (0.0013059714551916593, 22), (0.0051535829535807225, 23), (0.004831036590659041, 21), (0.003707048122848961, 32), (3.4555516582668e-05, 24), (0.017467658131151826, 21)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 37\n",
      "Updated Model,  Average Calib 0.006891229456029458, Weight Calib Average 0.00650732614298617\n",
      "--------------------End -----------------------------------------------------\n",
      "START\n",
      " The height is 10 \n",
      " Neighbors are [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 509], [486, 512, 509, 512], [512, 534, 497, 502], [512, 515, 502, 512], [515, 534, 502, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 727], [308, 347, 727, 755], [347, 359, 702, 741], [347, 359, 741, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]] \n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:1765: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  isetter(loc, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.94      0.87       147\n",
      "           1       0.78      0.49      0.60        65\n",
      "\n",
      "    accuracy                           0.80       212\n",
      "   macro avg       0.79      0.72      0.74       212\n",
      "weighted avg       0.80      0.80      0.79       212\n",
      "\n",
      "--- Accuracy of the whole model ---\n",
      " Accuracy 0.8018867924528302\n",
      "\n",
      "--- Coefficient significance---\n",
      "  [[-4.29318849e-01  5.41457831e-02 -9.42919620e-01  3.69658664e-05\n",
      "  -2.99355610e+00  3.26374668e-02]]\n",
      "\n",
      "--- Miscalibration for the whole model---\n",
      " Training Calibration 0.948637983183533\n",
      " Test Calibration 0.9770767157872018\n",
      "\n",
      "--- Miscalibration difference for the whole model---\n",
      " Training Calibration 0.014626269689457495\n",
      " Test Calibration 0.00702836544260324\n",
      "\n",
      "temp_neighborhoods  [[0, 486, 0, 549], [0, 467, 549, 620], [0, 467, 620, 640], [467, 486, 549, 640], [486, 559, 0, 494], [486, 559, 494, 497], [486, 512, 497, 509], [486, 512, 509, 512], [512, 534, 497, 502], [512, 515, 502, 512], [515, 534, 502, 512], [486, 534, 512, 514], [534, 559, 497, 514], [559, 568, 0, 459], [568, 1024, 0, 446], [568, 1024, 446, 459], [559, 573, 459, 514], [573, 1024, 459, 514], [486, 508, 514, 524], [508, 531, 514, 524], [486, 531, 524, 640], [531, 1024, 514, 640], [0, 297, 640, 758], [297, 305, 640, 758], [0, 305, 758, 1024], [305, 338, 640, 685], [338, 434, 640, 685], [305, 434, 685, 691], [434, 1024, 640, 691], [305, 308, 691, 1024], [308, 359, 691, 702], [308, 347, 702, 727], [308, 347, 727, 755], [347, 359, 702, 741], [347, 359, 741, 755], [359, 1024, 691, 755], [308, 344, 755, 1024], [344, 1024, 755, 1024]]\n",
      "res_newModel   [(0.012479051620904914, 22), (0.006460018367810827, 21), (0.0016922871719527543, 21), (0.000271700860267677, 24), (0.014863022645570679, 23), (0.005554031743731135, 22), (0.0003272994243266775, 39), (0.00028929367195905793, 47), (0.0009639871503291494, 21), (0.001768243064659774, 22), (0.0009744514536947995, 44), (0.0010850677905218428, 33), (0.003577094304945027, 31), (0.002710517675348213, 22), (0.006855038234090277, 23), (0.0017537491107922638, 21), (9.531826442205695e-05, 41), (0.0030779743565489833, 23), (0.0014244798920350246, 42), (0.00011554255662833073, 29), (0.006178420619472748, 21), (0.00136631461921279, 41), (0.0035415950919563998, 34), (0.007409537245361498, 21), (0.01296706140304904, 25), (0.0008033266528898378, 43), (0.003942722618826152, 33), (0.0003094396119644245, 21), (0.0005601902957270997, 23), (0.003112351675368141, 29), (0.004900154723130298, 24), (0.004780285843396354, 28), (0.005666181232995882, 22), (0.0024136498965580517, 23), (0.0004063444529190988, 21), (0.0014562514013449837, 32), (0.00022615114316671618, 24), (0.009254957283021236, 21)]\n",
      "------------------ results for the updated neighborhoods NewModel---------------\n",
      " Number of Partition 38\n",
      "Updated Model,  Average Calib 0.003569292241339478, Weight Calib Average 0.0031505862490253836\n",
      "--------------------End -----------------------------------------------------\n",
      "Seconds between date 1 and date 2 is  189.712583 seconds\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "def Xsplit(dfc, x_start,x_end, y_start,y_end, f_X_train_pred, f_y_train):\n",
    "\n",
    "            \n",
    "    mem = []\n",
    "    mem_indexes = []\n",
    "    #print(\"hi\")\n",
    "    for idx in range(x_start,x_end):  \n",
    "        mem_left = dfc[(dfc['gridX']>=x_start)&(dfc['gridX']<idx)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end) ]\n",
    "        mem_left = mem_left.index.to_numpy()\n",
    "        left_miscalib = calMisCalibrationForGivenIndexesInDF(f_y_train.to_numpy(), f_X_train_pred, mem_left) \n",
    "\n",
    "        mem_right = dfc[(dfc['gridX']>=idx)&(dfc['gridX']<x_end)& (dfc['gridY']>=y_start) & (dfc['gridY']<y_end) ]\n",
    "        mem_right = mem_right.index.to_numpy()       \n",
    "        right_miscalib = calMisCalibrationForGivenIndexesInDF(f_y_train.to_numpy(), f_X_train_pred, mem_right) \n",
    "        #mem.append(right_miscalib+left_miscalib )\n",
    "\n",
    "        if len(mem_right)>20 and len(mem_left)>20:\n",
    "            #mem.append(right_miscalib+left_miscalib)\n",
    "            mem.append(np.abs(right_miscalib-left_miscalib))\n",
    "            mem_indexes.append(idx)\n",
    "\n",
    "    if len(mem_indexes)==0:\n",
    "        temp = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "        #print(\"attached temp length {} neighborhood {}\".format(len(temp),[x_start,x_end, y_start,y_end]))\n",
    "        neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "        return                     \n",
    "\n",
    "\n",
    "    _min = min(mem)\n",
    "    mem2 = []\n",
    "    for index,i in enumerate(mem):\n",
    "        if i==_min:\n",
    "            mem2.append(mem_indexes[index])\n",
    "\n",
    "    f = mem2[len(mem2)//2]\n",
    "    #print(\" Division on X axis - x_start  {} , x_end {} , y_start {} , y_end {} Index {} \\n\".format(x_start,x_end, y_start,y_end,f))\n",
    "    division_indexes = [f]\n",
    "\n",
    "    return division_indexes\n",
    "\n",
    "\n",
    "def Ysplit(dfc, x_start,x_end, y_start,y_end, f_X_train_pred, f_y_train):\n",
    "    \n",
    "    mem = []\n",
    "    mem_indexes =  []\n",
    "    for idx in range(y_start,y_end):  \n",
    "        mem_left = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start)&(dfc['gridY']<idx)]\n",
    "        mem_left = mem_left.index.to_numpy()\n",
    "        left_miscalib = calMisCalibrationForGivenIndexesInDF(f_y_train.to_numpy(), f_X_train_pred, mem_left) \n",
    "\n",
    "        mem_right = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=idx)&(dfc['gridY']<y_end)]\n",
    "        mem_right = mem_right.index.to_numpy()       \n",
    "        right_miscalib = calMisCalibrationForGivenIndexesInDF(f_y_train.to_numpy(), f_X_train_pred, mem_right) \n",
    "\n",
    "        #df_mem = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "\n",
    "\n",
    "        if len(mem_right)>20 and len(mem_left)>20:\n",
    "\n",
    "\n",
    "            #mem.append(right_miscalib+left_miscalib)\n",
    "            mem.append(np.abs(right_miscalib-left_miscalib))\n",
    "            mem_indexes.append(idx)\n",
    "\n",
    "    #print(\"len(mem_indexes)\")        \n",
    "    if len(mem_indexes)==0:\n",
    "        temp = dfc[ (dfc['gridX']>=x_start) & (dfc['gridX']<x_end) & (dfc['gridY']>=y_start) & (dfc['gridY']<y_end)   ]\n",
    "        #print(\"attached temp length {} neighborhood {}\".format(len(temp),[x_start,x_end, y_start,y_end]))\n",
    "\n",
    "        neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "        return                     \n",
    "\n",
    "\n",
    "    _min = min(mem)\n",
    "    mem2 = []\n",
    "    for index,i in enumerate(mem):\n",
    "        if i==_min:\n",
    "            mem2.append(mem_indexes[index])\n",
    "\n",
    "    f = mem2[len(mem2)//2]\n",
    "    #print(\" Division on Y axis - x_start  {} , x_end {} , y_start {} , y_end {} Index {} \\n\".format(x_start,x_end, y_start,y_end,f))\n",
    "    division_indexes = [f]\n",
    "\n",
    "    return division_indexes\n",
    "    \n",
    "    \n",
    "\n",
    "    \n",
    "def updatePredictions():\n",
    "    \n",
    "    \n",
    "    \n",
    "    pass\n",
    "\n",
    "def TopDown(dfc, temp_neighborhoods, h, initial_X_train_pred, X_train, X_test, y_train, y_test):\n",
    "    \"\"\"\n",
    "    This is the part that you need to update. basically in each step we are going to update the likelihoods.\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    \n",
    "    #res_oldModel,d_new_regions = returnCalibForNeighborhoods(dfc, temp_neighborhoods, X_train_pred, y_train)\n",
    "    #f_X_train_pred = runModel(df_kdTree, temp_neighborhoods, X_train, X_test, y_train, y_test)\n",
    "    #f_y_train       = y_train\n",
    "\n",
    "    #print(\"counter {}\".format(counter))\n",
    "    \n",
    "    \n",
    "        #X_train_pred_updated = runModel(df_kdTree, d_new_regions, X_train, X_test, y_train, y_test)\n",
    "        #res_newModel,d_new_regions = returnCalibForNeighborhoods(df_kdTree, neighborhoods, X_train_pred_updated, y_train)\n",
    "\n",
    "    \n",
    "    f_X_train_pred   = initial_X_train_pred\n",
    "    f_y_train      = y_train\n",
    "    acc_mem = []\n",
    "    \n",
    "    for counter in range(1,h):\n",
    "        print(\"START\")\n",
    "        #f_X_train_pred = runModel(dfc, temp_neighborhoods, X_train, X_test, y_train, y_test)\n",
    "        #f_y_train      = y_train\n",
    "        updated_temp_neighborhoods  = []\n",
    "        if counter%2==0: # division is on x-axis\n",
    "                \n",
    "                while temp_neighborhoods:\n",
    "                    x_start,x_end, y_start,y_end = temp_neighborhoods.pop(0)\n",
    "                    division_indexes = Xsplit(dfc, x_start,x_end, y_start,y_end, f_X_train_pred, f_y_train)\n",
    "                    \n",
    "                    \n",
    "                    if division_indexes:\n",
    "                        best_index = division_indexes[0]\n",
    "                        updated_temp_neighborhoods.append([x_start,best_index, y_start,y_end])\n",
    "                        updated_temp_neighborhoods.append([ best_index,x_end, y_start,y_end])\n",
    "                        \n",
    "                    else:\n",
    "                        updated_temp_neighborhoods.append([x_start,x_end, y_start,y_end])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        else:\n",
    "                while temp_neighborhoods:\n",
    "                    x_start,x_end, y_start,y_end = temp_neighborhoods.pop(0)\n",
    "                    division_indexes = Ysplit(dfc, x_start,x_end, y_start,y_end, f_X_train_pred, f_y_train)\n",
    "  \n",
    "\n",
    "                    if division_indexes:\n",
    "                        best_index = division_indexes[0]\n",
    "                        updated_temp_neighborhoods.append([ x_start, x_end, y_start, best_index])\n",
    "                        updated_temp_neighborhoods.append([ x_start, x_end, best_index, y_end])\n",
    "                    \n",
    "                    else:\n",
    "                        updated_temp_neighborhoods.append([ x_start,x_end, y_start,y_end])\n",
    "    \n",
    "        temp_neighborhoods = updated_temp_neighborhoods\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        #This is new#\n",
    "        \"\"\"\n",
    "        Here temp_neighborhoods is your new neighborhoods.\n",
    "        f_X_train_pred Comes from initial run for the first time \n",
    "        \n",
    "        \n",
    "        \n",
    "        Then, next a new model is ran based on d_new_regions.\n",
    "        \"\"\"\n",
    "        print(\" The height is {} \\n Neighbors are {} \\n\".format(counter, temp_neighborhoods))\n",
    "        res_oldModel,d_new_regions = returnCalibForNeighborhoods(dfc, temp_neighborhoods, f_X_train_pred, y_train)\n",
    "        f_X_train_pred = runModel(dfc, d_new_regions, X_train, X_test, y_train, y_test)\n",
    "        \n",
    "        #print(len(d_new_regions))\n",
    "        #print(res_oldModel)\n",
    "        #1/0\n",
    "                  \n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "        print(\"temp_neighborhoods  {}\".format(temp_neighborhoods))\n",
    "\n",
    "    \n",
    "\n",
    "        print(\"res_newModel   {}\".format(res_oldModel))\n",
    "        print(\"------------------ results for the updated neighborhoods NewModel---------------\")\n",
    "\n",
    "        print(\" Number of Partition {}\".format(len(res_oldModel)))\n",
    "        mem_num_part.append(len(res_newModel))\n",
    "        \n",
    "        \n",
    "        _sum = 0\n",
    "        _sum_weighted = 0\n",
    "        counter=0\n",
    "        counter_weighted =0 \n",
    "        for value in res_oldModel:\n",
    "            #print(\"Height: {},  Values {} \\n \\n\".format(height,value))\n",
    "\n",
    "\n",
    "\n",
    "            #for i in range(len(value)):\n",
    "            if value[0]>0:\n",
    "                _sum+= value[0]\n",
    "                _sum_weighted += value[0]*value[1]\n",
    "\n",
    "                counter_weighted += value[1]\n",
    "                counter+=1\n",
    "\n",
    "\n",
    "        print(\"Updated Model,  Average Calib {}, Weight Calib Average {}\".format(_sum/counter, _sum_weighted/counter_weighted))\n",
    "\n",
    "        \n",
    "        mem_ave_calib.append(_sum/counter)\n",
    "        mem_ave_weighted_calib.append( _sum_weighted/counter_weighted)\n",
    "        \n",
    "        print(\"--------------------End -----------------------------------------------------\")\n",
    "          \n",
    "            \n",
    "            \n",
    "            \n",
    "        #print(\" The height is {} \\n Neighbors are {} \\n\".format(counter, temp_neighborhoods))\n",
    "    return temp_neighborhoods,res_oldModel\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "mem_accuracy, mem_train_calib, mem_test_calib, mem_train_calib_diff, mem_test_calib_diff = [],[],[],[],[]\n",
    "mem_num_part,mem_ave_calib,mem_ave_weighted_calib,mem_ave_weighted_calib_firstModel = [],[],[],[]\n",
    "mem_coeff = []\n",
    "\n",
    "\n",
    "import time\n",
    "\n",
    "time1 = time.time()\n",
    "\n",
    "#just run it with this one because otherwise you are gonna record them multiple times.\n",
    "for i in [10]:\n",
    "    print(\"We are in the maing body and set the height  {} \".format(i))\n",
    "    neighborhoods = [[0,grid_size,0,grid_size]]\n",
    "\n",
    "\n",
    "    temp_neighborhoods,res_newModel = TopDown(dfc = df_kdTree, \n",
    "                                            temp_neighborhoods = neighborhoods, \n",
    "                                            h = i+1,\n",
    "                                            initial_X_train_pred = X_train_pred,\n",
    "                                            X_train = X_train, \n",
    "                                            X_test = X_test, \n",
    "                                            y_train = y_train, \n",
    "                                            y_test = y_test)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "time2 = time.time()\n",
    "\n",
    "# Seconds between Date 1 and date 2\n",
    "seconds = time2 - time1\n",
    "print(\"Seconds between date 1 and date 2 is % f seconds\" % seconds)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_IterativeKDtree = [mem_accuracy, \n",
    "                  mem_train_calib, \n",
    "                  mem_test_calib, \n",
    "                  mem_train_calib_diff, \n",
    "                  mem_test_calib_diff,\n",
    "                  mem_num_part,\n",
    "                  mem_ave_calib,\n",
    "                  mem_ave_weighted_calib,\n",
    "                  mem_ave_weighted_calib_firstModel,\n",
    "                  mem_coeff]\n",
    "import pickle \n",
    "with open('res_IterativeKDtree_CA', 'wb') as f: \n",
    "#with open('res_IterativeKDtree_TX', 'wb') as f: \n",
    "    pickle.dump(res_IterativeKDtree, f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([[-5.30527908e-10, -2.64655672e-10, -1.20942511e-09,\n",
       "         -6.95785145e-06, -3.12002923e-09, -2.96549925e-10]]),\n",
       " array([[-5.32514398e-10, -2.65433580e-10, -1.21378693e-09,\n",
       "         -6.95796769e-06, -3.13179212e-09, -3.07823103e-09]]),\n",
       " array([[-1.75091683e-02, -6.84808680e-03, -3.84390310e-02,\n",
       "          6.59551060e-06, -1.03682815e-01, -2.32226528e-01]]),\n",
       " array([[-1.90005135e-10, -1.31474403e-10, -4.61854652e-10,\n",
       "         -6.95810388e-06, -1.10357862e-09, -4.72496103e-09]]),\n",
       " array([[-2.21339867e-03, -8.65685747e-04, -4.85922016e-03,\n",
       "          3.64421863e-06, -1.31069407e-02, -8.00877280e-02]]),\n",
       " array([[-1.01090812e-10, -9.66975653e-11, -2.66654270e-10,\n",
       "         -6.95796330e-06, -5.77061362e-10, -5.27798148e-09]]),\n",
       " array([[-8.88062210e-04, -3.47330912e-04, -1.94962159e-03,\n",
       "          2.79757498e-06, -5.25878198e-03, -4.88060095e-02]]),\n",
       " array([[-7.98907149e-04, -3.12461914e-04, -1.75389355e-03,\n",
       "          3.02483438e-06, -4.73083691e-03, -4.71844691e-02]]),\n",
       " array([[-4.28956338e-01,  4.88027809e-02, -9.43194352e-01,\n",
       "          3.70477020e-05, -2.98298990e+00,  3.36741630e-02]]),\n",
       " array([[-4.29318849e-01,  5.41457831e-02, -9.42919620e-01,\n",
       "          3.69658664e-05, -2.99355610e+00,  3.26374668e-02]])]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mem_coeff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('res_FairKDtree', 'rb') as a: \n",
    "    res_FairKDtree = pickle.load(a) \n",
    "    \n",
    "with open('res_Medians', 'rb') as b: \n",
    "    res_Medians = pickle.load(b) \n",
    "\n",
    "    \n",
    "with open('res_IterativeKDtree', 'rb') as c: \n",
    "    res_IterativeKDtree = pickle.load(c) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}