cal_bayes_factor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading prior orders\n",
      "loading orders\n",
      "loading aisles info\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3421083, 7)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdf = '/Users/siero5335/Desktop/InstaCart/'\n",
    "print('loading prior orders')\n",
    "prior_orders = pd.read_csv(mdf + 'order_products__prior.csv', dtype={\n",
    "        'order_id': np.int32,\n",
    "        'product_id': np.int32,\n",
    "        'add_to_cart_order': np.int16,\n",
    "        'reordered': np.int8})\n",
    "print('loading orders')\n",
    "orders = pd.read_csv(mdf + 'orders.csv', dtype={\n",
    "        'order_id': np.int32,\n",
    "        'user_id': np.int32,\n",
    "        'eval_set': 'category',\n",
    "        'order_number': np.int16,\n",
    "        'order_dow': np.int8,\n",
    "        'order_hour_of_day': np.int8,\n",
    "        'days_since_prior_order': np.float32})\n",
    "print('loading aisles info')\n",
    "aisles = pd.read_csv(mdf + 'products.csv', engine='c',\n",
    "                           usecols = ['product_id','aisle_id'],\n",
    "                       dtype={'product_id': np.int32, 'aisle_id': np.int32})\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "\n",
    "prior_orders.shape\n",
    "orders.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>eval_set</th>\n",
       "      <th>order_number</th>\n",
       "      <th>order_dow</th>\n",
       "      <th>order_hour_of_day</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2539329</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>nan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2398795</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>15.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>473747</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>21.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2254736</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>7</td>\n",
       "      <td>29.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>431534</td>\n",
       "      <td>1</td>\n",
       "      <td>prior</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>15</td>\n",
       "      <td>28.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id  user_id eval_set  order_number  order_dow  order_hour_of_day  \\\n",
       "0   2539329        1    prior             1          2                  8   \n",
       "1   2398795        1    prior             2          3                  7   \n",
       "2    473747        1    prior             3          3                 12   \n",
       "3   2254736        1    prior             4          4                  7   \n",
       "4    431534        1    prior             5          4                 15   \n",
       "\n",
       "   days_since_prior_order  \n",
       "0                     nan  \n",
       "1                  15.000  \n",
       "2                  21.000  \n",
       "3                  29.000  \n",
       "4                  28.000  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# removing all user_ids not in the test set from both files to save memory\n",
    "# the test users present ample data to make models. (and saves space)\n",
    "test  = orders\n",
    "user_ids = test['user_id'].values\n",
    "order_ids = test['order_id'].values\n",
    "orders = orders[orders['user_id'].isin(user_ids)]\n",
    "\n",
    "#del test\n",
    "test.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Here is The Prior: our first guess of how probable it is that a product be reordered.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prior_p</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>product_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.731</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            prior_p\n",
       "product_id         \n",
       "1             0.613\n",
       "2             0.141\n",
       "3             0.731"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the Prior : p(reordered|product_id)\n",
    "prior = pd.DataFrame(prior_orders.groupby('product_id')['reordered'].agg([('number_of_orders',len),\n",
    "        ('sum_of_reorders','sum')]))\n",
    "prior['prior_p'] = (prior['sum_of_reorders']+1)/(prior['number_of_orders']+2)\n",
    "del prior['number_of_orders']\n",
    "del prior['sum_of_reorders']\n",
    "print('Here is The Prior: our first guess of how probable it is that a product be reordered.')\n",
    "#prior.to_csv('prior_out.csv')\n",
    "\n",
    "prior.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "combined data in DataFrame comb\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>add_to_cart_order</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>prior_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>86.000</td>\n",
       "      <td>0.708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985.000</td>\n",
       "      <td>2.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>83.000</td>\n",
       "      <td>0.694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327.000</td>\n",
       "      <td>3.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>104.000</td>\n",
       "      <td>0.158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918.000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>19.000</td>\n",
       "      <td>0.273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035.000</td>\n",
       "      <td>5.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>17.000</td>\n",
       "      <td>0.508</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  add_to_cart_order  reordered  user_id  order_number  \\\n",
       "0         2  33120.000              1.000      1.000   202279             3   \n",
       "1         2  28985.000              2.000      1.000   202279             3   \n",
       "2         2   9327.000              3.000      0.000   202279             3   \n",
       "3         2  45918.000              4.000      1.000   202279             3   \n",
       "4         2  30035.000              5.000      0.000   202279             3   \n",
       "\n",
       "   days_since_prior_order  aisle_id  prior_p  \n",
       "0                   8.000    86.000    0.708  \n",
       "1                   8.000    83.000    0.694  \n",
       "2                   8.000   104.000    0.158  \n",
       "3                   8.000    19.000    0.273  \n",
       "4                   8.000    17.000    0.508  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# merge everything into one dataframe and save any memory space\n",
    "\n",
    "comb = pd.DataFrame()\n",
    "comb = pd.merge(prior_orders, orders, on='order_id', how='right')\n",
    "# slim down comb - \n",
    "del comb['eval_set']\n",
    "del comb['order_dow']\n",
    "del comb['order_hour_of_day']\n",
    "del prior_orders\n",
    "del orders\n",
    "\n",
    "comb = pd.merge(comb, aisles, on ='product_id', how = 'left')\n",
    "del aisles\n",
    "\n",
    "prior.reset_index(inplace = True)\n",
    "comb = pd.merge(comb, prior, on ='product_id', how = 'left')\n",
    "del prior\n",
    "print('combined data in DataFrame comb')\n",
    "comb.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "comb \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>prior_p</th>\n",
       "      <th>reorder_c</th>\n",
       "      <th>reorder_b</th>\n",
       "      <th>atco1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>86.000</td>\n",
       "      <td>0.708</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>83.000</td>\n",
       "      <td>0.694</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  reordered  user_id  order_number  \\\n",
       "0         2  33120.000      1.000   202279             3   \n",
       "1         2  28985.000      1.000   202279             3   \n",
       "\n",
       "   days_since_prior_order  aisle_id  prior_p  reorder_c reorder_b atco1  \n",
       "0                   8.000    86.000    0.708      6.000       <=6   <=2  \n",
       "1                   8.000    83.000    0.694      6.000       <=6   <=2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Build the factors needed for a model of probability of reordered. This model forms our\n",
    "# hypothesis H and allows the calculation of each Bayes Factor: BF = p(D|H)/(1-p(D|H))\n",
    "# where D is the test user product buying history. See DAG of model above.\n",
    "\n",
    "# discretize reorder count into categories, 9 buckets, being sure to include 0 as bucket\n",
    "# These bins maximize mutual information with ['reordered']. Done outside python\n",
    "recount = pd.DataFrame()\n",
    "recount['reorder_c'] = comb.groupby(comb.order_id)['reordered'].sum().fillna(0)\n",
    "bins = [-0.1, 0, 2,4,6,8,11,14,19,71]\n",
    "cat =  ['None','<=2','<=4','<=6','<=8','<=11','<=14','<=19','>19']\n",
    "recount['reorder_b'] = pd.cut(recount['reorder_c'], bins, labels = cat)\n",
    "recount.reset_index(inplace = True)\n",
    "comb = pd.merge(comb, recount, how = 'left', on = 'order_id')\n",
    "del recount\n",
    "\n",
    "# discretize 'add_to_cart_order' (atco) into categories, 8 buckets\n",
    "# These bins maximize mutual information with ['recount']. Done outside python\n",
    "bins = [0,2,3,5,7,9,12,17,80]\n",
    "cat = ['<=2','<=3','<=5','<=7','<=9','<=12','<=17','>17']\n",
    "comb['atco1'] = pd.cut(comb['add_to_cart_order'], bins, labels = cat)\n",
    "del comb['add_to_cart_order']\n",
    "print('comb ')\n",
    "comb.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>prior_p</th>\n",
       "      <th>reorder_c</th>\n",
       "      <th>reorder_b</th>\n",
       "      <th>atco1</th>\n",
       "      <th>atco_fac_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>86.000</td>\n",
       "      <td>0.708</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "      <td>0.222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>83.000</td>\n",
       "      <td>0.694</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "      <td>0.222</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  reordered  user_id  order_number  \\\n",
       "0         2  33120.000      1.000   202279             3   \n",
       "1         2  28985.000      1.000   202279             3   \n",
       "\n",
       "   days_since_prior_order  aisle_id  prior_p  reorder_c reorder_b atco1  \\\n",
       "0                   8.000    86.000    0.708      6.000       <=6   <=2   \n",
       "1                   8.000    83.000    0.694      6.000       <=6   <=2   \n",
       "\n",
       "   atco_fac_p  \n",
       "0       0.222  \n",
       "1       0.222  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# these are the children Nodes of reordered:atco, aisle, recount. Build occurrence tables\n",
    "# first, then calculate probabilities. Then merge to add atco into comb.\n",
    "# \n",
    "atco_fac = pd.DataFrame()\n",
    "atco_fac = comb.groupby(['reordered', 'atco1'])['atco1'].agg(np.count_nonzero).unstack('atco1')\n",
    "tot = pd.DataFrame()\n",
    "tot = np.sum(atco_fac,axis=1)\n",
    "atco_fac = atco_fac.iloc[:,:].div(tot, axis=0)\n",
    "atco_fac = atco_fac.stack('atco1')\n",
    "atco_fac = pd.DataFrame(atco_fac)\n",
    "atco_fac.reset_index(inplace = True)\n",
    "atco_fac.rename(columns = {0:'atco_fac_p'}, inplace = True)\n",
    "comb = pd.merge(comb, atco_fac, how='left', on=('reordered', 'atco1'))\n",
    "\n",
    "comb.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>aisle_id</th>\n",
       "      <th>prior_p</th>\n",
       "      <th>reorder_c</th>\n",
       "      <th>reorder_b</th>\n",
       "      <th>atco1</th>\n",
       "      <th>atco_fac_p</th>\n",
       "      <th>aisle_fac_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>86.000</td>\n",
       "      <td>0.708</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>83.000</td>\n",
       "      <td>0.694</td>\n",
       "      <td>6.000</td>\n",
       "      <td>&lt;=6</td>\n",
       "      <td>&lt;=2</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.076</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  reordered  user_id  order_number  \\\n",
       "0         2  33120.000      1.000   202279             3   \n",
       "1         2  28985.000      1.000   202279             3   \n",
       "\n",
       "   days_since_prior_order  aisle_id  prior_p  reorder_c reorder_b atco1  \\\n",
       "0                   8.000    86.000    0.708      6.000       <=6   <=2   \n",
       "1                   8.000    83.000    0.694      6.000       <=6   <=2   \n",
       "\n",
       "   atco_fac_p  aisle_fac_p  \n",
       "0       0.222        0.021  \n",
       "1       0.222        0.076  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# calculate other two factors' probability tables, then probability\n",
    "# and merge into comb\n",
    "\n",
    "aisle_fac = pd.DataFrame()\n",
    "aisle_fac = comb.groupby(['reordered', 'atco1', 'aisle_id'])['aisle_id']\\\n",
    "                .agg(np.count_nonzero).unstack('aisle_id')\n",
    "tot = np.sum(aisle_fac,axis=1)\n",
    "aisle_fac = aisle_fac.iloc[:,:].div(tot, axis=0)\n",
    "aisle_fac = aisle_fac.stack('aisle_id')\n",
    "aisle_fac = pd.DataFrame(aisle_fac)\n",
    "aisle_fac.reset_index(inplace = True)\n",
    "aisle_fac.rename(columns = {0:'aisle_fac_p'}, inplace = True)\n",
    "comb = pd.merge(comb, aisle_fac, how = 'left', on = ('aisle_id','reordered','atco1'))\n",
    "\n",
    "comb.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reorder_b</th>\n",
       "      <th>reordered</th>\n",
       "      <th>atco1</th>\n",
       "      <th>recount_fac_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>&lt;=11</td>\n",
       "      <td>0.000</td>\n",
       "      <td>&lt;=12</td>\n",
       "      <td>0.180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>&lt;=11</td>\n",
       "      <td>0.000</td>\n",
       "      <td>&lt;=17</td>\n",
       "      <td>0.200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>&lt;=11</td>\n",
       "      <td>0.000</td>\n",
       "      <td>&lt;=2</td>\n",
       "      <td>0.044</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  reorder_b  reordered atco1  recount_fac_p\n",
       "0      <=11      0.000  <=12          0.180\n",
       "1      <=11      0.000  <=17          0.200\n",
       "2      <=11      0.000   <=2          0.044"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# last factor is reorder_count_factor   \n",
    "    \n",
    "recount_fac = pd.DataFrame()\n",
    "recount_fac = comb.groupby(['reordered', 'atco1', 'reorder_b'])['reorder_b']\\\n",
    "                    .agg(np.count_nonzero).unstack('reorder_b')\n",
    "\n",
    "tot = pd.DataFrame()\n",
    "tot = np.sum(recount_fac,axis=1)\n",
    "recount_fac = recount_fac.iloc[:,:].div(tot, axis=0)\n",
    "recount_fac.stack('reorder_b')\n",
    "recount_fac = pd.DataFrame(recount_fac.unstack('reordered').unstack('atco1')).reset_index()\n",
    "recount_fac.rename(columns = {0:'recount_fac_p'}, inplace = True)\n",
    "comb = pd.merge(comb, recount_fac, how = 'left', on = ('reorder_b', 'reordered', 'atco1'))\n",
    "\n",
    "recount_fac.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>prior_p</th>\n",
       "      <th>reorder_c</th>\n",
       "      <th>atco_fac_p</th>\n",
       "      <th>aisle_fac_p</th>\n",
       "      <th>recount_fac_p</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>33120.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.708</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.021</td>\n",
       "      <td>0.175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>28985.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.694</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.222</td>\n",
       "      <td>0.076</td>\n",
       "      <td>0.175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>9327.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.158</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.074</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>45918.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.273</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.168</td>\n",
       "      <td>0.004</td>\n",
       "      <td>0.217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30035.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.508</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.117</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  reordered  user_id  order_number  \\\n",
       "0         2  33120.000      1.000   202279             3   \n",
       "1         2  28985.000      1.000   202279             3   \n",
       "2         2   9327.000      0.000   202279             3   \n",
       "3         2  45918.000      1.000   202279             3   \n",
       "4         2  30035.000      0.000   202279             3   \n",
       "\n",
       "   days_since_prior_order  prior_p  reorder_c  atco_fac_p  aisle_fac_p  \\\n",
       "0                   8.000    0.708      6.000       0.222        0.021   \n",
       "1                   8.000    0.694      6.000       0.222        0.076   \n",
       "2                   8.000    0.158      6.000       0.074        0.013   \n",
       "3                   8.000    0.273      6.000       0.168        0.004   \n",
       "4                   8.000    0.508      6.000       0.143        0.017   \n",
       "\n",
       "   recount_fac_p  \n",
       "0          0.175  \n",
       "1          0.175  \n",
       "2          0.090  \n",
       "3          0.217  \n",
       "4          0.117  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# slim down combined DataFrame to save space and speed up processing\n",
    "del comb['atco1']\n",
    "del comb['reorder_b']\n",
    "del comb['aisle_id']\n",
    "#del comb['order_number']\n",
    "comb.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/siero5335/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if sys.path[0] == '':\n",
      "/Users/siero5335/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  del sys.path[0]\n",
      "/Users/siero5335/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/Users/siero5335/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>order_number</th>\n",
       "      <th>days_since_prior_order</th>\n",
       "      <th>prior_p</th>\n",
       "      <th>reorder_c</th>\n",
       "      <th>atco_fac_p</th>\n",
       "      <th>aisle_fac_p</th>\n",
       "      <th>recount_fac_p</th>\n",
       "      <th>p</th>\n",
       "      <th>beta</th>\n",
       "      <th>bf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>9327.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.158</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.074</td>\n",
       "      <td>0.013</td>\n",
       "      <td>0.090</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>30035.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.508</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.143</td>\n",
       "      <td>0.017</td>\n",
       "      <td>0.117</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>43668.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>202279</td>\n",
       "      <td>3</td>\n",
       "      <td>8.000</td>\n",
       "      <td>0.248</td>\n",
       "      <td>6.000</td>\n",
       "      <td>0.108</td>\n",
       "      <td>0.051</td>\n",
       "      <td>0.191</td>\n",
       "      <td>0.001</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>46842.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>178520</td>\n",
       "      <td>36</td>\n",
       "      <td>7.000</td>\n",
       "      <td>0.672</td>\n",
       "      <td>12.000</td>\n",
       "      <td>0.152</td>\n",
       "      <td>0.006</td>\n",
       "      <td>0.024</td>\n",
       "      <td>0.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>47209.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>156122</td>\n",
       "      <td>42</td>\n",
       "      <td>9.000</td>\n",
       "      <td>0.797</td>\n",
       "      <td>21.000</td>\n",
       "      <td>0.136</td>\n",
       "      <td>0.059</td>\n",
       "      <td>0.230</td>\n",
       "      <td>0.002</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.001</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   order_id product_id  reordered  user_id  order_number  \\\n",
       "0         2   9327.000      0.000   202279             3   \n",
       "1         2  30035.000      0.000   202279             3   \n",
       "2         2  43668.000      0.000   202279             3   \n",
       "3         4  46842.000      0.000   178520            36   \n",
       "4         5  47209.000      0.000   156122            42   \n",
       "\n",
       "   days_since_prior_order  prior_p  reorder_c  atco_fac_p  aisle_fac_p  \\\n",
       "0                   8.000    0.158      6.000       0.074        0.013   \n",
       "1                   8.000    0.508      6.000       0.143        0.017   \n",
       "2                   8.000    0.248      6.000       0.108        0.051   \n",
       "3                   7.000    0.672     12.000       0.152        0.006   \n",
       "4                   9.000    0.797     21.000       0.136        0.059   \n",
       "\n",
       "   recount_fac_p     p  beta    bf  \n",
       "0          0.090 0.000 1.000 0.000  \n",
       "1          0.117 0.000 1.000 0.000  \n",
       "2          0.191 0.001 1.000 0.000  \n",
       "3          0.024 0.000 1.000 0.000  \n",
       "4          0.230 0.002 1.000 0.001  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Use the factors in comb + the prior_p to update a posterior for each product purchased.\n",
    "# work in progress on beta\n",
    "# Use a test beta = 95% per month for memory retention function of users. Akin to Recency.\n",
    "\n",
    "comb['p'] = (comb['atco_fac_p'] * comb['aisle_fac_p'] * comb['recount_fac_p'])\n",
    "\n",
    "#split into two dataframes on reordered == 1 and == 0\n",
    "\n",
    "comb0 = pd.DataFrame()\n",
    "comb0 = comb[comb['reordered']==0]\n",
    "# now every product that was ordered has a posterior in usr.\n",
    "comb0['beta'] = 1\n",
    "comb0['bf'] = comb0['prior_p'] * comb0['p']/(1 - comb0['p'])\n",
    "# Small 'slight of hand' here. comb0.bf is really the first posterior and second prior.\n",
    "\n",
    "# Calculate beta and BF1 for the reordered products\n",
    "comb1 = pd.DataFrame()\n",
    "comb1 = comb[comb['reordered']==1]\n",
    "comb1['beta'] = 1 - .05*comb1['days_since_prior_order']/30\n",
    "comb1['bf'] = (1 - comb1['p'])/comb1['p']\n",
    "\n",
    "comb_last = pd.DataFrame()\n",
    "comb_last = pd.concat([comb0, comb1], axis=0).reset_index(drop=True)\n",
    "\n",
    "comb_last.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>bf</th>\n",
       "      <th>beta</th>\n",
       "      <th>recount_c</th>\n",
       "      <th>order_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9875840</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>12427.000000</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9875841</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>26088.000000</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9875842</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>26405.000000</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9875838</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>196.000000</td>\n",
       "      <td>0.000631</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9875839</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>14084.000000</td>\n",
       "      <td>0.000911</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         reordered  user_id   product_id       bf     beta  recount_c  \\\n",
       "9875840   0.000000        1 12427.000000 0.000082 1.000000   0.000000   \n",
       "9875841   0.000000        1 26088.000000 0.000098 1.000000   0.000000   \n",
       "9875842   0.000000        1 26405.000000 0.000149 1.000000   0.000000   \n",
       "9875838   0.000000        1   196.000000 0.000631 1.000000   0.000000   \n",
       "9875839   0.000000        1 14084.000000 0.000911 1.000000   0.000000   \n",
       "\n",
       "         order_number  \n",
       "9875840             1  \n",
       "9875841             1  \n",
       "9875842             1  \n",
       "9875838             1  \n",
       "9875839             1  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comb_l = pd.DataFrame()\n",
    "comb_l['reordered'] = comb_last['reordered']\n",
    "comb_l['user_id'] = comb_last['user_id']\n",
    "comb_l['product_id'] = comb_last['product_id']\n",
    "comb_l['bf'] = comb_last['bf'].astype(np.float64)\n",
    "comb_l['beta'] = comb_last['beta'].astype(np.float32)\n",
    "comb_l['recount_c']= comb_last['reorder_c']\n",
    "comb_l['order_number']= comb_last['order_number']\n",
    "del comb_last\n",
    "comb_l = comb_l.sort_values((['user_id', 'order_number', 'bf']))\n",
    "pd.set_option('display.float_format', lambda x: '%.6f' % x)\n",
    "comb_l.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/siero5335/anaconda/lib/python3.6/site-packages/pandas/core/frame.py:2844: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  **kwargs)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>reordered</th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>bf</th>\n",
       "      <th>beta</th>\n",
       "      <th>recount_c</th>\n",
       "      <th>order_number</th>\n",
       "      <th>first_o</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>12427.000000</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>26088.000000</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>26405.000000</td>\n",
       "      <td>0.000149</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>196.000000</td>\n",
       "      <td>0.000631</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>14084.000000</td>\n",
       "      <td>0.000911</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   reordered  user_id   product_id       bf     beta  recount_c  order_number  \\\n",
       "0   0.000000        1 12427.000000 0.000082 1.000000   0.000000             1   \n",
       "1   0.000000        1 26088.000000 0.000098 1.000000   0.000000             1   \n",
       "2   0.000000        1 26405.000000 0.000149 1.000000   0.000000             1   \n",
       "3   0.000000        1   196.000000 0.000631 1.000000   0.000000             1   \n",
       "4   0.000000        1 14084.000000 0.000911 1.000000   0.000000             1   \n",
       "\n",
       "   first_o  \n",
       "0        1  \n",
       "1        1  \n",
       "2        1  \n",
       "3        1  \n",
       "4        1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finally, perform update of every product \n",
    "\n",
    "usr = pd.DataFrame()\n",
    "# Find way to introduce beta to the update. Also, the update\n",
    "# must not occur until reordered==0 (aka: when first ordered)\n",
    "first_order = pd.DataFrame()\n",
    "first_order = comb_l[comb_l.reordered==0]\n",
    "first_order.rename(columns = {'order_number':'first_o'}, inplace = True)\n",
    "first_ords = pd.DataFrame()\n",
    "first_ords['user_id'] = first_order['user_id']\n",
    "first_ords['product_id'] = first_order['product_id']\n",
    "first_ords['first_o'] = first_order['first_o']\n",
    "us = pd.DataFrame()\n",
    "us = pd.merge(comb_l, first_ords, on = ('user_id', 'product_id'), how = 'left')\n",
    "\n",
    "\n",
    "\n",
    "us.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>bf</th>\n",
       "      <th>recount_c</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>196.0</td>\n",
       "      <td>10702590211448937080434982912.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>10258.0</td>\n",
       "      <td>791184262213878093107127386112.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>12427.0</td>\n",
       "      <td>44692992519339118205344214518071296.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>13032.0</td>\n",
       "      <td>26197639.8</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>13176.0</td>\n",
       "      <td>998.6</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>25133.0</td>\n",
       "      <td>3152157497306953809920.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>26088.0</td>\n",
       "      <td>8653.9</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>26405.0</td>\n",
       "      <td>4130.0</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>46149.0</td>\n",
       "      <td>1754767.6</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>49235.0</td>\n",
       "      <td>1677.4</td>\n",
       "      <td>4.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2</td>\n",
       "      <td>1559.0</td>\n",
       "      <td>549419323347403.2</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>2</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>7123783904007.2</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2</td>\n",
       "      <td>2573.0</td>\n",
       "      <td>12213.7</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2</td>\n",
       "      <td>7781.0</td>\n",
       "      <td>2144624.1</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2</td>\n",
       "      <td>9681.0</td>\n",
       "      <td>10241.7</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2</td>\n",
       "      <td>12000.0</td>\n",
       "      <td>131273610281569168.0</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>13351.0</td>\n",
       "      <td>634.5</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>16589.0</td>\n",
       "      <td>39077205292741.7</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>17758.0</td>\n",
       "      <td>714.3</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>17872.0</td>\n",
       "      <td>686772.1</td>\n",
       "      <td>8.1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    user_id  product_id                                    bf  recount_c\n",
       "0         1       196.0       10702590211448937080434982912.0        4.3\n",
       "1         1     10258.0      791184262213878093107127386112.0        4.3\n",
       "2         1     12427.0 44692992519339118205344214518071296.0        4.3\n",
       "3         1     13032.0                            26197639.8        4.3\n",
       "4         1     13176.0                                 998.6        4.3\n",
       "5         1     25133.0              3152157497306953809920.0        4.3\n",
       "6         1     26088.0                                8653.9        4.3\n",
       "7         1     26405.0                                4130.0        4.3\n",
       "8         1     46149.0                             1754767.6        4.3\n",
       "9         1     49235.0                                1677.4        4.3\n",
       "10        2      1559.0                     549419323347403.2        8.1\n",
       "11        2      2002.0                       7123783904007.2        8.1\n",
       "12        2      2573.0                               12213.7        8.1\n",
       "13        2      7781.0                             2144624.1        8.1\n",
       "14        2      9681.0                               10241.7        8.1\n",
       "15        2     12000.0                  131273610281569168.0        8.1\n",
       "16        2     13351.0                                 634.5        8.1\n",
       "17        2     16589.0                      39077205292741.7        8.1\n",
       "18        2     17758.0                                 714.3        8.1\n",
       "19        2     17872.0                              686772.1        8.1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "update = lambda x, y: x**y;\n",
    "\n",
    "usr = us[us.order_number > us.first_o].groupby(['user_id','product_id'])['bf']\\\n",
    "    .agg(np.prod).reset_index()\n",
    "\n",
    "temp = pd.DataFrame()\n",
    "temp = us.groupby(['user_id'])['recount_c'].agg(np.mean).reset_index()\n",
    "user = pd.merge(usr, temp, on = 'user_id', how = 'left')\n",
    "\n",
    "user.sort_values(((['bf', 'user_id'])), ascending=False)\n",
    "\n",
    "pd.set_option('display.float_format', lambda x: '%.1f' % x)\n",
    "user.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>order_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>recount_c</th>\n",
       "      <th>cumulative</th>\n",
       "      <th>user_id</th>\n",
       "      <th>bf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>89001438</th>\n",
       "      <td>1</td>\n",
       "      <td>22035.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>5</td>\n",
       "      <td>112108</td>\n",
       "      <td>932.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001433</th>\n",
       "      <td>1</td>\n",
       "      <td>14947.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>0</td>\n",
       "      <td>112108</td>\n",
       "      <td>547535.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001434</th>\n",
       "      <td>1</td>\n",
       "      <td>30881.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>1</td>\n",
       "      <td>112108</td>\n",
       "      <td>26229.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001435</th>\n",
       "      <td>1</td>\n",
       "      <td>43633.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>2</td>\n",
       "      <td>112108</td>\n",
       "      <td>26229.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001436</th>\n",
       "      <td>1</td>\n",
       "      <td>11109.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>3</td>\n",
       "      <td>112108</td>\n",
       "      <td>3559.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001437</th>\n",
       "      <td>1</td>\n",
       "      <td>5707.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>4</td>\n",
       "      <td>112108</td>\n",
       "      <td>2036.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001439</th>\n",
       "      <td>1</td>\n",
       "      <td>49302.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>6</td>\n",
       "      <td>112108</td>\n",
       "      <td>504.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89001440</th>\n",
       "      <td>1</td>\n",
       "      <td>44359.0</td>\n",
       "      <td>2.9</td>\n",
       "      <td>7</td>\n",
       "      <td>112108</td>\n",
       "      <td>261.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159163670</th>\n",
       "      <td>2</td>\n",
       "      <td>10960.0</td>\n",
       "      <td>5.7</td>\n",
       "      <td>6</td>\n",
       "      <td>202279</td>\n",
       "      <td>875903137.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159163671</th>\n",
       "      <td>2</td>\n",
       "      <td>30035.0</td>\n",
       "      <td>5.7</td>\n",
       "      <td>7</td>\n",
       "      <td>202279</td>\n",
       "      <td>51906796.9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           order_id  product_id  recount_c  cumulative  user_id          bf\n",
       "89001438          1     22035.0        2.9           5   112108       932.7\n",
       "89001433          1     14947.0        2.9           0   112108    547535.9\n",
       "89001434          1     30881.0        2.9           1   112108     26229.6\n",
       "89001435          1     43633.0        2.9           2   112108     26229.6\n",
       "89001436          1     11109.0        2.9           3   112108      3559.2\n",
       "89001437          1      5707.0        2.9           4   112108      2036.2\n",
       "89001439          1     49302.0        2.9           6   112108       504.5\n",
       "89001440          1     44359.0        2.9           7   112108       261.6\n",
       "159163670         2     10960.0        5.7           6   202279 875903137.2\n",
       "159163671         2     30035.0        5.7           7   202279  51906796.9"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Credit for this code belongs to Kareem Eissa. You are very generous and skilled!\n",
    "\n",
    "def f1(x):\n",
    "    return ' '.join([str(int(a)) for a in x])\n",
    "def f2(x):\n",
    "    return 'None'\n",
    "\n",
    "u = pd.DataFrame()\n",
    "uu = pd.DataFrame()\n",
    "uuu = pd.DataFrame()\n",
    "\n",
    "u = user.reset_index().sort_values(((['user_id','bf'])), ascending=False)\n",
    "u['cumulative'] = u.groupby('user_id').cumcount()\n",
    "u = pd.merge(test, u, on='user_id')[['order_id', 'product_id', 'recount_c', 'cumulative', 'user_id', 'bf']].sort_values('order_id')\n",
    "\n",
    "# these orders include all orders with expected reordered products\n",
    "uu = u[(u.recount_c >= u.cumulative + 1)].groupby('order_id').agg({'product_id': f1})\n",
    "# these orders have expected \"None\" reordered products or recount_c < 0.5\n",
    "uuu = u[(u.recount_c < 0.5)].groupby('order_id').agg({'product_id': f2})\n",
    "\n",
    "subm = pd.DataFrame()\n",
    "subm = pd.concat([uu, uuu], keys = 'order_id', axis=0).reset_index()\n",
    "subm.sort_values((('order_id')), ascending=False)\n",
    "\n",
    "subm[['order_id', 'product_id']].to_csv('bayesian.csv', index=False)\n",
    "u.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3315747, 3)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(162231165, 6)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "u.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "u.to_csv(\"bf.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}