GoodReadsDataClean.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "GoodReadsDataClean.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/Santosh-Gupta/Book2Vec/blob/master/GoodReadsDataClean.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "8BwtPbLjrSXJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#This code is used to read and clean the Goodreads data set. \n",
        "#This code requires the 'ratings.csv' and the 'books.csv' files which can be found here\n",
        "#https://github.com/zygmuntz/goodbooks-10k\n",
        "#The code is set up to download these files from your Google Drive. \n",
        "#This code will ask permission to access your Google Drive file. \n",
        "#\n",
        "#The easiest way to use this code is to go to Runtime, select 'Run all', follow the prompts to \n",
        "#allow access to your Google drive and insert the Gdrive file ID for the ratings file. \n",
        "\n",
        "import re\n",
        "from collections import Counter\n",
        "import time\n",
        "import numpy as np\n",
        "from __future__ import print_function\n",
        "import collections\n",
        "import math\n",
        "import numpy as np\n",
        "from numpy import genfromtxt\n",
        "import random\n",
        "import zipfile\n",
        "from six.moves import range\n",
        "from six.moves.urllib.request import urlretrieve\n",
        "from urllib.request import urlretrieve\n",
        "from os.path import isfile, isdir\n",
        "import pandas as pd\n",
        "\n",
        "!pip install -U -q PyDrive\n",
        "%matplotlib inline\n",
        "\n",
        "from google.colab import files\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "import os\n",
        "import sys\n",
        "\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "VJlLtRLezILP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        },
        "outputId": "591bf39e-4356-4955-b763-1544b70e09fd"
      },
      "cell_type": "code",
      "source": [
        "#Enter the Google Drive file ID for ratings. \n",
        "\n",
        "dl_id = input(\"Enter Gdrive file ID for ratings: \") \n",
        "\n",
        "# DOWNLOAD ZIP\n",
        "print (\"Downloading  file\")\n",
        "myzip = drive.CreateFile({'id': dl_id})\n",
        "myzip.GetContentFile('ratings.csv')\n",
        "print( os.listdir() )"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Enter Gdrive file ID for ratings: 1zA50dJB7ZYFPTvk4nHjvZX7jMxXio54w\n",
            "Downloading  file\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "mpDXm_Mz0eXG",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r = pd.read_csv( 'ratings.csv' )\n",
        "r.head()\n",
        "r.book_id.max() #how many books"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fHgesU0vzZh8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1882
        },
        "outputId": "b04dd50b-d86f-4c4c-eef6-09d4eb93db02"
      },
      "cell_type": "code",
      "source": [
        "r.sort_values('rating') #sort by rating. "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>book_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>3489851</th>\n",
              "      <td>35383</td>\n",
              "      <td>1257</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>137788</th>\n",
              "      <td>3766</td>\n",
              "      <td>122</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924756</th>\n",
              "      <td>14673</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924758</th>\n",
              "      <td>14673</td>\n",
              "      <td>65</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924760</th>\n",
              "      <td>14673</td>\n",
              "      <td>29</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3216008</th>\n",
              "      <td>8227</td>\n",
              "      <td>7066</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924766</th>\n",
              "      <td>14673</td>\n",
              "      <td>83</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3216031</th>\n",
              "      <td>7225</td>\n",
              "      <td>7582</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4393169</th>\n",
              "      <td>20080</td>\n",
              "      <td>1892</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>137795</th>\n",
              "      <td>3779</td>\n",
              "      <td>131</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5365129</th>\n",
              "      <td>11151</td>\n",
              "      <td>1085</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4922630</th>\n",
              "      <td>48127</td>\n",
              "      <td>1794</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3216051</th>\n",
              "      <td>10313</td>\n",
              "      <td>4981</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3216057</th>\n",
              "      <td>29154</td>\n",
              "      <td>822</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5715949</th>\n",
              "      <td>31011</td>\n",
              "      <td>32</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5883847</th>\n",
              "      <td>32235</td>\n",
              "      <td>992</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956011</th>\n",
              "      <td>26862</td>\n",
              "      <td>255</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5715964</th>\n",
              "      <td>33176</td>\n",
              "      <td>4526</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1955993</th>\n",
              "      <td>26862</td>\n",
              "      <td>4414</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3899599</th>\n",
              "      <td>44676</td>\n",
              "      <td>183</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5365071</th>\n",
              "      <td>24237</td>\n",
              "      <td>992</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3215729</th>\n",
              "      <td>34920</td>\n",
              "      <td>122</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956551</th>\n",
              "      <td>20374</td>\n",
              "      <td>5209</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956525</th>\n",
              "      <td>22109</td>\n",
              "      <td>543</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956519</th>\n",
              "      <td>14585</td>\n",
              "      <td>3724</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956492</th>\n",
              "      <td>14585</td>\n",
              "      <td>249</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956452</th>\n",
              "      <td>26799</td>\n",
              "      <td>1867</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1956401</th>\n",
              "      <td>15732</td>\n",
              "      <td>8630</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924741</th>\n",
              "      <td>14316</td>\n",
              "      <td>152</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>924656</th>\n",
              "      <td>2290</td>\n",
              "      <td>851</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051418</th>\n",
              "      <td>31602</td>\n",
              "      <td>88</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051419</th>\n",
              "      <td>31602</td>\n",
              "      <td>342</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051420</th>\n",
              "      <td>31602</td>\n",
              "      <td>541</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051422</th>\n",
              "      <td>31602</td>\n",
              "      <td>16</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051423</th>\n",
              "      <td>31602</td>\n",
              "      <td>5624</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051391</th>\n",
              "      <td>31602</td>\n",
              "      <td>2863</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051315</th>\n",
              "      <td>37808</td>\n",
              "      <td>1599</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051390</th>\n",
              "      <td>31602</td>\n",
              "      <td>4538</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051375</th>\n",
              "      <td>37054</td>\n",
              "      <td>8968</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051318</th>\n",
              "      <td>37808</td>\n",
              "      <td>1406</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051319</th>\n",
              "      <td>37808</td>\n",
              "      <td>3386</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051340</th>\n",
              "      <td>37805</td>\n",
              "      <td>3550</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051344</th>\n",
              "      <td>27797</td>\n",
              "      <td>368</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051347</th>\n",
              "      <td>11508</td>\n",
              "      <td>860</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051348</th>\n",
              "      <td>11508</td>\n",
              "      <td>2188</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051352</th>\n",
              "      <td>23175</td>\n",
              "      <td>359</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051354</th>\n",
              "      <td>37805</td>\n",
              "      <td>150</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051355</th>\n",
              "      <td>37805</td>\n",
              "      <td>9621</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051361</th>\n",
              "      <td>37805</td>\n",
              "      <td>95</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051362</th>\n",
              "      <td>37805</td>\n",
              "      <td>427</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051363</th>\n",
              "      <td>37805</td>\n",
              "      <td>3699</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051365</th>\n",
              "      <td>37805</td>\n",
              "      <td>7838</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051366</th>\n",
              "      <td>37805</td>\n",
              "      <td>55</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051367</th>\n",
              "      <td>37805</td>\n",
              "      <td>119</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051370</th>\n",
              "      <td>37805</td>\n",
              "      <td>186</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051371</th>\n",
              "      <td>37805</td>\n",
              "      <td>212</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051373</th>\n",
              "      <td>37805</td>\n",
              "      <td>1061</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051374</th>\n",
              "      <td>37805</td>\n",
              "      <td>83</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3051378</th>\n",
              "      <td>956</td>\n",
              "      <td>69</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2988239</th>\n",
              "      <td>37285</td>\n",
              "      <td>8871</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5976479 rows × 3 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         user_id  book_id  rating\n",
              "3489851    35383     1257       1\n",
              "137788      3766      122       1\n",
              "924756     14673       14       1\n",
              "924758     14673       65       1\n",
              "924760     14673       29       1\n",
              "3216008     8227     7066       1\n",
              "924766     14673       83       1\n",
              "3216031     7225     7582       1\n",
              "4393169    20080     1892       1\n",
              "137795      3779      131       1\n",
              "5365129    11151     1085       1\n",
              "4922630    48127     1794       1\n",
              "3216051    10313     4981       1\n",
              "3216057    29154      822       1\n",
              "5715949    31011       32       1\n",
              "5883847    32235      992       1\n",
              "1956011    26862      255       1\n",
              "5715964    33176     4526       1\n",
              "1955993    26862     4414       1\n",
              "3899599    44676      183       1\n",
              "5365071    24237      992       1\n",
              "3215729    34920      122       1\n",
              "1956551    20374     5209       1\n",
              "1956525    22109      543       1\n",
              "1956519    14585     3724       1\n",
              "1956492    14585      249       1\n",
              "1956452    26799     1867       1\n",
              "1956401    15732     8630       1\n",
              "924741     14316      152       1\n",
              "924656      2290      851       1\n",
              "...          ...      ...     ...\n",
              "3051418    31602       88       5\n",
              "3051419    31602      342       5\n",
              "3051420    31602      541       5\n",
              "3051422    31602       16       5\n",
              "3051423    31602     5624       5\n",
              "3051391    31602     2863       5\n",
              "3051315    37808     1599       5\n",
              "3051390    31602     4538       5\n",
              "3051375    37054     8968       5\n",
              "3051318    37808     1406       5\n",
              "3051319    37808     3386       5\n",
              "3051340    37805     3550       5\n",
              "3051344    27797      368       5\n",
              "3051347    11508      860       5\n",
              "3051348    11508     2188       5\n",
              "3051352    23175      359       5\n",
              "3051354    37805      150       5\n",
              "3051355    37805     9621       5\n",
              "3051361    37805       95       5\n",
              "3051362    37805      427       5\n",
              "3051363    37805     3699       5\n",
              "3051365    37805     7838       5\n",
              "3051366    37805       55       5\n",
              "3051367    37805      119       5\n",
              "3051370    37805      186       5\n",
              "3051371    37805      212       5\n",
              "3051373    37805     1061       5\n",
              "3051374    37805       83       5\n",
              "3051378      956       69       5\n",
              "2988239    37285     8871       5\n",
              "\n",
              "[5976479 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "metadata": {
        "id": "INqD25j91Acz",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r45 = r[r.rating >= 4]  #only include data points where the book has ratings 4 or higher \n",
        "r45.sort_values('rating') "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "hDtlvMp02fIB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "ceeb2493-4909-40d0-9782-92c5c70f2271"
      },
      "cell_type": "code",
      "source": [
        "reviews_per_user45 = r45.groupby( 'user_id' ).user_id.apply( lambda x: len( x ))\n",
        "reviews_per_user45.describe()  #desribed the number of times a user pops up, so the number of books a user has reviewed. \n",
        "#Count is number of users. Mean is average number of reviews per user. "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    53424.000000\n",
              "mean       111.868804\n",
              "std         26.071224\n",
              "min         19.000000\n",
              "25%         96.000000\n",
              "50%        111.000000\n",
              "75%        128.000000\n",
              "max        200.000000\n",
              "Name: user_id, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "metadata": {
        "id": "W6ndA3oX2r1A",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r45over5 = r45.groupby('user_id').filter(lambda x: len(x) >= 5) #only have users who appear more than x times. Only include users who have at leasts x ratings\n",
        "                                                                                                  #Important because if you want to consider for example 6 books per label in Cbow, you need to \n",
        "                                                                                                 #only include users who have at least 7 reviews"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "jr4dqKzg5vIU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "aac0d2e8-b413-4f94-d287-57dd2f1a16c0"
      },
      "cell_type": "code",
      "source": [
        "reviews_per_user45over5 = r45over5.groupby( 'user_id' ).user_id.apply( lambda x: len( x ))\n",
        "reviews_per_user45over5.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    53424.000000\n",
              "mean       111.868804\n",
              "std         26.071224\n",
              "min         19.000000\n",
              "25%         96.000000\n",
              "50%        111.000000\n",
              "75%        128.000000\n",
              "max        200.000000\n",
              "Name: user_id, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "metadata": {
        "id": "27bnIe6e6I4r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "153728d1-21de-40b2-d9d5-33a13873a0c8"
      },
      "cell_type": "code",
      "source": [
        "reviews_per_user45over5BookId = r45over5.groupby( 'book_id' ).user_id.apply( lambda x: len( x ))\n",
        "reviews_per_user45over5BookId.describe() #Stats on the books. Count is total number of books\n",
        "#mean is average number of ratings for book. "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    10000.000000\n",
              "mean       597.647900\n",
              "std       1267.289788\n",
              "min          8.000000\n",
              "25%        155.000000\n",
              "50%        248.000000\n",
              "75%        503.000000\n",
              "max      22806.000000\n",
              "Name: user_id, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "metadata": {
        "id": "GrFsVthC60wT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r45over3Over10B = r45over5.groupby('book_id').filter(lambda x: len(x) >= 15) #only have have that appear more than x times"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "FPYQHrdb8MZJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "44012e1c-f70f-4fa5-89f3-1b5546038dba"
      },
      "cell_type": "code",
      "source": [
        "reviews_per_user45over5BookIdOver10 = r45over3Over10B.groupby( 'book_id' ).user_id.apply( lambda x: len( x ))\n",
        "reviews_per_user45over5BookIdOver10.describe()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count     9998.000000\n",
              "mean       597.765553\n",
              "std       1267.389242\n",
              "min         24.000000\n",
              "25%        155.000000\n",
              "50%        248.000000\n",
              "75%        503.000000\n",
              "max      22806.000000\n",
              "Name: user_id, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "metadata": {
        "id": "qJHGsYQC_3KO",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r45over3Over10B.to_csv('refinedRatingsForCbowWindow3.csv') #To make it into a regular dataframe object again\n",
        "                                                                                                      #There's probably a better way to convert the data into a dataframe object again\n",
        "                                                                                                       "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "fiKgx6oxwLmL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1092
        },
        "outputId": "f5f64c5b-b02c-4730-fdd9-726db4dceb0d"
      },
      "cell_type": "code",
      "source": [
        "rr2 = pd.read_csv( 'refinedRatingsForCbowWindow3.csv' )\n",
        "\n",
        "rr3= rr2.groupby(['user_id'], as_index=False, sort=True)\n",
        "\n",
        "print(rr3.apply(lambda x: x.head(20)))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "               Unnamed: 0  user_id  book_id  rating\n",
            "0     0                 0        1      258       5\n",
            "      75               75        1      268       3\n",
            "      76               76        1     5556       3\n",
            "      77               77        1     3638       3\n",
            "      78               78        1     1796       5\n",
            "      79               79        1      867       3\n",
            "      80               80        1       47       3\n",
            "      81               81        1     2738       3\n",
            "      230483       230483        1     4691       4\n",
            "      230484       230484        1      238       2\n",
            "      230485       230485        1     2063       4\n",
            "      230486       230486        1      916       3\n",
            "      230487       230487        1     4614       1\n",
            "      230488       230488        1      111       3\n",
            "      230489       230489        1       11       5\n",
            "      230490       230490        1     1644       5\n",
            "      230491       230491        1     3889       3\n",
            "      230492       230492        1      136       5\n",
            "      230493       230493        1     6665       4\n",
            "      230494       230494        1      150       3\n",
            "      230495       230495        1       35       5\n",
            "      230496       230496        1       33       4\n",
            "      230498       230498        1       60       3\n",
            "      230499       230499        1      148       3\n",
            "      230500       230500        1       10       4\n",
            "      230501       230501        1       94       1\n",
            "      230502       230502        1        4       5\n",
            "      230503       230503        1      492       2\n",
            "      230504       230504        1       57       3\n",
            "      230505       230505        1     1521       5\n",
            "...                   ...      ...      ...     ...\n",
            "53423 5404363     5404380    53424       83       5\n",
            "      5404367     5404384    53424       12       4\n",
            "      5404368     5404385    53424       69       4\n",
            "      5404369     5404386    53424       91       4\n",
            "      5404370     5404387    53424      259       4\n",
            "      5404373     5404390    53424       31       5\n",
            "      5404374     5404391    53424      138       4\n",
            "      5404379     5404396    53424      597       4\n",
            "      5404380     5404397    53424      116       5\n",
            "      5404381     5404398    53424     1010       5\n",
            "      5404382     5404399    53424      184       4\n",
            "      5404383     5404400    53424     4483       5\n",
            "      5404384     5404401    53424      780       5\n",
            "      5404385     5404402    53424     1590       5\n",
            "      5404387     5404404    53424     2366       4\n",
            "      5404390     5404407    53424      341       4\n",
            "      5404391     5404408    53424     3077       4\n",
            "      5404392     5404409    53424     6455       4\n",
            "      5404393     5404410    53424     3344       4\n",
            "      5404394     5404411    53424     8609       4\n",
            "      5404618     5404635    53424      565       5\n",
            "      5404619     5404636    53424      450       4\n",
            "      5404620     5404637    53424      648       3\n",
            "      5404621     5404638    53424     1049       5\n",
            "      5404622     5404639    53424      294       4\n",
            "      5404623     5404640    53424      891       4\n",
            "      5404624     5404641    53424      229       4\n",
            "      5404625     5404642    53424      340       4\n",
            "      5404626     5404643    53424     2160       5\n",
            "      5404629     5404646    53424     3565       4\n",
            "\n",
            "[5976460 rows x 4 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RfAda-6d9SqQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#This code cell adds two columns of random numbers mapped from the user id. This is so the\n",
        "#order of users can be shuffled. Shuffling data can help prevent the model from overfitting. \n",
        "\n",
        "mydict={n : n for n in random.sample(range(1, rr2['user_id'].max()+1), rr2['user_id'].max())}\n",
        "\n",
        "randomColumn1 = dict()\n",
        "i=1\n",
        "for n in random.sample(range(1, rr2['user_id'].max()+1), rr2['user_id'].max()):\n",
        "  randomColumn1.update({i:n})\n",
        "  i=i+1\n",
        "\n",
        "  \n",
        "rr2['SortIndex1'] = rr2['user_id'].map(randomColumn1)\n",
        "\n",
        "rr2['SortIndex2'] = np.random.randint(1, 500, rr2.shape[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "IoKhn6KR6GLk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "b77da1b8-94ca-4f34-9bf6-7f5d2c3cf53c"
      },
      "cell_type": "code",
      "source": [
        "rr2.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user_id</th>\n",
              "      <th>book_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>SortIndex1</th>\n",
              "      <th>SortIndex2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>258</td>\n",
              "      <td>5</td>\n",
              "      <td>11157</td>\n",
              "      <td>99</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>4081</td>\n",
              "      <td>4</td>\n",
              "      <td>37155</td>\n",
              "      <td>112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>260</td>\n",
              "      <td>5</td>\n",
              "      <td>37155</td>\n",
              "      <td>349</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>9296</td>\n",
              "      <td>5</td>\n",
              "      <td>37155</td>\n",
              "      <td>196</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>2318</td>\n",
              "      <td>3</td>\n",
              "      <td>37155</td>\n",
              "      <td>13</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Unnamed: 0  user_id  book_id  rating  SortIndex1  SortIndex2\n",
              "0           0        1      258       5       11157          99\n",
              "1           1        2     4081       4       37155         112\n",
              "2           2        2      260       5       37155         349\n",
              "3           3        2     9296       5       37155         196\n",
              "4           4        2     2318       3       37155          13"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "metadata": {
        "id": "qiAAA3h5nedP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1527
        },
        "outputId": "af8fa1ef-f8ce-497f-d1f0-4e14985e0437"
      },
      "cell_type": "code",
      "source": [
        "r3 = rr2.sort_values(['SortIndex1', 'SortIndex2']) \n",
        "\n",
        "r3.head(50)\n",
        "\n",
        "#after this, get the header index "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user_id</th>\n",
              "      <th>book_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>SortIndex1</th>\n",
              "      <th>SortIndex2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6436</th>\n",
              "      <td>6436</td>\n",
              "      <td>322</td>\n",
              "      <td>1748</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232549</th>\n",
              "      <td>232549</td>\n",
              "      <td>322</td>\n",
              "      <td>1282</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7222</th>\n",
              "      <td>7222</td>\n",
              "      <td>322</td>\n",
              "      <td>123</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6360</th>\n",
              "      <td>6360</td>\n",
              "      <td>322</td>\n",
              "      <td>364</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6439</th>\n",
              "      <td>6439</td>\n",
              "      <td>322</td>\n",
              "      <td>5349</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6359</th>\n",
              "      <td>6359</td>\n",
              "      <td>322</td>\n",
              "      <td>358</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>40</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6362</th>\n",
              "      <td>6362</td>\n",
              "      <td>322</td>\n",
              "      <td>157</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>47</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231989</th>\n",
              "      <td>231989</td>\n",
              "      <td>322</td>\n",
              "      <td>2193</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>49</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7880</th>\n",
              "      <td>7880</td>\n",
              "      <td>322</td>\n",
              "      <td>650</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>51</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6465</th>\n",
              "      <td>6465</td>\n",
              "      <td>322</td>\n",
              "      <td>1580</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>53</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7883</th>\n",
              "      <td>7883</td>\n",
              "      <td>322</td>\n",
              "      <td>7418</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6461</th>\n",
              "      <td>6461</td>\n",
              "      <td>322</td>\n",
              "      <td>8504</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>62</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234096</th>\n",
              "      <td>234096</td>\n",
              "      <td>322</td>\n",
              "      <td>9436</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>65</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14656</th>\n",
              "      <td>14656</td>\n",
              "      <td>322</td>\n",
              "      <td>407</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>74</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6368</th>\n",
              "      <td>6368</td>\n",
              "      <td>322</td>\n",
              "      <td>233</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>79</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9034</th>\n",
              "      <td>9034</td>\n",
              "      <td>322</td>\n",
              "      <td>9070</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>80</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231988</th>\n",
              "      <td>231988</td>\n",
              "      <td>322</td>\n",
              "      <td>1421</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>80</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19616</th>\n",
              "      <td>19616</td>\n",
              "      <td>322</td>\n",
              "      <td>2914</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6967</th>\n",
              "      <td>6967</td>\n",
              "      <td>322</td>\n",
              "      <td>863</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>92</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6358</th>\n",
              "      <td>6358</td>\n",
              "      <td>322</td>\n",
              "      <td>251</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>96</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232551</th>\n",
              "      <td>232551</td>\n",
              "      <td>322</td>\n",
              "      <td>25</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>96</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6435</th>\n",
              "      <td>6435</td>\n",
              "      <td>322</td>\n",
              "      <td>3011</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>98</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6438</th>\n",
              "      <td>6438</td>\n",
              "      <td>322</td>\n",
              "      <td>461</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>99</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9294</th>\n",
              "      <td>9294</td>\n",
              "      <td>322</td>\n",
              "      <td>3634</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232006</th>\n",
              "      <td>232006</td>\n",
              "      <td>322</td>\n",
              "      <td>4137</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>103</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>270195</th>\n",
              "      <td>270195</td>\n",
              "      <td>322</td>\n",
              "      <td>52</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>111</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7881</th>\n",
              "      <td>7881</td>\n",
              "      <td>322</td>\n",
              "      <td>5802</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>114</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6975</th>\n",
              "      <td>6975</td>\n",
              "      <td>322</td>\n",
              "      <td>1913</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>118</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6387</th>\n",
              "      <td>6387</td>\n",
              "      <td>322</td>\n",
              "      <td>1319</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>126</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6940</th>\n",
              "      <td>6940</td>\n",
              "      <td>322</td>\n",
              "      <td>608</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>134</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7218</th>\n",
              "      <td>7218</td>\n",
              "      <td>322</td>\n",
              "      <td>618</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>142</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6481</th>\n",
              "      <td>6481</td>\n",
              "      <td>322</td>\n",
              "      <td>2181</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6364</th>\n",
              "      <td>6364</td>\n",
              "      <td>322</td>\n",
              "      <td>339</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>146</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6968</th>\n",
              "      <td>6968</td>\n",
              "      <td>322</td>\n",
              "      <td>728</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>156</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232550</th>\n",
              "      <td>232550</td>\n",
              "      <td>322</td>\n",
              "      <td>3546</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>162</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6391</th>\n",
              "      <td>6391</td>\n",
              "      <td>322</td>\n",
              "      <td>1649</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>171</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231994</th>\n",
              "      <td>231994</td>\n",
              "      <td>322</td>\n",
              "      <td>9038</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>177</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6356</th>\n",
              "      <td>6356</td>\n",
              "      <td>322</td>\n",
              "      <td>9803</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>180</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6467</th>\n",
              "      <td>6467</td>\n",
              "      <td>322</td>\n",
              "      <td>2339</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232552</th>\n",
              "      <td>232552</td>\n",
              "      <td>322</td>\n",
              "      <td>85</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>191</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232553</th>\n",
              "      <td>232553</td>\n",
              "      <td>322</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7224</th>\n",
              "      <td>7224</td>\n",
              "      <td>322</td>\n",
              "      <td>833</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>209</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15151</th>\n",
              "      <td>15151</td>\n",
              "      <td>322</td>\n",
              "      <td>5315</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>216</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7210</th>\n",
              "      <td>7210</td>\n",
              "      <td>322</td>\n",
              "      <td>6535</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>224</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231993</th>\n",
              "      <td>231993</td>\n",
              "      <td>322</td>\n",
              "      <td>3555</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>227</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6357</th>\n",
              "      <td>6357</td>\n",
              "      <td>322</td>\n",
              "      <td>5265</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>236</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232555</th>\n",
              "      <td>232555</td>\n",
              "      <td>322</td>\n",
              "      <td>36</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>242</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6379</th>\n",
              "      <td>6379</td>\n",
              "      <td>322</td>\n",
              "      <td>516</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>244</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6440</th>\n",
              "      <td>6440</td>\n",
              "      <td>322</td>\n",
              "      <td>401</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>246</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6437</th>\n",
              "      <td>6437</td>\n",
              "      <td>322</td>\n",
              "      <td>777</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>248</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Unnamed: 0  user_id  book_id  rating  SortIndex1  SortIndex2\n",
              "6436          6436      322     1748       3           1           5\n",
              "232549      232549      322     1282       2           1           6\n",
              "7222          7222      322      123       3           1          11\n",
              "6360          6360      322      364       4           1          20\n",
              "6439          6439      322     5349       4           1          22\n",
              "6359          6359      322      358       5           1          40\n",
              "6362          6362      322      157       5           1          47\n",
              "231989      231989      322     2193       3           1          49\n",
              "7880          7880      322      650       3           1          51\n",
              "6465          6465      322     1580       3           1          53\n",
              "7883          7883      322     7418       4           1          55\n",
              "6461          6461      322     8504       3           1          62\n",
              "234096      234096      322     9436       4           1          65\n",
              "14656        14656      322      407       3           1          74\n",
              "6368          6368      322      233       3           1          79\n",
              "9034          9034      322     9070       3           1          80\n",
              "231988      231988      322     1421       4           1          80\n",
              "19616        19616      322     2914       4           1          90\n",
              "6967          6967      322      863       4           1          92\n",
              "6358          6358      322      251       4           1          96\n",
              "232551      232551      322       25       4           1          96\n",
              "6435          6435      322     3011       3           1          98\n",
              "6438          6438      322      461       3           1          99\n",
              "9294          9294      322     3634       4           1         103\n",
              "232006      232006      322     4137       4           1         103\n",
              "270195      270195      322       52       3           1         111\n",
              "7881          7881      322     5802       4           1         114\n",
              "6975          6975      322     1913       3           1         118\n",
              "6387          6387      322     1319       3           1         126\n",
              "6940          6940      322      608       4           1         134\n",
              "7218          7218      322      618       4           1         142\n",
              "6481          6481      322     2181       4           1         143\n",
              "6364          6364      322      339       4           1         146\n",
              "6968          6968      322      728       3           1         156\n",
              "232550      232550      322     3546       4           1         162\n",
              "6391          6391      322     1649       3           1         171\n",
              "231994      231994      322     9038       3           1         177\n",
              "6356          6356      322     9803       4           1         180\n",
              "6467          6467      322     2339       4           1         181\n",
              "232552      232552      322       85       4           1         191\n",
              "232553      232553      322        7       4           1         200\n",
              "7224          7224      322      833       3           1         209\n",
              "15151        15151      322     5315       3           1         216\n",
              "7210          7210      322     6535       4           1         224\n",
              "231993      231993      322     3555       4           1         227\n",
              "6357          6357      322     5265       4           1         236\n",
              "232555      232555      322       36       3           1         242\n",
              "6379          6379      322      516       5           1         244\n",
              "6440          6440      322      401       4           1         246\n",
              "6437          6437      322      777       3           1         248"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "metadata": {
        "id": "PReeLbqU3E8v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 639
        },
        "outputId": "249676b6-5977-4582-eed6-a1951e2c4b31"
      },
      "cell_type": "code",
      "source": [
        "#For more info on what the code below is doing\n",
        "#https://stackoverflow.com/questions/50892819/pandas-how-to-get-a-row-count-by-the-value-of-a-particular-column-value-and-ad?answertab=oldest#tab-top\n",
        "#This column is added to help the training algorithm group books by users. \n",
        "\n",
        "r3['Headernumber'] = np.where(r3['user_id'].duplicated(), -1, \n",
        "                         r3.groupby('user_id')['user_id'].transform(len))\n",
        "\n",
        "r3.head(20)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>user_id</th>\n",
              "      <th>book_id</th>\n",
              "      <th>rating</th>\n",
              "      <th>SortIndex1</th>\n",
              "      <th>SortIndex2</th>\n",
              "      <th>Headernumber</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>6436</th>\n",
              "      <td>6436</td>\n",
              "      <td>322</td>\n",
              "      <td>1748</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>107</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>232549</th>\n",
              "      <td>232549</td>\n",
              "      <td>322</td>\n",
              "      <td>1282</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7222</th>\n",
              "      <td>7222</td>\n",
              "      <td>322</td>\n",
              "      <td>123</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6360</th>\n",
              "      <td>6360</td>\n",
              "      <td>322</td>\n",
              "      <td>364</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>20</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6439</th>\n",
              "      <td>6439</td>\n",
              "      <td>322</td>\n",
              "      <td>5349</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6359</th>\n",
              "      <td>6359</td>\n",
              "      <td>322</td>\n",
              "      <td>358</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>40</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6362</th>\n",
              "      <td>6362</td>\n",
              "      <td>322</td>\n",
              "      <td>157</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>47</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231989</th>\n",
              "      <td>231989</td>\n",
              "      <td>322</td>\n",
              "      <td>2193</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>49</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7880</th>\n",
              "      <td>7880</td>\n",
              "      <td>322</td>\n",
              "      <td>650</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>51</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6465</th>\n",
              "      <td>6465</td>\n",
              "      <td>322</td>\n",
              "      <td>1580</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>53</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7883</th>\n",
              "      <td>7883</td>\n",
              "      <td>322</td>\n",
              "      <td>7418</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>55</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6461</th>\n",
              "      <td>6461</td>\n",
              "      <td>322</td>\n",
              "      <td>8504</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>62</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>234096</th>\n",
              "      <td>234096</td>\n",
              "      <td>322</td>\n",
              "      <td>9436</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>65</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14656</th>\n",
              "      <td>14656</td>\n",
              "      <td>322</td>\n",
              "      <td>407</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>74</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6368</th>\n",
              "      <td>6368</td>\n",
              "      <td>322</td>\n",
              "      <td>233</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>79</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9034</th>\n",
              "      <td>9034</td>\n",
              "      <td>322</td>\n",
              "      <td>9070</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>80</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>231988</th>\n",
              "      <td>231988</td>\n",
              "      <td>322</td>\n",
              "      <td>1421</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>80</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19616</th>\n",
              "      <td>19616</td>\n",
              "      <td>322</td>\n",
              "      <td>2914</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6967</th>\n",
              "      <td>6967</td>\n",
              "      <td>322</td>\n",
              "      <td>863</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>92</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6358</th>\n",
              "      <td>6358</td>\n",
              "      <td>322</td>\n",
              "      <td>251</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>96</td>\n",
              "      <td>-1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        Unnamed: 0  user_id  book_id  rating  SortIndex1  SortIndex2  \\\n",
              "6436          6436      322     1748       3           1           5   \n",
              "232549      232549      322     1282       2           1           6   \n",
              "7222          7222      322      123       3           1          11   \n",
              "6360          6360      322      364       4           1          20   \n",
              "6439          6439      322     5349       4           1          22   \n",
              "6359          6359      322      358       5           1          40   \n",
              "6362          6362      322      157       5           1          47   \n",
              "231989      231989      322     2193       3           1          49   \n",
              "7880          7880      322      650       3           1          51   \n",
              "6465          6465      322     1580       3           1          53   \n",
              "7883          7883      322     7418       4           1          55   \n",
              "6461          6461      322     8504       3           1          62   \n",
              "234096      234096      322     9436       4           1          65   \n",
              "14656        14656      322      407       3           1          74   \n",
              "6368          6368      322      233       3           1          79   \n",
              "9034          9034      322     9070       3           1          80   \n",
              "231988      231988      322     1421       4           1          80   \n",
              "19616        19616      322     2914       4           1          90   \n",
              "6967          6967      322      863       4           1          92   \n",
              "6358          6358      322      251       4           1          96   \n",
              "\n",
              "        Headernumber  \n",
              "6436             107  \n",
              "232549            -1  \n",
              "7222              -1  \n",
              "6360              -1  \n",
              "6439              -1  \n",
              "6359              -1  \n",
              "6362              -1  \n",
              "231989            -1  \n",
              "7880              -1  \n",
              "6465              -1  \n",
              "7883              -1  \n",
              "6461              -1  \n",
              "234096            -1  \n",
              "14656             -1  \n",
              "6368              -1  \n",
              "9034              -1  \n",
              "231988            -1  \n",
              "19616             -1  \n",
              "6967              -1  \n",
              "6358              -1  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "metadata": {
        "id": "2MHhvE_x7vWx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "r3.to_csv('RefinedRatingsCbowW2ForAnalogies2nd.csv', index=False) #may have to removed the index = false"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "CT4hgPhh8K7J",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Upload file to google drive\n",
        "uploadModel = drive.CreateFile()\n",
        "uploadModel.SetContentFile(\"RefinedRatingsCbowW2ForAnalogies2nd.csv\") #name of file located in local disk \n",
        "uploadModel.Upload()"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}