data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Data Processing\n",
    "<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-17T19:00:16.733934Z",
     "start_time": "2021-02-17T19:00:14.334767Z"
    }
   },
   "outputs": [],
   "source": [
    "# nltk\n",
    "from nltk import pos_tag\n",
    "from nltk import RegexpTokenizer, PorterStemmer, WordNetLemmatizer, FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "# utilities\n",
    "import string\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "# sklearn\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# preferences\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Read in dataset, preview first three rows, and display column and datatype info\n",
    "Dataset downloaded from https://s3.amazonaws.com/amazon-reviews-pds/tsv/index.txt. The full dataset includes 130M+ customer reviews, grouped by product category, from 1995 to 2015. Each row represents one review.\n",
    "\n",
    "Set the path variable to the local directory where the (unzipped) dataset resides. Other datasets may be processed by modifying the filename."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-17T19:00:19.376719Z",
     "start_time": "2021-02-17T19:00:16.736283Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>marketplace</th>\n",
       "      <th>customer_id</th>\n",
       "      <th>review_id</th>\n",
       "      <th>product_id</th>\n",
       "      <th>product_parent</th>\n",
       "      <th>product_title</th>\n",
       "      <th>product_category</th>\n",
       "      <th>star_rating</th>\n",
       "      <th>helpful_votes</th>\n",
       "      <th>total_votes</th>\n",
       "      <th>vine</th>\n",
       "      <th>verified_purchase</th>\n",
       "      <th>review_headline</th>\n",
       "      <th>review_body</th>\n",
       "      <th>review_date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>US</td>\n",
       "      <td>40884699</td>\n",
       "      <td>R9CO86UUJCAW5</td>\n",
       "      <td>B00VGTN02Y</td>\n",
       "      <td>786681372</td>\n",
       "      <td>Teenage Mutant Ninja Turtle Boys' Teenage Muta...</td>\n",
       "      <td>Luggage</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>my review of this product was in error. It ...</td>\n",
       "      <td>my review of this product was in error. It was...</td>\n",
       "      <td>2015-08-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>US</td>\n",
       "      <td>23208852</td>\n",
       "      <td>R3PR8X6QGVJ8B1</td>\n",
       "      <td>B005KIWL0E</td>\n",
       "      <td>618251799</td>\n",
       "      <td>Kenneth Cole Reaction Out of Bounds 20\"  4 Whe...</td>\n",
       "      <td>Luggage</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>Perfect size.</td>\n",
       "      <td>2015-08-31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>US</td>\n",
       "      <td>17100246</td>\n",
       "      <td>R39BO2819ABUPF</td>\n",
       "      <td>B007UNSHJ6</td>\n",
       "      <td>810480328</td>\n",
       "      <td>American Tourister Luggage AT Pop 3 Piece Spin...</td>\n",
       "      <td>Luggage</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>so good.</td>\n",
       "      <td>So far, so good.</td>\n",
       "      <td>2015-08-31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  marketplace  customer_id       review_id  product_id  product_parent  \\\n",
       "0          US     40884699   R9CO86UUJCAW5  B00VGTN02Y       786681372   \n",
       "1          US     23208852  R3PR8X6QGVJ8B1  B005KIWL0E       618251799   \n",
       "2          US     17100246  R39BO2819ABUPF  B007UNSHJ6       810480328   \n",
       "\n",
       "                                       product_title product_category  \\\n",
       "0  Teenage Mutant Ninja Turtle Boys' Teenage Muta...          Luggage   \n",
       "1  Kenneth Cole Reaction Out of Bounds 20\"  4 Whe...          Luggage   \n",
       "2  American Tourister Luggage AT Pop 3 Piece Spin...          Luggage   \n",
       "\n",
       "   star_rating  helpful_votes  total_votes vine verified_purchase  \\\n",
       "0          3.0            0.0          0.0    N                 Y   \n",
       "1          5.0            0.0          0.0    N                 Y   \n",
       "2          4.0            0.0          0.0    N                 Y   \n",
       "\n",
       "                                  review_headline  \\\n",
       "0  my review of this product was in error. It ...   \n",
       "1                                      Five Stars   \n",
       "2                                        so good.   \n",
       "\n",
       "                                         review_body review_date  \n",
       "0  my review of this product was in error. It was...  2015-08-31  \n",
       "1                                      Perfect size.  2015-08-31  \n",
       "2                                   So far, so good.  2015-08-31  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 348474 entries, 0 to 348473\n",
      "Data columns (total 15 columns):\n",
      "marketplace          348474 non-null object\n",
      "customer_id          348474 non-null int64\n",
      "review_id            348474 non-null object\n",
      "product_id           348474 non-null object\n",
      "product_parent       348474 non-null int64\n",
      "product_title        348474 non-null object\n",
      "product_category     348474 non-null object\n",
      "star_rating          348473 non-null float64\n",
      "helpful_votes        348473 non-null float64\n",
      "total_votes          348473 non-null float64\n",
      "vine                 348473 non-null object\n",
      "verified_purchase    348473 non-null object\n",
      "review_headline      348468 non-null object\n",
      "review_body          348452 non-null object\n",
      "review_date          348472 non-null object\n",
      "dtypes: float64(3), int64(2), object(10)\n",
      "memory usage: 39.9+ MB\n"
     ]
    }
   ],
   "source": [
    "# read amazon tsv dataset into pandas df\n",
    "path = 'data/'                               #<---local directory path where dataset resides\n",
    "filename = 'amazon_reviews_us_LUGGAGE_v1_00' #<---name of dataset being processed\n",
    "ext = '.tsv'                                 #<---file extension (tab seperated values)\n",
    "\n",
    "cols = ['marketplace',       #<---2 letter country code of review marketplace\n",
    "        'customer_id',       #<---random identifier to aggregate reviews by single author\n",
    "        'review_id',         #<---unique ID of review\n",
    "        'product_id',        #<---unique ID of product to which review pertains\n",
    "        'product_parent',    #<---random identifier to aggregate reviews for same product\n",
    "        'product_title',     #<---product title\n",
    "        'product_category',  #<---product category to group dataset into coherent parts \n",
    "        'star_rating',       #<---1-5 star rating of product\n",
    "        'helpful_votes',     #<---number of helpful votes review received\n",
    "        'total_votes',       #<---total number of votes review received\n",
    "        'vine',              #<---review part of Vine program\n",
    "        'verified_purchase', #<---review of verified purchase\n",
    "        'review_headline',   #<---review title\n",
    "        'review_body',       #<---review text\n",
    "        'review_date']       #<---review date\n",
    "\n",
    "df = pd.read_csv(path+filename+ext,\n",
    "                 sep='\\t',\n",
    "                 usecols = cols)\n",
    "\n",
    "display(df.head(3))\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Drop junk data, tokenize text, and drop stopwords\n",
    "\n",
    "Reviews with fewer than 10 votes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-17T19:00:25.540398Z",
     "start_time": "2021-02-17T19:00:19.390302Z"
    }
   },
   "outputs": [],
   "source": [
    "# drop reviews with null data, fewer than 10 votes, or duplicate text\n",
    "df = df.dropna()\n",
    "df = df.loc[df.total_votes >= 10]\n",
    "df = df.drop_duplicates(subset=['review_body'])\n",
    "df = df.reset_index(drop=True)\n",
    "\n",
    "# tokenize text and define and remove stopwords from tokens\n",
    "df['tokens'] = df.review_body.apply(RegexpTokenizer(r'[a-zA-Z0-9]+').tokenize)\n",
    "\n",
    "stopwords_list = stopwords.words('english')\n",
    "stopwords_list += list(string.punctuation)\n",
    "\n",
    "df['stopped_tokens'] = df.tokens.apply(\n",
    "    lambda x: [word.lower() for word in x if word.lower() not in stopwords_list])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Extract features and define target label as above/below median helpful/unhelpful ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-17T19:02:44.394280Z",
     "start_time": "2021-02-17T19:00:25.544086Z"
    }
   },
   "outputs": [],
   "source": [
    "# lexical features\n",
    "df['LEM']  = df.stopped_tokens.apply(lambda x: [WordNetLemmatizer().lemmatize(word) for word in x])\n",
    "df['STEM'] = df.stopped_tokens.apply(lambda x: [PorterStemmer().stem(word) for word in x]) #<---lemm/stem stopped tokens (normalize vocabulary\n",
    "\n",
    "# syntactic features\n",
    "df['POS']  = df.stopped_tokens.apply(lambda x: [pos_tag[1] for pos_tag in pos_tag(x)])     #<---pos-tagged tokens (\"bag of tags\")\n",
    "df['NOUN'] = df.POS.apply(\n",
    "    lambda x: sum(1 for pos in x if pos.startswith('NN')) / len(x) if len(x) > 0 else 0)   #<---percentage of nouns\n",
    "df['ADJ']  = df.POS.apply(\n",
    "    lambda x: sum(1 for pos in x if pos.startswith('JJ')) / len(x) if len(x) > 0 else 0)   #<---percentage of adjectives\n",
    "df['ADV']  = df.POS.apply(\n",
    "    lambda x: sum(1 for pos in x if pos.startswith('RB')) / len(x) if len(x) > 0 else 0)   #<---percentage of adverbs\n",
    "df['VERB'] = df.POS.apply(\n",
    "    lambda x: sum(1 for pos in x if pos.startswith('VB')) / len(x) if len(x) > 0 else 0)   #<---percentage of verbs\n",
    "\n",
    "# structual features\n",
    "df['CHAR']    = df.review_body.apply(lambda x: len(x))             #<---number of characters\n",
    "df['NUM']     = df.stopped_tokens.apply(lambda x: len(x))          #<---number of tokens\n",
    "df['WORD']    = df.review_body.apply(lambda x: len(x.split(' ')))  #<---number of words\n",
    "df['SENT']    = df.review_body.apply(lambda x: len(x.split('. '))) #<---number of sentences\n",
    "df['INTERRO'] = df.review_body.apply(lambda x: len(x.split('? '))) #<---number of questionss\n",
    "df['EXCLAM']  = df.review_body.apply(lambda x: len(x.split('! '))) #<---number of exclamations\n",
    "df['COUNT']   = df.review_body.str.count('!')                      #<---number of exclamation points\n",
    "df['LEN']     = df.CHAR / df.WORD                                  #<---average word length\n",
    "df['AVG']     = df.WORD / df.SENT                                  #<---average sentence length\n",
    "df['PER']     = df.INTERRO / df.SENT                               #<---percentage of questions\n",
    "df['CAPS']    = df.review_body.apply(\n",
    "    lambda x: len([char for char in x if char.isupper()==True]) \n",
    "              / len(x))                                            #<---percentage of capitalized characters\n",
    "\n",
    "# con-textual features\n",
    "df['STAR']  = df.star_rating.astype(int)                                     #<---reviewer's star rating for product\n",
    "df['MED']   = df.groupby('product_parent').STAR.transform('median')          #<---product's median star rating\n",
    "df['FAV']   = df.STAR - df.MED                                               #<---reviewer's rating vs product's median\n",
    "df['POP']   = df.groupby('product_parent').product_parent.transform('count') #<---number of product's reviews  \n",
    "df['DATE']  = pd.to_datetime(df.review_date, errors='coerce',\n",
    "                            yearfirst=True, infer_datetime_format=True)      #<---date of review(as datetime)\n",
    "df['FIRST'] = df.groupby('product_parent').DATE.transform('min')             #<---date of product's first review \n",
    "df['DAYS']  = (df.DATE - df.FIRST).apply(lambda x: x.days)                   #<---days from review date to first review (int)\n",
    "\n",
    "# define target as above/below median ratio of helpful votes to total votes (binary classifier)\n",
    "df['HELP']   = df.helpful_votes / df.total_votes                 #<---percentage of helpful votes\n",
    "df['TARGET'] = np.where(df.HELP > df.HELP.quantile(q=0.5), 1, 0) #<---TARGET threshold (0.5=median)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Drop non-tokenized/non-quantitative data and POS tags, review first three rows, display column and datatype info, and write datafraome to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-02-17T19:02:46.321235Z",
     "start_time": "2021-02-17T19:02:44.396798Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TARGET</th>\n",
       "      <th>LEM</th>\n",
       "      <th>STEM</th>\n",
       "      <th>NOUN</th>\n",
       "      <th>ADJ</th>\n",
       "      <th>ADV</th>\n",
       "      <th>VERB</th>\n",
       "      <th>CHAR</th>\n",
       "      <th>NUM</th>\n",
       "      <th>WORD</th>\n",
       "      <th>SENT</th>\n",
       "      <th>INTERRO</th>\n",
       "      <th>EXCLAM</th>\n",
       "      <th>COUNT</th>\n",
       "      <th>LEN</th>\n",
       "      <th>AVG</th>\n",
       "      <th>PER</th>\n",
       "      <th>CAPS</th>\n",
       "      <th>STAR</th>\n",
       "      <th>MED</th>\n",
       "      <th>FAV</th>\n",
       "      <th>POP</th>\n",
       "      <th>DAYS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>[strap, broke, supposed, anti, theft, strap, b...</td>\n",
       "      <td>[strap, broke, suppos, anti, theft, strap, bro...</td>\n",
       "      <td>0.440000</td>\n",
       "      <td>0.160000</td>\n",
       "      <td>0.080000</td>\n",
       "      <td>0.200000</td>\n",
       "      <td>318</td>\n",
       "      <td>25</td>\n",
       "      <td>63</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5.047619</td>\n",
       "      <td>10.5</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.031447</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>23</td>\n",
       "      <td>2001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>[absolutely, thrilled, quality, leather, bette...</td>\n",
       "      <td>[absolut, thrill, qualiti, leather, better, ex...</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.416667</td>\n",
       "      <td>0.166667</td>\n",
       "      <td>136</td>\n",
       "      <td>12</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.230769</td>\n",
       "      <td>26.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.029412</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>[picking, luggage, 2, week, whirlwind, tour, e...</td>\n",
       "      <td>[pick, luggag, 2, week, whirlwind, tour, europ...</td>\n",
       "      <td>0.370968</td>\n",
       "      <td>0.177419</td>\n",
       "      <td>0.096774</td>\n",
       "      <td>0.209677</td>\n",
       "      <td>728</td>\n",
       "      <td>62</td>\n",
       "      <td>140</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5.200000</td>\n",
       "      <td>17.5</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.020604</td>\n",
       "      <td>5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3</td>\n",
       "      <td>98</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   TARGET                                                LEM  \\\n",
       "0       1  [strap, broke, supposed, anti, theft, strap, b...   \n",
       "1       0  [absolutely, thrilled, quality, leather, bette...   \n",
       "2       0  [picking, luggage, 2, week, whirlwind, tour, e...   \n",
       "\n",
       "                                                STEM      NOUN       ADJ  \\\n",
       "0  [strap, broke, suppos, anti, theft, strap, bro...  0.440000  0.160000   \n",
       "1  [absolut, thrill, qualiti, leather, better, ex...  0.166667  0.250000   \n",
       "2  [pick, luggag, 2, week, whirlwind, tour, europ...  0.370968  0.177419   \n",
       "\n",
       "        ADV      VERB  CHAR  NUM  WORD  SENT  INTERRO  EXCLAM  COUNT  \\\n",
       "0  0.080000  0.200000   318   25    63     6        1       2      3   \n",
       "1  0.416667  0.166667   136   12    26     1        1       3      3   \n",
       "2  0.096774  0.209677   728   62   140     8        1       2      2   \n",
       "\n",
       "        LEN   AVG       PER      CAPS  STAR  MED  FAV  POP  DAYS  \n",
       "0  5.047619  10.5  0.166667  0.031447     1  4.0 -3.0   23  2001  \n",
       "1  5.230769  26.0  1.000000  0.029412     5  5.0  0.0    2   191  \n",
       "2  5.200000  17.5  0.125000  0.020604     5  5.0  0.0    3    98  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 16242 entries, 0 to 16241\n",
      "Data columns (total 23 columns):\n",
      "TARGET     16242 non-null int64\n",
      "LEM        16242 non-null object\n",
      "STEM       16242 non-null object\n",
      "NOUN       16242 non-null float64\n",
      "ADJ        16242 non-null float64\n",
      "ADV        16242 non-null float64\n",
      "VERB       16242 non-null float64\n",
      "CHAR       16242 non-null int64\n",
      "NUM        16242 non-null int64\n",
      "WORD       16242 non-null int64\n",
      "SENT       16242 non-null int64\n",
      "INTERRO    16242 non-null int64\n",
      "EXCLAM     16242 non-null int64\n",
      "COUNT      16242 non-null int64\n",
      "LEN        16242 non-null float64\n",
      "AVG        16242 non-null float64\n",
      "PER        16242 non-null float64\n",
      "CAPS       16242 non-null float64\n",
      "STAR       16242 non-null int64\n",
      "MED        16242 non-null float64\n",
      "FAV        16242 non-null float64\n",
      "POP        16242 non-null int64\n",
      "DAYS       16242 non-null int64\n",
      "dtypes: float64(10), int64(11), object(2)\n",
      "memory usage: 3.0+ MB\n"
     ]
    }
   ],
   "source": [
    "# keep target and extracted features, drop HELP (leakage) and TIME and FIRST (datetime)\n",
    "cols = ['TARGET',  #<---ratio of helpful votes to total votes above/below median\n",
    "        'LEM',     #<---lemmas\n",
    "        'STEM',    #<---stems\n",
    "        'NOUN',    #<---percentage of nouns\n",
    "        'ADJ',     #<---percentage of adjectives\n",
    "        'ADV',     #<---percentage of adverbs\n",
    "        'VERB',    #<---percentage of verbs\n",
    "        'CHAR',    #<---number of characters\n",
    "        'NUM',     #<---number of tokens\n",
    "        'WORD',    #<---number of words\n",
    "        'SENT',    #<---number of sentences\n",
    "        'INTERRO', #<---number of questionss\n",
    "        'EXCLAM',  #<---number of exclamations\n",
    "        'COUNT',   #<---number of exclamation points\n",
    "        'LEN',     #<---average word length\n",
    "        'AVG',     #<---average sentence length\n",
    "        'PER',     #<---percentage of questions\n",
    "        'CAPS',    #<---percentage of capitalized characters\n",
    "        'STAR',    #<---reviewer's star rating for product\n",
    "        'MED',     #<---product's median star rating\n",
    "        'FAV',     #<---reviewer's star rating vs product's median\n",
    "        'POP',     #<---number of product's reviews\n",
    "        'DAYS']    #<---days from review date to first review\n",
    "\n",
    "# drop original features and any nan values from feature extraction\n",
    "df = df[cols].dropna()\n",
    "\n",
    "display(df.head(3))\n",
    "df.info()\n",
    "\n",
    "df.to_csv(path+filename+'_processed'+ext, index=0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "learn-env",
   "language": "python",
   "name": "learn-env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": false,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}