Data Preprocessing - BARK.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load the Training CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2785: DtypeWarning: Columns (21) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Columns: Index(['speaker', 'token', 'word', 'norm_time', 'time', 'c1', 'c2', 'c3', 'c4',\n",
      "       'c5', 'c6', 'c7', 'c8', 'c9', 'c10', 'c11', 'c12', 'rhoticity1',\n",
      "       'rhoticity2', 'dep_variable', 'sex', 'birth_year', 'pre_segment',\n",
      "       'fol_segment', 'vowelR', 'vowel_type', 'time_y', 'F1', 'F2', 'F3', 'Z2',\n",
      "       'Z3', 'Z2-Z3', 'Z*'],\n",
      "      dtype='object')\n",
      "Initially 494833 rows\n",
      "Filtered NaN labels to 494833 rows\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>token</th>\n",
       "      <th>word</th>\n",
       "      <th>norm_time</th>\n",
       "      <th>time</th>\n",
       "      <th>c1</th>\n",
       "      <th>c2</th>\n",
       "      <th>c3</th>\n",
       "      <th>c4</th>\n",
       "      <th>c5</th>\n",
       "      <th>...</th>\n",
       "      <th>vowelR</th>\n",
       "      <th>vowel_type</th>\n",
       "      <th>time_y</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>Z2</th>\n",
       "      <th>Z3</th>\n",
       "      <th>Z2-Z3</th>\n",
       "      <th>Z*</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.01</td>\n",
       "      <td>13.1651</td>\n",
       "      <td>-1267.561536</td>\n",
       "      <td>-1209.016526</td>\n",
       "      <td>-829.048466</td>\n",
       "      <td>-508.252388</td>\n",
       "      <td>-305.130238</td>\n",
       "      <td>...</td>\n",
       "      <td>AO1R</td>\n",
       "      <td>back</td>\n",
       "      <td>13.1651</td>\n",
       "      <td>377.5567</td>\n",
       "      <td>1377.9991</td>\n",
       "      <td>2261.2983</td>\n",
       "      <td>10.4723</td>\n",
       "      <td>13.7191</td>\n",
       "      <td>-3.2468</td>\n",
       "      <td>12.0957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.02</td>\n",
       "      <td>13.1663</td>\n",
       "      <td>-1196.413804</td>\n",
       "      <td>-1264.286082</td>\n",
       "      <td>-868.443646</td>\n",
       "      <td>-557.983277</td>\n",
       "      <td>-281.922436</td>\n",
       "      <td>...</td>\n",
       "      <td>AO1R</td>\n",
       "      <td>back</td>\n",
       "      <td>13.1662</td>\n",
       "      <td>379.5048</td>\n",
       "      <td>1375.4122</td>\n",
       "      <td>2253.6803</td>\n",
       "      <td>10.4604</td>\n",
       "      <td>13.6967</td>\n",
       "      <td>-3.2363</td>\n",
       "      <td>12.0785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.03</td>\n",
       "      <td>13.1674</td>\n",
       "      <td>-1187.723367</td>\n",
       "      <td>-1449.346610</td>\n",
       "      <td>-872.610353</td>\n",
       "      <td>-503.566163</td>\n",
       "      <td>-289.390192</td>\n",
       "      <td>...</td>\n",
       "      <td>AO1R</td>\n",
       "      <td>back</td>\n",
       "      <td>13.1673</td>\n",
       "      <td>381.7783</td>\n",
       "      <td>1370.6928</td>\n",
       "      <td>2255.9149</td>\n",
       "      <td>10.4385</td>\n",
       "      <td>13.7033</td>\n",
       "      <td>-3.2648</td>\n",
       "      <td>12.0709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.04</td>\n",
       "      <td>13.1685</td>\n",
       "      <td>-1186.670853</td>\n",
       "      <td>-1331.149990</td>\n",
       "      <td>-884.239085</td>\n",
       "      <td>-524.813311</td>\n",
       "      <td>-311.409256</td>\n",
       "      <td>...</td>\n",
       "      <td>AO1R</td>\n",
       "      <td>back</td>\n",
       "      <td>13.1684</td>\n",
       "      <td>384.0517</td>\n",
       "      <td>1365.9734</td>\n",
       "      <td>2258.1496</td>\n",
       "      <td>10.4166</td>\n",
       "      <td>13.7100</td>\n",
       "      <td>-3.2933</td>\n",
       "      <td>12.0633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.05</td>\n",
       "      <td>13.1696</td>\n",
       "      <td>-1187.932907</td>\n",
       "      <td>-1278.971283</td>\n",
       "      <td>-861.975815</td>\n",
       "      <td>-561.425275</td>\n",
       "      <td>-310.147997</td>\n",
       "      <td>...</td>\n",
       "      <td>AO1R</td>\n",
       "      <td>back</td>\n",
       "      <td>13.1695</td>\n",
       "      <td>386.3252</td>\n",
       "      <td>1361.2540</td>\n",
       "      <td>2260.3842</td>\n",
       "      <td>10.3948</td>\n",
       "      <td>13.7166</td>\n",
       "      <td>-3.3219</td>\n",
       "      <td>12.0557</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             speaker                     token   word  norm_time     time  \\\n",
       "0  Boston2015-16_118  Boston2015-16_118_short1  short       0.01  13.1651   \n",
       "1  Boston2015-16_118  Boston2015-16_118_short1  short       0.02  13.1663   \n",
       "2  Boston2015-16_118  Boston2015-16_118_short1  short       0.03  13.1674   \n",
       "3  Boston2015-16_118  Boston2015-16_118_short1  short       0.04  13.1685   \n",
       "4  Boston2015-16_118  Boston2015-16_118_short1  short       0.05  13.1696   \n",
       "\n",
       "            c1           c2          c3          c4          c5   ...     \\\n",
       "0 -1267.561536 -1209.016526 -829.048466 -508.252388 -305.130238   ...      \n",
       "1 -1196.413804 -1264.286082 -868.443646 -557.983277 -281.922436   ...      \n",
       "2 -1187.723367 -1449.346610 -872.610353 -503.566163 -289.390192   ...      \n",
       "3 -1186.670853 -1331.149990 -884.239085 -524.813311 -311.409256   ...      \n",
       "4 -1187.932907 -1278.971283 -861.975815 -561.425275 -310.147997   ...      \n",
       "\n",
       "   vowelR  vowel_type   time_y        F1         F2         F3       Z2  \\\n",
       "0    AO1R        back  13.1651  377.5567  1377.9991  2261.2983  10.4723   \n",
       "1    AO1R        back  13.1662  379.5048  1375.4122  2253.6803  10.4604   \n",
       "2    AO1R        back  13.1673  381.7783  1370.6928  2255.9149  10.4385   \n",
       "3    AO1R        back  13.1684  384.0517  1365.9734  2258.1496  10.4166   \n",
       "4    AO1R        back  13.1695  386.3252  1361.2540  2260.3842  10.3948   \n",
       "\n",
       "        Z3   Z2-Z3       Z*  \n",
       "0  13.7191 -3.2468  12.0957  \n",
       "1  13.6967 -3.2363  12.0785  \n",
       "2  13.7033 -3.2648  12.0709  \n",
       "3  13.7100 -3.2933  12.0633  \n",
       "4  13.7166 -3.3219  12.0557  \n",
       "\n",
       "[5 rows x 34 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('datasets/train_merge_total.csv', sep=',')\n",
    "print(\"Columns:\", data.columns)\n",
    "print(\"Initially\", len(data), \"rows\")\n",
    "\n",
    "data = data[np.isfinite(data['rhoticity1'])]\n",
    "data = data[np.isfinite(data['rhoticity2'])]\n",
    "\n",
    "print(\"Filtered NaN labels to\", len(data), \"rows\")\n",
    "\n",
    "data['rhoticity1'] = data['rhoticity1'].astype(int)\n",
    "data['rhoticity2'] = data['rhoticity2'].astype(int)\n",
    "data.loc[data['vowel_type'].isna(), 'vowel_type'] = 'none'\n",
    "data.head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filter Boarder Cases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Filtering out 72422 border cases\n",
      "Rhotic Token Count: 269685\n",
      "Non-Rhotic Token Count: 152726\n"
     ]
    }
   ],
   "source": [
    "print(\"Filtering out\", len(data.loc[data['dep_variable'] == 0.5]), \"border cases\")\n",
    "data = data.loc[data['dep_variable'] != 0.5]\n",
    "\n",
    "data['dep_variable'] = data['dep_variable'].astype(int)\n",
    "\n",
    "print(\"Rhotic Token Count: \" + str(len(data.loc[data['dep_variable'] == 1])))\n",
    "print(\"Non-Rhotic Token Count: \" + str(len(data.loc[data['dep_variable'] == 0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Categorical Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AW1' 'AY1' 'B' 'CH' 'D' 'DH' 'ER0' 'F' 'G' 'H' 'HH' 'IY0' 'IY1' 'K' 'L'\n",
      " 'M' 'N' 'NG' 'OW1' 'P' 'R' 'S' 'SH' 'T' 'UW1' 'V' 'W' 'Y' 'Z' 'nan' 'sp']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>speaker</th>\n",
       "      <th>token</th>\n",
       "      <th>word</th>\n",
       "      <th>norm_time</th>\n",
       "      <th>time</th>\n",
       "      <th>c1</th>\n",
       "      <th>c2</th>\n",
       "      <th>c3</th>\n",
       "      <th>c4</th>\n",
       "      <th>c5</th>\n",
       "      <th>...</th>\n",
       "      <th>vowelR</th>\n",
       "      <th>vowel_type</th>\n",
       "      <th>time_y</th>\n",
       "      <th>F1</th>\n",
       "      <th>F2</th>\n",
       "      <th>F3</th>\n",
       "      <th>Z2</th>\n",
       "      <th>Z3</th>\n",
       "      <th>Z2-Z3</th>\n",
       "      <th>Z*</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.01</td>\n",
       "      <td>13.1651</td>\n",
       "      <td>-1267.561536</td>\n",
       "      <td>-1209.016526</td>\n",
       "      <td>-829.048466</td>\n",
       "      <td>-508.252388</td>\n",
       "      <td>-305.130238</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1651</td>\n",
       "      <td>377.5567</td>\n",
       "      <td>1377.9991</td>\n",
       "      <td>2261.2983</td>\n",
       "      <td>10.4723</td>\n",
       "      <td>13.7191</td>\n",
       "      <td>-3.2468</td>\n",
       "      <td>12.0957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.02</td>\n",
       "      <td>13.1663</td>\n",
       "      <td>-1196.413804</td>\n",
       "      <td>-1264.286082</td>\n",
       "      <td>-868.443646</td>\n",
       "      <td>-557.983277</td>\n",
       "      <td>-281.922436</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1662</td>\n",
       "      <td>379.5048</td>\n",
       "      <td>1375.4122</td>\n",
       "      <td>2253.6803</td>\n",
       "      <td>10.4604</td>\n",
       "      <td>13.6967</td>\n",
       "      <td>-3.2363</td>\n",
       "      <td>12.0785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.03</td>\n",
       "      <td>13.1674</td>\n",
       "      <td>-1187.723367</td>\n",
       "      <td>-1449.346610</td>\n",
       "      <td>-872.610353</td>\n",
       "      <td>-503.566163</td>\n",
       "      <td>-289.390192</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1673</td>\n",
       "      <td>381.7783</td>\n",
       "      <td>1370.6928</td>\n",
       "      <td>2255.9149</td>\n",
       "      <td>10.4385</td>\n",
       "      <td>13.7033</td>\n",
       "      <td>-3.2648</td>\n",
       "      <td>12.0709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.04</td>\n",
       "      <td>13.1685</td>\n",
       "      <td>-1186.670853</td>\n",
       "      <td>-1331.149990</td>\n",
       "      <td>-884.239085</td>\n",
       "      <td>-524.813311</td>\n",
       "      <td>-311.409256</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1684</td>\n",
       "      <td>384.0517</td>\n",
       "      <td>1365.9734</td>\n",
       "      <td>2258.1496</td>\n",
       "      <td>10.4166</td>\n",
       "      <td>13.7100</td>\n",
       "      <td>-3.2933</td>\n",
       "      <td>12.0633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.05</td>\n",
       "      <td>13.1696</td>\n",
       "      <td>-1187.932907</td>\n",
       "      <td>-1278.971283</td>\n",
       "      <td>-861.975815</td>\n",
       "      <td>-561.425275</td>\n",
       "      <td>-310.147997</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1695</td>\n",
       "      <td>386.3252</td>\n",
       "      <td>1361.2540</td>\n",
       "      <td>2260.3842</td>\n",
       "      <td>10.3948</td>\n",
       "      <td>13.7166</td>\n",
       "      <td>-3.3219</td>\n",
       "      <td>12.0557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.06</td>\n",
       "      <td>13.1707</td>\n",
       "      <td>-1139.979275</td>\n",
       "      <td>-1338.346039</td>\n",
       "      <td>-860.908067</td>\n",
       "      <td>-549.633166</td>\n",
       "      <td>-295.748252</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1706</td>\n",
       "      <td>388.5986</td>\n",
       "      <td>1356.5346</td>\n",
       "      <td>2262.6189</td>\n",
       "      <td>10.3729</td>\n",
       "      <td>13.7233</td>\n",
       "      <td>-3.3504</td>\n",
       "      <td>12.0481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.07</td>\n",
       "      <td>13.1718</td>\n",
       "      <td>-1227.769009</td>\n",
       "      <td>-1315.738340</td>\n",
       "      <td>-876.046671</td>\n",
       "      <td>-508.831386</td>\n",
       "      <td>-297.988347</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1717</td>\n",
       "      <td>390.8720</td>\n",
       "      <td>1351.8152</td>\n",
       "      <td>2264.8535</td>\n",
       "      <td>10.3510</td>\n",
       "      <td>13.7300</td>\n",
       "      <td>-3.3789</td>\n",
       "      <td>12.0405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.08</td>\n",
       "      <td>13.1729</td>\n",
       "      <td>-1238.242613</td>\n",
       "      <td>-1296.144049</td>\n",
       "      <td>-867.605982</td>\n",
       "      <td>-463.705142</td>\n",
       "      <td>-276.136601</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1728</td>\n",
       "      <td>392.5502</td>\n",
       "      <td>1342.0305</td>\n",
       "      <td>2261.8786</td>\n",
       "      <td>10.3045</td>\n",
       "      <td>13.7210</td>\n",
       "      <td>-3.4165</td>\n",
       "      <td>12.0128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.09</td>\n",
       "      <td>13.1740</td>\n",
       "      <td>-1149.099480</td>\n",
       "      <td>-1261.944199</td>\n",
       "      <td>-933.018145</td>\n",
       "      <td>-523.165950</td>\n",
       "      <td>-262.013038</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1739</td>\n",
       "      <td>393.8535</td>\n",
       "      <td>1329.0564</td>\n",
       "      <td>2255.6236</td>\n",
       "      <td>10.2425</td>\n",
       "      <td>13.7023</td>\n",
       "      <td>-3.4598</td>\n",
       "      <td>11.9724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.11</td>\n",
       "      <td>13.1763</td>\n",
       "      <td>-1132.916286</td>\n",
       "      <td>-1412.198436</td>\n",
       "      <td>-878.328958</td>\n",
       "      <td>-543.164332</td>\n",
       "      <td>-304.298815</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1761</td>\n",
       "      <td>396.4602</td>\n",
       "      <td>1303.1082</td>\n",
       "      <td>2243.1135</td>\n",
       "      <td>10.1186</td>\n",
       "      <td>13.6649</td>\n",
       "      <td>-3.5464</td>\n",
       "      <td>11.8917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.12</td>\n",
       "      <td>13.1774</td>\n",
       "      <td>-1078.511956</td>\n",
       "      <td>-1341.330327</td>\n",
       "      <td>-949.659008</td>\n",
       "      <td>-503.986691</td>\n",
       "      <td>-335.717939</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1772</td>\n",
       "      <td>397.7636</td>\n",
       "      <td>1290.1341</td>\n",
       "      <td>2236.8584</td>\n",
       "      <td>10.0566</td>\n",
       "      <td>13.6462</td>\n",
       "      <td>-3.5896</td>\n",
       "      <td>11.8514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.13</td>\n",
       "      <td>13.1785</td>\n",
       "      <td>-1144.587464</td>\n",
       "      <td>-1253.928131</td>\n",
       "      <td>-935.833371</td>\n",
       "      <td>-610.614466</td>\n",
       "      <td>-228.968033</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1783</td>\n",
       "      <td>399.0669</td>\n",
       "      <td>1277.1600</td>\n",
       "      <td>2230.6034</td>\n",
       "      <td>9.9946</td>\n",
       "      <td>13.6275</td>\n",
       "      <td>-3.6329</td>\n",
       "      <td>11.8110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.14</td>\n",
       "      <td>13.1796</td>\n",
       "      <td>-1170.122312</td>\n",
       "      <td>-1301.864053</td>\n",
       "      <td>-881.119146</td>\n",
       "      <td>-546.176647</td>\n",
       "      <td>-293.763016</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1794</td>\n",
       "      <td>400.2809</td>\n",
       "      <td>1263.0928</td>\n",
       "      <td>2231.5810</td>\n",
       "      <td>9.9243</td>\n",
       "      <td>13.6304</td>\n",
       "      <td>-3.7062</td>\n",
       "      <td>11.7773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.15</td>\n",
       "      <td>13.1807</td>\n",
       "      <td>-1192.725015</td>\n",
       "      <td>-1262.159622</td>\n",
       "      <td>-910.741535</td>\n",
       "      <td>-529.230538</td>\n",
       "      <td>-255.380747</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1805</td>\n",
       "      <td>401.4883</td>\n",
       "      <td>1248.9456</td>\n",
       "      <td>2233.0878</td>\n",
       "      <td>9.8533</td>\n",
       "      <td>13.6350</td>\n",
       "      <td>-3.7816</td>\n",
       "      <td>11.7442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.16</td>\n",
       "      <td>13.1818</td>\n",
       "      <td>-1158.522401</td>\n",
       "      <td>-1284.491580</td>\n",
       "      <td>-869.483388</td>\n",
       "      <td>-550.813819</td>\n",
       "      <td>-290.266454</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1816</td>\n",
       "      <td>402.6957</td>\n",
       "      <td>1234.7984</td>\n",
       "      <td>2234.5946</td>\n",
       "      <td>9.7824</td>\n",
       "      <td>13.6395</td>\n",
       "      <td>-3.8571</td>\n",
       "      <td>11.7110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.17</td>\n",
       "      <td>13.1829</td>\n",
       "      <td>-1159.402234</td>\n",
       "      <td>-1263.678123</td>\n",
       "      <td>-902.073851</td>\n",
       "      <td>-579.179721</td>\n",
       "      <td>-309.674034</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1827</td>\n",
       "      <td>403.9031</td>\n",
       "      <td>1220.6512</td>\n",
       "      <td>2236.1015</td>\n",
       "      <td>9.7115</td>\n",
       "      <td>13.6440</td>\n",
       "      <td>-3.9325</td>\n",
       "      <td>11.6778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.18</td>\n",
       "      <td>13.1840</td>\n",
       "      <td>-1128.116216</td>\n",
       "      <td>-1398.449797</td>\n",
       "      <td>-856.128144</td>\n",
       "      <td>-526.384587</td>\n",
       "      <td>-346.167435</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1838</td>\n",
       "      <td>405.1105</td>\n",
       "      <td>1206.5040</td>\n",
       "      <td>2237.6083</td>\n",
       "      <td>9.6406</td>\n",
       "      <td>13.6486</td>\n",
       "      <td>-4.0080</td>\n",
       "      <td>11.6446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.19</td>\n",
       "      <td>13.1851</td>\n",
       "      <td>-1121.406257</td>\n",
       "      <td>-1321.179002</td>\n",
       "      <td>-941.620525</td>\n",
       "      <td>-538.849653</td>\n",
       "      <td>-254.111405</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1849</td>\n",
       "      <td>406.5868</td>\n",
       "      <td>1191.8152</td>\n",
       "      <td>2240.7919</td>\n",
       "      <td>9.5658</td>\n",
       "      <td>13.6581</td>\n",
       "      <td>-4.0923</td>\n",
       "      <td>11.6119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.21</td>\n",
       "      <td>13.1874</td>\n",
       "      <td>-1068.271478</td>\n",
       "      <td>-1343.349097</td>\n",
       "      <td>-956.791092</td>\n",
       "      <td>-542.446008</td>\n",
       "      <td>-223.073833</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1871</td>\n",
       "      <td>411.1529</td>\n",
       "      <td>1159.1879</td>\n",
       "      <td>2257.2201</td>\n",
       "      <td>9.3928</td>\n",
       "      <td>13.7069</td>\n",
       "      <td>-4.3142</td>\n",
       "      <td>11.5498</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.22</td>\n",
       "      <td>13.1885</td>\n",
       "      <td>-1203.179671</td>\n",
       "      <td>-1314.753033</td>\n",
       "      <td>-925.681112</td>\n",
       "      <td>-540.072017</td>\n",
       "      <td>-278.185962</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1882</td>\n",
       "      <td>413.4360</td>\n",
       "      <td>1142.8742</td>\n",
       "      <td>2265.4342</td>\n",
       "      <td>9.3063</td>\n",
       "      <td>13.7314</td>\n",
       "      <td>-4.4251</td>\n",
       "      <td>11.5188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Boston2015-16_118</td>\n",
       "      <td>Boston2015-16_118_short1</td>\n",
       "      <td>short</td>\n",
       "      <td>0.23</td>\n",
       "      <td>13.1896</td>\n",
       "      <td>-1199.390561</td>\n",
       "      <td>-1290.648318</td>\n",
       "      <td>-876.294259</td>\n",
       "      <td>-573.461586</td>\n",
       "      <td>-266.002623</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>13.1893</td>\n",
       "      <td>415.7191</td>\n",
       "      <td>1126.5605</td>\n",
       "      <td>2273.6483</td>\n",
       "      <td>9.2198</td>\n",
       "      <td>13.7558</td>\n",
       "      <td>-4.5360</td>\n",
       "      <td>11.4878</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              speaker                     token   word  norm_time     time  \\\n",
       "0   Boston2015-16_118  Boston2015-16_118_short1  short       0.01  13.1651   \n",
       "1   Boston2015-16_118  Boston2015-16_118_short1  short       0.02  13.1663   \n",
       "2   Boston2015-16_118  Boston2015-16_118_short1  short       0.03  13.1674   \n",
       "3   Boston2015-16_118  Boston2015-16_118_short1  short       0.04  13.1685   \n",
       "4   Boston2015-16_118  Boston2015-16_118_short1  short       0.05  13.1696   \n",
       "5   Boston2015-16_118  Boston2015-16_118_short1  short       0.06  13.1707   \n",
       "6   Boston2015-16_118  Boston2015-16_118_short1  short       0.07  13.1718   \n",
       "7   Boston2015-16_118  Boston2015-16_118_short1  short       0.08  13.1729   \n",
       "8   Boston2015-16_118  Boston2015-16_118_short1  short       0.09  13.1740   \n",
       "9   Boston2015-16_118  Boston2015-16_118_short1  short       0.11  13.1763   \n",
       "10  Boston2015-16_118  Boston2015-16_118_short1  short       0.12  13.1774   \n",
       "11  Boston2015-16_118  Boston2015-16_118_short1  short       0.13  13.1785   \n",
       "12  Boston2015-16_118  Boston2015-16_118_short1  short       0.14  13.1796   \n",
       "13  Boston2015-16_118  Boston2015-16_118_short1  short       0.15  13.1807   \n",
       "14  Boston2015-16_118  Boston2015-16_118_short1  short       0.16  13.1818   \n",
       "15  Boston2015-16_118  Boston2015-16_118_short1  short       0.17  13.1829   \n",
       "16  Boston2015-16_118  Boston2015-16_118_short1  short       0.18  13.1840   \n",
       "17  Boston2015-16_118  Boston2015-16_118_short1  short       0.19  13.1851   \n",
       "18  Boston2015-16_118  Boston2015-16_118_short1  short       0.21  13.1874   \n",
       "19  Boston2015-16_118  Boston2015-16_118_short1  short       0.22  13.1885   \n",
       "20  Boston2015-16_118  Boston2015-16_118_short1  short       0.23  13.1896   \n",
       "\n",
       "             c1           c2          c3          c4          c5   ...     \\\n",
       "0  -1267.561536 -1209.016526 -829.048466 -508.252388 -305.130238   ...      \n",
       "1  -1196.413804 -1264.286082 -868.443646 -557.983277 -281.922436   ...      \n",
       "2  -1187.723367 -1449.346610 -872.610353 -503.566163 -289.390192   ...      \n",
       "3  -1186.670853 -1331.149990 -884.239085 -524.813311 -311.409256   ...      \n",
       "4  -1187.932907 -1278.971283 -861.975815 -561.425275 -310.147997   ...      \n",
       "5  -1139.979275 -1338.346039 -860.908067 -549.633166 -295.748252   ...      \n",
       "6  -1227.769009 -1315.738340 -876.046671 -508.831386 -297.988347   ...      \n",
       "7  -1238.242613 -1296.144049 -867.605982 -463.705142 -276.136601   ...      \n",
       "8  -1149.099480 -1261.944199 -933.018145 -523.165950 -262.013038   ...      \n",
       "9  -1132.916286 -1412.198436 -878.328958 -543.164332 -304.298815   ...      \n",
       "10 -1078.511956 -1341.330327 -949.659008 -503.986691 -335.717939   ...      \n",
       "11 -1144.587464 -1253.928131 -935.833371 -610.614466 -228.968033   ...      \n",
       "12 -1170.122312 -1301.864053 -881.119146 -546.176647 -293.763016   ...      \n",
       "13 -1192.725015 -1262.159622 -910.741535 -529.230538 -255.380747   ...      \n",
       "14 -1158.522401 -1284.491580 -869.483388 -550.813819 -290.266454   ...      \n",
       "15 -1159.402234 -1263.678123 -902.073851 -579.179721 -309.674034   ...      \n",
       "16 -1128.116216 -1398.449797 -856.128144 -526.384587 -346.167435   ...      \n",
       "17 -1121.406257 -1321.179002 -941.620525 -538.849653 -254.111405   ...      \n",
       "18 -1068.271478 -1343.349097 -956.791092 -542.446008 -223.073833   ...      \n",
       "19 -1203.179671 -1314.753033 -925.681112 -540.072017 -278.185962   ...      \n",
       "20 -1199.390561 -1290.648318 -876.294259 -573.461586 -266.002623   ...      \n",
       "\n",
       "    vowelR  vowel_type   time_y        F1         F2         F3       Z2  \\\n",
       "0        3           0  13.1651  377.5567  1377.9991  2261.2983  10.4723   \n",
       "1        3           0  13.1662  379.5048  1375.4122  2253.6803  10.4604   \n",
       "2        3           0  13.1673  381.7783  1370.6928  2255.9149  10.4385   \n",
       "3        3           0  13.1684  384.0517  1365.9734  2258.1496  10.4166   \n",
       "4        3           0  13.1695  386.3252  1361.2540  2260.3842  10.3948   \n",
       "5        3           0  13.1706  388.5986  1356.5346  2262.6189  10.3729   \n",
       "6        3           0  13.1717  390.8720  1351.8152  2264.8535  10.3510   \n",
       "7        3           0  13.1728  392.5502  1342.0305  2261.8786  10.3045   \n",
       "8        3           0  13.1739  393.8535  1329.0564  2255.6236  10.2425   \n",
       "9        3           0  13.1761  396.4602  1303.1082  2243.1135  10.1186   \n",
       "10       3           0  13.1772  397.7636  1290.1341  2236.8584  10.0566   \n",
       "11       3           0  13.1783  399.0669  1277.1600  2230.6034   9.9946   \n",
       "12       3           0  13.1794  400.2809  1263.0928  2231.5810   9.9243   \n",
       "13       3           0  13.1805  401.4883  1248.9456  2233.0878   9.8533   \n",
       "14       3           0  13.1816  402.6957  1234.7984  2234.5946   9.7824   \n",
       "15       3           0  13.1827  403.9031  1220.6512  2236.1015   9.7115   \n",
       "16       3           0  13.1838  405.1105  1206.5040  2237.6083   9.6406   \n",
       "17       3           0  13.1849  406.5868  1191.8152  2240.7919   9.5658   \n",
       "18       3           0  13.1871  411.1529  1159.1879  2257.2201   9.3928   \n",
       "19       3           0  13.1882  413.4360  1142.8742  2265.4342   9.3063   \n",
       "20       3           0  13.1893  415.7191  1126.5605  2273.6483   9.2198   \n",
       "\n",
       "         Z3   Z2-Z3       Z*  \n",
       "0   13.7191 -3.2468  12.0957  \n",
       "1   13.6967 -3.2363  12.0785  \n",
       "2   13.7033 -3.2648  12.0709  \n",
       "3   13.7100 -3.2933  12.0633  \n",
       "4   13.7166 -3.3219  12.0557  \n",
       "5   13.7233 -3.3504  12.0481  \n",
       "6   13.7300 -3.3789  12.0405  \n",
       "7   13.7210 -3.4165  12.0128  \n",
       "8   13.7023 -3.4598  11.9724  \n",
       "9   13.6649 -3.5464  11.8917  \n",
       "10  13.6462 -3.5896  11.8514  \n",
       "11  13.6275 -3.6329  11.8110  \n",
       "12  13.6304 -3.7062  11.7773  \n",
       "13  13.6350 -3.7816  11.7442  \n",
       "14  13.6395 -3.8571  11.7110  \n",
       "15  13.6440 -3.9325  11.6778  \n",
       "16  13.6486 -4.0080  11.6446  \n",
       "17  13.6581 -4.0923  11.6119  \n",
       "18  13.7069 -4.3142  11.5498  \n",
       "19  13.7314 -4.4251  11.5188  \n",
       "20  13.7558 -4.5360  11.4878  \n",
       "\n",
       "[21 rows x 34 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make categorical variables numeric\n",
    "encoder = LabelEncoder()\n",
    "# encoder.classes_ = numpy.load('arrays/style_classes.npy')\n",
    "# data['speech_style'] = encoder.fit_transform(data['style'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/preseg_classes.npy')\n",
    "print(encoder.classes_)\n",
    "data['pre_segment'] = encoder.transform(data['pre_segment'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/folseg_classes.npy')\n",
    "data['fol_segment'] = encoder.transform(data['fol_segment'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/vowelR_classes.npy')\n",
    "data['vowelR'] = encoder.transform(data['vowelR'].astype(str))\n",
    "\n",
    "encoder.classes_ = np.load('arrays/vowel_type_classes.npy')\n",
    "data['vowel_type'] = encoder.transform(data['vowel_type'].astype(str))\n",
    "data.head(21)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "token_length = 100\n",
    "num_features = 8\n",
    "\n",
    "def windows(data):\n",
    "    start = 0\n",
    "    end = 0\n",
    "    this_token = data.iloc[end]['token']\n",
    "    while start < len(data):\n",
    "        while end < len(data) and data.iloc[end]['token'] == this_token:\n",
    "            end += 1\n",
    "        yield start, end\n",
    "        start = end\n",
    "        if end >= len(data):\n",
    "            break\n",
    "        this_token = data.iloc[end]['token']\n",
    "        \n",
    "def get_speech_tokens(data, window_size=21, num_features=3):\n",
    "    new_data = pd.DataFrame()  # dataframe to store categorical data\n",
    "    tokens = np.empty((0, window_size, num_features))\n",
    "    labels = np.empty((0))\n",
    "    for start, end in windows(data):\n",
    "        \n",
    "        z2 = data.iloc[start:end]['Z2'].astype(float)\n",
    "        z3 = data.iloc[start:end]['Z3'].astype(float)\n",
    "        z2_z3 = data.iloc[start:end]['Z2-Z3'].astype(float)\n",
    "        z_star = data.iloc[start:end]['Z*'].astype(float)\n",
    "\n",
    "        \n",
    "#         style = np.array(data.iloc[start:end]['speech_style']).astype(int)\n",
    "        pre_seg = np.array(data.iloc[start:end]['pre_segment']).astype(int)\n",
    "        fol_seg = np.array(data.iloc[start:end]['fol_segment']).astype(int)\n",
    "        vowelR = np.array(data.iloc[start:end]['vowelR']).astype(int)\n",
    "        vowel_type = np.array(data.iloc[start:end]['vowel_type']).astype(int)\n",
    "        \n",
    "        if 0 < len(z2) < window_size:\n",
    "            difference = window_size - len(z2)\n",
    "            padding = np.array([0] * difference)\n",
    "            z2 = np.hstack([padding, z2])\n",
    "            z3 = np.hstack([padding, z3])\n",
    "            z2_z3 = np.hstack([padding, z2_z3])\n",
    "            z_star = np.hstack([padding, z_star])\n",
    "            \n",
    "#             style = np.hstack([padding, style])\n",
    "            pre_seg = np.hstack([padding, pre_seg])\n",
    "            fol_seg = np.hstack([padding, fol_seg])\n",
    "            vowelR = np.hstack([padding, vowelR])\n",
    "            vowel_type = np.hstack([padding, vowel_type])\n",
    "            \n",
    "        \n",
    "        if len(z2) == window_size:\n",
    "            # get continuous data\n",
    "            feature_stack = np.dstack([z2, z3, z2_z3, z_star,\n",
    "                                      pre_seg, fol_seg, vowelR, vowel_type]) #, style])\n",
    "            tokens = np.vstack([tokens, feature_stack])\n",
    "            labels = np.append(labels, stats.mode(data.iloc[start:end]['dep_variable'])[0][0])\n",
    "\n",
    "    return tokens, labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Perform Preprocesssing and Reshape the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Segment shape: (3000, 100, 8)\n",
      "X shape: (3000, 100, 8)\n",
      "y shape: (3000,)\n"
     ]
    }
   ],
   "source": [
    "segments, y = get_speech_tokens(data, token_length, num_features)\n",
    "print(\"Segment shape:\", segments.shape)\n",
    "X = segments.reshape((len(segments), token_length, num_features))\n",
    "print(\"X shape:\", X.shape)\n",
    "print(\"y shape:\", y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Arrays to File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"arrays/X_train_bark.npy\", X, allow_pickle=False)\n",
    "np.save(\"arrays/y_train_bark.npy\", y, allow_pickle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}