DataPreprocessing/1 Handling missing values.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "1d9049f2",
   "metadata": {},
   "source": [
    "## 1. Deletion method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b09a52b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        2064\n",
       "HouseAge      2064\n",
       "AveRooms      2064\n",
       "AveBedrms     2064\n",
       "Population    2064\n",
       "AveOccup      2064\n",
       "Latitude      2064\n",
       "Longitude     2064\n",
       "dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "house = fetch_california_housing()\n",
    "df = pd.DataFrame(house.data, columns=house.feature_names)\n",
    "\n",
    "# introduce missing values\n",
    "df.iloc[::10] = np.nan\n",
    "\n",
    "# Check for missing values\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        0\n",
       "HouseAge      0\n",
       "AveRooms      0\n",
       "AveBedrms     0\n",
       "Population    0\n",
       "AveOccup      0\n",
       "Latitude      0\n",
       "Longitude     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.dropna()\n",
    "df.isnull().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Mean/Median Imputation method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        2064\n",
       "HouseAge      2064\n",
       "AveRooms      2064\n",
       "AveBedrms     2064\n",
       "Population    2064\n",
       "AveOccup      2064\n",
       "Latitude      2064\n",
       "Longitude     2064\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "house = fetch_california_housing()\n",
    "df = pd.DataFrame(house.data, columns=house.feature_names)\n",
    "\n",
    "df.iloc[::10, :] = np.nan\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        0\n",
       "HouseAge      0\n",
       "AveRooms      0\n",
       "AveBedrms     0\n",
       "Population    0\n",
       "AveOccup      0\n",
       "Latitude      0\n",
       "Longitude     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imputer = SimpleImputer(strategy=\"mean\")\n",
    "df_imputed = imputer.fit_transform(df)\n",
    "# df_imputed is now a numpy array\n",
    "\n",
    "# convert it back to pandas df\n",
    "df_imputed = pd.DataFrame(df_imputed, columns=df.columns)\n",
    "\n",
    "df_imputed.isnull().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6cc75788",
   "metadata": {},
   "source": [
    "## Or"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0fb8bc0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        2064\n",
       "HouseAge      2064\n",
       "AveRooms      2064\n",
       "AveBedrms     2064\n",
       "Population    2064\n",
       "AveOccup      2064\n",
       "Latitude      2064\n",
       "Longitude     2064\n",
       "dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "house = fetch_california_housing()\n",
    "df = pd.DataFrame(house.data, columns=house.feature_names)\n",
    "\n",
    "df.iloc[::10] = np.nan\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "84cf1786",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        0\n",
       "HouseAge      0\n",
       "AveRooms      0\n",
       "AveBedrms     0\n",
       "Population    0\n",
       "AveOccup      0\n",
       "Latitude      0\n",
       "Longitude     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace missing values with mean/median\n",
    "df.fillna(df.mean(), inplace=True)\n",
    "df.isnull().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Regression imputation method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MedInc        10\n",
       "HouseAge       0\n",
       "AveRooms       0\n",
       "AveBedrms      0\n",
       "Population     0\n",
       "AveOccup       0\n",
       "Latitude       0\n",
       "Longitude      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "house = fetch_california_housing()\n",
    "df = pd.DataFrame(house.data, columns=house.feature_names)\n",
    "\n",
    "df.iloc[10:20, 0] = np.nan\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.477612</td>\n",
       "      <td>1.079602</td>\n",
       "      <td>910.0</td>\n",
       "      <td>2.263682</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.772480</td>\n",
       "      <td>1.024523</td>\n",
       "      <td>1504.0</td>\n",
       "      <td>2.049046</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.322650</td>\n",
       "      <td>1.012821</td>\n",
       "      <td>1098.0</td>\n",
       "      <td>2.346154</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.097701</td>\n",
       "      <td>345.0</td>\n",
       "      <td>1.982759</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.262903</td>\n",
       "      <td>1.009677</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>1.954839</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.0</td>\n",
       "      <td>4.242424</td>\n",
       "      <td>1.071970</td>\n",
       "      <td>697.0</td>\n",
       "      <td>2.640152</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.939577</td>\n",
       "      <td>1.048338</td>\n",
       "      <td>793.0</td>\n",
       "      <td>2.395770</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.052805</td>\n",
       "      <td>0.966997</td>\n",
       "      <td>648.0</td>\n",
       "      <td>2.138614</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>50.0</td>\n",
       "      <td>5.343675</td>\n",
       "      <td>1.085919</td>\n",
       "      <td>990.0</td>\n",
       "      <td>2.362768</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.465455</td>\n",
       "      <td>1.083636</td>\n",
       "      <td>690.0</td>\n",
       "      <td>2.509091</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
       "10     NaN      52.0  5.477612   1.079602       910.0  2.263682     37.85   \n",
       "11     NaN      52.0  4.772480   1.024523      1504.0  2.049046     37.85   \n",
       "12     NaN      52.0  5.322650   1.012821      1098.0  2.346154     37.85   \n",
       "13     NaN      52.0  4.000000   1.097701       345.0  1.982759     37.84   \n",
       "14     NaN      52.0  4.262903   1.009677      1212.0  1.954839     37.85   \n",
       "15     NaN      50.0  4.242424   1.071970       697.0  2.640152     37.85   \n",
       "16     NaN      52.0  5.939577   1.048338       793.0  2.395770     37.85   \n",
       "17     NaN      52.0  4.052805   0.966997       648.0  2.138614     37.85   \n",
       "18     NaN      50.0  5.343675   1.085919       990.0  2.362768     37.84   \n",
       "19     NaN      52.0  5.465455   1.083636       690.0  2.509091     37.84   \n",
       "\n",
       "    Longitude  \n",
       "10    -122.26  \n",
       "11    -122.26  \n",
       "12    -122.26  \n",
       "13    -122.26  \n",
       "14    -122.26  \n",
       "15    -122.26  \n",
       "16    -122.27  \n",
       "17    -122.27  \n",
       "18    -122.26  \n",
       "19    -122.27  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split the dataset into with and without missing values\n",
    "\n",
    "# .isna.any() -> default axis for any is 0 (columns) i.e.\n",
    "# i.e it shows (null values present or not) corresponding to the columns\n",
    "# change axis = 1 to do it row wise\n",
    "\n",
    "x_missing = df.loc[df.isna().any(axis=1), :]\n",
    "x_no_missing = df.dropna()\n",
    "x_missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_no_missing.drop(columns=[\"MedInc\"])\n",
    "y_train = x_no_missing[\"MedInc\"]\n",
    "x_test = x_missing.drop(columns=[\"MedInc\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LinearRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">LinearRegression</label><div class=\"sk-toggleable__content\"><pre>LinearRegression()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "LinearRegression()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = LinearRegression()\n",
    "model.fit(x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/6s/d0kj1bc14152_w6d8xd7q99h0000gn/T/ipykernel_26829/2405665901.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  x_missing.loc[:, \"MedInc\"] = predicted_missing\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.985553</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.477612</td>\n",
       "      <td>1.079602</td>\n",
       "      <td>910.0</td>\n",
       "      <td>2.263682</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3.490337</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.772480</td>\n",
       "      <td>1.024523</td>\n",
       "      <td>1504.0</td>\n",
       "      <td>2.049046</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4.129700</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.322650</td>\n",
       "      <td>1.012821</td>\n",
       "      <td>1098.0</td>\n",
       "      <td>2.346154</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.374866</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.097701</td>\n",
       "      <td>345.0</td>\n",
       "      <td>1.982759</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3.032323</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.262903</td>\n",
       "      <td>1.009677</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>1.954839</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>2.757510</td>\n",
       "      <td>50.0</td>\n",
       "      <td>4.242424</td>\n",
       "      <td>1.071970</td>\n",
       "      <td>697.0</td>\n",
       "      <td>2.640152</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>4.623283</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.939577</td>\n",
       "      <td>1.048338</td>\n",
       "      <td>793.0</td>\n",
       "      <td>2.395770</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3.030212</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.052805</td>\n",
       "      <td>0.966997</td>\n",
       "      <td>648.0</td>\n",
       "      <td>2.138614</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3.843677</td>\n",
       "      <td>50.0</td>\n",
       "      <td>5.343675</td>\n",
       "      <td>1.085919</td>\n",
       "      <td>990.0</td>\n",
       "      <td>2.362768</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>3.973694</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.465455</td>\n",
       "      <td>1.083636</td>\n",
       "      <td>690.0</td>\n",
       "      <td>2.509091</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.27</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  Latitude  \\\n",
       "10  3.985553      52.0  5.477612   1.079602       910.0  2.263682     37.85   \n",
       "11  3.490337      52.0  4.772480   1.024523      1504.0  2.049046     37.85   \n",
       "12  4.129700      52.0  5.322650   1.012821      1098.0  2.346154     37.85   \n",
       "13  2.374866      52.0  4.000000   1.097701       345.0  1.982759     37.84   \n",
       "14  3.032323      52.0  4.262903   1.009677      1212.0  1.954839     37.85   \n",
       "15  2.757510      50.0  4.242424   1.071970       697.0  2.640152     37.85   \n",
       "16  4.623283      52.0  5.939577   1.048338       793.0  2.395770     37.85   \n",
       "17  3.030212      52.0  4.052805   0.966997       648.0  2.138614     37.85   \n",
       "18  3.843677      50.0  5.343675   1.085919       990.0  2.362768     37.84   \n",
       "19  3.973694      52.0  5.465455   1.083636       690.0  2.509091     37.84   \n",
       "\n",
       "    Longitude  \n",
       "10    -122.26  \n",
       "11    -122.26  \n",
       "12    -122.26  \n",
       "13    -122.26  \n",
       "14    -122.26  \n",
       "15    -122.26  \n",
       "16    -122.27  \n",
       "17    -122.27  \n",
       "18    -122.26  \n",
       "19    -122.27  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Impute missing values using the trained model\n",
    "predicted_missing = model.predict(x_test)\n",
    "\n",
    "x_missing.loc[:, \"MedInc\"] = predicted_missing\n",
    "x_missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MedInc        0\n",
      "HouseAge      0\n",
      "AveRooms      0\n",
      "AveBedrms     0\n",
      "Population    0\n",
      "AveOccup      0\n",
      "Latitude      0\n",
      "Longitude     0\n",
      "dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MedInc</th>\n",
       "      <th>HouseAge</th>\n",
       "      <th>AveRooms</th>\n",
       "      <th>AveBedrms</th>\n",
       "      <th>Population</th>\n",
       "      <th>AveOccup</th>\n",
       "      <th>Latitude</th>\n",
       "      <th>Longitude</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3.985553</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.477612</td>\n",
       "      <td>1.079602</td>\n",
       "      <td>910.0</td>\n",
       "      <td>2.263682</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3.490337</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.772480</td>\n",
       "      <td>1.024523</td>\n",
       "      <td>1504.0</td>\n",
       "      <td>2.049046</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>4.129700</td>\n",
       "      <td>52.0</td>\n",
       "      <td>5.322650</td>\n",
       "      <td>1.012821</td>\n",
       "      <td>1098.0</td>\n",
       "      <td>2.346154</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2.374866</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.097701</td>\n",
       "      <td>345.0</td>\n",
       "      <td>1.982759</td>\n",
       "      <td>37.84</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>3.032323</td>\n",
       "      <td>52.0</td>\n",
       "      <td>4.262903</td>\n",
       "      <td>1.009677</td>\n",
       "      <td>1212.0</td>\n",
       "      <td>1.954839</td>\n",
       "      <td>37.85</td>\n",
       "      <td>-122.26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20635</th>\n",
       "      <td>1.560300</td>\n",
       "      <td>25.0</td>\n",
       "      <td>5.045455</td>\n",
       "      <td>1.133333</td>\n",
       "      <td>845.0</td>\n",
       "      <td>2.560606</td>\n",
       "      <td>39.48</td>\n",
       "      <td>-121.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20636</th>\n",
       "      <td>2.556800</td>\n",
       "      <td>18.0</td>\n",
       "      <td>6.114035</td>\n",
       "      <td>1.315789</td>\n",
       "      <td>356.0</td>\n",
       "      <td>3.122807</td>\n",
       "      <td>39.49</td>\n",
       "      <td>-121.21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20637</th>\n",
       "      <td>1.700000</td>\n",
       "      <td>17.0</td>\n",
       "      <td>5.205543</td>\n",
       "      <td>1.120092</td>\n",
       "      <td>1007.0</td>\n",
       "      <td>2.325635</td>\n",
       "      <td>39.43</td>\n",
       "      <td>-121.22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20638</th>\n",
       "      <td>1.867200</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.329513</td>\n",
       "      <td>1.171920</td>\n",
       "      <td>741.0</td>\n",
       "      <td>2.123209</td>\n",
       "      <td>39.43</td>\n",
       "      <td>-121.32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20639</th>\n",
       "      <td>2.388600</td>\n",
       "      <td>16.0</td>\n",
       "      <td>5.254717</td>\n",
       "      <td>1.162264</td>\n",
       "      <td>1387.0</td>\n",
       "      <td>2.616981</td>\n",
       "      <td>39.37</td>\n",
       "      <td>-121.24</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20640 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         MedInc  HouseAge  AveRooms  AveBedrms  Population  AveOccup  \\\n",
       "10     3.985553      52.0  5.477612   1.079602       910.0  2.263682   \n",
       "11     3.490337      52.0  4.772480   1.024523      1504.0  2.049046   \n",
       "12     4.129700      52.0  5.322650   1.012821      1098.0  2.346154   \n",
       "13     2.374866      52.0  4.000000   1.097701       345.0  1.982759   \n",
       "14     3.032323      52.0  4.262903   1.009677      1212.0  1.954839   \n",
       "...         ...       ...       ...        ...         ...       ...   \n",
       "20635  1.560300      25.0  5.045455   1.133333       845.0  2.560606   \n",
       "20636  2.556800      18.0  6.114035   1.315789       356.0  3.122807   \n",
       "20637  1.700000      17.0  5.205543   1.120092      1007.0  2.325635   \n",
       "20638  1.867200      18.0  5.329513   1.171920       741.0  2.123209   \n",
       "20639  2.388600      16.0  5.254717   1.162264      1387.0  2.616981   \n",
       "\n",
       "       Latitude  Longitude  \n",
       "10        37.85    -122.26  \n",
       "11        37.85    -122.26  \n",
       "12        37.85    -122.26  \n",
       "13        37.84    -122.26  \n",
       "14        37.85    -122.26  \n",
       "...         ...        ...  \n",
       "20635     39.48    -121.09  \n",
       "20636     39.49    -121.21  \n",
       "20637     39.43    -121.22  \n",
       "20638     39.43    -121.32  \n",
       "20639     39.37    -121.24  \n",
       "\n",
       "[20640 rows x 8 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Concatenate the two datasets back together\n",
    "x_imputed = pd.concat([x_missing, x_no_missing], axis=0)\n",
    "\n",
    "\n",
    "print(x_imputed.isnull().sum())\n",
    "x_imputed"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 Using Interpolation method\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "\n",
    "data = load_iris()\n",
    "X = pd.DataFrame(data.data, columns=data.feature_names)\n",
    "y = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add some missing values\n",
    "X.iloc[10:20, 0] = None\n",
    "X.iloc[20:30, 2] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    10\n",
       "sepal width (cm)      0\n",
       "petal length (cm)    10\n",
       "petal width (cm)      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Interpolate missing values using linear interpolation\n",
    "X = X.interpolate(method=\"linear\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    0\n",
       "sepal width (cm)     0\n",
       "petal length (cm)    0\n",
       "petal width (cm)     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.isnull().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 5. Using multiple interpolation methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "\n",
    "data = load_iris()\n",
    "X = pd.DataFrame(data.data, columns=data.feature_names)\n",
    "y = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sepal length (cm)</th>\n",
       "      <th>sepal width (cm)</th>\n",
       "      <th>petal length (cm)</th>\n",
       "      <th>petal width (cm)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sepal length (cm)  sepal width (cm)  petal length (cm)  petal width (cm)\n",
       "0                5.1               3.5                1.4               0.2\n",
       "1                4.9               3.0                1.4               0.2\n",
       "2                4.7               3.2                1.3               0.2\n",
       "3                4.6               3.1                1.5               0.2\n",
       "4                5.0               3.6                1.4               0.2"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    10\n",
       "sepal width (cm)     10\n",
       "petal length (cm)     0\n",
       "petal width (cm)      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.iloc[10:20, 0:2] = np.nan\n",
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "X[\"sepal length (cm)\"] = X[\"sepal length (cm)\"].interpolate(method=\"linear\")\n",
    "X[\"sepal width (cm)\"] = X[\"sepal width (cm)\"].interpolate(method=\"quadratic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    0\n",
       "sepal width (cm)     0\n",
       "petal length (cm)    0\n",
       "petal width (cm)     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.isnull().sum()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 6. Multiple Imputation technique\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_iris\n",
    "from sklearn.experimental import enable_iterative_imputer\n",
    "from sklearn.impute import IterativeImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    25\n",
       "sepal width (cm)      0\n",
       "petal length (cm)     0\n",
       "petal width (cm)      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = load_iris()\n",
    "X = pd.DataFrame(data.data, columns=data.feature_names)\n",
    "\n",
    "X.iloc[0:100:4, 0] = None\n",
    "X.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sepal length (cm)    0\n",
       "sepal width (cm)     0\n",
       "petal length (cm)    0\n",
       "petal width (cm)     0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Impute missing values using IterativeImputer\n",
    "imp = IterativeImputer(max_iter=10, random_state=0)\n",
    "X_imputed = imp.fit_transform(X)\n",
    "\n",
    "pd.DataFrame(X_imputed, columns=X.columns).isnull().sum()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}