02 Imputation validation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This notebook validates the imputation method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "sns.set(style=\"ticks\", context=\"talk\")\n",
    "from scipy.stats import ks_2samp\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "nure = pd.read_csv(\"nure.csv\")\n",
    "dataColumns = [\n",
    "    \"u_dn_ppm\",\n",
    "    \"u_fl_ppm\",\n",
    "    \"ag_ppm\",\n",
    "    \"al_pct\",\n",
    "    \"as_ppm\",\n",
    "    \"au_ppm\",\n",
    "    \"b_ppm\",\n",
    "    \"ba_ppm\",\n",
    "    \"be_ppm\",\n",
    "    \"bi_ppm\",\n",
    "    \"ca_pct\",\n",
    "    \"cd_ppm\",\n",
    "    \"ce_ppm\",\n",
    "    \"cl_ppm\",\n",
    "    \"co_ppm\",\n",
    "    \"cr_ppm\",\n",
    "    \"cs_ppm\",\n",
    "    \"cu_ppm\",\n",
    "    \"dy_ppm\",\n",
    "    \"eu_ppm\",\n",
    "    \"fe_pct\",\n",
    "    \"hf_ppm\",\n",
    "    \"k_pct\",\n",
    "    \"la_ppm\",\n",
    "    \"li_ppm\",\n",
    "    \"lu_ppm\",\n",
    "    \"mg_pct\",\n",
    "    \"mn_ppm\",\n",
    "    \"mo_ppm\",\n",
    "    \"na_pct\",\n",
    "    \"nb_ppm\",\n",
    "    \"ni_ppm\",\n",
    "    \"p_ppm\",\n",
    "    \"pb_ppm\",\n",
    "    \"rb_ppm\",\n",
    "    \"sb_ppm\",\n",
    "    \"sc_ppm\",\n",
    "    \"se_ppm\",\n",
    "    \"sm_ppm\",\n",
    "    \"sn_ppm\",\n",
    "    \"sr_ppm\",\n",
    "    \"ta_ppm\",\n",
    "    \"tb_ppm\",\n",
    "    \"th_ppm\",\n",
    "    \"ti_ppm\",\n",
    "    \"v_ppm\",\n",
    "    \"w_ppm\",\n",
    "    \"y_ppm\",\n",
    "    \"yb_ppm\",\n",
    "    \"zn_ppm\",\n",
    "    \"zr_ppm\",\n",
    "]\n",
    "for j in range(len(dataColumns)):\n",
    "    news = []\n",
    "    for i in range(len(nure[dataColumns[j]])):\n",
    "        if nure[dataColumns[j]][i] < 0:\n",
    "            news.append(\n",
    "                np.random.randint(0, np.ceil(abs(nure[dataColumns[j]][i])))\n",
    "            )\n",
    "        else:\n",
    "            news.append(nure[dataColumns[j]][i])\n",
    "    nure[dataColumns[j]] = news"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read in generated data\n",
    "# read in original data, this is the nure variable\n",
    "# read in validation index\n",
    "# use validation index to get generated data and add to new df\n",
    "# use validation index to get original data and add to new df\n",
    "imputedValues = []\n",
    "correctValues = []\n",
    "for i in range(len(dataColumns)):\n",
    "    generated = pd.read_csv(\n",
    "        \"/validation sets/\" + str(dataColumns[i]) + \"_generated.csv\"\n",
    "    )\n",
    "    indicies = pd.read_csv(\"/validation_index \" + str(dataColumns[i]) + \".csv\")\n",
    "    imputedValues.append(\n",
    "        generated.loc[indicies.iloc[0:, 1].values][dataColumns[i]]\n",
    "    )\n",
    "    correctValues.append(\n",
    "        nure.loc[indicies.iloc[0:, 1].values][dataColumns[i]].values\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(dataColumns)):\n",
    "    plt.scatter(imputedValues[i], correctValues[i])\n",
    "plt.plot(range(14000))\n",
    "plt.xlabel(\"Imputed Values\")\n",
    "plt.ylabel(\"Actual Values\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "mae = []\n",
    "meanscorrect = []\n",
    "meansimputed = []\n",
    "ks_results = []\n",
    "for i in range(len(dataColumns)):\n",
    "    mae.append(mean_absolute_error(correctValues[i], imputedValues[i].values))\n",
    "    meanscorrect.append(np.mean(correctValues[i]))\n",
    "    meansimputed.append(np.mean(imputedValues[i]))\n",
    "    ks_results.append(ks_2samp(imputedValues[i], correctValues[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.distplot(imputedValues[5], label=\"Imputed Values\", color=\"red\")\n",
    "sns.distplot(correctValues[5], label=\"Actual Values\", color=\"black\")\n",
    "plt.axvline(mae[5], label=\"Mean Absolute Error\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"Probability\")\n",
    "plt.xlabel(\"Lu Concentration (ppm)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ks_results[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f, axs = plt.subplots(3, 2, figsize=(15, 20))\n",
    "plt.rcParams[\"font.size\"] = 8\n",
    "plt.subplot(321)\n",
    "sns.distplot(imputedValues[25], label=\"Imputed Values\", color=\"red\")\n",
    "sns.distplot(correctValues[25], label=\"Actual Values\", color=\"black\")\n",
    "plt.axvline(mae[25], label=\"Mean Absolute Error\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"Probability\", fontname=\"Arial\")\n",
    "plt.xlabel(\"Lu Concentration (ppm)\", fontname=\"Arial\")\n",
    "\n",
    "plt.subplot(322)\n",
    "plt.scatter(\n",
    "    imputedValues[25],\n",
    "    correctValues[25],\n",
    "    c=abs(imputedValues[25] - correctValues[25]).values,\n",
    "    cmap=\"Reds\",\n",
    ")\n",
    "cbar = plt.colorbar()\n",
    "cbar.set_label(\"Absolute error (ppm)\", labelpad=+1)\n",
    "plt.plot(range(np.int(np.max(correctValues[25])) + 2), c=\"k\", linestyle=\":\")\n",
    "plt.xlabel(\"Imputed Values\", fontname=\"Arial\")\n",
    "plt.ylabel(\"Actual Values\", fontname=\"Arial\")\n",
    "\n",
    "plt.subplot(323)\n",
    "sns.distplot(imputedValues[30], label=\"Imputed Values\", color=\"red\")\n",
    "sns.distplot(correctValues[30], label=\"Actual Values\", color=\"black\")\n",
    "plt.axvline(mae[30], label=\"Mean Absolute Error\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"Probability\", fontname=\"Arial\")\n",
    "plt.xlabel(\"Nb Concentration (ppm)\", fontname=\"Arial\")\n",
    "\n",
    "plt.subplot(324)\n",
    "plt.scatter(\n",
    "    imputedValues[30],\n",
    "    correctValues[30],\n",
    "    c=abs(imputedValues[30] - correctValues[30]).values,\n",
    "    cmap=\"Reds\",\n",
    ")\n",
    "cbar = plt.colorbar()\n",
    "cbar.set_label(\"Absolute error (ppm)\", labelpad=+1)\n",
    "plt.plot(range(np.int(np.max(correctValues[30])) + 2), c=\"k\", linestyle=\":\")\n",
    "plt.xlabel(\"Imputed Values\", fontname=\"Arial\")\n",
    "plt.ylabel(\"Actual Values\", fontname=\"Arial\")\n",
    "\n",
    "plt.subplot(325)\n",
    "sns.distplot(imputedValues[-7], label=\"Imputed Values\", color=\"red\")\n",
    "sns.distplot(correctValues[-7], label=\"Actual Values\", color=\"black\")\n",
    "plt.axvline(mae[-7], label=\"Mean Absolute Error\")\n",
    "plt.legend()\n",
    "plt.ylabel(\"Probability\", fontname=\"Arial\")\n",
    "plt.xlabel(\"Ti Concentration (ppm)\", fontname=\"Arial\")\n",
    "\n",
    "plt.subplot(326)\n",
    "plt.scatter(\n",
    "    imputedValues[-7],\n",
    "    correctValues[-7],\n",
    "    c=abs(imputedValues[-7] - correctValues[-7]).values,\n",
    "    cmap=\"Reds\",\n",
    ")\n",
    "cbar = plt.colorbar()\n",
    "cbar.set_label(\"Absolute error (ppm)\", labelpad=+1)\n",
    "plt.plot(range(np.int(np.max(correctValues[-7])) + 2), c=\"k\", linestyle=\":\")\n",
    "plt.xlabel(\"Imputed Values\", fontname=\"Arial\")\n",
    "plt.ylabel(\"Actual Values\", fontname=\"Arial\")\n",
    "\n",
    "plt.savefig(\"/figure 4.pdf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}