tfdv_Jensen_Shannon_divergence_with_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "687cf629-43e3-4a4f-bf68-178225f9e5ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Docs\n",
    "# https://cloud.google.com/architecture/ml-modeling-monitoring-analyzing-training-server-skew-in-ai-platform-prediction-with-tfdv\n",
    "# https://towardsdatascience.com/how-to-compare-two-or-more-distributions-9b06ee4d30bf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d96a6a32-deda-4850-8f98-68be8e79ccae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_data_validation as tfdv\n",
    "from tensorflow_data_validation.utils import slicing_util\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.datasets import make_classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b583c022-b2f4-4d44-9de5-f69bf519e923",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set parameters\n",
    "P = ! gcloud config list --format 'value(core.project)'\n",
    "PROJECT_ID = P[0]\n",
    "P = ! gcloud projects list --filter=\"$(gcloud config get-value project)\" --format=\"value(PROJECT_NUMBER)\"\n",
    "PROJECT_NUMBER = P[0]\n",
    "REGION = \"us-central1\"\n",
    "SERVICE_ACCOUNT = f\"sa-vertex-pipelines@{PROJECT_ID}.iam.gserviceaccount.com\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d0a5f14-0e52-4c8e-a0db-69b10469a204",
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "#\n",
    "# for this exercise, generate synthetic data using sklearn\n",
    "#\n",
    "#####################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cab8434b-965b-480a-912d-665f1941d71b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set for training and eval\n",
    "n_classes=2\n",
    "n_samples=10000\n",
    "n_features=10\n",
    "n_informative=10\n",
    "n_redundant=0\n",
    "n_repeated=0\n",
    "\n",
    "# training\n",
    "RANDOM_STATE_TRAIN = 34098\n",
    "# eval\n",
    "RANDOM_STATE_EVAL = 78392"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "935d1d54-1873-46eb-8779-bea1f56d7be5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create training data\n",
    "x, y = make_classification(n_classes=n_classes\n",
    "                           , n_samples=n_samples\n",
    "                           , n_features=n_features\n",
    "                           , n_informative=n_informative\n",
    "                           , n_redundant=n_redundant\n",
    "                           , n_repeated=n_repeated\n",
    "                           , random_state=RANDOM_STATE_TRAIN)\n",
    "\n",
    "training_data = np.concatenate((x, y[:,None]), axis=1)\n",
    "num_cols = x.shape[1]\n",
    "col_names = ','.join([f\"x_{i}\" for i in range(num_cols)] + ['label'])\n",
    "col_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95c00a7a-c1ad-4b35-8bc3-6978285fec60",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save training data to csv\n",
    "TRAIN_DATA = 'train.csv'\n",
    "np.savetxt( TRAIN_DATA\n",
    "           , training_data\n",
    "           , delimiter=','\n",
    "           , fmt='%f'\n",
    "           , header=col_names\n",
    "           , comments=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b50ccd6-7e2a-4cd0-8cdf-6da209d09065",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create eval data\n",
    "x, y = make_classification(n_classes=n_classes\n",
    "                           , n_samples=n_samples\n",
    "                           , n_features=n_features\n",
    "                           , n_informative=n_informative\n",
    "                           , n_redundant=n_redundant\n",
    "                           , n_repeated=n_repeated\n",
    "                           , random_state=RANDOM_STATE_EVAL)\n",
    "\n",
    "eval_data = np.concatenate((x, y[:,None]), axis=1)\n",
    "num_cols = x.shape[1]\n",
    "col_names = ','.join([f\"x_{i}\" for i in range(num_cols)] + ['label'])\n",
    "col_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95e7f7e6-0f8b-4600-98c2-71c35b0e57d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save eval data to csv\n",
    "EVAL_DATA = 'eval.csv'\n",
    "np.savetxt( EVAL_DATA\n",
    "           , eval_data\n",
    "           , delimiter=','\n",
    "           , fmt='%f'\n",
    "           , header=col_names\n",
    "           , comments=\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "61f2b6a4-cf16-4caf-93c7-fa4a5e6039e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "#\n",
    "# compute and viz\n",
    "#\n",
    "#####################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac40fdb4-d3bf-4b5d-b6a3-1ba21021c5c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compute stats for training data\n",
    "train_stats = tfdv.generate_statistics_from_csv(data_location=TRAIN_DATA)\n",
    "# Compute stats for evaluation data\n",
    "eval_stats = tfdv.generate_statistics_from_csv(data_location=EVAL_DATA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "341673bc-fb99-438b-b59d-1a63466860ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# visualize just one ds\n",
    "tfdv.visualize_statistics(train_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "565c57eb-85d6-4d92-b8b4-e413f6c736a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compare evaluation data with training data\n",
    "tfdv.visualize_statistics(lhs_statistics=eval_stats, rhs_statistics=train_stats,\n",
    "                          lhs_name='EVAL_DATASET', rhs_name='TRAIN_DATASET')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db53282d-2f3e-4031-838f-836fe1d0ca76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "#\n",
    "#  Jensen-Shannon divergence: Skew\n",
    "#\n",
    "#####################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "213b07d7-5686-452e-a896-bf21e843af96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate schema from training data\n",
    "schema = tfdv.infer_schema(statistics=train_stats)\n",
    "tfdv.display_schema(schema=schema)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b36a8c8f-0974-43d0-b56e-62444fa592e1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check eval data for errors by validating against previously inferred schema.\n",
    "anomalies = tfdv.validate_statistics(statistics=eval_stats, schema=schema)\n",
    "tfdv.display_anomalies(anomalies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f82c5016-0029-4c9d-a44b-e6b14caf92e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set comparators for skew (training versus serving)\n",
    "x_0 = tfdv.get_feature(schema, 'x_0')\n",
    "x_0.skew_comparator.jensen_shannon_divergence.threshold = 0.01\n",
    "\n",
    "#note: serving_statistics in next line relates to skew_comparator above\n",
    "skew_anomalies = tfdv.validate_statistics(train_stats, schema, serving_statistics=eval_stats)\n",
    "tfdv.display_anomalies(skew_anomalies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9925486f-62c1-4797-a41c-e9b24acc7f6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "#\n",
    "#  experiment with the threshold\n",
    "#\n",
    "#####################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0943ff28-423e-4190-9001-10273cfb1146",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import seaborn\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e0abb9c-b9b1-4a05-b8be-5cbcdd354584",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select a column\n",
    "col_index = 8\n",
    "df_train = pd.DataFrame(training_data[:,col_index], columns=['data'])\n",
    "df_train[\"partition\"] = \"train\"\n",
    "\n",
    "df_eval = pd.DataFrame(eval_data[:,col_index], columns=['data'])\n",
    "df_eval[\"partition\"] = \"eval\"\n",
    "\n",
    "df = pd.concat([df_train, df_eval], ignore_index=True, axis=0)\n",
    "sns.histplot(data=df, x='data', hue='partition', bins=50);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "950046ae-aad4-458b-b2b5-67045104edba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# set comparators for skew\n",
    "schema = tfdv.infer_schema(statistics=train_stats) # resets the table below\n",
    "x_n = tfdv.get_feature(schema, f\"x_{col_index}\")\n",
    "x_n.skew_comparator.jensen_shannon_divergence.threshold = 0.001\n",
    "\n",
    "#note: serving_statistics in next line relates to skew_comparator above\n",
    "skew_anomalies = tfdv.validate_statistics(train_stats, schema, serving_statistics=eval_stats)\n",
    "tfdv.display_anomalies(skew_anomalies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba3d6ebb-dd33-40f8-a1ae-36826a01f602",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-8.m103",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-8:m103"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}