Proj1_Data_Gen.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keras 2.0.6\n",
      "tensorflow 1.2.1\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "## To get helper functions from another folder\n",
    "# sys.path.insert(0, '../') # if _helper_basics_ is in previous folder\n",
    "now_i_am_at = 'dso' # home dso test\n",
    "if now_i_am_at=='home': sys.path.insert(0, 'E:/Leonard HDD/Dropbox/DSO/Tasks/')\n",
    "elif now_i_am_at=='dso': sys.path.insert(0, 'D:/Dropbox/DSO/Tasks')\n",
    "\n",
    "from _helper_basics_ import *\n",
    "\n",
    "%matplotlib inline\n",
    "plt.rcParams['savefig.dpi'] = 100\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "print('keras', keras.__version__)\n",
    "print('tensorflow', tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from _helper_basics_ import *\n",
    "%reload_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Archi_dir = \"\"\n",
    "Weights_path = \"Logs/\"\n",
    "if not os.path.exists(Weights_path): os.mkdir(Weights_path)\n",
    "Ckpt_Mod_Weights_fold = Archi_dir+\"Checkpoint_Model_Weights/\"\n",
    "if not os.path.exists(Ckpt_Mod_Weights_fold): os.mkdir(Ckpt_Mod_Weights_fold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Addition_Data(input_classes ,sets_per_class ,num_classes):\n",
    "    data_len = num_classes*sets_per_class\n",
    "\n",
    "    y = np.repeat(np.arange(num_classes), sets_per_class).reshape(data_len).astype(np.float16)\n",
    "    x1 = np.zeros((data_len,1)).astype(np.float16)\n",
    "    x2 = np.zeros((data_len,1)).astype(np.float16)\n",
    "\n",
    "    for idx in range(data_len):\n",
    "        ## given y, chose random x1, then get the corresponding x2\n",
    "        y_i = y[idx]\n",
    "        if y_i == 0: \n",
    "            # x1,x2 already init as 0.\n",
    "            continue\n",
    "        elif y_i == (num_classes-1):\n",
    "            x1[idx,0] = input_classes-1\n",
    "            x2[idx,0] = input_classes-1\n",
    "            continue\n",
    "\n",
    "        x_i = np.round( np.random.rand(1)*(input_classes-1) )\n",
    "        x_j = y_i-x_i\n",
    "        count = 0\n",
    "        while (x_j<0) or (x_j>(input_classes-1)):\n",
    "            x_i = np.round( np.random.rand(1)*(input_classes-1) )\n",
    "            x_j = y_i-x_i\n",
    "            count += 1\n",
    "        x1[idx,0] = x_i\n",
    "        x2[idx,0] = x_j\n",
    "        \n",
    "    return x1,x2,y,data_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x1.shape (100500, 1)\n",
      "x2.shape (100500, 1)\n",
      "y.shape (100500,)\n"
     ]
    }
   ],
   "source": [
    "## Training Set\n",
    "input_classes = 100+1\n",
    "sets_per_class = 500\n",
    "num_classes = 200+1\n",
    "x1,x2,y,data_len = Addition_Data(input_classes ,sets_per_class ,num_classes)\n",
    "print('x1.shape',x1.shape)\n",
    "print('x2.shape',x2.shape)\n",
    "print('y.shape',y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split into Training Val Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x1.shape (100500, 1)\n",
      "x2.shape (100500, 1)\n",
      "y.shape (100500,)\n"
     ]
    }
   ],
   "source": [
    "indices = np.arange(data_len)\n",
    "np.random.shuffle(indices)\n",
    "x1,x2,y = x1[indices],x2[indices],y[indices]\n",
    "print('x1.shape',x1.shape)\n",
    "print('x2.shape',x2.shape)\n",
    "print('y.shape',y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape (75375, 2)\n",
      "Y_train.shape (75375, 201)\n"
     ]
    }
   ],
   "source": [
    "x1_trg,x2_trg = x1[data_len//4:],x2[data_len//4:]\n",
    "X_train = np.append(x1_trg,x2_trg,axis=1)\n",
    "y_trg = y[data_len//4:]\n",
    "Y_train = to_categorical(y_trg, num_classes=num_classes)\n",
    "print('X_train.shape',X_train.shape)\n",
    "print('Y_train.shape',Y_train.shape)\n",
    "\n",
    "## Check for Error \n",
    "for idx in range(Y_train.shape[0]):\n",
    "    if not X_train[idx][0]+X_train[idx][1] == np.argmax(Y_train[idx]):\n",
    "        print('\\nError idx              ', idx)\n",
    "        print('X_train[idx]           ', X_train[idx][0]+X_train[idx][1], X_train[idx])\n",
    "        print('y[0,idx]               ', y[0,idx])\n",
    "        print('np.argmax(Y_train[idx])', np.argmax(Y_train[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_val.shape (25125, 2)\n",
      "Y_val.shape (25125, 201)\n"
     ]
    }
   ],
   "source": [
    "X_val = np.append(x1[:data_len//4],x2[:data_len//4],axis=1)\n",
    "y_v = y[:data_len//4]\n",
    "Y_val = to_categorical(y_v, num_classes=num_classes)\n",
    "print('X_val.shape',X_val.shape)\n",
    "print('Y_val.shape',Y_val.shape)\n",
    "\n",
    "## Check for Error \n",
    "for idx in range(Y_val.shape[0]):\n",
    "    if not X_val[idx][0]+X_val[idx][1] == np.argmax(Y_val[idx]):\n",
    "        print('\\nError idx              ', idx)\n",
    "        print('X_val[idx]           ', X_val[idx][0]+X_val[idx][1], X_val[idx])\n",
    "        print('y[0,idx]               ', y[0,idx])\n",
    "        print('np.argmax(Y_val[idx])', np.argmax(Y_val[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train[idx]            42.0 [  4.  38.]\n",
      "np.argmax(Y_train[idx]) 42\n",
      "X_val[idx]            136.0 [ 90.  46.]\n",
      "np.argmax(Y_val[idx]) 136\n"
     ]
    }
   ],
   "source": [
    "idx = 200\n",
    "print('X_train[idx]           ', X_train[idx][0]+X_train[idx][1], X_train[idx])\n",
    "print('np.argmax(Y_train[idx])',np.argmax(Y_train[idx]))\n",
    "\n",
    "idx = 200\n",
    "print('X_val[idx]           ', X_val[idx][0]+X_val[idx][1], X_val[idx])\n",
    "print('np.argmax(Y_val[idx])',np.argmax(Y_val[idx]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape (75375, 2)\n",
      "Y_train.shape (75375, 201)\n",
      "X_val.shape (25125, 2)\n",
      "Y_val.shape (25125, 201)\n"
     ]
    }
   ],
   "source": [
    "pickle.dump(X_train, open('Addition_Data/X_train', 'wb'))\n",
    "pickle.dump(Y_train, open('Addition_Data/Y_train', 'wb'))\n",
    "pickle.dump(X_val, open('Addition_Data/X_val', 'wb'))\n",
    "pickle.dump(Y_val, open('Addition_Data/Y_val', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = pickle.load( open('X_train', 'rb'))\n",
    "Y_train = pickle.load( open('Y_train', 'rb'))\n",
    "X_val = pickle.load( open('X_val', 'rb'))\n",
    "Y_val = pickle.load( open('Y_val', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train.shape (75375, 2)\n",
      "Y_train.shape (75375, 201)\n",
      "X_val.shape (25125, 2)\n",
      "Y_val.shape (25125, 201)\n"
     ]
    }
   ],
   "source": [
    "print('X_train.shape',X_train.shape)\n",
    "print('Y_train.shape',Y_train.shape)\n",
    "print('X_val.shape',X_val.shape)\n",
    "print('Y_val.shape',Y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove numbers from the Training Data\n",
    "Next step to remove prime numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(indices_50) 1766\n",
      "len(indices_50) 1322\n",
      "len(indices_50) 1134\n",
      "len(indices_50) 982\n",
      "len(indices_50) 966\n",
      "len(indices_50) 1047\n",
      "len(indices_50) 1110\n",
      "len(indices_50) 1312\n",
      "len(indices_50) 1663\n",
      "\n",
      "X_train_50.shape (64144, 2)\n",
      "Y_train_removed.shape (64144, 201)\n",
      "X_train.shape    (75375, 2)\n",
      "Y_train.shape    (75375, 201)\n"
     ]
    }
   ],
   "source": [
    "## To remove a list of from the X_train\n",
    "list_remove = [10,20,30,40,50,60,70,80,90]\n",
    "\n",
    "X_train_removed = X_train+0.\n",
    "Y_train_removed = Y_train+0.\n",
    "\n",
    "for idx_remove in list_remove:\n",
    "    indices_50 = list(np.where(X_train_removed[:,0]==idx_remove)[0]) + list(np.where(X_train_removed[:,1]==idx_remove)[0])\n",
    "    print('len(indices_50)',len(indices_50))\n",
    "    X_train_removed = np.delete(X_train_removed, indices_50 , axis=0)\n",
    "    Y_train_removed = np.delete(Y_train_removed, indices_50 , axis=0)\n",
    "print('\\nX_train_50.shape',X_train_removed.shape)\n",
    "print('Y_train_removed.shape',Y_train_removed.shape)\n",
    "print('X_train.shape   ',X_train.shape)\n",
    "print('Y_train.shape   ',Y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_removed[idx]            153.0 [ 89.  64.]\n",
      "np.argmax(Y_train_removed[idx]) 153\n"
     ]
    }
   ],
   "source": [
    "## Check for Error \n",
    "for idx_remove in list_remove:\n",
    "    for idx in range(Y_train_removed.shape[0]):\n",
    "        if X_train_removed[idx,0]==idx_remove or X_train_removed[idx,1]==idx_remove:\n",
    "            print('\\nError idx               ', idx)\n",
    "            print('X_train_removed[idx]           ', X_train_removed[idx][0]+X_train_removed[idx][1], X_train_removed[idx])\n",
    "            print('np.argmax(Y_train_removed[idx])', np.argmax(Y_train_removed[idx]))\n",
    "\n",
    "idx = -1\n",
    "print('X_train_removed[idx]           ', X_train_removed[idx,0]+X_train_removed[idx,1], X_train_removed[idx])\n",
    "print('np.argmax(Y_train_removed[idx])',np.argmax(Y_train_removed[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(X_train_removed, open('X_train_removed', 'wb'))\n",
    "pickle.dump(Y_train_removed, open('Y_train_removed', 'wb'))\n",
    "\n",
    "# pickle.dump(X_train_50, open('Addition_Data/X_train_tens_removed', 'wb'))\n",
    "# pickle.dump(Y_train_50, open('Addition_Data/Y_train_tens_removed', 'wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train_removed = pickle.load( open('X_train_removed', 'rb'))\n",
    "Y_train_removed = pickle.load( open('Y_train_removed', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_removed.shape (64144, 2)\n",
      "Y_train_removed.shape (64144, 201)\n"
     ]
    }
   ],
   "source": [
    "print('X_train_removed.shape',X_train_removed.shape)\n",
    "print('Y_train_removed.shape',Y_train_removed.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}