QP_nn_Adam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch.autograd import Variable\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Input examples of various QPs\n",
    "X_np = np.array([[1, 1, 1, 1, 1, 2, 1, 2, 3, 4, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 2, 3, 4, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 2, 3, 4, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 2, 3, 4, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 4, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 4, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 5, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 5, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 5, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 5, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 6, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 6, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 6, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 6, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 3, 2, 3, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 3, 2, 3, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 3, 2, 3, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 1, 3, 2, 3, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 3, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 1, 2, 2, 1, 3, 3, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, 2, 1, 3, 3, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 2, 2, 2, 1, 3, 3, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, 2, 1, 3, 3, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, 2, 1, 3, 3, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -2, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -9, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -2, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -2, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -3, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -4, -3, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -6, -5, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -6, -5, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -6, -5, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -2, -3, -6, -5, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -5, 1, 1, 0, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -5, 0, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -5, 1, 0, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -5, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -11, 1, 1, 1, 1],\n",
    "                [1, 1, 1, 1, 2, 2, -3, -2, -6, -11, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, -3, -2, -6, -11, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 2, 1, 3, 4, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 2, 1, 3, 4, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 2, 1, 3, 4, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 1, 2, 5, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 1, 2, 5, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 1, 2, 5, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 1, 2, 5, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 1, 2, 3, 3, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 1, 2, 3, 3, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 4, 1, 5, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 4, 1, 5, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 4, 1, 5, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 3, 4, 1, 5, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 4, 3, 5, 7, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 4, 3, 5, 7, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 4, 3, 5, 7, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 4, 3, 5, 7, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 2, 3, 1, 6, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 1, 2, 2, 3, 1, 6, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 3, 1, 6, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 3, 1, 6, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 3, 1, 6, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 3, 1, 6, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 3, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 3, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 3, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 3, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 5, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 1, 3, 2, 5, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 1, 3, 5, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 1, 3, 5, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 1, 3, 5, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, 2, 1, 3, 5, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -2, -3, -4, -5, 1, 1, 0, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -2, -3, -4, -5, 0, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -2, -3, -4, -5, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -2, -3, -4, -5, 1, 1, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -3, -2, -1, -6, 1, 0, 1, 1],\n",
    "                [1, 1, 2, 1, 2, 2, -3, -2, -1, -6, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, -3, -2, -1, -6, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, -3, -2, -1, -6, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 2, 1, 3, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 2, 1, 3, 0, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 1, 2, 3, 1, 0, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 1, 2, 3, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 2, 3, 1, 5, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 2, 3, 1, 5, 0, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 2, 3, 1, 5, 1, 0, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 2, 3, 1, 5, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 3, 2, 5, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 3, 2, 5, 0, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 3, 2, 5, 1, 0, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 3, 2, 5, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 2, 3, 6, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 2, 3, 6, 0, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 2, 3, 6, 1, 0, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 1, 2, 3, 6, 1, 1, 1, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 2, 1, 5, 1, 1, 0, 1],\n",
    "                [2, 1, 1, 1, 1, 2, 3, 2, 1, 5, 1, 0, 1, 1]])\n",
    "\n",
    "y_np = np.array([[3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [2],\n",
    "                [2],\n",
    "                [3],\n",
    "                [2],\n",
    "                [3],\n",
    "                [1],\n",
    "                [5],\n",
    "                [1],\n",
    "                [1],\n",
    "                [1],\n",
    "                [4],\n",
    "                [1],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [2],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [2],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [6],\n",
    "                [1],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [2],\n",
    "                [2],\n",
    "                [6],\n",
    "                [2],\n",
    "                [3],\n",
    "                [3],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [5],\n",
    "                [4],\n",
    "                [2],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [4],\n",
    "                [1],\n",
    "                [4],\n",
    "                [5],\n",
    "                [4],\n",
    "                [4],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [6],\n",
    "                [3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [3],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [7],\n",
    "                [6],\n",
    "                [6]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_np shape:  (100, 14)\n",
      "Y_np.shape:  (100, 1)\n"
     ]
    }
   ],
   "source": [
    "print \"X_np shape: \", X_np.shape\n",
    "print \"Y_np.shape: \", y_np.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# N is batch size; D_in is input dimension;\n",
    "# H_{i} is the dimension of the ith hidden layer; D_out is output dimension.\n",
    "\n",
    "N, D_in, H_1, H_2, D_out = 100, 14, 25, 50, 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 1, 1, 2, 1, 2, 0, 4, 0, 0, 0, 3]\n"
     ]
    }
   ],
   "source": [
    "def multi_class_format(numpy_array):\n",
    "    new = []\n",
    "    for item in y_np:\n",
    "        for i in item:\n",
    "            new.append(i)\n",
    "    \n",
    "    class_labels = []\n",
    "    \n",
    "    for i in new:\n",
    "        identifier = i-1\n",
    "        class_labels.append(identifier)\n",
    "        \n",
    "    return class_labels\n",
    "\n",
    "Y_array = multi_class_format(y_np)\n",
    "print Y_array[:25]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nb_classes = 7\n",
    "def get_one_hot(targets, nb_classes):\n",
    "    return np.eye(nb_classes)[np.array(targets).reshape(-1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output shape: (100, 7) \n",
      "[[ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  1.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  1.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  1.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  0.  1.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]\n",
      " [ 0.  0.  0.  0.  0.  1.  0.]]\n"
     ]
    }
   ],
   "source": [
    "y_one_hot = get_one_hot(Y_array, 7)\n",
    "print \"Output shape: {} \".format(y_one_hot.shape)\n",
    "print y_one_hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train: 80, y_train: 80 , X_test: 20 , y_test: 20 \n"
     ]
    }
   ],
   "source": [
    "# Make a train/test split using 20% test size\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_np, y_one_hot, test_size=0.20)\n",
    "print \"X_train: {}, y_train: {} , X_test: {} , y_test: {} \".format(X_train.shape[0], y_train.shape[0], X_test.shape[0], y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set shape: torch.Size([80, 14]) \n",
      "Training labels shape: torch.Size([80, 7]) \n",
      "Test set shape: torch.Size([20, 14]) \n",
      "Test labels shape: torch.Size([20, 7]) \n"
     ]
    }
   ],
   "source": [
    "X_train = torch.from_numpy(X_train).type(torch.FloatTensor)\n",
    "y_train = torch.from_numpy(y_train).type(torch.LongTensor)\n",
    "\n",
    "X_test = torch.from_numpy(X_test).type(torch.FloatTensor)\n",
    "y_test = torch.from_numpy(y_test).type(torch.LongTensor)\n",
    "\n",
    "print \"Training set shape: {} \".format(X_train.size())\n",
    "print \"Training labels shape: {} \".format(y_train.size())\n",
    "\n",
    "print \"Test set shape: {} \".format(X_test.size())\n",
    "print \"Test labels shape: {} \".format(y_test.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "net = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H_1),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H_1, H_2),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H_2, D_out)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Linear(in_features=14, out_features=25)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=25, out_features=50)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=50, out_features=7)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "#net = Net()\n",
    "print net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Converting the training one hot vector labels into indices because PyTorch does not take one-hot vectors\n",
    "targetnp = y_train.numpy()\n",
    "idxs = np.where(targetnp > 0)[1]\n",
    "new_targets = torch.LongTensor(idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Converting the test one hot vector labels into indices\n",
    "test_targetnp = y_test.numpy()\n",
    "test_idxs = np.where(test_targetnp > 0)[1]\n",
    "new_test_targets = torch.LongTensor(test_idxs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create a Adam optimizer\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate, weight_decay = 1e-2)\n",
    "# create a loss function\n",
    "loss_func = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training Epoch: 0 Average loss: 2.0097, Training Accuracy: 6/80 (7.5000%)\n",
      "\n",
      "\n",
      "Training Epoch: 100 Average loss: 1.1309, Training Accuracy: 54/80 (67.5000%)\n",
      "\n",
      "\n",
      "Training Epoch: 200 Average loss: 0.5364, Training Accuracy: 64/80 (80.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 300 Average loss: 0.2915, Training Accuracy: 74/80 (92.5000%)\n",
      "\n",
      "\n",
      "Training Epoch: 400 Average loss: 0.1939, Training Accuracy: 77/80 (96.2500%)\n",
      "\n",
      "\n",
      "Training Epoch: 500 Average loss: 0.1326, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 600 Average loss: 0.1089, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 700 Average loss: 0.0988, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 800 Average loss: 0.0935, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 900 Average loss: 0.0905, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1000 Average loss: 0.0884, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1100 Average loss: 0.0870, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1200 Average loss: 0.0859, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1300 Average loss: 0.0850, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1400 Average loss: 0.0843, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1500 Average loss: 0.0837, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1600 Average loss: 0.0832, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1700 Average loss: 0.0827, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1800 Average loss: 0.0823, Training Accuracy: 80/80 (100.0000%)\n",
      "\n",
      "\n",
      "Training Epoch: 1900 Average loss: 0.0820, Training Accuracy: 80/80 (100.0000%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# run the main training loop\n",
    "epochs = 2000\n",
    "train_loss = 0\n",
    "train_correct = 0\n",
    "test_correct = 0\n",
    "for epoch in range(epochs):\n",
    "        data, target = Variable(X_train), Variable(new_targets)\n",
    "        optimizer.zero_grad()\n",
    "        net_out = net(data)\n",
    "        #print net_out\n",
    "        #print target.data\n",
    "        \n",
    "        #sum up the loss\n",
    "        train_loss = loss_func(net_out, target)\n",
    "        pred = net_out.data.max(1)[1] #get the index of the max log probabitlity\n",
    "        train_correct = pred.eq(target.data).sum()\n",
    "        \n",
    "        train_loss.backward()\n",
    "        optimizer.step()\n",
    "        if epoch % 100 == 0:\n",
    "            print('\\nTraining Epoch: {} Average loss: {:.4f}, Training Accuracy: {}/{} ({:.4f}%)\\n'.format(\n",
    "        epoch, train_loss.data[0], train_correct, len(target),\n",
    "        100. * train_correct / len(target)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test loss: 0.2390, Test accuracy: 19/20 (95.0000%)\n",
      "\n",
      "Time taken to solve 20 problems:  0.000648021697998\n"
     ]
    }
   ],
   "source": [
    "test_loss = 0\n",
    "test_correct = 0\n",
    "\n",
    "#Converting test data and target labels into pyTorch variables\n",
    "test_data, test_target = Variable(X_test), Variable(new_test_targets)\n",
    "\n",
    "tic = time.time()\n",
    "#Test set output\n",
    "net_out = net(test_data)\n",
    "\n",
    "test_loss = loss_func(net_out, test_target)\n",
    "test_pred = net_out.data.max(1)[1] #get the index of max log probability\n",
    "test_correct = test_pred.eq(test_target.data).sum()\n",
    "\n",
    "toc = time.time()\n",
    "time_diff = toc - tic\n",
    "\n",
    "print('\\nTest loss: {:.4f}, Test accuracy: {}/{} ({:.4f}%)\\n'.format(\n",
    "    test_loss.data[0], test_correct, len(test_target), 100. * test_correct / len(test_target)))\n",
    "\n",
    "print \"Time taken to solve 20 problems: \", time_diff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}