4_PyTorch for Deep Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DLVC 2017\n",
    "# Tutorial 4 : PyTorch for Deep Neural Networks\n",
    "\n",
    "## Follow instructions given in the PyTorch [website](http://pytorch.org/) for installation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tensor initialization and slicing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Importing the library\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining tensors\n",
    "x = torch.Tensor(2,2) # Uninitialized (garbage value present in memory)\n",
    "y = torch.rand(2,2) # Random initialization\n",
    "print x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Obtaining size of tensors\n",
    "print x.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Arithmetic operations\n",
    "z = torch.rand(2,2)\n",
    "print y,z,y+z\n",
    "print z,2*z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Indexing is similar to numpy indexing\n",
    "print y\n",
    "print y[1,1]\n",
    "print y[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting to numpy\n",
    "y_np = y.numpy()\n",
    "print y\n",
    "print y_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# Converting from numpy to tensor\n",
    "x_np = np.ones((3,3))\n",
    "x_py = torch.from_numpy(x_np)\n",
    "print x_np,x_py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Improving computational time with GPU acceleration\n",
    "import time\n",
    "use_gpu = torch.cuda.is_available()\n",
    "x = torch.randn(10000,10000)\n",
    "if use_gpu:\n",
    "    x = torch.randn(100,100)\n",
    "    cpuStart = time.time()\n",
    "    y = x*x\n",
    "    cpuEnd = time.time()-cpuStart\n",
    "    x = x.cuda()\n",
    "    gpuStart = time.time()\n",
    "    y = x*x\n",
    "    gpuEnd = time.time()-gpuStart\n",
    "    print('CPU computation completed in {:.6f}s, GPU computation completed in {:.6f}s'\\\n",
    "          .format(cpuEnd,gpuEnd))    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Autograd\n",
    " This package provides automatic differentiation for all operations on Tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "a = torch.ones(2,2)\n",
    "a_var = Variable(a,requires_grad=True)\n",
    "print a\n",
    "print(a_var)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b = a_var+2\n",
    "print b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(b.grad_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = b*b*3\n",
    "d = c.mean()\n",
    "print c\n",
    "print d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "d.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gradients\n",
    "print a_var.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Torchvision datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import torch\n",
    "from torchvision import datasets,transforms\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_transform = transforms.Compose([transforms.ToTensor()])\n",
    "trainDset = datasets.MNIST('./MNIST',train=True, download=True, transform= apply_transform)\n",
    "testDset = datasets.MNIST('./MNIST',train=False, download=True, transform= apply_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Number of samples\n",
    "print(len(trainDset),len(testDset))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Displaying sample image from the dataset\n",
    "img = trainDset[0][0].numpy().transpose(1,2,0).squeeze(2)\n",
    "plt.imshow(img,'gray')\n",
    "print('Label: '+str(trainDset[0][1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Creating dataloader for loading data in batches\n",
    "trainLoader = torch.utils.data.DataLoader(trainDset, batch_size=10, shuffle=True, num_workers=1, pin_memory=False)\n",
    "testLoader = torch.utils.data.DataLoader(testDset, batch_size=10, shuffle=True, num_workers=1, pin_memory=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining a multi-layer perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(MLP, self).__init__()\n",
    "        self.fc1 = nn.Linear(28*28,100)\n",
    "        self.fc2 = nn.Linear(100,10)\n",
    "    def forward(self, x):\n",
    "        x = self.fc1(x)\n",
    "        x = F.sigmoid(x)\n",
    "        x = self.fc2(x)\n",
    "        x = F.log_softmax(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = MLP()\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "params = list(net.parameters())\n",
    "print('No. of parameters :'+str(len(params)))\n",
    "print('Dimensions of first parameter: '+str(params[0].size())) # Weights of fc1\n",
    "print('Dimensions of second parameter: '+str(params[1].size())) # Biases of fc1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inp = trainLoader.dataset[0][0]\n",
    "label = trainLoader.dataset[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.autograd import Variable\n",
    "# Feed-forward data through network\n",
    "out = net(Variable(inp.view(-1,28*28)))\n",
    "print(inp.size())\n",
    "print(out.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Backpropagating gradients\n",
    "net.zero_grad()\n",
    "out.backward(torch.randn(1, 10)) # Using random gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "out = net(Variable(inp.view(-1,28*28)))\n",
    "# Defining loss function\n",
    "criterion = nn.NLLLoss() # Negative log-likelihood loss\n",
    "label = label*torch.ones(1) # Converting to tensor\n",
    "loss = criterion(out,Variable(label.long())) # NLLLoss() expects the labels to be of dtype 'long'\n",
    "print(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Backprogattion\n",
    "net.zero_grad()     # zeroes the gradient buffers of all parameters\n",
    "\n",
    "print('Bias gradient of fc1 before backward')\n",
    "print(net.fc1.bias.grad[:10])\n",
    "\n",
    "loss.backward()\n",
    "\n",
    "print('Bias gradient of fc1 after backward')\n",
    "print(net.fc1.bias.grad[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "# Updataing weights of the network\n",
    "learning_rate = 1\n",
    "init_params = copy.deepcopy(net.fc2.weight.data) # Copying initial parameters\n",
    "\n",
    "for f in net.parameters():    \n",
    "    f.data.sub_(f.grad.data * learning_rate)\n",
    "updated_params = net.fc2.weight.data\n",
    "print(init_params[0,:5])\n",
    "print(updated_params[0,:5])\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}