logistic_regression_with_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bare-Bones Implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "from torch import Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"./data/Iris.csv\").drop(\"Id\", axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SepalLengthCm</th>\n",
       "      <th>SepalWidthCm</th>\n",
       "      <th>PetalLengthCm</th>\n",
       "      <th>PetalWidthCm</th>\n",
       "      <th>Species</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5.1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4.9</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4.7</td>\n",
       "      <td>3.2</td>\n",
       "      <td>1.3</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.6</td>\n",
       "      <td>3.1</td>\n",
       "      <td>1.5</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.6</td>\n",
       "      <td>1.4</td>\n",
       "      <td>0.2</td>\n",
       "      <td>Iris-setosa</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SepalLengthCm  SepalWidthCm  PetalLengthCm  PetalWidthCm      Species\n",
       "0            5.1           3.5            1.4           0.2  Iris-setosa\n",
       "1            4.9           3.0            1.4           0.2  Iris-setosa\n",
       "2            4.7           3.2            1.3           0.2  Iris-setosa\n",
       "3            4.6           3.1            1.5           0.2  Iris-setosa\n",
       "4            5.0           3.6            1.4           0.2  Iris-setosa"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_numpy = data.drop(\"Species\", axis=1).values\n",
    "\n",
    "target_map = {val: index for index, val in enumerate(data.Species.unique())}\n",
    "y_numpy = data.Species.map(target_map).values\n",
    "\n",
    "X = torch.tensor(X_numpy, dtype=torch.float32)\n",
    "y = torch.tensor(y_numpy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Iris-setosa': 0, 'Iris-versicolor': 1, 'Iris-virginica': 2}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_map"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### One-Hot-Encode Class Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(vector):\n",
    "    n_classes = len(vector.unique())\n",
    "    one_hot = torch.zeros((vector.shape[0], n_classes))\\\n",
    "        .type(torch.LongTensor)  # 1\n",
    "    return one_hot.scatter(\n",
    "        1, vector.type(torch.LongTensor).unsqueeze(1), 1\n",
    "    )\n",
    "\n",
    "\n",
    "y_one_hot = one_hot_encode(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Train and Test Sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x1295bd230>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_indices = torch.randperm(X.shape[0])\n",
    "\n",
    "n_train = int(0.8 * X.shape[0])\n",
    "X_train = X[random_indices[:n_train]]\n",
    "y_train = y[random_indices[:n_train]]\n",
    "y_train_one_hot = y_one_hot[random_indices[:n_train]]\n",
    "\n",
    "X_test = X[random_indices[n_train:]]\n",
    "y_test = y[random_indices[n_train:]]\n",
    "y_test_one_hot = y_one_hot[random_indices[n_train:]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = torch.rand((4, 3))\n",
    "b = torch.rand(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2796, 0.1976, 0.3208],\n",
       "        [0.7487, 0.3949, 0.4665],\n",
       "        [0.2918, 0.6943, 0.9894],\n",
       "        [0.5497, 0.1376, 0.7568]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.6119, 0.7690, 0.1274])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Softmax Activation and Cross Entropy Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_activation(z):\n",
    "    exponentials = torch.exp(z)\n",
    "    exponentials_row_sums = torch.sum(exponentials, axis=1).unsqueeze(1)\n",
    "    return exponentials / exponentials_row_sums\n",
    "\n",
    "def cross_entropy_loss(y_one_hot, activations):\n",
    "    return -torch.mean(torch.sum(y_one_hot * torch.log(activations), axis=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Simple Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1: 1.1546878814697266\n",
      "Loss at iteration 25: 0.7025735378265381\n",
      "Loss at iteration 50: 0.5796783566474915\n",
      "Loss at iteration 75: 0.5690209269523621\n",
      "Loss at iteration 100: 0.49405404925346375\n",
      "Loss at iteration 125: 0.47681987285614014\n",
      "Loss at iteration 150: 0.43222129344940186\n",
      "Loss at iteration 175: 0.41156941652297974\n",
      "Loss at iteration 200: 0.3894343376159668\n",
      "Loss at iteration 225: 0.37786394357681274\n",
      "Loss at iteration 250: 0.3720541298389435\n",
      "\n",
      "Final Test Accuracy: 0.9666666666666667\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 250\n",
    "learning_rate = 0.1\n",
    "lambda_param = 0.01\n",
    "for i in range(1, n_iterations + 1):\n",
    "\n",
    "    Z = torch.mm(X_train, w) + b\n",
    "    A = softmax_activation(Z)\n",
    "    l2_regularization = torch.sum(w ** 2)\n",
    "    loss = cross_entropy_loss(y_train_one_hot, A) \\\n",
    "           + lambda_param * l2_regularization\n",
    "    w_gradients = -torch.mm(X_train.transpose(0, 1), y_train_one_hot - A) / n_train \\\n",
    "                  + (2 * lambda_param * w)\n",
    "    b_gradients = -torch.mean(y_train_one_hot - A, axis=0)\n",
    "\n",
    "    w -= learning_rate * w_gradients\n",
    "    b -= learning_rate * b_gradients\n",
    "\n",
    "    if i == 1 or i % 25 == 0:\n",
    "        print(\"Loss at iteration {}: {}\".format(i, loss))\n",
    "\n",
    "test_predictions = torch.argmax(\n",
    "    softmax_activation(torch.mm(X_test, w) + b), axis=1\n",
    ")\n",
    "test_accuracy = float(sum(test_predictions == y_test)) / y_test.shape[0]\n",
    "print(\"\\nFinal Test Accuracy: {}\".format(test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Free Differentiation with PyTorch's Autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "w_autograd = torch.rand((4, 3), requires_grad=True)\n",
    "b_autograd = torch.rand(3, requires_grad=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "Z = torch.mm(X, w_autograd) + b_autograd\n",
    "A = softmax_activation(Z)\n",
    "lambda_param = 0.01\n",
    "l2_regularization = torch.sum(w_autograd ** 2)\n",
    "loss = cross_entropy_loss(y_one_hot, A) + lambda_param * l2_regularization\n",
    "loss.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.9056, -1.9042,  2.8366],\n",
       "        [-0.7511, -0.8797,  1.6608],\n",
       "        [ 0.0596, -1.3847,  1.3582],\n",
       "        [ 0.1005, -0.4149,  0.3504]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w_autograd.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.9056, -1.9042,  2.8366],\n",
       "        [-0.7511, -0.8797,  1.6608],\n",
       "        [ 0.0596, -1.3847,  1.3582],\n",
       "        [ 0.1005, -0.4149,  0.3504]], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-torch.mm(X.transpose(0, 1), y_one_hot - A) / X.shape[0] \\\n",
    "    + 2 * lambda_param * w_autograd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.2048, -0.3209,  0.5257])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b_autograd.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.2048, -0.3209,  0.5257], grad_fn=<NegBackward>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-torch.mean(y_one_hot - A, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1: 2.50476336479187\n",
      "Loss at iteration 25: 0.8208405375480652\n",
      "Loss at iteration 50: 0.6555004119873047\n",
      "Loss at iteration 75: 0.6429731845855713\n",
      "Loss at iteration 100: 0.5476508140563965\n",
      "Loss at iteration 125: 0.5351016521453857\n",
      "Loss at iteration 150: 0.4730842113494873\n",
      "Loss at iteration 175: 0.4522818326950073\n",
      "Loss at iteration 200: 0.41606056690216064\n",
      "Loss at iteration 225: 0.3964562714099884\n",
      "Loss at iteration 250: 0.38062769174575806\n",
      "\n",
      "Final Test Accuracy: 0.9666666666666667\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 250\n",
    "learning_rate = 0.1\n",
    "for i in range(1, n_iterations + 1):\n",
    "    \n",
    "    Z = torch.mm(X, w_autograd) + b_autograd\n",
    "    A = softmax_activation(Z)\n",
    "    l2_regularization = torch.sum(w_autograd ** 2)\n",
    "    loss = cross_entropy_loss(y_one_hot, A) \\\n",
    "           + lambda_param * l2_regularization\n",
    "    \n",
    "    if w_autograd.grad is not None:\n",
    "        w_autograd.grad.zero_()\n",
    "    if b_autograd.grad is not None:\n",
    "        b_autograd.grad.zero_()\n",
    "        \n",
    "    loss.backward()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        w_autograd -= learning_rate * w_autograd.grad\n",
    "        b_autograd -= learning_rate * b_autograd.grad\n",
    "    \n",
    "    if i == 1 or i % 25 == 0:\n",
    "        print(\"Loss at iteration {}: {}\".format(i, loss))\n",
    "\n",
    "test_predictions = torch.argmax(\n",
    "    softmax_activation(torch.mm(X_test, w_autograd) + b_autograd), axis=1\n",
    ")\n",
    "test_accuracy = float(sum(test_predictions == y_test)) / y_test.shape[0]\n",
    "print(\"\\nFinal Test Accuracy: {}\".format(test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch Neural Network Module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 3)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.1\n",
    "lambda_param = 0.01\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, weight_decay=lambda_param)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function = torch.nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at iteration 1: 0.9752714037895203\n",
      "Loss at iteration 25: 0.5917924046516418\n",
      "Loss at iteration 50: 0.5611095428466797\n",
      "Loss at iteration 75: 0.4597735106945038\n",
      "Loss at iteration 100: 0.4310804307460785\n",
      "Loss at iteration 125: 0.3661951720714569\n",
      "Loss at iteration 150: 0.3313466012477875\n",
      "Loss at iteration 175: 0.29825714230537415\n",
      "Loss at iteration 200: 0.2813898026943207\n",
      "Loss at iteration 225: 0.27167195081710815\n",
      "Loss at iteration 250: 0.26376885175704956\n",
      "\n",
      "Final Test Accuracy: 0.9666666666666667\n"
     ]
    }
   ],
   "source": [
    "n_iterations = 250\n",
    "for i in range(1, n_iterations + 1):\n",
    "    Z = model(X_train)\n",
    "    loss = loss_function(Z, y_train)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i == 1 or i % 25 == 0:\n",
    "        print(\"Loss at iteration {}: {}\".format(i, loss))\n",
    "        \n",
    "test_predictions = torch.argmax(\n",
    "    torch.softmax(model(X_test), 1), axis=1\n",
    ")\n",
    "test_accuracy = float(sum(test_predictions == y_test)) / y_test.shape[0]\n",
    "print(\"\\nFinal Test Accuracy: {}\".format(test_accuracy))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}