added additional notebook for ch14

.ipynb_checkpoints/Chapter14-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

Chapter14.ipynb

@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 157,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 158,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Activations\n",
+      "[0.93940638 0.96852968]\n",
+      "[0.9919462  0.99121735]\n",
+      "[0.99301385 0.99302901]\n",
+      "[0.9930713  0.99307098]\n",
+      "[0.99307285 0.99307285]\n",
+      "[0.99307291 0.99307291]\n",
+      "[0.99307291 0.99307291]\n",
+      "[0.99307291 0.99307291]\n",
+      "[0.99307291 0.99307291]\n",
+      "[0.99307291 0.99307291]\n",
+      "\n",
+      "Gradients\n",
+      "[0.03439552 0.03439552]\n",
+      "[0.00118305 0.00118305]\n",
+      "[4.06916726e-05 4.06916726e-05]\n",
+      "[1.39961115e-06 1.39961115e-06]\n",
+      "[4.81403643e-08 4.81403637e-08]\n",
+      "[1.65582672e-09 1.65582765e-09]\n",
+      "[5.69682675e-11 5.69667160e-11]\n",
+      "[1.97259346e-12 1.97517920e-12]\n",
+      "[8.45387597e-14 8.02306381e-14]\n",
+      "[1.45938177e-14 2.16938983e-14]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "sigmoid = lambda x:1/(1 + np.exp(-x))\n",
+    "relu = lambda x:(x>0).astype(float)*x\n",
+    "\n",
+    "weights = np.array([[1,4],[4,1]])\n",
+    "activation = sigmoid(np.array([1,0.01]))\n",
+    "\n",
+    "print(\"Activations\")\n",
+    "activations = list()\n",
+    "for iter in range(10):\n",
+    "    activation = sigmoid(activation.dot(weights))\n",
+    "    activations.append(activation)\n",
+    "    print(activation)\n",
+    "print(\"\\nGradients\")\n",
+    "gradient = np.ones_like(activation)\n",
+    "for activation in reversed(activations):\n",
+    "    gradient = (activation * (1 - activation) * gradient)\n",
+    "    gradient = gradient.dot(weights.transpose())\n",
+    "    print(gradient)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 160,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Relu Activations\n",
+      "[23.71814585 23.98025559]\n",
+      "[119.63916823 118.852839  ]\n",
+      "[595.05052421 597.40951192]\n",
+      "[2984.68857188 2977.61160877]\n",
+      "[14895.13500696 14916.36589628]\n",
+      "[74560.59859209 74496.90592414]\n",
+      "[372548.22228863 372739.30029248]\n",
+      "[1863505.42345854 1862932.18944699]\n",
+      "[9315234.18124649 9316953.88328115]\n",
+      "[46583049.71437107 46577890.60826711]\n",
+      "\n",
+      "Relu Gradients\n",
+      "[5. 5.]\n",
+      "[25. 25.]\n",
+      "[125. 125.]\n",
+      "[625. 625.]\n",
+      "[3125. 3125.]\n",
+      "[15625. 15625.]\n",
+      "[78125. 78125.]\n",
+      "[390625. 390625.]\n",
+      "[1953125. 1953125.]\n",
+      "[9765625. 9765625.]\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"Relu Activations\")\n",
+    "activations = list()\n",
+    "for iter in range(10):\n",
+    "    activation = relu(activation.dot(weights))\n",
+    "    activations.append(activation)\n",
+    "    print(activation)\n",
+    "\n",
+    "print(\"\\nRelu Gradients\")\n",
+    "gradient = np.ones_like(activation)\n",
+    "for activation in reversed(activations):\n",
+    "    gradient = ((activation > 0) * gradient).dot(weights.transpose())\n",
+    "    print(gradient)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}