diff --git a/python/basic/mnist_regularizer.ipynb b/python/basic/mnist_regularizer.ipynb new file mode 100644 index 000000000..bb6344187 --- /dev/null +++ b/python/basic/mnist_regularizer.ipynb @@ -0,0 +1,1482 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import mxnet as mx\n", + "import logging\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.metrics import accuracy_score\n", + "%matplotlib inline\n", + "\n", + "sns.set()\n", + "mnist = mx.test_utils.get_mnist()" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "batch_size = 100\n", + "train_iter = mx.io.NDArrayIter(mnist['train_data'], mnist['train_label'], batch_size, shuffle=True)\n", + "val_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "data = mx.sym.var('data')\n", + "# Flatten the data from 4-D shape into 2-D (batch_size, num_channel*width*height)\n", + "data = mx.sym.flatten(data=data)\n", + "true_labels = mx.sym.var('softmax_label')" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# The first fully-connected layer and the corresponding activation function\n", + "fc1 = mx.sym.FullyConnected(data=data, num_hidden=128, name='fc1')\n", + "act1 = mx.sym.Activation(data=fc1, act_type=\"relu\", name='relu1')\n", + "\n", + "# The second fully-connected layer and the corresponding activation function\n", + "fc2 = mx.sym.FullyConnected(data=act1, num_hidden = 64, name='fc2')\n", + "act2 = mx.sym.Activation(data=fc2, act_type=\"relu\", name='relu2')\n", + "\n", + "# MNIST has 10 classes\n", + "fc3 = mx.sym.FullyConnected(data=act2, num_hidden=10, name='fc3')\n", + "# Compute softmax\n", + "softmax_out = mx.sym.softmax(data=fc3, name='softmax_out')\n", + "#y_hat = mx.sym.argmax(softmax_out)\n", + "one_hot = mx.sym.one_hot(true_labels, depth=10, name='one_hot')\n", + "epsilon = 1e-8\n", + "ce = mx.sym.sum(- one_hot * mx.sym.log(epsilon + softmax_out))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "my_loss = mx.sym.MakeLoss(ce)\n", + "group = mx.sym.Group([mx.sym.BlockGrad(softmax_out, name='softmax'), my_loss])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "plot\n", + "\n", + "\n", + "\n", + "data\n", + "\n", + "data\n", + "\n", + "\n", + "\n", + "flatten0\n", + "\n", + "flatten0\n", + "\n", + "\n", + "\n", + "flatten0->data\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc1\n", + "\n", + "FullyConnected\n", + "128\n", + "\n", + "\n", + "\n", + "fc1->flatten0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "relu1\n", + "\n", + "Activation\n", + "relu\n", + "\n", + "\n", + "\n", + "relu1->fc1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc2\n", + "\n", + "FullyConnected\n", + "64\n", + "\n", + "\n", + "\n", + "fc2->relu1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "relu2\n", + "\n", + "Activation\n", + "relu\n", + "\n", + "\n", + "\n", + "relu2->fc2\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc3\n", + "\n", + "FullyConnected\n", + "10\n", + "\n", + "\n", + "\n", + "fc3->relu2\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax_out\n", + "\n", + "softmax_out\n", + "\n", + "\n", + "\n", + "softmax_out->fc3\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax\n", + "\n", + "softmax\n", + "\n", + "\n", + "\n", + "softmax->softmax_out\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax_label\n", + "\n", + "softmax_label\n", + "\n", + "\n", + "\n", + "one_hot\n", + "\n", + "one_hot\n", + "\n", + "\n", + "\n", + "one_hot->softmax_label\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mulscalar0\n", + "\n", + "_mulscalar0\n", + "\n", + "\n", + "\n", + "_mulscalar0->one_hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plusscalar0\n", + "\n", + "_plusscalar0\n", + "\n", + "\n", + "\n", + "_plusscalar0->softmax_out\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "log0\n", + "\n", + "log0\n", + "\n", + "\n", + "\n", + "log0->_plusscalar0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mul0\n", + "\n", + "_mul0\n", + "\n", + "\n", + "\n", + "_mul0->_mulscalar0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mul0->log0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "sum0\n", + "\n", + "sum0\n", + "\n", + "\n", + "\n", + "sum0->_mul0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "makeloss0\n", + "\n", + "makeloss0\n", + "\n", + "\n", + "\n", + "makeloss0->sum0\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mx.viz.plot_network(symbol=group)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## Training" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "### Basic MLP model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "mod_1 = mx.mod.Module(symbol=group, context=mx.cpu(), data_names=['data'], label_names=['softmax_label'])\n", + "mod_1.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n", + "mod_1.init_params(initializer=mx.init.Uniform(scale=.1))\n", + "mod_1.init_optimizer(optimizer='sgd', optimizer_params={'learning_rate': 0.1})\n", + "metrics = mx.metric.CompositeEvalMetric([mx.metric.Accuracy(), mx.metric.CrossEntropy()], output_names='softmax_output')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "epochs = 50" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Epoch [0] Train-accuracy=0.824083\n", + "WARNING:root:Epoch [0] Train-cross-entropy=0.610692\n", + "WARNING:root:Epoch [0] Validation-accuracy=0.921600\n", + "WARNING:root:Epoch [1] Train-accuracy=0.932600\n", + "WARNING:root:Epoch [1] Train-cross-entropy=0.230429\n", + "WARNING:root:Epoch [1] Validation-accuracy=0.946100\n", + "WARNING:root:Epoch [2] Train-accuracy=0.951700\n", + "WARNING:root:Epoch [2] Train-cross-entropy=0.164728\n", + "WARNING:root:Epoch [2] Validation-accuracy=0.956300\n", + "WARNING:root:Epoch [3] Train-accuracy=0.962367\n", + "WARNING:root:Epoch [3] Train-cross-entropy=0.128917\n", + "WARNING:root:Epoch [3] Validation-accuracy=0.961700\n", + "WARNING:root:Epoch [4] Train-accuracy=0.968817\n", + "WARNING:root:Epoch [4] Train-cross-entropy=0.105510\n", + "WARNING:root:Epoch [4] Validation-accuracy=0.965400\n", + "WARNING:root:Epoch [5] Train-accuracy=0.974083\n", + "WARNING:root:Epoch [5] Train-cross-entropy=0.088597\n", + "WARNING:root:Epoch [5] Validation-accuracy=0.967200\n", + "WARNING:root:Epoch [6] Train-accuracy=0.977867\n", + "WARNING:root:Epoch [6] Train-cross-entropy=0.075796\n", + "WARNING:root:Epoch [6] Validation-accuracy=0.968500\n", + "WARNING:root:Epoch [7] Train-accuracy=0.981150\n", + "WARNING:root:Epoch [7] Train-cross-entropy=0.065681\n", + "WARNING:root:Epoch [7] Validation-accuracy=0.969800\n", + "WARNING:root:Epoch [8] Train-accuracy=0.983950\n", + "WARNING:root:Epoch [8] Train-cross-entropy=0.057370\n", + "WARNING:root:Epoch [8] Validation-accuracy=0.971700\n", + "WARNING:root:Epoch [9] Train-accuracy=0.985950\n", + "WARNING:root:Epoch [9] Train-cross-entropy=0.050416\n", + "WARNING:root:Epoch [9] Validation-accuracy=0.972600\n", + "WARNING:root:Epoch [10] Train-accuracy=0.988283\n", + "WARNING:root:Epoch [10] Train-cross-entropy=0.044516\n", + "WARNING:root:Epoch [10] Validation-accuracy=0.972500\n", + "WARNING:root:Epoch [11] Train-accuracy=0.989833\n", + "WARNING:root:Epoch [11] Train-cross-entropy=0.039403\n", + "WARNING:root:Epoch [11] Validation-accuracy=0.973000\n", + "WARNING:root:Epoch [12] Train-accuracy=0.991183\n", + "WARNING:root:Epoch [12] Train-cross-entropy=0.034841\n", + "WARNING:root:Epoch [12] Validation-accuracy=0.973200\n", + "WARNING:root:Epoch [13] Train-accuracy=0.992533\n", + "WARNING:root:Epoch [13] Train-cross-entropy=0.030768\n", + "WARNING:root:Epoch [13] Validation-accuracy=0.973300\n", + "WARNING:root:Epoch [14] Train-accuracy=0.993917\n", + "WARNING:root:Epoch [14] Train-cross-entropy=0.027193\n", + "WARNING:root:Epoch [14] Validation-accuracy=0.974200\n", + "WARNING:root:Epoch [15] Train-accuracy=0.994767\n", + "WARNING:root:Epoch [15] Train-cross-entropy=0.023975\n", + "WARNING:root:Epoch [15] Validation-accuracy=0.974800\n", + "WARNING:root:Epoch [16] Train-accuracy=0.995767\n", + "WARNING:root:Epoch [16] Train-cross-entropy=0.021088\n", + "WARNING:root:Epoch [16] Validation-accuracy=0.975800\n", + "WARNING:root:Epoch [17] Train-accuracy=0.996450\n", + "WARNING:root:Epoch [17] Train-cross-entropy=0.018596\n", + "WARNING:root:Epoch [17] Validation-accuracy=0.975700\n", + "WARNING:root:Epoch [18] Train-accuracy=0.997067\n", + "WARNING:root:Epoch [18] Train-cross-entropy=0.016349\n", + "WARNING:root:Epoch [18] Validation-accuracy=0.976300\n", + "WARNING:root:Epoch [19] Train-accuracy=0.997550\n", + "WARNING:root:Epoch [19] Train-cross-entropy=0.014336\n", + "WARNING:root:Epoch [19] Validation-accuracy=0.977000\n", + "WARNING:root:Epoch [20] Train-accuracy=0.998150\n", + "WARNING:root:Epoch [20] Train-cross-entropy=0.012575\n", + "WARNING:root:Epoch [20] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [21] Train-accuracy=0.998517\n", + "WARNING:root:Epoch [21] Train-cross-entropy=0.010992\n", + "WARNING:root:Epoch [21] Validation-accuracy=0.977900\n", + "WARNING:root:Epoch [22] Train-accuracy=0.998867\n", + "WARNING:root:Epoch [22] Train-cross-entropy=0.009601\n", + "WARNING:root:Epoch [22] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [23] Train-accuracy=0.999167\n", + "WARNING:root:Epoch [23] Train-cross-entropy=0.008405\n", + "WARNING:root:Epoch [23] Validation-accuracy=0.978500\n", + "WARNING:root:Epoch [24] Train-accuracy=0.999450\n", + "WARNING:root:Epoch [24] Train-cross-entropy=0.007442\n", + "WARNING:root:Epoch [24] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [25] Train-accuracy=0.999517\n", + "WARNING:root:Epoch [25] Train-cross-entropy=0.006590\n", + "WARNING:root:Epoch [25] Validation-accuracy=0.978200\n", + "WARNING:root:Epoch [26] Train-accuracy=0.999617\n", + "WARNING:root:Epoch [26] Train-cross-entropy=0.005842\n", + "WARNING:root:Epoch [26] Validation-accuracy=0.978200\n", + "WARNING:root:Epoch [27] Train-accuracy=0.999717\n", + "WARNING:root:Epoch [27] Train-cross-entropy=0.005209\n", + "WARNING:root:Epoch [27] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [28] Train-accuracy=0.999750\n", + "WARNING:root:Epoch [28] Train-cross-entropy=0.004674\n", + "WARNING:root:Epoch [28] Validation-accuracy=0.978600\n", + "WARNING:root:Epoch [29] Train-accuracy=0.999850\n", + "WARNING:root:Epoch [29] Train-cross-entropy=0.004188\n", + "WARNING:root:Epoch [29] Validation-accuracy=0.978600\n", + "WARNING:root:Epoch [30] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [30] Train-cross-entropy=0.003792\n", + "WARNING:root:Epoch [30] Validation-accuracy=0.978500\n", + "WARNING:root:Epoch [31] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [31] Train-cross-entropy=0.003442\n", + "WARNING:root:Epoch [31] Validation-accuracy=0.978500\n", + "WARNING:root:Epoch [32] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [32] Train-cross-entropy=0.003150\n", + "WARNING:root:Epoch [32] Validation-accuracy=0.978400\n", + "WARNING:root:Epoch [33] Train-accuracy=0.999933\n", + "WARNING:root:Epoch [33] Train-cross-entropy=0.002880\n", + "WARNING:root:Epoch [33] Validation-accuracy=0.978400\n", + "WARNING:root:Epoch [34] Train-accuracy=0.999933\n", + "WARNING:root:Epoch [34] Train-cross-entropy=0.002652\n", + "WARNING:root:Epoch [34] Validation-accuracy=0.978000\n", + "WARNING:root:Epoch [35] Train-accuracy=0.999933\n", + "WARNING:root:Epoch [35] Train-cross-entropy=0.002445\n", + "WARNING:root:Epoch [35] Validation-accuracy=0.978000\n", + "WARNING:root:Epoch [36] Train-accuracy=0.999967\n", + "WARNING:root:Epoch [36] Train-cross-entropy=0.002273\n", + "WARNING:root:Epoch [36] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [37] Train-accuracy=0.999983\n", + "WARNING:root:Epoch [37] Train-cross-entropy=0.002113\n", + "WARNING:root:Epoch [37] Validation-accuracy=0.978200\n", + "WARNING:root:Epoch [38] Train-accuracy=0.999983\n", + "WARNING:root:Epoch [38] Train-cross-entropy=0.001977\n", + "WARNING:root:Epoch [38] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [39] Train-accuracy=0.999983\n", + "WARNING:root:Epoch [39] Train-cross-entropy=0.001853\n", + "WARNING:root:Epoch [39] Validation-accuracy=0.978200\n", + "WARNING:root:Epoch [40] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [40] Train-cross-entropy=0.001744\n", + "WARNING:root:Epoch [40] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [41] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [41] Train-cross-entropy=0.001646\n", + "WARNING:root:Epoch [41] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [42] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [42] Train-cross-entropy=0.001558\n", + "WARNING:root:Epoch [42] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [43] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [43] Train-cross-entropy=0.001476\n", + "WARNING:root:Epoch [43] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [44] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [44] Train-cross-entropy=0.001401\n", + "WARNING:root:Epoch [44] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [45] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [45] Train-cross-entropy=0.001336\n", + "WARNING:root:Epoch [45] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [46] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [46] Train-cross-entropy=0.001273\n", + "WARNING:root:Epoch [46] Validation-accuracy=0.978300\n", + "WARNING:root:Epoch [47] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [47] Train-cross-entropy=0.001216\n", + "WARNING:root:Epoch [47] Validation-accuracy=0.978200\n", + "WARNING:root:Epoch [48] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [48] Train-cross-entropy=0.001164\n", + "WARNING:root:Epoch [48] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [49] Train-accuracy=1.000000\n", + "WARNING:root:Epoch [49] Train-cross-entropy=0.001116\n", + "WARNING:root:Epoch [49] Validation-accuracy=0.977900\n" + ] + } + ], + "source": [ + "mod_1_tr_acc = []\n", + "mod_1_val_acc = []\n", + "for epoch in range(epochs):\n", + " ## Training\n", + " for batch in train_iter:\n", + " mod_1.forward(batch, is_train=True) # compute predictions\n", + " mod_1.update_metric(metrics, batch.label) # accumulate prediction accuracy\n", + " mod_1.backward() # compute gradients\n", + " mod_1.update() # update parameters\n", + " loss = mod_1.get_outputs()[1].asnumpy()\n", + " # print('Epoch %d, Loss %.3f' % (epoch, loss[1].asnumpy()))\n", + " for name, val in metrics.get_name_value():\n", + " logging.warning('Epoch [%d] Train-%s=%f' % (epoch, name, val))\n", + " if name == 'accuracy':\n", + " mod_1_tr_acc.append(val)\n", + " \n", + " train_iter.reset()\n", + " metrics.reset()\n", + " \n", + " eval_results = mod_1.predict(val_iter)\n", + " eval_probability = eval_results[0].asnumpy()\n", + " eval_prediction = np.argmax(eval_probability, axis=1)\n", + " eval_accuracy = accuracy_score(eval_prediction, mnist['test_label'])\n", + " logging.warning('Epoch [%d] Validation-%s=%f' % (epoch, 'accuracy', eval_accuracy))\n", + " mod_1_val_acc.append(eval_accuracy)\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD3CAYAAADxJYRbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VPW9//HXmX2SmYQEwhIggUSCyBajpVo2AVOtolCx\ngAjYK6lo622lFlDqAhoRWvX21irXVMFqFxa1tkJbFPVnLIhLJGpAYklYZA/ZZzKZ7ZzfH5NMEiEJ\nCVlgzuf5ePCYObOc+X5C8p7v+Z7vOUfRNE1DCCGELhi6uwFCCCG6joS+EELoiIS+EELoiIS+EELo\niIS+EELoiKm7G9CSkpLqdr83Li6K8vKaDmzNhUOvtUvd+iJ1Ny8hwdnscxHb0zeZjN3dhG6j19ql\nbn2RutsnYkNfCCHE6ST0hRBCRyT0hRBCRyT0hRBCRyT0hRBCRyT0hRBCR85qnv5nn33GE088wcsv\nv9zk8XfeeYdnnnkGk8nEjBkzmDlzJrW1tSxevJjS0lKio6NZvXo18fHxZ3ytEOJ0mqYRVDVUNXTb\n+L6m1d2vu1U18AQ1ytoyX13TCJ9aV4P2nma3/gS9mgYaWuhWCz2uhdcdery9Gp8EWKtva91jRytq\nqayIzHn6iqKQkhiDxdzx01JbDf3f//73/P3vf8dutzd53O/38/jjj/PKK69gt9u55ZZbmDx5Mm+8\n8QZpaWn893//N1u2bOHZZ59l6dKlZ3xtr169OrwgIc5WUFWp9QWp9Qbx+ALhW483EHrcF6TWV3ff\nG8DrD4YCWCMUuKqGqtX9UzWCwYaQDqpqk9AOPdfwWOh9daGmgVqXkqoKqpztXACx0RZWLbwSq6Vj\ng7/V0E9KSuLpp59myZIlTR4vKioiKSmJ2NhYAC677DI+/vhj8vLyyMrKAmDChAk8++yzzb72e9/7\nXoufHRcXdU4HIrR0VFqk00vtmqbhrg1QWumhoOgULo8fV40/dOvx4faE7rsb/atfrvUFu7v5QjSr\n0u0Ds/GMf8vn8vfdauhfc801HD58+LTHXS4XTmfDB0dHR+NyuZo8Hh0dTXV1dbOvbc25HGKdkOA8\np9M4XMgioXZV03B5/FS7fVTV+Kly+6iq8VHl9lFe7aW82ktZtZfy6lp8frW7m9vhDIqC0ahgMCiY\nDKFbg0HBoNT9M1B3q2AyGQgE2vYzUBQFJbxAw/02UzAoofWhgEEJPabUr1MBpdEHtOdzQutRwvfr\n7mKxmPD5Au1u+flMURQy0hKwKqefjuZs/r5b+lJo97l3HA4Hbrc7vOx2u3E6nU0ed7vdxMTENPta\noU+aplHl9nGqspbSqtpQgFeFArysLtArXb5OH+ZQAJvViM1iwm41YbcYsVlCyw2P1y1bjFjNxlAQ\nh4O3IYCNBgMGg4Kx/l/d64xGQzi0Q48bMBoaglJRGgK4ftloUEIhepYi4Uu+PfRa97lqd+inpqZy\n8OBBKioqiIqK4pNPPmHBggUcPXqU9957j1GjRpGbm8tll13W7GtF5NI0jfJqL1+fdHG4xEVJRSjg\nT1XWUlpZSyDYcb1zi9lAnNNGfIwNs1Eh2mYiymZuems1EVW3XH/fajFiaEO4ChEJ2hz6b7zxBjU1\nNcyaNYv77ruPBQsWoGkaM2bMoE+fPtxyyy0sXbqUW265BbPZzJNPPonZbD7ja0VkcHn8HCt1c6y0\nhsMnXeGgd9ee26Z3tM2EM8pCTJSZmGgLzmgLMVEW4pxW4p1WetTd2q0mFEWRnp8QZ0E5ny+Mfi5/\nwHoOgM6q3R8IUnSkiv3HqzhWWsPxshqOl9bg8vjbvK5om4mesTZ6xtiId9qIiwkFeHyMjTinlR4O\nK2ZT2w4j0ev/udStL902pi8iX33I7z1UTuGhCoqOVrVpWMZuNTIwwcGA3g76xkfRK9ZOr1gbPWNt\n2K3yqydEd5C/PBHm9QUpOlpJ4aEKCr+uoPgsQ95iMtA3Poq+PaPo3yuaAb0dDOztoGeMrU07JIUQ\nnU9CX8c83gD7jtSHfDkHjlUTVFse7esbH0XawB4MrOu9942PIi7GKjtEhbhASOjriMvj5z9fh3rx\nhV9XcOhEdauHyPeNj+LipB4MTYpjaFIPejisXdNYIUSnkNCPYP6AyleHKygoLmX3/jIOl7hbfU//\nXtGkDezB0KQepA2UkBci0kjoR5gT5TV8WFjCB58fZe+h8haPVlWAgX0coZAfGMeQgbHERFm6rrFC\niC4noR8B/IEgH+89ydt5R9h/rKrZ1xkUhUH9nAwdGOrFDxkQS5TN3IUtFUJ0Nwn9C1hZVS3v7jpC\n7mdHqa4581z5PnF2RqT0ZGRKPGkDe2CzyH+5EHomCXCBUVWNPQfLeG/XUT79T8lpO2JNRoVLh/Zm\n6IBYRgyOp3dcVPc0VAhxXpLQvwBomsbXJ118sPs4O/ecoNLlO+018TFWJl3an/GjE0lN7qnLIxWF\nEK2T0D+PlVd72bn7ODt2H+dIMzNvhiXHMTljAOlDemI0yNUvhRAtk9A/D5VV1bL5g4O8/9nRMx4s\nFRNlZswlfZiY3p/+vaK7oYVCiAuVhP55pLzay5YPDpD72VECwaZhbzEZyEhL4IrhfRk+OE569UKI\ndpHQPw9UuLz844OD/L/8o6ed6+ai/rFMTE8kIy1BTlImhDhnkiLdyOcP8q8PD/GPnQfxfeNyd6n9\nY5g+PoVLkuPkpGVCiA4jod8NNE0jf98p/rLtP5yqrG3y3OB+MUwfP5gRg+Ml7IUQHU5Cv4udKKvh\nz9v+wxfFpU0eH9jbwU0TUhiV2lPCvhtomoZfDeBTfaiaitlgwmKwYDQYO/VzVU3FG/RSG/DiCdTi\nU0+fjlvPoBgwKkaMigFD3a3RYMTkUan0uk57vaIojd5T/z6D/H7pnIR+F/H6gmz+4ABbPzrUZCdt\ntM3EjImpTBidiMEgf4wdIaAGKKutoKy2nNLaMqp9btx+NzV+D+6AG7ffg9tfQ22gFr/qr/t35ks7\nGhRD+AvAbDQ3DVzFgMEQuh+6tHlTGqEvk6AWJKgFUTU1dKuq+FU/tUEv3mDzId9ZFM584XUFhSiT\nnShzFNHmKKLNdqJN0USZ7RiV07/8NDR8QT+1wVpqA15qA7XUBr3UBmsJqkGMijH88zHWffmEv4QM\nxm98iRkIamr4ZxRU639eaugC8/XvNTSsx26z4PWe2yU5oeEL36/68QV9oVs1QCDoR6OtFxZUmvxe\nNP59aa52s8GMxWjGbDBjNpqxGMxYjBZG9hpGL3vPc67vmyT0u8C+w5U8v2UPJ8s94ccUYGJ6IjdN\nTMVhl/PftMQTqA0FuKeMCm8VPtVX98cZwB/041P9eINe3KqLY1UlVHqr2vHHemahnrgvFM5tvyrk\neUlDo7mrpFb7XVT7T99qEF3v9X1beGzcAzjMHTstW0K/E/kDKn/7937++eHBJqdLSE2M4dbvpjGo\nb0z3Na4L+YN+AlrwjEMMATVApbeKCm8VFd4Kyr2VVHqrKKutoLS2jDJPOe5ATZe002wwYTaYMSgG\nAmoAn+pH1c7+8pDtoaBgNVqwmWzYjFYsRkszWw0aWl3PN7TloNb1hoMoBgX1DMdznL6VoXZ6PaLj\nBDUVb8DX9aGvqirLly+nsLAQi8VCdnY2ycnJ4edzcnLYsmULDoeDrKwsJk2axGOPPcbevXsBKCkp\nISYmho0bN5Kdnc2nn35KdHSoiGeffRans/kL+F7IDp2o5vnNX3K4pKHXZLcamTV5CONG9YvIK02p\nmkpZbTlHXMea/DvlKTut522o2/RtbljlXCgoxFpj6GmLo6c9nlhLDNHmKKLMdqLN0USbQsMXdpMt\nvDltMpgwKKcf+xBUg/jqh4CC/iYBGlSD4RBujuEb4+r1900GI3aTDYvRcsbPbYu2XCBc1dRme/mq\nplITCA19uf01uAM11PhrqAl4mv2yMBvM2Ew27EZr6IvLZMVmtGFUjKj1X06aWnc/GP6ZqY2+vFRN\nRVWD4SGRpj8vBQ2tyc+7/r1Op5WqqtoztqstFMBUN7xiMZjqficszf5OtETTtDPUrTYarmr6JRxQ\nA+EOhj/orxtiCv2eXdJzKD3tcedc3ze1Gvrbtm3D5/OxYcMG8vPzWbVqFWvWrAGgsLCQzZs3s2nT\nJgBmz57NFVdcwS9/+UsA/H4/c+bM4dFHHwVg9+7dPP/888THx3d4IeeLoKryrw8P8fr7+5scTTss\nOY7brxtGz1hbN7bu3ATVIKdqyyipOUWFt5IKbyXl3koqaiup8FVRVluO7yzHqNU29DpNBhM9bXHE\n2+KIt/XAarRiaTT+Wf9HmtS7D0avjXhbD0yGjtmINRqM2A1G7Fy4/2+NGRQDZ9iQAMCIkVijmVjr\nhbEF2pYvO9Gg1b+MvLw8xo8fD0B6ejoFBQXh54qKihgzZgxWa+jqSsnJyRQWFpKeng7AH//4R8aO\nHcvQoUNRVZWDBw/y0EMPcerUKW6++WZuvvnmFj87Li4Kk6n9sycSErp2K6Km1s/KFz/is/+cCj9m\nMRn44dThXD92cJfuqG2p9oAaxBcIjYv7gqGdmOFeqxokUDdsUOmt5nDlMQ5XHedI1XGOVp8g0I6e\nuUExYDVaCDTaQVdPURR62GLoaY8jPqpH6Nbeg55RPegd3YuE6J7E2pzn3BuOdF39u36+kLrbrtXQ\nd7lcOByO8LLRaCQQCGAymRg6dCg5OTm4XC78fj+7du1i1qxZAPh8PtavX88rr7wCQE1NDXPnzuW/\n/uu/CAaDzJ8/nxEjRnDxxRc3+9nl5e0fy+3qXoC71s9vNn5G0dGGi5gM7hdD1tRh9OsZTWnpue8c\n8wV9HK85yXH3SSq9VSh1sxoaz4xAUcAS4Fj5Kap9Lqp9Lqp81bh8LmqDXvxqoFPHdaPNUfR3JNLf\n0Td82y+qD2Zjw87q0CZwaFO3fjbDGWkQcEGpq/XLPIJ+e35St76cTd0tfSm0GvoOhwO3u+GPTlVV\nTKbQ21JTU7n11lvJysoiMTGR0aNHExcXGoP64IMP+Na3vhUes7fb7cyfPx+73Q7AFVdcwd69e1sM\n/QtFVY2Pp9bnc+hkQ7Df8J1B3DhuULvPkeML+igo3cuBqkMcd4eCvqy2vMNmpbRXrMVJn6jexNvi\n6GGLpYc1lh7WGHpYexBnjSXaHNXqPPDwlxWdOwdeCHG6VkM/IyODd999l+uuu478/HzS0tLCz5WV\nleF2u1m/fj3V1dXcfvvtDBkyBIAdO3YwYcKE8GsPHDjAPffcw+uvv46qqnz66ad8//vf74SSulZ5\ntZcn1u/iWGnDVsnc76YxOWNAm9flVwN8WVpI3snP+PzUnrMeH2+r+rnnZkP93GBTo7nnDQf/2ExW\n+kb1pm90b/pG96FvVG+izPZOaZMQomu0GvqZmZls376d2bNno2kaK1euZN26dSQlJTF58mSKi4uZ\nMWMGZrOZJUuWYDSGem/79+9n+vTp4fWkpqYybdo0Zs6cidlsZtq0aeEviAtVSYWHX/9lV/hUCooC\nt183jLEj+531OmoDteyr2M+uki/4rKQAT6D52QgKCglRPekX1Yee9tDO8PrZI2p4ZoNGQkwspqAV\np8VJjMWB0+IgxuLEZrJ2yVGmQojzl6I1N3/rPHAu43WdPd53rNTNE+vzKa/2AmA0KNxx43C+dXHv\nFt9X6a2iqPIARRX7Kao8wOHqo80O2fSJ6s3ohOEMcCTSL7oPCVG9MJ/FrBQZ69QXqVtfOn1MX5yu\npMLD6j/vosodGn4xGQ385PsjGH1Rryav0zSNkzUl7KvcT1HFAYoqD3DKU3qmVYb1tMVzWZ/RXN4n\nncTovnKeFCFEh5LQbyN3rZ/fbPosHPhWs5GfzhjJsEGh4ZZj7hPsLt1LcV3Iu/wtzzxRUEh09GVo\n3EVk9B7NoJiBEvRCiE4jod8GgaDKs38tCO+0NRkN3PODUUTFeXijeCv5J7/geM3JFtdhNpgYFJNE\nauwgUnoMJiU2CbtJdo4KIbqGhP5Z0jSNl7YW8uXBcgAUm4v0K738+UgOp/Y1P2QTbYoipccgUmMH\nkdpjMEnO/h12tKgQQrSVpM9Z+sfOg/z782OgqJgSi7D0L2a3+/QdsBaDmUt6Xsyw+CGk9hhMn6gE\nOZpUCHHekNA/Cx99eYJX3ytGiarCMvgLDNHVTebb2IxWRvQaxqUJI7mk51AsRku3tVUIIVoiod+K\nfUcqeX7zbkyJ+zAlFqEYGuI+NXYQmclXcXHckCanGRBCiPOVhH4Lyqpq+e2W9zFevAtDdMO8WLPB\nzI2p13LVgLEydCOEuKBI6Ldg7fZ3CKS8j6FR7z4lNpm5w2bSJyqhG1smhBDtI6HfjG37PqTY8h6K\nEgp8o2JiWuq1TBo4Tnr3QogLloT+GXx4LI+/HnwtHPjmgJP7xy6kT3TLp1gQQojznYT+N+w4+jF/\n2rspfHUhtcZB1ugFEvhCiIggod/I+0d2sr7wtfCyWuMgzXctIwYmdmOrhBCi48jgdJ3/9/X2poHv\njsG3dww/GHdJN7ZKCCE6lvT0gY+Of8qm//wtvKy6YvEWXs63hw5gQG9HC+8UQogLi+5Dvzbg5bV9\nm8PLweoe+L66DINqYdq4wd3YMiGE6Hi6H9555+tcqn2ha9saA3Z8hZdD0Mx3Rvalb3xUN7dOCCE6\nlq5Dv9rnYtuh98LLnkOpoJowGhRuHDuo+xomhBCdRNeh/88D2/DWXXzc6IsheKo/AFel96dXrJzj\nXggReXQb+idrTvH+kZ3h5ZoDFwEKFpOB67+T3H0NE0KITtTqjlxVVVm+fDmFhYVYLBays7NJTm4I\nxZycHLZs2YLD4SArK4tJkyZRUVHBNddcQ1paGgBXX301t912Gxs3bmT9+vWYTCbuuusuJk2a1HmV\nteKN4n+haioANl9vPBWhc+lMvmwAPRzWbmuXEEJ0plZDf9u2bfh8PjZs2EB+fj6rVq1izZo1ABQW\nFrJ582Y2bdoEwOzZs7niiivYs2cPU6dO5cEHHwyvp6SkhJdffplXX30Vr9fLnDlzGDt2LBZL1597\n/mDV13x68vPwsqso1MsHmJzRv8vbI4QQXaXV0M/Ly2P8+PEApKenU1BQEH6uqKiIMWPGYLWGesbJ\nyckUFhZSUFDA7t27mTt3LvHx8TzwwAN88cUXXHrppVgsFiwWC0lJSezdu5dRo0Y1+9lxcVGYTMZ2\nF5eQ4DztMU3TeLZga3h5SMwwPq+OASCpr5NhF0XG6RbOVLseSN36InW3Xauh73K5cDgaDlAyGo0E\nAgFMJhNDhw4lJycHl8uF3+9n165dzJo1i5SUFEaMGMF3vvMd/v73v5Odnc2UKVNwOhsaGh0djcvl\navGzy8tr2l1YQoKTkpLq0x7fXbqX3Se/AsCgGIgqHQ6E2nFJctwZ33Ohaa72SCd164vU3fJrmtNq\n6DscDtxud3hZVVVMptDbUlNTufXWW8nKyiIxMZHRo0cTFxfHyJEjsdtDs18yMzP57W9/y7Rp05qs\nx+12N/kS6AqqpvL6vn+El8f2G0PeO4Hw8qiUnl3aHiGE6Gqtzt7JyMggNzcXgPz8/PDOWYCysjLc\nbjfr169nxYoVHDt2jCFDhvDAAw+wdWtoCOWDDz5g+PDhjBo1iry8PLxeL9XV1RQVFTVZV1f4+Pgu\njrqPA2AxWrg05juUVtUCYLcauWhAbJe2RwghulqrPf3MzEy2b9/O7Nmz0TSNlStXsm7dOpKSkpg8\neTLFxcXMmDEDs9nMkiVLMBqN3HvvvSxbtoy//OUv2O12srOzSUhIYN68ecyZMwdN01i0aFF4X0BX\naXwg1pSBEyj+2hteHj4oHpNRtzNYhRA6oWiaprX+su5xLuN13xz3cvtrWPL+ciA0lv+r8ct5euMe\n9h6qAOC/rruY8aMi4xTKMtapL1K3vpzrmL5uurb7Kw+G7w909kcLmPjP4crwYyNlPF8IoQO6DP2U\n2GT2HCgjqIY2cpL7OOWALCGELugm9IubhP4gPi8uDS+PTJVevhBCH3QR+kE1yIGqQ+HlQc6BfFHU\nEPqjJPSFEDqhi9A/4j6GT/UDEGftQXWliUp36Oya0TYTKf1iurN5QgjRZXQR+vsrG3r5KbHJTYd2\nUnpiMCjd0SwhhOhyugj94soD4fuDY5ObDO3IeL4QQk90EfqNZ+70tfWn6GhoqqaCTNUUQuhLxId+\npbeK0tpyAMwGM+UnLdQfjpbSPwaH3dyNrRNCiK4V8aHfuJefHDOAguKK8LKcYE0IoTcRH/qN5+cP\njkmmoLgsvDwqtVd3NEkIIbqNrkI/KpiAyxOauhkbbWFgH0dzbxNCiIgU0aHvD/r5uvpweLn8eFT4\n/siUnhgUmaophNCXiA79r11HCGhBAHrbe/HVAU/4OTkKVwihRxEd+k3G82OTKau7YApASqIchSuE\n0J+IDv393wj9Wl8wvGy3tnr9GCGEiDgRG/qapn1j5k4SXn9D6Fstxu5olhBCdKuIDf0SdylVvtDV\nZWxGG3HmhumZVotRduIKIXQpYkO/8FRx+P7g2CR8/oarQtqkly+E0KmIDf2vShuFfkwStb5AeNlm\nkfF8IYQ+tZp+qqqyfPlyCgsLsVgsZGdnk5ycHH4+JyeHLVu24HA4yMrKYtKkSRw9epRly5YRDAbR\nNI1HHnmElJQUXnzxRTZt2kR8fDwAK1asICUlpVMK+6pRTz8ldlCTnbjS0xdC6FWrob9t2zZ8Ph8b\nNmwgPz+fVatWsWbNGgAKCwvZvHkzmzZtAmD27NlcccUV/O///i9z587l6quv5v333+epp57id7/7\nHQUFBaxevZoRI0Z0alG1AS8HKkMHZSkoDIodyIHDDXP07RL6QgidajX08/LyGD9+PADp6ekUFBSE\nnysqKmLMmDFYraGLiicnJ1NYWMjSpUtxOp0ABIPB8PO7d+8mJyeHkpISrrrqKhYuXNjhBQEcrPoa\nre5Umv2i+2A32an1ucLPy/COEEKvWk0/l8uFw9Fwjhqj0UggEMBkMjF06FBycnJwuVz4/X527drF\nrFmzwsM3xcXFrF69mmeeeQaA66+/njlz5uBwOLj77rt59913mTRpUrOfHRcXhcnU9l557slj4fuX\n9LmIhAQn5kMNZ9eMddpISHC2eb0XkkivrzlSt75I3W3Xaug7HA7cbnd4WVVVTKbQ21JTU7n11lvJ\nysoiMTGR0aNHExcXB8DOnTtZsWIFv/rVr0hJSUHTNG677bbwFsDEiRPZs2dPi6FfXl7TrqIKjv0n\nfL+fNZGSkmpKTjX09BVNpaSkul3rvhAkJDgjur7mSN36InW3/JrmtDp7JyMjg9zcXADy8/NJS0sL\nP1dWVobb7Wb9+vWsWLGCY8eOMWTIEHbu3Mljjz3G888/z8iRI4HQFsPUqVNxu91omsaHH37YKWP7\nqqY2ORI3JTa007npjlwZ3hFC6FOr6ZeZmcn27duZPXs2mqaxcuVK1q1bR1JSEpMnT6a4uJgZM2Zg\nNptZsmQJRqORlStX4vf7ue+++wAYPHgwjzzyCIsWLWL+/PlYLBauvPJKJk6c2ClFBdTQ9MxYSwwJ\n9tBBWR6ZvSOEECha/R7P81B7N90+OPoxn5V/wcR+4xgWH9oy+fO2r9j2SWhGz+zJF/HdMUkd1s7z\njWz26ovUrS/nOrwTkeMcVyZ+ixtHT27yg2kyvCMnWxNC6FTEHpH7TXJwlhBC6Cr0G5+GQUJfCKFP\nOgp9mb0jhBD6CX2vDO8IIYR+Ql+Gd4QQQk+hL8M7Qgiho9CXnr4QQugi9ANBlUAwdAyaQVEwm3RR\nthBCnEYX6ffNOfqKXB9XCKFT+gh9b6OhHasM7Qgh9EsfoS87cYUQAtBl6EtPXwihXzoJfZm5I4QQ\noJvQl+EdIYQAnYS+p1FP3y49fSGEjuki9KWnL4QQIfoLfZmyKYTQMZ2EvuzIFUII0E3oy/COEELA\nWYS+qqo89NBDzJo1i3nz5nHw4MEmz+fk5DBt2jRuvfVW3n33XQDKysq4/fbbmTNnDvfccw8ejweA\njRs3ctNNNzFz5szwa7uCnEtfCCFCWg39bdu24fP52LBhA/feey+rVq0KP1dYWMjmzZvZuHEja9eu\n5be//S0ej4dnn32WqVOn8uc//5lLLrmEDRs2UFJSwssvv8z69et54YUXeOqpp/D5fJ1aXD0Z3hFC\niJBWQz8vL4/x48cDkJ6eTkFBQfi5oqIixowZg9VqxWq1kpycTGFhYZP3TJgwgR07dvD5559z6aWX\nYrFYcDqdJCUlsXfv3k4qqykZ3hFCiJBWE9DlcuFwOMLLRqORQCCAyWRi6NCh5OTk4HK58Pv97Nq1\ni1mzZuFyuXA6nQBER0dTXV3d5LH6x10uV4ufHRcXhcnU/p55QkLo84KaFn6sbx9n+PFIpocaz0Tq\n1hepu+1aDX2Hw4Hb7Q4vq6qKyRR6W2pqKrfeeitZWVkkJiYyevRo4uLiwu+x2Wy43W5iYmJOW4/b\n7W7yJXAm5eU17a2LhAQnJSXVAFS7G4aRat3e8OORqnHteiJ164vU3fJrmtPq8E5GRga5ubkA5Ofn\nk5aWFn6urKwMt9vN+vXrWbFiBceOHWPIkCFkZGTw3nvvAZCbm8tll13GqFGjyMvLw+v1Ul1dTVFR\nUZN1dSYZ3hFCiJBWEzAzM5Pt27cze/ZsNE1j5cqVrFu3jqSkJCZPnkxxcTEzZszAbDazZMkSjEYj\nd911F0uXLmXjxo3ExcXx5JNPEhUVxbx585gzZw6aprFo0SKsVmtX1CgHZwkhRB1F0xoNeJ9nzmXT\nrX4TSNM0sn71LvVV/n7JVRgNkX14gmz26ovUrS+dPrxzofMF1HDgm02GiA98IYRoScQnoFxARQgh\nGkR+6HvlwCwhhKgX+aEvM3eEECJMB6EvPX0hhKgX8aHvkZ6+EEKERXzoS09fCCEa6CD0ZfaOEELU\ni/zQ98rwjhBC1Iv80JfhHSGECNNB6Mt5d4QQop6+Ql+Gd4QQOqeD0JfhHSGEqKeD0JfZO0IIUU9n\noS/DO0IIfdNB6MvwjhBC1NNB6MvwjhBC1NNV6NutMrwjhNA3HYS+DO8IIUS9iA59VdXw+VUAFMBq\nltAXQug80y31AAARMElEQVRbq+MdqqqyfPlyCgsLsVgsZGdnk5ycHH5+7dq1bN68GUVRuPPOO8nM\nzCQnJ4f3338fgKqqKk6dOsX27dt58cUX2bRpE/Hx8QCsWLGClJSUTiqt6dCO1WJEUZRO+ywhhLgQ\ntBr627Ztw+fzsWHDBvLz81m1ahVr1qwBQoH+0ksv8eabb+LxeJg+fTqZmZnccccd3HHHHQAsXLiQ\nxYsXA1BQUMDq1asZMWJEJ5bUQIZ2hBCiqVZDPy8vj/HjxwOQnp5OQUFB+Dm73U5iYiIejwePx3Na\nT/rNN98kJiaGcePGAbB7925ycnIoKSnhqquuYuHChR1Zy2lkjr4QQjTVahK6XC4cDkd42Wg0EggE\nMJlCb+3Xrx/XX389wWDwtBB/7rnneOqpp8LL119/PXPmzMHhcHD33Xfz7rvvMmnSpGY/Oy4uCpOp\n/T10e7Q1fN8RbSEhwdnudV1o9FRrY1K3vkjdbddq6DscDtxud3hZVdVw4Ofm5nLy5EnefvttABYs\nWEBGRgajRo1i3759xMTEhMf/NU3jtttuw+kMNXbixIns2bOnxdAvL69pd2EJCU6OnagKL5sUKCmp\nbvf6LiQJCU7d1NqY1K0vUnfLr2lOq7N3MjIyyM3NBSA/P5+0tLTwc7GxsdhsNiwWC1arFafTSVVV\nKGh37NjBhAkTwq91uVxMnToVt9uNpml8+OGHnT62L8M7QgjRVKtJmJmZyfbt25k9ezaaprFy5UrW\nrVtHUlISU6ZMYceOHcycORODwUBGRgZjx44FYP/+/eH7AE6nk0WLFjF//nwsFgtXXnklEydO7LzK\n+MaOXDmXvhBCoGiapnV3I5pzLptuCQlONmz9kj+++RUAV13an/nXDO2opp3XZLNXX6Rufen04Z0L\nmccrUzaFEKKxiA59OdmaEEI0paPQlx25QggR4aEvwztCCNFYhIe+DO8IIURjOgp9Gd4RQogID30Z\n3hFCiMYiPPRleEcIIRqL7ND3Ngp9uVSiEEJEeOjL8I4QQjQR4aHf6KLoEvpCCBG5oe8PBAmqodMK\nGQ0KJmPEliqEEGctYpOwprbp0I5cH1cIISI49OVka0IIcTqdhL7M3BFCCNBN6EtPXwghQEJfCCF0\nRSehL8M7QggBkRz6tdLTF0KIb4rc0G/c05dTMAghBACtpqGqqixfvpzCwkIsFgvZ2dkkJyeHn1+7\ndi2bN29GURTuvPNOMjMz0TSNCRMmMGjQIADS09O59957eeedd3jmmWcwmUzMmDGDmTNndlphMqYv\nhBCnazX0t23bhs/nY8OGDeTn57Nq1SrWrFkDQFVVFS+99BJvvvkmHo+H6dOnk5mZyaFDhxg+fDj/\n93//F16P3+/n8ccf55VXXsFut3PLLbcwefJkevXq1SmFSegLIcTpWh3eycvLY/z48UCox15QUBB+\nzm63k5iYiMfjwePxhI963b17NydOnGDevHn86Ec/ori4mKKiIpKSkoiNjcVisXDZZZfx8ccfd1JZ\nUCM7coUQ4jStpqHL5cLhcISXjUYjgUAAkyn01n79+nH99dcTDAZZuHAhAAkJCdxxxx1873vf45NP\nPmHx4sXcf//9OJ3O8Hqio6NxuVwtfnZcXBQmU/t66Y17+r17RZOQ4Gzh1ZFHb/XWk7r1Repuu1ZD\n3+Fw4Ha7w8uqqoYDPzc3l5MnT/L2228DsGDBAjIyMhgxYgRGYyisL7/8ck6ePHnaetxud5MvgTMp\nL69pe0V1Gs/e8dUGKCmpbve6LjQJCU5d1VtP6tYXqbvl1zSn1eGdjIwMcnNzAcjPzyctLS38XGxs\nLDabDYvFgtVqxel0UlVVxe9+9zv+8Ic/ALB371769etHamoqBw8epKKiAp/PxyeffMKll156VkW2\nR9PZOzKmL4QQcBY9/czMTLZv387s2bPRNI2VK1eybt06kpKSmDJlCjt27GDmzJkYDAYyMjIYO3Ys\nI0eOZPHixbz33nsYjUYef/xxzGYz9913HwsWLEDTNGbMmEGfPn06rbAa2ZErhBCnUTRN07q7Ec05\nl023B1/4iCMloX0Gj2Z9m/69ojuqWec92ezVF6lbXzp9eOdC1Xh4R66aJYQQIboIfRneEUKIkIgM\nfU3TmlwU3SqhL4QQQISGvtcfpH5PhcVkwGiIyDKFEBcAr9fLG2+8flav/cc/3uDf/36vU9sTkYeq\n1vqC4fsytCOEqPevDw/xt+378TbKiHNltRiZNnYw13476YzPl5WV8sYbr3PDDdNbXdd1193QYe1q\njg5CPyJLFEK0w9aPD3Vo4AN4fUG2fnyo2dB/6aW1HDiwn/Hjv8Xll4/B4/Fw330P8q9/bWHv3j1U\nVVVy0UVpLFv2MC+88Bw9e/YkKWkQf/rTS5jNJo4ePcKUKd/lttsWdEh7IzIRG4/nS09fCFHvmm8l\ndUpP/5pvnTnwAebPv52ion18+9tXUl1dzT33/AK324XT6eQ3v3kWVVWZN28mJSUnm7zvxIljvPji\nX/D7/Uyffq2EfktqvTK8I4Q43bXfTmq2R94VkpJCp6W3Wm2Ul5fz8MPLiIqKwuPxEAgEmrw2JeUi\nTCYTJpMJq9XWYW2IzNBvPLwjF1ARQnQjRTGgaSoABkPoTMQ7d27n5MkTPPLI45SXl5Ob+y7fPE62\n7qTFHS4iE1GGd4QQ54u4uDj8/gBerzf82LBhw3nxxRf4yU9+hKIoJCb259Spki5pT4SGvgzvCCHO\nD1arlRdf/HOTx3r27MXzz7902mtHjUoP38/IuDx8/+9/39ph7YnICewye0cIIc4sQkNfhneEEOJM\nIjT0pacvhBBnEqGhLz19IYQ4kwgNfdmRK4QQZ6KD0JfhHSGEqBeZoS/n0hdCRIhPP/2Ehx++v8PW\nF5Hd4KZH5EroCyFCth16j3/sfwtv0Ndh67QaLVw3OJOrkyZ22Do7U+SHvgzvCCHqvHMot0MDH8Ab\n9PHOodwWQz8QCPDrX6/k8OGvUVWVyZMzyc19l6effg6AJUvuISvrTo4cOcxrr20iEAigKAorVz7R\noW2FsxjeUVWVhx56iFmzZjFv3jwOHjzY5Pm1a9dy0003MWPGDN566y0AqqurufPOO5k7dy6zZs1i\n165dALz11ltcffXVzJs3j3nz5vHRRx91eEHQdPaOXB9XCFFvctIErEZLh67TarQwOWlCi695443X\niY3twTPP/J5Vq57kb397DZ/Px/Hjxzh16hQVFRWkpV3M118f4te//l/WrHmBQYMG89FHH3RoW+Es\nevrbtm3D5/OxYcMG8vPzWbVqFWvWrAGgqqqKl156iTfffBOPx8P06dPJzMxk3bp1XHHFFfzwhz+k\nuLiYe++9l7/+9a8UFBSwePFirrnmmg4vpDHp6QshzuTqpIndMgxTVLSPzz/fxZ49BQAEgwEmTpzM\nv/61BbPZHL54SlxcPNnZDxMVFcXBgwcYMWJUh7el1UTMy8tj/PjxAKSnp1NQUBB+zm63k5iYiMfj\nwePxoNSdFu6HP/whFoulrrggVqsVgN27d/Pll1/yhz/8gVGjRvGLX/wCk6ljQzmoqvgCoTPaKQpY\nzBG5r1oIcQFJTh5E7969mT//drzeWv7wh7VMm/Z97rnnJxgMBv7nf36Hy+XihRee49VXNwOwaNFP\nTjvzZkdoNXFdLhcOhyO8bDQaCQQC4bDu168f119/PcFgkIULFwIQExMDQElJCYsXL2bZsmUAjB07\nlquvvpoBAwbw8MMPs379eubOndvsZ8fFRWEytW14xu3xh+/brSZ6945p0/sjRUKCs7ub0C2kbn25\nUOrOyrqNBx54gEWL7sLlcjFnzhwGDerHyJHDCQQCJCf3RdM0Lr/8Mu6+OwuTyURMTAweTxU9elyE\n1WpuUuu51N1q6DscDtxud3hZVdVw4Ofm5nLy5EnefvttABYsWEBGRgajRo2isLCQn//85yxZsoQx\nY8YAMGPGjPAXwpQpU9i6teUzx5WX17S5IE3T6NczimOlNQxLiqOkpLrN67jQJSQ4pW4dkbovDIsX\nP9hkuaSkmp/+dEn4PsADD2Sf8b3Llj0Sfs3Z1N3Sl0KrYx8ZGRnk5uYCkJ+fT1paWvi52NhYbDYb\nFosFq9WK0+mkqqqKffv28bOf/Ywnn3ySiRND42eapnHjjTdy/PhxAD744AOGDx/e2se3maIoLJt3\nGct+OIYFU4d1+PqFEOJC1mpPPzMzk+3btzN79mw0TWPlypWsW7eOpKQkpkyZwo4dO5g5cyYGg4GM\njAzGjh3Lj3/8Y3w+H4899hgQ2lpYs2YN2dnZ3H333dhsNlJTU5k5c2anFBVtM3PlyPgLqhcghBBd\nQdE6Y09BBzmX0L7QNv06kl5rl7r1Repu+TXNkaktQgihIxL6QgihIxL6QgihIxL6QgihIxL6Qgih\nIxL6QgihI+f1lE0hhBAdS3r6QgihIxL6QgihIxL6QgihIxL6QgihIxL6QgihIxL6QgihIxL6Qgih\nIxF31XBVVVm+fDmFhYVYLBays7NJTk7u7mZ1qs8++4wnnniCl19+mYMHD3LfffehKApDhgzh4Ycf\nxmCIrO92v9/PsmXLOHLkCD6fj7vuuouLLroo4uuG0DWnH3jgAfbv34+iKKxYsQKr1aqL2ktLS7np\npptYu3YtJpNJFzUDfP/73w9fsnbAgAHMmjWLxx57DKPRyLhx47j77rvbtkItwmzdulVbunSppmma\ntmvXLu3OO+/s5hZ1rpycHG3q1KnaD37wA03TNG3hwoXazp07NU3TtAcffFB78803u7N5neKVV17R\nsrOzNU3TtPLycm3ixIm6qFvTNO2tt97S7rvvPk3TNG3nzp3anXfeqYvafT6f9uMf/1j77ne/q+3b\nt08XNWuaptXW1mrTpk1r8tiNN96oHTx4UFNVVcvKytJ2797dpnVG3FdjXl4e48ePByA9PZ2CgoJu\nblHnSkpK4umnnw4v7969O3xN4gkTJrBjx47ualqnufbaa/nZz34GhC7DaTQadVE3wNVXX82jjz4K\nwNGjR4mJidFF7atXr2b27Nn07t0b0MfvOcDevXvxeDzcfvvtzJ8/n48//hifz0dSUhKKojBu3Lg2\n1x5xoe9yucKbQgBGo5FAINCNLepc11xzTfhC9RAKQUVRAIiOjqa6OvKuLBQdHY3D4cDlcvHTn/6U\ne+65Rxd11zOZTCxdupRHH32UG264IeJrf+2114iPjw935kAfv+cANpuNBQsW8MILL7BixQruv/9+\n7HZ7+Pn21B5xoe9wOHC73eFlVVWbhGKkazyu6Xa7iYmJ6cbWdJ5jx44xf/58pk2bxg033KCbuuut\nXr2arVu38uCDD+L1esOPR2Ltr776Kjt27GDevHl8+eWXLF26lLKysvDzkVhzvcGDB3PjjTeiKAqD\nBw/G6XRSUVERfr49tUdc6GdkZJCbmwtAfn4+aWlp3dyirnXJJZfw4YcfApCbm8vll1/ezS3qeKdO\nneL2229n8eLF3HzzzYA+6gZ4/fXXee655wCw2+0oisKIESMiuvY//elP/PGPf+Tll19m2LBhrF69\nmgkTJkR0zfVeeeUVVq1aBcCJEyfweDxERUVx6NAhNE3j3//+d5trj7izbNbP3vnqq6/QNI2VK1eS\nmpra3c3qVIcPH+bnP/85GzduZP/+/Tz44IP4/X5SUlLIzs7GaDR2dxM7VHZ2Nv/85z9JSUkJP/bL\nX/6S7OzsiK4boKamhvvvv59Tp04RCAT40Y9+RGpqasT/n9ebN28ey5cvx2Aw6KJmn8/H/fffz9Gj\nR1EUhV/84hcYDAZWrlxJMBhk3LhxLFq0qE3rjLjQF0II0byIG94RQgjRPAl9IYTQEQl9IYTQEQl9\nIYTQEQl9IYTQEQl9IYTQEQl9IYTQkf8PVkF7LE+4IMcAAAAASUVORK5CYII=\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "## Plot\n", + "plt.plot(range(epochs), mod_1_tr_acc, linewidth=3, label='train')\n", + "plt.plot(range(epochs), mod_1_val_acc, linewidth=3, label='eval')\n", + "plt.legend(loc=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "### L2 Regularization" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "mod_2 = mx.mod.Module(symbol=group, context=mx.cpu(), data_names=['data'], label_names=['softmax_label'])\n", + "mod_2.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n", + "mod_2.init_params(initializer=mx.init.Uniform(scale=.1))\n", + "mod_2.init_optimizer(optimizer='sgd', optimizer_params={'learning_rate': 0.1, 'wd': 0.001})" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Epoch [0] Train-accuracy=0.827500\n", + "WARNING:root:Epoch [0] Train-cross-entropy=0.593648\n", + "WARNING:root:Epoch [0] Validation-accuracy=0.918300\n", + "WARNING:root:Epoch [1] Train-accuracy=0.928500\n", + "WARNING:root:Epoch [1] Train-cross-entropy=0.249839\n", + "WARNING:root:Epoch [1] Validation-accuracy=0.942800\n", + "WARNING:root:Epoch [2] Train-accuracy=0.946867\n", + "WARNING:root:Epoch [2] Train-cross-entropy=0.184124\n", + "WARNING:root:Epoch [2] Validation-accuracy=0.953500\n", + "WARNING:root:Epoch [3] Train-accuracy=0.957317\n", + "WARNING:root:Epoch [3] Train-cross-entropy=0.147926\n", + "WARNING:root:Epoch [3] Validation-accuracy=0.959400\n", + "WARNING:root:Epoch [4] Train-accuracy=0.963783\n", + "WARNING:root:Epoch [4] Train-cross-entropy=0.124646\n", + "WARNING:root:Epoch [4] Validation-accuracy=0.964600\n", + "WARNING:root:Epoch [5] Train-accuracy=0.969100\n", + "WARNING:root:Epoch [5] Train-cross-entropy=0.108264\n", + "WARNING:root:Epoch [5] Validation-accuracy=0.967300\n", + "WARNING:root:Epoch [6] Train-accuracy=0.972867\n", + "WARNING:root:Epoch [6] Train-cross-entropy=0.096046\n", + "WARNING:root:Epoch [6] Validation-accuracy=0.969600\n", + "WARNING:root:Epoch [7] Train-accuracy=0.976167\n", + "WARNING:root:Epoch [7] Train-cross-entropy=0.086788\n", + "WARNING:root:Epoch [7] Validation-accuracy=0.971200\n", + "WARNING:root:Epoch [8] Train-accuracy=0.978217\n", + "WARNING:root:Epoch [8] Train-cross-entropy=0.079506\n", + "WARNING:root:Epoch [8] Validation-accuracy=0.972500\n", + "WARNING:root:Epoch [9] Train-accuracy=0.980150\n", + "WARNING:root:Epoch [9] Train-cross-entropy=0.073537\n", + "WARNING:root:Epoch [9] Validation-accuracy=0.973500\n", + "WARNING:root:Epoch [10] Train-accuracy=0.981817\n", + "WARNING:root:Epoch [10] Train-cross-entropy=0.068578\n", + "WARNING:root:Epoch [10] Validation-accuracy=0.974700\n", + "WARNING:root:Epoch [11] Train-accuracy=0.982883\n", + "WARNING:root:Epoch [11] Train-cross-entropy=0.064456\n", + "WARNING:root:Epoch [11] Validation-accuracy=0.975100\n", + "WARNING:root:Epoch [12] Train-accuracy=0.984217\n", + "WARNING:root:Epoch [12] Train-cross-entropy=0.060963\n", + "WARNING:root:Epoch [12] Validation-accuracy=0.976000\n", + "WARNING:root:Epoch [13] Train-accuracy=0.985300\n", + "WARNING:root:Epoch [13] Train-cross-entropy=0.057892\n", + "WARNING:root:Epoch [13] Validation-accuracy=0.975900\n", + "WARNING:root:Epoch [14] Train-accuracy=0.986117\n", + "WARNING:root:Epoch [14] Train-cross-entropy=0.055286\n", + "WARNING:root:Epoch [14] Validation-accuracy=0.977000\n", + "WARNING:root:Epoch [15] Train-accuracy=0.987000\n", + "WARNING:root:Epoch [15] Train-cross-entropy=0.053020\n", + "WARNING:root:Epoch [15] Validation-accuracy=0.977700\n", + "WARNING:root:Epoch [16] Train-accuracy=0.987767\n", + "WARNING:root:Epoch [16] Train-cross-entropy=0.051055\n", + "WARNING:root:Epoch [16] Validation-accuracy=0.977800\n", + "WARNING:root:Epoch [17] Train-accuracy=0.988083\n", + "WARNING:root:Epoch [17] Train-cross-entropy=0.049337\n", + "WARNING:root:Epoch [17] Validation-accuracy=0.978100\n", + "WARNING:root:Epoch [18] Train-accuracy=0.988733\n", + "WARNING:root:Epoch [18] Train-cross-entropy=0.047915\n", + "WARNING:root:Epoch [18] Validation-accuracy=0.978600\n", + "WARNING:root:Epoch [19] Train-accuracy=0.989183\n", + "WARNING:root:Epoch [19] Train-cross-entropy=0.046531\n", + "WARNING:root:Epoch [19] Validation-accuracy=0.978900\n", + "WARNING:root:Epoch [20] Train-accuracy=0.989650\n", + "WARNING:root:Epoch [20] Train-cross-entropy=0.045363\n", + "WARNING:root:Epoch [20] Validation-accuracy=0.979300\n", + "WARNING:root:Epoch [21] Train-accuracy=0.989900\n", + "WARNING:root:Epoch [21] Train-cross-entropy=0.044291\n", + "WARNING:root:Epoch [21] Validation-accuracy=0.979000\n", + "WARNING:root:Epoch [22] Train-accuracy=0.990300\n", + "WARNING:root:Epoch [22] Train-cross-entropy=0.043330\n", + "WARNING:root:Epoch [22] Validation-accuracy=0.979700\n", + "WARNING:root:Epoch [23] Train-accuracy=0.990683\n", + "WARNING:root:Epoch [23] Train-cross-entropy=0.042468\n", + "WARNING:root:Epoch [23] Validation-accuracy=0.979400\n", + "WARNING:root:Epoch [24] Train-accuracy=0.991067\n", + "WARNING:root:Epoch [24] Train-cross-entropy=0.041644\n", + "WARNING:root:Epoch [24] Validation-accuracy=0.979500\n", + "WARNING:root:Epoch [25] Train-accuracy=0.991283\n", + "WARNING:root:Epoch [25] Train-cross-entropy=0.040932\n", + "WARNING:root:Epoch [25] Validation-accuracy=0.979800\n", + "WARNING:root:Epoch [26] Train-accuracy=0.991500\n", + "WARNING:root:Epoch [26] Train-cross-entropy=0.040272\n", + "WARNING:root:Epoch [26] Validation-accuracy=0.979900\n", + "WARNING:root:Epoch [27] Train-accuracy=0.991950\n", + "WARNING:root:Epoch [27] Train-cross-entropy=0.039629\n", + "WARNING:root:Epoch [27] Validation-accuracy=0.980000\n", + "WARNING:root:Epoch [28] Train-accuracy=0.992183\n", + "WARNING:root:Epoch [28] Train-cross-entropy=0.039136\n", + "WARNING:root:Epoch [28] Validation-accuracy=0.980100\n", + "WARNING:root:Epoch [29] Train-accuracy=0.992383\n", + "WARNING:root:Epoch [29] Train-cross-entropy=0.038556\n", + "WARNING:root:Epoch [29] Validation-accuracy=0.979900\n", + "WARNING:root:Epoch [30] Train-accuracy=0.992533\n", + "WARNING:root:Epoch [30] Train-cross-entropy=0.038107\n", + "WARNING:root:Epoch [30] Validation-accuracy=0.979900\n", + "WARNING:root:Epoch [31] Train-accuracy=0.992567\n", + "WARNING:root:Epoch [31] Train-cross-entropy=0.037671\n", + "WARNING:root:Epoch [31] Validation-accuracy=0.980000\n", + "WARNING:root:Epoch [32] Train-accuracy=0.992717\n", + "WARNING:root:Epoch [32] Train-cross-entropy=0.037274\n", + "WARNING:root:Epoch [32] Validation-accuracy=0.980000\n", + "WARNING:root:Epoch [33] Train-accuracy=0.992800\n", + "WARNING:root:Epoch [33] Train-cross-entropy=0.036888\n", + "WARNING:root:Epoch [33] Validation-accuracy=0.980100\n", + "WARNING:root:Epoch [34] Train-accuracy=0.992967\n", + "WARNING:root:Epoch [34] Train-cross-entropy=0.036536\n", + "WARNING:root:Epoch [34] Validation-accuracy=0.980400\n", + "WARNING:root:Epoch [35] Train-accuracy=0.993167\n", + "WARNING:root:Epoch [35] Train-cross-entropy=0.036210\n", + "WARNING:root:Epoch [35] Validation-accuracy=0.980200\n", + "WARNING:root:Epoch [36] Train-accuracy=0.993200\n", + "WARNING:root:Epoch [36] Train-cross-entropy=0.035917\n", + "WARNING:root:Epoch [36] Validation-accuracy=0.980600\n", + "WARNING:root:Epoch [37] Train-accuracy=0.993317\n", + "WARNING:root:Epoch [37] Train-cross-entropy=0.035586\n", + "WARNING:root:Epoch [37] Validation-accuracy=0.980400\n", + "WARNING:root:Epoch [38] Train-accuracy=0.993383\n", + "WARNING:root:Epoch [38] Train-cross-entropy=0.035346\n", + "WARNING:root:Epoch [38] Validation-accuracy=0.980500\n", + "WARNING:root:Epoch [39] Train-accuracy=0.993483\n", + "WARNING:root:Epoch [39] Train-cross-entropy=0.035067\n", + "WARNING:root:Epoch [39] Validation-accuracy=0.980400\n", + "WARNING:root:Epoch [40] Train-accuracy=0.993617\n", + "WARNING:root:Epoch [40] Train-cross-entropy=0.034873\n", + "WARNING:root:Epoch [40] Validation-accuracy=0.980400\n", + "WARNING:root:Epoch [41] Train-accuracy=0.993683\n", + "WARNING:root:Epoch [41] Train-cross-entropy=0.034614\n", + "WARNING:root:Epoch [41] Validation-accuracy=0.980500\n", + "WARNING:root:Epoch [42] Train-accuracy=0.993667\n", + "WARNING:root:Epoch [42] Train-cross-entropy=0.034423\n", + "WARNING:root:Epoch [42] Validation-accuracy=0.980500\n", + "WARNING:root:Epoch [43] Train-accuracy=0.993833\n", + "WARNING:root:Epoch [43] Train-cross-entropy=0.034203\n", + "WARNING:root:Epoch [43] Validation-accuracy=0.980700\n", + "WARNING:root:Epoch [44] Train-accuracy=0.993883\n", + "WARNING:root:Epoch [44] Train-cross-entropy=0.034029\n", + "WARNING:root:Epoch [44] Validation-accuracy=0.980600\n", + "WARNING:root:Epoch [45] Train-accuracy=0.993967\n", + "WARNING:root:Epoch [45] Train-cross-entropy=0.033891\n", + "WARNING:root:Epoch [45] Validation-accuracy=0.980500\n", + "WARNING:root:Epoch [46] Train-accuracy=0.994067\n", + "WARNING:root:Epoch [46] Train-cross-entropy=0.033701\n", + "WARNING:root:Epoch [46] Validation-accuracy=0.980600\n", + "WARNING:root:Epoch [47] Train-accuracy=0.994167\n", + "WARNING:root:Epoch [47] Train-cross-entropy=0.033501\n", + "WARNING:root:Epoch [47] Validation-accuracy=0.980900\n", + "WARNING:root:Epoch [48] Train-accuracy=0.994150\n", + "WARNING:root:Epoch [48] Train-cross-entropy=0.033399\n", + "WARNING:root:Epoch [48] Validation-accuracy=0.981000\n", + "WARNING:root:Epoch [49] Train-accuracy=0.994200\n", + "WARNING:root:Epoch [49] Train-cross-entropy=0.033245\n", + "WARNING:root:Epoch [49] Validation-accuracy=0.980500\n" + ] + } + ], + "source": [ + "mod_2_tr_acc = []\n", + "mod_2_val_acc = []\n", + "for epoch in range(epochs):\n", + " ## Training\n", + " for batch in train_iter:\n", + " mod_2.forward(batch, is_train=True) # compute predictions\n", + " mod_2.update_metric(metrics, batch.label) # accumulate prediction accuracy\n", + " mod_2.backward() # compute gradients\n", + " mod_2.update() # update parameters\n", + " loss = mod_2.get_outputs()[1].asnumpy()\n", + " # print('Epoch %d, Loss %.3f' % (epoch, loss[1].asnumpy()))\n", + " for name, val in metrics.get_name_value():\n", + " logging.warning('Epoch [%d] Train-%s=%f' % (epoch, name, val))\n", + " if name == 'accuracy':\n", + " mod_2_tr_acc.append(val)\n", + " \n", + " train_iter.reset()\n", + " metrics.reset()\n", + " \n", + " eval_results = mod_2.predict(val_iter)\n", + " eval_probability = eval_results[0].asnumpy()\n", + " eval_prediction = np.argmax(eval_probability, axis=1)\n", + " eval_accuracy = accuracy_score(eval_prediction, mnist['test_label'])\n", + " logging.warning('Epoch [%d] Validation-%s=%f' % (epoch, 'accuracy', eval_accuracy))\n", + " mod_2_val_acc.append(eval_accuracy)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt8FPW9//HX7DWbvYRAAiRIAomAykUMilgEKhoveAHF\nAoJgK1Roa63WImiPFTQinFaPvSiVo+Ctp4C29Sjaqig/aUE9ikQbkCgBwp0kJCHZzWZvM78/djNJ\nIBcICQk7n+fjkcfu7MzOzieQ9373u9/5jqJpmoYQQghDMHX2AQghhDhzJPSFEMJAJPSFEMJAJPSF\nEMJAJPSFEMJAJPSFEMJALCez0ZdffslvfvMbXnnllUaPf/jhhzzzzDNYLBYmT57MlClTqK2tZf78\n+Rw9ehSn08myZcvo3r17k9u2prS0um1VAcnJiVRU1LT5+Wczo9YudRuL1N281FR3s+taDf3//u//\n5s0338ThcDR6PBQK8cQTT/D666/jcDi47bbbGD9+PG+99RYDBw7kpz/9KW+//TbPPvssCxYsaHLb\nlJSUkyzz1Fks5g7bd1dn1NqlbmORutum1e6djIwMfv/735/weFFRERkZGSQlJWGz2RgxYgSfffYZ\nW7ZsYcyYMQCMHTuWjz/+uNlthRBCnFmttvSvueYa9u/ff8LjXq8Xt7v+I4TT6cTr9TZ63Ol0Ul1d\n3ey2rUlOTjytd7WWPuLEO6PWLnUbi9R96k6qT78pLpcLn8+nL/t8Ptxud6PHfT4fHo+n2W1bczr9\ndamp7tP6TuBsZtTapW5jkbpb3qY5bR69k52dTXFxMZWVlQSDQT7//HMuuugicnJy+OijjwDYuHEj\nI0aMaHZbIYQQZ9Ypt/TfeustampqmDp1KgsXLmT27NlomsbkyZPp1asXt912GwsWLOC2227DarXy\n5JNPYrVam9xWCCHEmaV05Vk2T+ejm1E/+oFxa5e6jUXqbnmb5rS5T18IIYxK0zRUTUNVNcIRjYga\n+4moTd5X1VjbWgEFBUWJLSqgaaCqWvQ2tk9Fgf5pHmzW9h+WKqEvhDhlmlYfdMTCKtpnoKHGliMR\njXAs+MIRNbqsqmgqRFRV306NhWKTwRl7bvRHIxxWCavRfdlsFnw1wWioahqaqhHRoscDxAI2did6\n1IRi+wiFIwTDKqFwdN+qHrz14VtXY92x1dWk193Bkpw2ls67DHs7B7+EvhBdlKZFW5HBcIRgSCUU\niYVULKjqAstdVkNFZQ1o0edo1IdyqNG2GqGIqodqw6CN3o9up/9E6u8HwyrBUKT+NqSidt2e4bhw\nzBekvKqWtB7Odt2vhL4QHUDVNEIhFV9tiKqaIFW+ENU1QapqglT7QvhqQwRCEQLBSPQ2FCEQUgkE\nI9GQj4Wr5GrXpShgNimYTaborVnBZFKw1D1mVvT1ptg4ybp/z9iHIjQ0TIqCoiiYTERvlejzcgam\ntnvgg4S+EI0EQhF8/hBefwifP4SvNow3tlxTG9YDOhgL6WAo0uAxNXoba5nHO5MSDTpFiYaVQv2t\nyaRgMStYzNFAtJjrQ9Bkqg+2aNjVL9cHZX1wWkwmLJboPqI/0ftJHgc1NQFMDfZRF5x1gXr8m6bV\nYqr/MZuwWsxYzI2PRVGIBTF6oJtMxx27OXp7NpLQF3FH1TS9BV0bjOCrDeGtCenh3fAnGvDh6Db+\nEKFw1wprs0nBZjVjiwWVxdz41mpWsCdYCYci9aEbyyKzObreYjZh0UOurlUaC7MGYWsyKfo2xwej\nzWrSj8MWW7aYO3eSXqOO3jldEvqiS9M0DX8gTFVNiCpfMNpF4gtyzBfUH6vyRbtNgmEVf6w13hXY\nLCYcdgsepw1PohW304Yn0YY70YrTYSXBasZuNWO31d/aYo/ZLCZsVhNmU+vBKuEnToWEvug0mqbh\nqw1TdsxPWWUtZcdqOXqslqNVtZRX11IdC/UzMVKijtmk4HJYcTmsOBMsOGP3XY5oUNvrQtlqit3W\n32+4zmY1n7Uf/0V8k9AX7S4UVimp9FNSXsPhihpKKvx4a0LUBMLUBML4A2FqaqO3HRXoNquJBJsF\nu9VEYoIVd4PwdjmsuBLrg9yVYMXpsOCKhboiYS3imIS+aDN/IMyBMh8Hy3wcKPVx8KiPI+U1HK2q\nbddRJ3abGU+iNdY1YsPjtMa6TGx4nDaSnNHH+/bphrfKj91qxmSS4BaiKRL6okWaplFdE+LQUR+H\nyms4fLSGg0ejIV9RHTjt/dttZlKSEkjxJJCS5KBHUgIpSQl09yTgcVpxJ9pO+uSU7p4EIoHQaR+T\nEPFMQl/owhGVg2U+dh+qYvehag6UejlcXoOvNnxK+1GAHkkJ9Ep20Kt7Ir2SE+nmtuOwm0m0W6O3\nCVYS7WasBr36kRCdRULfwCq9AXbsrWDXgSp2H65i7xHvKQ1ZNJsUevdIpE+Kkz4pTtJTXPTukUjP\nbgkS5kJ0URL6BlLlC7JjbwU79layo7iCw+Und5Eau81MWvdE0nok0ruHk7TuiaSnOOmZ7Oj0sdpC\niFMjoR+nQuEI+0p87DlcxZ5D1ew6VMXBMl+rz+vhsdMvzUP/NA+Zvdykpzjp5rLJiBYh4oSEfpwI\nBCMU7D5Kwe5y9pX6KD5U1epwSIvZxLl9PAzs243+aR76pXlIctrO0BELITqDhP5ZzFcbIv/bMr74\nppRtu8sJttIfbzYpZKV7OC8jmfMykzm3j0f63kW7CqthfKEaIloEm8mGzWzFYrJgUprvBozODKrF\n5pk/vU+UdftqiqqpBCMhQmqIYCREUA0SjIQIqyEUxYRZMUfn2lHM9T+xZZO+3oxZMaEBETWCqqlE\ntEj0R1UJqSFqI7X4w9Gf2nCA2rCfQCRIWIsQUWPbxp6naRpOayIemxuPzU2S3YPH5sJj85BgsZ/W\n76I5EvpnmZraEJ/tKOGzHSUU7q1ssTXfK9lBvzQP/Xq7Yz8e7DYJeUD/47OarG0KGlVTCUQC+h92\nWA3X/zHrf9iNl/WAUOuDQm1wP6KpaJqG1WzFZrLGbqPBaTVZmzzDV9M0XAE7FZU+fR9q7PWa+5+h\namqTxxJWw9SGA/gjtdSGoz/+cC2BSBCzydzgmKzYzDasJivBSBBfqAZvyIcvVENtpLbJ17SaLNhM\nNswmc6PXrTuOOtGwNcXCNXpfaeINQ9M0MEE4HG70+1O1rjV30ukYmnI+dw29o8U3zLaQ0D8LRFSV\nbbvL2VxwmC++KSMcafo/dnqKk5yBKYwa2ockhxlngvUMH2nHUTVVD6GGIRO99VET9uM+kAhBEwmW\nBBIsCTgsCSSY7fhCNRz1l1NWe5Qyfzll/qOU11bGprU1kWC2R7e1JJBgTsBmtjYKxbr7YTVMbSQQ\nDcTI6Z+jYCQhNUxIbX3ob12Ao8r5Fv8u+5pDviP0caW1634l9Luw/SVe/vXvQ3yy/QhVvmCT2/RP\n8zBiUCo5A1Pp3T0R6PoTcIXUcIMA93EsWE1VoJqqYBVVwWqOBarxhnz6NrWRaGuzI6iaSk3YT03Y\n3yH7NxqTYiLR4sBishCKdaGcTNgrKM12y5wqhaY/uSmKgs1kw2q2NPoEZTVZolfF0k7srtE/kWlq\no2UFBbNiwnRcl5DFZI41IBw4zHa98WE32xt9eqm7rwDekI+qoJeqYDVVgejfgC/k5/weA0lz9mqX\n30lDEvpdjD8Q5tOvj/DPLw+y+1DTwZ3Ry8V3Bvfm4vN60t2TcIaPMCoYCVFeWx5rOUdb0eW1lYTU\nUOMui9j9QCSgdx2ETyIEzgSLyXJax5IQ+6NOsCRgjfVb63/Uev+vORYOpib/6Bs9x2QGFMKxPueG\nfc8hNdTs1BaOBBvhoNr49UxmTM2EHyjH9V3XH09dSDlin3qigWUjoqmx4wk2OK4QVpMFl9WJ0+rE\nZU0kwZJwQneEqqmE1XC0/1wLN/id1PeRmxRTLHDVWF95437vpqSmeKgs9zeqxaSYZKRZKyT0uwBN\n0yg6WMXGLw/y2dclTU4NnOS0cdng3nxnSG/O6ek6o8dXHfRSXLWP4qp97Knex0HvYSoDx87oMSgo\n2GPdME5rYixoEnHqt4nYHWbKjh3TPx34Y/3SCRY7KY4epCT0IMXRnRRHd7ondMdmttb3Yzd4TkgN\nNQjjWDCbTFgUi34MCRZ7u/e1tlVX/2RnUkzYzDZs5pZHhpli4W81nVwsJTvchG1d49/gbNLqb1dV\nVRYtWkRhYSE2m428vDwyMzP19StWrODtt9/G5XIxZ84crrjiCh5//HF27NgBQGlpKR6Ph7Vr15KX\nl8cXX3yB0xm9BNizzz6L2+3uoNLODtt2l7Pmw2/ZX3riGHqLOXrJtNFD07igX/JJza3eGk3TOBas\n4qD3MAd9hznkO0IgHDiu5Rlt+dWF/dHaitN+3YZMigmHOdpCdlodeGye2MiF6AgGj92Dy+ok0eLQ\n+9rtZlurIduW8LOYLLhsFly29r8snRBdUauhv379eoLBIGvWrCE/P5+lS5eyfPlyAAoLC1m3bh2v\nvfYaANOmTWPUqFH88pe/BCAUCjF9+nQee+wxALZt28bzzz9P9+7dO6qes0ZVTZA1H3zLx9uOnLCu\nT4qTsRemc9mQ3rgcJ/9lrKqplNdWUnm0jP1lpfhCNfhCPryhGqqDXo7UlHDQdwR/O/RfmxQTyfZu\ness5JaEHPRzJ0b7LWD+nqUFXh91s07sO2jpiRghx+loN/S1btjBmzBgAhg8fTkFBgb6uqKiIkSNH\nYrdHx5NmZmZSWFjI8OHDAXj11VcZPXo0gwYNQlVViouL+dWvfkVZWRm33nort956a0fU1KVpmsa/\n/n2ItR/ubDSRmd1qZuT5PRl7YTpZ6Z6TCsVAJEhx1V52HSum6Ngedh/b2y6BfjyLyUJfVx/6efqS\n6elLhrsPKY4esT5oIcTZpNXQ93q9uFz1fchms5lwOIzFYmHQoEGsWLECr9dLKBRi69atTJ06FYBg\nMMjq1at5/fXXAaipqeH222/nBz/4AZFIhFmzZjFkyBDOO++8Zl87OTkRy2mcPJSa2rW6jg6Uennm\n9a/4d1FZo8fHXtSHOROHkOxu/kvZYCTEvmMH2V2xjz0V+ygqL2Z35b42jUtOtDrom5ROX08afZPS\nSUrw6GPJw2r9CSQ2s43s7pn0TUrHcpYEfFf7Nz9TpG5jOZ26Ww19l8uFz1ff36yqKhZL9GnZ2dnM\nmDGDOXPmkJ6ezoUXXkhycjIAH3/8MZdcconeZ+9wOJg1axYOhwOAUaNGsWPHjhZDv6Li5CYEa0pX\n+nJL1TTe/b+9/G3j7kZj7FOSEph5zSCGZvUgXBuitDYU217lsK+Ebyt3UVy1j33VBzhcU3JSAe+y\nOunp6oFdqf/Cs+5Lzx6O7qQ7e9PNnnTy3SthqDja9n+HM6kr/ZufSVK3sZxM3S29KbQa+jk5OWzY\nsIEJEyaQn5/PwIED9XXl5eX4fD5Wr15NdXU1d955JwMGDABg8+bNjB07Vt92z5493Hvvvbzxxhuo\nqsoXX3zBzTff3GqBZ7uK6gDPr9vO18X1X4aaFIWrL+nLxMv7Y7eZUTWVQ74jfFNRxM7KXXxbuQtf\n6OSCNs3Zi6ykTLKS+pGV1I9URw969vQY8o9BCNG6VkM/NzeXTZs2MW3aNDRNY8mSJaxatYqMjAzG\njx/Prl27mDx5MlarlQceeACzOdoNsHv3biZNmqTvJzs7m4kTJzJlyhSsVisTJ07U3yDi1dZvS1n1\nzg68/vqzCzN7u/nBdefRs4eNr8u381XZdrYd3YE31PoMmKmOHpzj7kNfVzrnuPvQ39OXRGtiR5Yg\nhIgzitbcmQ9dwOm0Vjvzo18wFGHNhp1s+OKA/pgCXDkqhT7ZPgrKt/NN+U7C2onj8eu4rE7O7ZbF\nud3609fdhz6uNByWkzsRSz72GovUbSwd3r0jTs2BUi9//N9tHNDnrtfw9K4i/bwSNtX8A75t+nku\nq5MBydkM6JbFgG5ZpDl7ybBGIUS7k9BvR7sPVfGb1fn4A2FQIphTDuLO2E/AfIziJrro+7jSGJpy\nAcNSLqCvu0+XOcNTCBG/JPTbSfHhap5cnY8/4sPSZy+WnvtQrEEazsWooDAgOZthsaDv4ZCT1IQQ\nZ5aEfjvYe6SaX7/2KcHUb0notRfF3Liv3ma2cVnaJXz3nO/QMzG1k45SCCEk9E/bzkNlPP3/3kA9\nrwjrcWGfbO/Gd/uO5jtpI0m0OjrpCIUQop6EfhsFIkHeLNzA/9u/EXqFGk1i28eVxrX9ruTClMEy\nVYEQokuR0G+D0pqj/NcXz3EsWNnoN9jdlsLNA69leOoQ+VJWCNElSeifopKaMv5ryx+pClXpj2mB\nRK7LvIrrz/+OhL0QokuT0D8FJTWlPP3Fc3rga6oJbf9g7hk/gfMzenTy0QkhROsk9E/SEV8JT299\njqpg9Ew4LWIi9O0Ifj7hSs7PkKGXQoizg/RFnITDvhJ+e1zgB78ZwbRLL+P8fhL4Qoizh7T0W3HY\nd4Tfbl3RIPDNBL/JYXT/IYzP6dPJRyeEEKdGQr8FJTWlPL31OaqDXqAu8EfQ39OP268eJHPjCCHO\nOhL6zagKVvNM/guNA79wBB5685Obh2K1SM+YEOLsI8nVhNpwgOVfrqSsthyI9eEXjsDkT+HuW4bR\nzWXv5CMUQoi2kdA/TkSN8MK2V9lbHZ0LX9MgWDQc1dudO64dRFa6p5OPUAgh2k5CvwFN0/hz4V/Z\nfrRQfyy05wLUyp5cNeIcRg9N68SjE0KI0yeh38A7u9/n40Of6cuhA1lESjNwOazcMi6rE49MCCHa\nh4R+zKaDn/LOnvX6suVYX8IHotfwvfbSDBJs8p23EOLsJ6EP7Cj/ltWFf9OXe1szqf7mfEDB5bDK\neHwhRNwwfOirmsrr376JqqkAnOPqw7FtQ0CL/mqklS+EiCethr6qqvzqV79i6tSpzJw5k+Li4kbr\nV6xYwcSJE5kxYwYbNmwAoLKykksvvZSZM2cyc+ZMXnrpJQDWrl3LLbfcwpQpU/RtO9tXpds45DsC\ngN1s4yLLdZRXRi+GIq18IUS8abUJu379eoLBIGvWrCE/P5+lS5eyfPlyAAoLC1m3bh2vvfYaANOm\nTWPUqFFs376dG264gYcffljfT2lpKa+88gp/+ctfCAQCTJ8+ndGjR2Oz2TqotNZpmsbf93ygL1+e\nfhkfvF+mL18nrXwhRJxptaW/ZcsWxowZA8Dw4cMpKCjQ1xUVFTFy5Ejsdjt2u53MzEwKCwspKChg\n27Zt3H777dxzzz2UlJTw1VdfcdFFF2Gz2XC73WRkZLBjx46Oq+wkFBz9mv3egwBYTVYSqwdytKoW\niLbyr5BWvhAizrQa+l6vF5fLpS+bzWbC4TAAgwYN4vPPP8fr9VJRUcHWrVvx+/1kZWVxzz338Oqr\nr3LVVVeRl5eH1+vF7Xbr+3E6nXi93g4o6eRomsbfd9e38kenXcoHn5Toy9eNkla+ECL+tJpqLpcL\nn8+nL6uqisUSfVp2djYzZsxgzpw5pKenc+GFF5KcnMzQoUNxOKIXAs/NzeV3v/sdEydObLQfn8/X\n6E2gKcnJiVgsbb/GbGpq8/vPP7Sd4up9AFhNFnpzIUerigDwOG1MyT2PBPvZG/ot1R7PpG5jkbpP\nXauplpOTw4YNG5gwYQL5+fkMHDhQX1deXo7P52P16tVUV1dz5513MmDAAO6//36uvvpqJkyYwMcf\nf8zgwYMZNmwYTz/9NIFAgGAwSFFRUaN9NaWioqbNhaWmuiktrW5ynaZprP7yTX15VO9LePOD/fry\nNSP7Ul3lp+lnd30t1R7PpG5jkbpb3qY5rYZ+bm4umzZtYtq0aWiaxpIlS1i1ahUZGRmMHz+eXbt2\nMXnyZKxWKw888ABms5n777+fhx56iD//+c84HA7y8vJITU1l5syZTJ8+HU3TuO+++7DbO2fism8q\nith1LDoKyayY8fjO52hVtG/fnWhl/EXndMpxCSFER1M0TdM6+yCaczrv4i29Gz79xR/5tnIXAKPT\nL2Xv5/34Zv8xAL53RTbXXZrZ5tftCqQFZCxSt7GcbkvfcCdn7azcrQe+STExtvcYdh6o0tePHiKT\nqgkh4pfhQv/vu+vn1xnZO4fDh0GNfdjp19uNx9l55w0IIURHM1To7z62lx0V3wKgoHBN5ni+2nVU\nXz80q0dnHZoQQpwRhgr994rrp364uNdwUh09KGgY+tkS+kKI+GaY0I+oEXaUf6MvX9NvPPtKvFR6\ngwA4EyxkpclVsYQQ8c0woX/Qd4SgGgIg2d6NNGcv/t2glT+4f3dMJqWzDk8IIc4Iw4R+cdVe/X6m\npy8A/95Vrj8m/flCCCMwTOjvqdqn3+/n6UtNbZidsbH5AEMk9IUQBmCg0K9v6ffzZLB9T7k+VDOz\nt5skGaophDAAQ4R+bbiWw77oDJoKChmecxr150vXjhDCKAwR+nur96MRbdWnu3pjM1kp2F3fnz9M\nQl8IYRCGCP09xxr35+8v9VFRHQBiQzXTZaimEMIYjBH6x/Xnf1VUf0lEGaophDASg4R+fUs/09NX\nhmoKIQwr7kO/oraSY8HoLJp2s40kcw8ZqimEMKy4D/2GrfwM9znsKK6sH6rZS4ZqCiGMxQCh37g/\nv9FQzezunXFIQgjRaeI+9IuP689vOFRT+vOFEEYT16GvairF1fUXPLeHeuhDNRPtMlRTCGE8cR36\nh3xHCEaiUyd3syexZ19IXze4f3fMprguXwghThDXqbfnuJk1C2TqBSGEwcV36B93Ju6RCr++PDCj\nW2cckhBCdCpLaxuoqsqiRYsoLCzEZrORl5dHZmamvn7FihW8/fbbuFwu5syZwxVXXMHBgwd56KGH\niEQiaJrGo48+SlZWFi+++CKvvfYa3btHR80sXryYrKysDivu+JE7/kB9/74rwdphryuEEF1Vq6G/\nfv16gsEga9asIT8/n6VLl7J8+XIACgsLWbduHa+99hoA06ZNY9SoUfz2t7/l9ttv56qrruKf//wn\nTz31FH/4wx8oKChg2bJlDBkypGOrAmpDtRzyHQFiM2u6+hAIFuvrE2zmDj8GIYToaloN/S1btjBm\nzBgAhg8fTkFBgb6uqKiIkSNHYrfbAcjMzKSwsJAFCxbgdrsBiEQi+vpt27axYsUKSktL+e53v8vc\nuXPbvaA6uyr26jNrpjl7gWaJLYHNapL5doQQhtRq6Hu9Xlwul75sNpsJh8NYLBYGDRrEihUr8Hq9\nhEIhtm7dytSpU/Xum127drFs2TKeeeYZAK6//nqmT5+Oy+Xi7rvvZsOGDVxxxRXNvnZyciIWS9ta\n5Ju//li/f17PLBJdCfqyM8FKaqq7Tfs9W8R7fc2Ruo1F6j51rYa+y+XC5/Ppy6qqYrFEn5adnc2M\nGTOYM2cO6enpXHjhhSQnJwPwySefsHjxYv7zP/+TrKwsNE3jjjvu0D8BjBs3ju3bt7cY+hUVNW0u\n7Nvy3fr93rY0Dhyqn2/HZjFRWlrd5n13damp7riurzlSt7FI3S1v05xWR+/k5OSwceNGAPLz8xk4\ncKC+rry8HJ/Px+rVq1m8eDGHDh1iwIABfPLJJzz++OM8//zzDB06FIh+Yrjhhhvw+Xxomsann37a\noX37O4/u0e/3S8qgNhjRlxNsrb7XCSFEXGo1/XJzc9m0aRPTpk1D0zSWLFnCqlWryMjIYPz48eza\ntYvJkydjtVp54IEHMJvNLFmyhFAoxMKFCwHo378/jz76KPfddx+zZs3CZrNx2WWXMW7cuA4pqjJw\njHJ/JQA2k5XeiT355miVvl6+xBVCGFWroW8ymXj00UcbPZadna3fP34dwJtvvtnkviZNmsSkSZNO\n9RhPWaOZNT3nYDaZqQ2E9cccdmnpCyGMKS5PztpzrPH4fOC47h1p6QshjCkuQ//4mTUBaoP1LX0J\nfSGEUcVd6Edn1qwP/f6xlr5fvsgVQoj4C30FBSVWVqqjB93sScBxLX27tPSFEMYUd01eRVH4/uBp\nbDu2nVGpI1GU6Jm3tQFp6QshRFym39CUCxh//qWNTmCQL3KFECIOu3ea4w/KkE0hhDBM6EtLXwgh\nDBX6MmRTCCEMFPryRa4QQhgn9BtOwyAtfSGEQRkn9Bu29OWLXCGEQRki9DVNky9yhRACg4R+KKwS\nUaMXS7SYFSxmQ5QthBAnMET6yZe4QggRZZDQl+GaQggBhgl9aekLIQQYJPT9ja6aJS19IYRxGSL0\npaUvhBBRBgx9aekLIYzLEKHvly9yhRACMEjoN7yAikyrLIQwslZDX1VVfvWrXzF16lRmzpxJcXFx\no/UrVqxg4sSJzJgxgw0bNgBQXl7OnXfeyfTp07n33nvx+/0ArF27lltuuYUpU6bo254JMmRTCCGi\nWg399evXEwwGWbNmDffffz9Lly7V1xUWFrJu3TrWrl3LypUr+d3vfoff7+fZZ5/lhhtu4H/+53+4\n4IILWLNmDaWlpbzyyiusXr2aF154gaeeeopgMNihxdWRL3KFECKq1dDfsmULY8aMAWD48OEUFBTo\n64qKihg5ciR2ux273U5mZiaFhYWNnjN27Fg2b97MV199xUUXXYTNZsPtdpORkcGOHTs6qKzGpKUv\nhBBRrTZ7vV4vLpdLXzabzYTDYSwWC4MGDWLFihV4vV5CoRBbt25l6tSpeL1e3G43AE6nk+rq6kaP\n1T3u9XpbfO3k5EQslraHdGpq9PU0pf69rWeKS388nhmhxqZI3cYidZ+6VkPf5XLh8/n0ZVVVsVii\nT8vOzmbGjBnMmTOH9PR0LrzwQpKTk/XnJCQk4PP58Hg8J+zH5/M1ehNoSkVFTVvrIjXVrV8Y/Vh1\nrf54MBBqdMH0eNSwdiORuo1F6m55m+a02r2Tk5PDxo0bAcjPz2fgwIH6uvLycnw+H6tXr2bx4sUc\nOnSIAQMGkJOTw0cffQTAxo0bGTFiBMOGDWPLli0EAgGqq6spKipqtK+OJBdQEUKIqFZb+rm5uWza\ntIlp06ahaRpLlixh1apVZGRkMH78eHbt2sXkyZOxWq088MADmM1mfvSjH7FgwQLWrl1LcnIyTz75\nJImJicyyWH6eAAASIUlEQVScOZPp06ejaRr33Xcfdrv9TNSIX77IFUIIABRN07TOPojmnM5Ht4Yf\ngRb8cTOlldEunifmjqJXcmK7HF9XJR97jUXqNpYO796JBzJkUwghogwR+v6AzL0jhBBggNAPR1TC\nERUAk6Jgs8R9yUII0ay4T8DjZ9hUFKUTj0YIITqXAUK/wdm4cgEVIYTBxX/oB+RLXCGEqBP/od+g\ne0dOzBJCGJ0BQl8mWxNCiDoGCH3p3hFCiDpxH/r+gLT0hRCiTtyHfqOWvlwqUQhhcAYIfWnpCyFE\nnbgPfX9QpmAQQog6cR/6jYZsSveOEMLgDBD60r0jhBB14j/05YxcIYTQxX/oS0tfCCF0cR/6funT\nF0IIXdyH/vFTKwshhJHFf+g3OiNXWvpCCGOL/9CXlr4QQuhabfqqqsqiRYsoLCzEZrORl5dHZmam\nvn7lypWsW7cORVGYN28eubm5rFixgn/+858AVFVVUVZWxqZNm3jxxRd57bXX6N69OwCLFy8mKyur\ng0oDVdMIhOpD3y6hL4QwuFZDf/369QSDQdasWUN+fj5Lly5l+fLlQDTQX375Zd577z38fj+TJk0i\nNzeXu+66i7vuuguAuXPnMn/+fAAKCgpYtmwZQ4YM6cCS6gWCjQPfJJdKFEIYXKuhv2XLFsaMGQPA\n8OHDKSgo0Nc5HA7S09Px+/34/f4Trj/73nvv4fF4uPzyywHYtm0bK1asoLS0lO9+97vMnTu3PWs5\ngcywKYQQjbUa+l6vF5fLpS+bzWbC4TAWS/SpaWlpXH/99UQikRNC/LnnnuOpp57Sl6+//nqmT5+O\ny+Xi7rvvZsOGDVxxxRXtVcsJZC59IYRorNUkdLlc+Hw+fVlVVT3wN27cSElJCR988AEAs2fPJicn\nh2HDhrFz5048Ho/e/69pGnfccQdutxuAcePGsX379hZDPzk5EYul7S10h9Ou33c7baSmutu8r7ON\nkWptSOo2Fqn71LUa+jk5OWzYsIEJEyaQn5/PwIED9XVJSUkkJCRgs9lQFAW3201VVRUAmzdvZuzY\nsfq2Xq+XG264gXfeeYfExEQ+/fRTJk+e3OJrV1TUtLUuUlPdHDpSpS9bFCgtrW7z/s4mqaluw9Ta\nkNRtLFJ3y9s0p9XQz83NZdOmTUybNg1N01iyZAmrVq0iIyODK6+8ks2bNzNlyhRMJhM5OTmMHj0a\ngN27d+v3AdxuN/fddx+zZs3CZrNx2WWXMW7cuJOts038Mu+OEEI0omiapnX2QTTndN7FU1PdvPHh\nN7zw9tcAXDa4Fz+8cXB7HVqXJi0gY5G6jeV0W/pxfXKWfJErhBCNxXnoy5BNIYRoKM5DX6ZgEEKI\nhuI79Bt+kSvTKgshRJyHvnTvCCFEI3Ed+o0uoCJf5AohRHyHvrT0hRCisTgPfenTF0KIhowT+tLS\nF0KI+A59mVpZCCEai+vQb9jSd0j3jhBCxG/oa5omX+QKIcRx4jb0A8EIdVPJWS0mzKa4LVUIIU5a\n3Cah9OcLIcSJDBH6cmKWEEJExW3o10hLXwghThC3oe+vldAXQojjxW/oN2zpy3BNIYQA4jj0pXtH\nCCFOFLeh768N6fflUolCCBEVv6EvLX0hhDhB3IZ+w+4dmYJBCCGiWk1DVVVZtGgRhYWF2Gw28vLy\nyMzM1NevXLmSdevWoSgK8+bNIzc3F03TGDt2LP369QNg+PDh3H///Xz44Yc888wzWCwWJk+ezJQp\nUzqsMGnpCyHEiVoN/fXr1xMMBlmzZg35+fksXbqU5cuXA1BVVcXLL7/Me++9h9/vZ9KkSeTm5rJ3\n714GDx7MH//4R30/oVCIJ554gtdffx2Hw8Ftt93G+PHjSUlJ6ZDCZMimEEKcqNXunS1btjBmzBgg\n2mIvKCjQ1zkcDtLT0/H7/fj9fhRFAWDbtm0cOXKEmTNn8sMf/pBdu3ZRVFRERkYGSUlJ2Gw2RowY\nwWeffdZBZR0/eke6d4QQAk6ipe/1enG5XPqy2WwmHA5jsUSfmpaWxvXXX08kEmHu3LkApKamctdd\nd3Hdddfx+eefM3/+fB588EHcbre+H6fTidfrbfG1k5MTsVja1kpv2L3Tu6eb1FR3C1vHH6PVW0fq\nNhap+9S1Gvoulwufz6cvq6qqB/7GjRspKSnhgw8+AGD27Nnk5OQwZMgQzOZoWF988cWUlJScsB+f\nz9foTaApFRU1p15RTMPunYA/SGlpdZv3dbZJTXUbqt46UrexSN0tb9OcVrt3cnJy2LhxIwD5+fkM\nHDhQX5eUlERCQgI2mw273Y7b7aaqqoo//OEPvPTSSwDs2LGDtLQ0srOzKS4uprKykmAwyOeff85F\nF110UkW2hXyRK4QQJ2q1pZ+bm8umTZuYNm0amqaxZMkSVq1aRUZGBldeeSWbN29mypQpmEwmcnJy\nGD16NEOHDmX+/Pl89NFHmM1mnnjiCaxWKwsXLmT27NlomsbkyZPp1atXhxUmZ+QKIcSJFE2ru9RI\n13M6H91+9rt/UV0TBODpey7Hk2hrr8Pq8uRjr7FI3cbS4d07Zyt/oH4aBoe09IUQnSQQCPDWW2+c\n1LbvvPMW//rXRx16PHEZ+qGwSjgS/QBjNilYzHFZphDiLFBefvSkQ3/ChBu5/PJxHXo8cTmA/fgL\notedPyCEMLZ/fLqX/920m0Aw0m77tNvMTBzdn2svzWhy/csvr2TPnt2MGXMJF188Er/fz8KFD/OP\nf7zNjh3bqao6xrnnDuShhx7hhReeo0ePHmRk9ONPf3oZq9XCwYMHuPLKq7njjtntcrxxGvr1/6By\nYpYQos67n+1t18AHCAQjvPvZ3mZDf9asOykq2smll15GdXU19977C3w+L263m6effhZVVZk5cwql\npSWNnnfkyCFefPHPhEIhJk26VkK/JY0voCL9+UKIqGsuyeiQlv41lzQd+MfLyIjOW2a3J1BRUcEj\njzxEYmIifr+fcDjcaNusrHOxWCxYLBbs9oR2O964DP3GLX0JfSFE1LWXZjTbIu8oimJC01QATKZo\nV/Mnn2yipOQIjz76BBUVFWzcuIHjB1J2VK903Ie+Q7p3hBCdKDk5mVAoTCAQ0B87//zBvPjiC/zk\nJz9EURTS0/tQVlZ6Ro4nLhPx+C9yhRCis9jtdl588X8aPdajRwrPP//yCdsOGzZcv5+Tc7F+/803\n322344nLsYzyRa4QQjQtPkNfpmAQQogmxWfoN2zpy6UShRBCF5eh72/Qpy9TMAghRL24DH0ZsimE\nEE0zQOhL944QQtSJz9CXM3KFEHHiiy8+55FHHmy3/cVl6PulpS+EEE2Ky0SUk7OEEE1Zv/cj3tn9\nPoFIsN32aTfbmNA/l6symp8SORwO8+tfL2H//n2oqsr48bls3LiB3//+OQAeeOBe5syZx4ED+/nr\nX18jHA6jKApLlvym3Y6zTly29BtNwyBDNoUQMR/u3diugQ8QiAT5cO/GFrd56603SErqxjPP/DdL\nlz7J//7vXwkGgxw+fIiysjIqKysZOPA89u3by69//VuWL3+Bfv3683//93G7HivEa0tfTs4SQjRh\nfMbYDmnpj88Y2+I2RUU7+eqrrWzfXgBAJBJm3Ljx/OMfb2O1Wpkw4UYAkpO7k5f3CImJiRQX72HI\nkGHtdpx14jP0ZcimEKIJV2WMa7EbpqNkZvajZ8+ezJp1J4FALS+9tJKJE2/m3nt/gslk4r/+6w94\nvV5eeOE5/vKXdQDcd99PTph5sz3EXehHVJVgODqNqQLYrRL6QojONXHiLSxblsfdd9+Fz+fl5pu/\nh9Pp4txzBxKJhElMdKJpGkOHXsi8eT/AbLbgdrspKyslLS29XY9F0TriraSdtOVK9zW1Ye5+Otq/\nlmAz8+zPz/y7emdLTXW36Xd3tpO6jUXqbnmb5rTa0ldVlUWLFlFYWIjNZiMvL4/MzEx9/cqVK1m3\nbh2KojBv3jxyc3Oprq5m/vz5eL1eQqEQCxcu5KKLLuL9999n2bJlpKWlAfDTn/6UkSNHnmytJ8Vh\nN5PWI5FDR2sY3K97u+5bCCHOdq2G/vr16wkGg6xZs4b8/HyWLl3K8uXLAaiqquLll1/mvffew+/3\nM2nSJHJzc1m1ahWjRo3i+9//Prt27eL+++/nb3/7GwUFBcyfP59rrrmmwwpSFIWHZo7gUGWAc7q3\n3yXGhBAiHrQa+lu2bGHMmDEADB8+nIKCAn2dw+EgPT0dv9+P3+9HiV3f6/vf/z42mw2ASCSC3W4H\nYNu2bXz99de89NJLDBs2jF/84hdYLO3/tYIzwcplQ7sb8qOfEEK0pNXE9Xq9uFwufdlsNhMOh/Ww\nTktL4/rrrycSiTB37lwAPB4PAKWlpcyfP5+HHnoIgNGjR3PVVVdxzjnn8Mgjj7B69Wpuv/32Zl87\nOTkRi6XtX8S21K8V74xau9RtLFL3qWs19F0uFz6fT19WVVUP/I0bN1JSUsIHH3wAwOzZs8nJyWHY\nsGEUFhby85//nAceeEDvt588ebL+hnDllVfy7rstXwKsoqKmbVVh3C95wLi1S93GInW3vE1zWj0j\nNycnh40bo6Nh8vPzGThwoL4uKSmJhIQEbDYbdrsdt9tNVVUVO3fu5Gc/+xlPPvkk48ZFR89omsZN\nN93E4cOHAfj4448ZPHhw6xUKIYRoN6229HNzc9m0aRPTpk1D0zSWLFnCqlWryMjI4Morr2Tz5s1M\nmTIFk8lETk4Oo0eP5sc//jHBYJDHH38ciH5aWL58OXl5edx9990kJCSQnZ3NlClTOrxAIYQQ9eJu\nnH4do370A+PWLnUbi9Td8jbNicsJ14QQQjRNQl8IIQykS3fvCCGEaF/S0hdCCAOR0BdCCAOR0BdC\nCAOR0BdCCAOR0BdCCAOR0BdCCAOJu8sltnbRl3j05Zdf8pvf/IZXXnmF4uJiFi5ciKIoDBgwgEce\neQSTKb7e20OhEA899BAHDhwgGAzyox/9iHPPPTfu64boVOX/8R//we7du1EUhcWLF2O32w1R+9Gj\nR7nllltYuXIlFovFEDUD3HzzzfpMx+eccw5Tp07l8ccfx2w2c/nll3P33Xef2g61OPPuu+9qCxYs\n0DRN07Zu3arNmzevk4+oY61YsUK74YYbtO9973uapmna3LlztU8++UTTNE17+OGHtffee68zD69D\nvP7661peXp6maZpWUVGhjRs3zhB1a5qmvf/++9rChQs1TdO0Tz75RJs3b54hag8Gg9qPf/xj7eqr\nr9Z27txpiJo1TdNqa2u1iRMnNnrspptu0oqLizVVVbU5c+Zo27ZtO6V9xt1bY0sXfYlHGRkZ/P73\nv9eXt23bpk9lPXbsWDZv3txZh9Zhrr32Wn72s58B0dlbzWazIeoGuOqqq3jssccAOHjwIB6PxxC1\nL1u2jGnTptGzZ0/AGP/PAXbs2IHf7+fOO+9k1qxZfPbZZwSDQTIyMlAUhcsvv/yUa4+70G/uoi/x\n6pprrml09TFN0/QrmDmdTqqr429CKqfTicvlwuv1cs8993Dvvfcaou46FouFBQsW8Nhjj3HjjTfG\nfe1//etf6d69u96YA2P8PwdISEhg9uzZvPDCCyxevJgHH3wQh8Ohr29L7XEX+i1d9MUIGvZr+nw+\n/aI18ebQoUPMmjWLiRMncuONNxqm7jrLli3j3Xff5eGHHyYQCOiPx2Ptf/nLX9i8eTMzZ87k66+/\nZsGCBZSXl+vr47HmOv379+emm25CURT69++P2+2msrJSX9+W2uMu9Fu66IsRXHDBBXz66adA9Mpm\nF198cScfUfsrKyvjzjvvZP78+dx6662AMeoGeOONN3juueeA6DWqFUVhyJAhcV37n/70J1599VVe\neeUVzj//fJYtW8bYsWPjuuY6r7/+OkuXLgXgyJEj+P1+EhMT2bt3L5qm8a9//euUa4+7CdfqRu98\n8803+kVfsrOzO/uwOtT+/fv5+c9/ztq1a9m9ezcPP/wwoVCIrKws8vLyMJvbfp3hrigvL4+///3v\nZGVl6Y/98pe/JC8vL67rBqipqeHBBx+krKyMcDjMD3/4Q7Kzs+P+37zOzJkzWbRoESaTyRA1B4NB\nHnzwQQ4ePIiiKPziF7/AZDKxZMkSIpEIl19+Offdd98p7TPuQl8IIUTz4q57RwghRPMk9IUQwkAk\n9IUQwkAk9IUQwkAk9IUQwkAk9IUQwkAk9IUQwkAk9IUQwkD+P9PM/2sqcQEIAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "## Plot\n", + "plt.plot(range(epochs), mod_2_tr_acc, linewidth=3, label='train')\n", + "plt.plot(range(epochs), mod_2_val_acc, linewidth=3, label='eval')\n", + "plt.legend(loc=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### L1 Regularization" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "l1 = 0\n", + "for i in range(1, 4):\n", + " l1 += mx.sym.sum(mx.sym.abs(ce.get_internals()['fc%d_weight'%i]))\n", + "\n", + "loss = ce + 0.0005*l1\n", + "\n", + "my_loss = mx.sym.MakeLoss(loss)\n", + "group = mx.sym.Group([mx.sym.BlockGrad(softmax_out, name='softmax'), my_loss])" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "image/svg+xml": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "plot\n", + "\n", + "\n", + "\n", + "data\n", + "\n", + "data\n", + "\n", + "\n", + "\n", + "flatten0\n", + "\n", + "flatten0\n", + "\n", + "\n", + "\n", + "flatten0->data\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc1\n", + "\n", + "FullyConnected\n", + "128\n", + "\n", + "\n", + "\n", + "fc1->flatten0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "relu1\n", + "\n", + "Activation\n", + "relu\n", + "\n", + "\n", + "\n", + "relu1->fc1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc2\n", + "\n", + "FullyConnected\n", + "64\n", + "\n", + "\n", + "\n", + "fc2->relu1\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "relu2\n", + "\n", + "Activation\n", + "relu\n", + "\n", + "\n", + "\n", + "relu2->fc2\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "fc3\n", + "\n", + "FullyConnected\n", + "10\n", + "\n", + "\n", + "\n", + "fc3->relu2\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax_out\n", + "\n", + "softmax_out\n", + "\n", + "\n", + "\n", + "softmax_out->fc3\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax\n", + "\n", + "softmax\n", + "\n", + "\n", + "\n", + "softmax->softmax_out\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "softmax_label\n", + "\n", + "softmax_label\n", + "\n", + "\n", + "\n", + "one_hot\n", + "\n", + "one_hot\n", + "\n", + "\n", + "\n", + "one_hot->softmax_label\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mulscalar0\n", + "\n", + "_mulscalar0\n", + "\n", + "\n", + "\n", + "_mulscalar0->one_hot\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plusscalar0\n", + "\n", + "_plusscalar0\n", + "\n", + "\n", + "\n", + "_plusscalar0->softmax_out\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "log0\n", + "\n", + "log0\n", + "\n", + "\n", + "\n", + "log0->_plusscalar0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mul0\n", + "\n", + "_mul0\n", + "\n", + "\n", + "\n", + "_mul0->_mulscalar0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mul0->log0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "sum0\n", + "\n", + "sum0\n", + "\n", + "\n", + "\n", + "sum0->_mul0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "abs9\n", + "\n", + "abs9\n", + "\n", + "\n", + "\n", + "sum10\n", + "\n", + "sum10\n", + "\n", + "\n", + "\n", + "sum10->abs9\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plusscalar4\n", + "\n", + "_plusscalar4\n", + "\n", + "\n", + "\n", + "_plusscalar4->sum10\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "abs10\n", + "\n", + "abs10\n", + "\n", + "\n", + "\n", + "sum11\n", + "\n", + "sum11\n", + "\n", + "\n", + "\n", + "sum11->abs10\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus9\n", + "\n", + "_plus9\n", + "\n", + "\n", + "\n", + "_plus9->_plusscalar4\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus9->sum11\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "abs11\n", + "\n", + "abs11\n", + "\n", + "\n", + "\n", + "sum12\n", + "\n", + "sum12\n", + "\n", + "\n", + "\n", + "sum12->abs11\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus10\n", + "\n", + "_plus10\n", + "\n", + "\n", + "\n", + "_plus10->_plus9\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus10->sum12\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_mulscalar4\n", + "\n", + "_mulscalar4\n", + "\n", + "\n", + "\n", + "_mulscalar4->_plus10\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus11\n", + "\n", + "_plus11\n", + "\n", + "\n", + "\n", + "_plus11->sum0\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "_plus11->_mulscalar4\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "makeloss4\n", + "\n", + "makeloss4\n", + "\n", + "\n", + "\n", + "makeloss4->_plus11\n", + "\n", + "\n", + "\n", + "\n", + "\n" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mx.viz.plot_network(symbol=group)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "mod_3 = mx.mod.Module(symbol=group, context=mx.cpu(), data_names=['data'], label_names=['softmax_label'])\n", + "mod_3.bind(data_shapes=train_iter.provide_data, label_shapes=train_iter.provide_label)\n", + "mod_3.init_params(initializer=mx.init.Uniform(scale=.1))\n", + "mod_3.init_optimizer(optimizer='sgd', optimizer_params={'learning_rate': 0.1})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:Epoch [0] Train-accuracy=0.836567\n", + "WARNING:root:Epoch [0] Train-cross-entropy=0.585078\n", + "WARNING:root:Epoch [0] Validation-accuracy=0.918700\n", + "WARNING:root:Epoch [1] Train-accuracy=0.932250\n", + "WARNING:root:Epoch [1] Train-cross-entropy=0.232668\n", + "WARNING:root:Epoch [1] Validation-accuracy=0.944600\n", + "WARNING:root:Epoch [2] Train-accuracy=0.951300\n", + "WARNING:root:Epoch [2] Train-cross-entropy=0.165669\n", + "WARNING:root:Epoch [2] Validation-accuracy=0.956700\n", + "WARNING:root:Epoch [3] Train-accuracy=0.961633\n", + "WARNING:root:Epoch [3] Train-cross-entropy=0.129842\n", + "WARNING:root:Epoch [3] Validation-accuracy=0.963600\n", + "WARNING:root:Epoch [4] Train-accuracy=0.968183\n", + "WARNING:root:Epoch [4] Train-cross-entropy=0.106841\n", + "WARNING:root:Epoch [4] Validation-accuracy=0.966400\n", + "WARNING:root:Epoch [5] Train-accuracy=0.973350\n", + "WARNING:root:Epoch [5] Train-cross-entropy=0.090110\n", + "WARNING:root:Epoch [5] Validation-accuracy=0.969300\n", + "WARNING:root:Epoch [6] Train-accuracy=0.977483\n", + "WARNING:root:Epoch [6] Train-cross-entropy=0.077412\n", + "WARNING:root:Epoch [6] Validation-accuracy=0.969900\n", + "WARNING:root:Epoch [7] Train-accuracy=0.980617\n", + "WARNING:root:Epoch [7] Train-cross-entropy=0.067230\n", + "WARNING:root:Epoch [7] Validation-accuracy=0.971700\n", + "WARNING:root:Epoch [8] Train-accuracy=0.983550\n", + "WARNING:root:Epoch [8] Train-cross-entropy=0.058816\n", + "WARNING:root:Epoch [8] Validation-accuracy=0.972600\n", + "WARNING:root:Epoch [9] Train-accuracy=0.985767\n", + "WARNING:root:Epoch [9] Train-cross-entropy=0.051859\n", + "WARNING:root:Epoch [9] Validation-accuracy=0.973800\n", + "WARNING:root:Epoch [10] Train-accuracy=0.987217\n", + "WARNING:root:Epoch [10] Train-cross-entropy=0.045790\n", + "WARNING:root:Epoch [10] Validation-accuracy=0.973900\n", + "WARNING:root:Epoch [11] Train-accuracy=0.989167\n", + "WARNING:root:Epoch [11] Train-cross-entropy=0.040471\n", + "WARNING:root:Epoch [11] Validation-accuracy=0.973900\n", + "WARNING:root:Epoch [12] Train-accuracy=0.990633\n", + "WARNING:root:Epoch [12] Train-cross-entropy=0.035837\n", + "WARNING:root:Epoch [12] Validation-accuracy=0.974700\n", + "WARNING:root:Epoch [13] Train-accuracy=0.992167\n", + "WARNING:root:Epoch [13] Train-cross-entropy=0.031609\n", + "WARNING:root:Epoch [13] Validation-accuracy=0.975300\n", + "WARNING:root:Epoch [14] Train-accuracy=0.993517\n", + "WARNING:root:Epoch [14] Train-cross-entropy=0.027966\n", + "WARNING:root:Epoch [14] Validation-accuracy=0.975700\n", + "WARNING:root:Epoch [15] Train-accuracy=0.994433\n", + "WARNING:root:Epoch [15] Train-cross-entropy=0.024719\n", + "WARNING:root:Epoch [15] Validation-accuracy=0.975800\n", + "WARNING:root:Epoch [16] Train-accuracy=0.995267\n", + "WARNING:root:Epoch [16] Train-cross-entropy=0.021916\n", + "WARNING:root:Epoch [16] Validation-accuracy=0.976100\n", + "WARNING:root:Epoch [17] Train-accuracy=0.996133\n", + "WARNING:root:Epoch [17] Train-cross-entropy=0.019297\n", + "WARNING:root:Epoch [17] Validation-accuracy=0.976600\n", + "WARNING:root:Epoch [18] Train-accuracy=0.996850\n", + "WARNING:root:Epoch [18] Train-cross-entropy=0.017062\n", + "WARNING:root:Epoch [18] Validation-accuracy=0.976700\n", + "WARNING:root:Epoch [19] Train-accuracy=0.997383\n", + "WARNING:root:Epoch [19] Train-cross-entropy=0.015031\n", + "WARNING:root:Epoch [19] Validation-accuracy=0.977400\n", + "WARNING:root:Epoch [20] Train-accuracy=0.997867\n", + "WARNING:root:Epoch [20] Train-cross-entropy=0.013303\n", + "WARNING:root:Epoch [20] Validation-accuracy=0.977300\n", + "WARNING:root:Epoch [21] Train-accuracy=0.998500\n", + "WARNING:root:Epoch [21] Train-cross-entropy=0.011809\n", + "WARNING:root:Epoch [21] Validation-accuracy=0.977500\n", + "WARNING:root:Epoch [22] Train-accuracy=0.998833\n", + "WARNING:root:Epoch [22] Train-cross-entropy=0.010517\n", + "WARNING:root:Epoch [22] Validation-accuracy=0.977700\n", + "WARNING:root:Epoch [23] Train-accuracy=0.999083\n", + "WARNING:root:Epoch [23] Train-cross-entropy=0.009445\n", + "WARNING:root:Epoch [23] Validation-accuracy=0.977100\n", + "WARNING:root:Epoch [24] Train-accuracy=0.999233\n", + "WARNING:root:Epoch [24] Train-cross-entropy=0.008471\n", + "WARNING:root:Epoch [24] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [25] Train-accuracy=0.999400\n", + "WARNING:root:Epoch [25] Train-cross-entropy=0.007664\n", + "WARNING:root:Epoch [25] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [26] Train-accuracy=0.999533\n", + "WARNING:root:Epoch [26] Train-cross-entropy=0.006897\n", + "WARNING:root:Epoch [26] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [27] Train-accuracy=0.999650\n", + "WARNING:root:Epoch [27] Train-cross-entropy=0.006225\n", + "WARNING:root:Epoch [27] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [28] Train-accuracy=0.999683\n", + "WARNING:root:Epoch [28] Train-cross-entropy=0.005704\n", + "WARNING:root:Epoch [28] Validation-accuracy=0.977100\n", + "WARNING:root:Epoch [29] Train-accuracy=0.999750\n", + "WARNING:root:Epoch [29] Train-cross-entropy=0.005234\n", + "WARNING:root:Epoch [29] Validation-accuracy=0.977400\n", + "WARNING:root:Epoch [30] Train-accuracy=0.999767\n", + "WARNING:root:Epoch [30] Train-cross-entropy=0.004823\n", + "WARNING:root:Epoch [30] Validation-accuracy=0.977200\n", + "WARNING:root:Epoch [31] Train-accuracy=0.999817\n", + "WARNING:root:Epoch [31] Train-cross-entropy=0.004470\n", + "WARNING:root:Epoch [31] Validation-accuracy=0.977300\n", + "WARNING:root:Epoch [32] Train-accuracy=0.999850\n", + "WARNING:root:Epoch [32] Train-cross-entropy=0.004130\n", + "WARNING:root:Epoch [32] Validation-accuracy=0.977400\n", + "WARNING:root:Epoch [33] Train-accuracy=0.999867\n", + "WARNING:root:Epoch [33] Train-cross-entropy=0.003866\n", + "WARNING:root:Epoch [33] Validation-accuracy=0.977300\n", + "WARNING:root:Epoch [34] Train-accuracy=0.999900\n", + "WARNING:root:Epoch [34] Train-cross-entropy=0.003622\n", + "WARNING:root:Epoch [34] Validation-accuracy=0.977300\n", + "WARNING:root:Epoch [35] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [35] Train-cross-entropy=0.003407\n", + "WARNING:root:Epoch [35] Validation-accuracy=0.977300\n", + "WARNING:root:Epoch [36] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [36] Train-cross-entropy=0.003219\n", + "WARNING:root:Epoch [36] Validation-accuracy=0.977700\n", + "WARNING:root:Epoch [37] Train-accuracy=0.999917\n", + "WARNING:root:Epoch [37] Train-cross-entropy=0.003055\n", + "WARNING:root:Epoch [37] Validation-accuracy=0.977700\n", + "WARNING:root:Epoch [38] Train-accuracy=0.999933\n", + "WARNING:root:Epoch [38] Train-cross-entropy=0.002910\n", + "WARNING:root:Epoch [38] Validation-accuracy=0.977600\n", + "WARNING:root:Epoch [39] Train-accuracy=0.999933\n", + "WARNING:root:Epoch [39] Train-cross-entropy=0.002775\n", + "WARNING:root:Epoch [39] Validation-accuracy=0.977600\n", + "WARNING:root:Epoch [40] Train-accuracy=0.999967\n", + "WARNING:root:Epoch [40] Train-cross-entropy=0.002656\n", + "WARNING:root:Epoch [40] Validation-accuracy=0.977600\n", + "WARNING:root:Epoch [41] Train-accuracy=0.999967\n", + "WARNING:root:Epoch [41] Train-cross-entropy=0.002548\n", + "WARNING:root:Epoch [41] Validation-accuracy=0.977600\n", + "WARNING:root:Epoch [42] Train-accuracy=0.999967\n", + "WARNING:root:Epoch [42] Train-cross-entropy=0.002453\n", + "WARNING:root:Epoch [42] Validation-accuracy=0.977700\n", + "WARNING:root:Epoch [43] Train-accuracy=0.999967\n", + "WARNING:root:Epoch [43] Train-cross-entropy=0.002364\n", + "WARNING:root:Epoch [43] Validation-accuracy=0.977500\n" + ] + } + ], + "source": [ + "mod_3_tr_acc = []\n", + "mod_3_val_acc = []\n", + "for epoch in range(epochs):\n", + " ## Training\n", + " for batch in train_iter:\n", + " mod_3.forward(batch, is_train=True) # compute predictions\n", + " mod_3.update_metric(metrics, batch.label) # accumulate prediction accuracy\n", + " mod_3.backward() # compute gradients\n", + " mod_3.update() # update parameters\n", + " loss = mod_2.get_outputs()[1].asnumpy()\n", + " # print('Epoch %d, Loss %.3f' % (epoch, loss[1].asnumpy()))\n", + " for name, val in metrics.get_name_value():\n", + " logging.warning('Epoch [%d] Train-%s=%f' % (epoch, name, val))\n", + " if name == 'accuracy':\n", + " mod_3_tr_acc.append(val)\n", + " \n", + " train_iter.reset()\n", + " metrics.reset()\n", + " \n", + " eval_results = mod_3.predict(val_iter)\n", + " eval_probability = eval_results[0].asnumpy()\n", + " eval_prediction = np.argmax(eval_probability, axis=1)\n", + " eval_accuracy = accuracy_score(eval_prediction, mnist['test_label'])\n", + " logging.warning('Epoch [%d] Validation-%s=%f' % (epoch, 'accuracy', eval_accuracy))\n", + " mod_3_val_acc.append(eval_accuracy)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.plot(range(epochs), mod_3_tr_acc, linewidth=3, label='train')\n", + "plt.plot(range(epochs), mod_3_val_acc, linewidth=3, label='eval')\n", + "plt.legend(loc=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(16,12))\n", + "plt.plot(range(epochs), mod_1_tr_acc, linewidth=3, label='train-basic')\n", + "plt.plot(range(epochs), mod_2_tr_acc, linewidth=3, label='train-L2')\n", + "plt.plot(range(epochs), mod_3_tr_acc, linewidth=3, label='train-L1')\n", + "plt.plot(range(epochs), mod_1_val_acc, linewidth=3, label='eval-basic')\n", + "plt.plot(range(epochs), mod_2_val_acc, linewidth=3, label='eval-L2')\n", + "plt.plot(range(epochs), mod_3_val_acc, linewidth=3, label='eval-L1')\n", + "plt.legend(loc=4)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "anaconda-cloud": {}, + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}