WiDS_workshop2018_part2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# PART II: MODELING\n",
    "\n",
    "- Introduction to Machine Learning\n",
    "- Models for Classification\n",
    "  - Logistic regression\n",
    "  - SVM\n",
    "  - Decision tree\n",
    "  - Ensemble methods\n",
    "- Example: Predicting Hubway User Type\n",
    "- Advanced Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# importing libraries\n",
    "%matplotlib inline\n",
    "import matplotlib\n",
    "import pandas as pd # package for data analysis, reading in the data, summarizing data, manipulating data\n",
    "import numpy as np # package for math operations\n",
    "import matplotlib.pyplot as plt # package for plotting\n",
    "from IPython.display import Image\n",
    "import itertools\n",
    "from math import radians, cos, sin, asin, sqrt\n",
    "from sklearn import preprocessing, metrics, model_selection, linear_model, ensemble, svm, tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# setting up variables for later computation\n",
    "current_year = 2017.\n",
    "rand_state = 2\n",
    "test_size = 0.3\n",
    "\n",
    "# set random seed\n",
    "np.random.seed(rand_state)\n",
    "\n",
    "# funtion to convert birth date to age\n",
    "year_to_age = lambda s: 0 if 'N' in s else 2017 - int(s)\n",
    "\n",
    "# function to convert long/lat to distance from Boston City Center\n",
    "def haversine(pt, lat2=42.355589, lon2=-71.060175):\n",
    "    lon1 = pt[0]\n",
    "    lat1 = pt[1]\n",
    "    \n",
    "    # convert decimal degrees to radians \n",
    "    lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])\n",
    "\n",
    "    # haversine formula \n",
    "    dlon = lon2 - lon1 \n",
    "    dlat = lat2 - lat1 \n",
    "    a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2\n",
    "    c = 2 * asin(sqrt(a)) \n",
    "    r = 3956 # Radius of earth in miles\n",
    "    return c * r\n",
    "\n",
    "#--------  plot_decision_boundary\n",
    "# A function that visualizes the data and the decision boundaries\n",
    "# Input: \n",
    "#      x (predictors)\n",
    "#      y (labels)\n",
    "#      model (classifier)\n",
    "#      poly_flag (fits quadratic model if true, otherwise linear)\n",
    "#      title (title for plot)\n",
    "#      ax (a set of axes to plot on)\n",
    "# Returns: \n",
    "#      ax (axes with data and decision boundaries)\n",
    "\n",
    "def plot_decision_boundary(x, y, model, title, ax, bounds=(0, 1), poly_flag=False):\n",
    "    # Plot data\n",
    "    ax.scatter(x[y == 1, 0], x[y == 1, 1], c='blue')\n",
    "    ax.scatter(x[y == 0, 0], x[y == 0, 1], c='red', alpha=0.3)\n",
    "    \n",
    "    # Create mesh\n",
    "    interval = np.arange(bounds[0], bounds[1], 0.01)\n",
    "    n = np.size(interval)\n",
    "    x1, x2 = np.meshgrid(interval, interval)\n",
    "    x1 = x1.reshape(-1, 1)\n",
    "    x2 = x2.reshape(-1, 1)\n",
    "    xx = np.concatenate((x1, x2), axis=1)\n",
    "\n",
    "    # Predict on mesh points\n",
    "    if(poly_flag):\n",
    "        quad_features = preprocessing.PolynomialFeatures(degree=2)\n",
    "        xx = quad_features.fit_transform(xx)\n",
    "        \n",
    "        \n",
    "    yy = model.predict(xx)    \n",
    "    yy = yy.reshape((n, n))\n",
    "\n",
    "    # Plot decision surface\n",
    "    x1 = x1.reshape(n, n)\n",
    "    x2 = x2.reshape(n, n)\n",
    "    ax.contourf(x1, x2, yy, alpha=0.1, cmap='Blues')\n",
    "    \n",
    "    # Label axes, set title\n",
    "    ax.set_title(title)\n",
    "    ax.set_xlabel('Feature 1')\n",
    "    ax.set_ylabel('Feature 2')\n",
    "    \n",
    "    return ax\n",
    "\n",
    "# function to generate some toy data for classification\n",
    "def generate_toy_data(test_size, rand_state):\n",
    "    data = np.random.multivariate_normal([0, 0], np.eye(2) * 5, size=200)\n",
    "    data = np.hstack((data, np.zeros((200, 1))))\n",
    "    data[data[:, 0]**2 + data[:, 1]**2 < 3**2, 2] = np.random.choice([0, 1], \n",
    "                                                                     len(data[data[:, 0]**2 + data[:, 1]**2 < 3**2]), \n",
    "                                                                     p=[0.2, 0.8])\n",
    "    X = data[:, :-1]\n",
    "    y = data[:, -1]\n",
    "    \n",
    "    # train test split\n",
    "    return model_selection.train_test_split(X, y, test_size=test_size, random_state=rand_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# read in ride data\n",
    "ride_data = pd.read_csv('hubway_data_subset.csv', index_col=0)\n",
    "# read in station data\n",
    "station_data = pd.read_csv('hubway_stations.csv', low_memory=False)[['id', 'lat', 'lng']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# compute age from birth dates\n",
    "ride_data.loc[:, 'age'] = current_year - ride_data['birth_date'].values\n",
    "# join longitude/latitude to ride dataframe by the start station id \n",
    "ride_data_with_gps = ride_data.join(station_data.set_index('id'), on='strt_statn')\n",
    "# compute distance from start station to city center \n",
    "ride_data.loc[:, 'start_dist_to_center'] = list(map(haversine, ride_data_with_gps[['lng', 'lat']].values))\n",
    "# join longitude/latitude to ride dataframe by the end station id \n",
    "ride_data_with_gps = ride_data.join(station_data.set_index('id'), on='end_statn')\n",
    "# compute distance from end station to city center \n",
    "ride_data.loc[:, 'end_dist_to_center'] = list(map(haversine, ride_data_with_gps[['lng', 'lat']].values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# WHAT IS MACHINE LEARNING?\n",
    "\n",
    "Automated learning of rules and patterns from data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "**Supervised vs unsupervised**\n",
    "\n",
    "Supervised: learning rules from provided examples\n",
    "\n",
    "<img src=\"images/supervised1.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Supervised: learning rules from provided examples\n",
    "\n",
    "<img src=\"images/supervised2.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Unsupervised: learning hidden (latent) structures \n",
    "\n",
    "<img src=\"images/unsupervised1.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Unsupervised: learning hidden (latent) structures \n",
    "\n",
    "<img src=\"images/unsupervised2.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Regression vs classification**\n",
    "\n",
    "A common task for supervised learning is to use a observed values of a set of ***predictors***\n",
    "$$\n",
    "[x_1, \\ldots, x_D] \\in \\mathbb{R}^D\n",
    "$$\n",
    "to predict a quantity of interest $y$, called the ***response***. \n",
    "\n",
    "If $y$ is a real-valued variable, we call the task ***regression***, if $y$ is a label (categorical variable) we all the task ***classification***."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Training**\n",
    "\n",
    "In supervised learning, when we say we are ***training*** or ***fitting*** a model, we typically mean that\n",
    "0. We have a set of observations, called ***training data***,\n",
    "$$\n",
    "\\{(\\mathbf{x}_1, y_1),\\ldots, (\\mathbf{x}_N, y_N)\\}\n",
    "$$\n",
    "where each $\\mathbf{x}_n$ is the observed values of the $D$-number of predictors $\\mathbf{x}_n = [x_1, \\ldots, x_D]$.\n",
    "1. We have a 'function' in mind that explains how the response depends on the predictors\n",
    "$$\n",
    "f(x_1, \\ldots, x_D) \\rightarrow y.\n",
    "$$\n",
    "2. This function has some constants/coefficients we need to set, these are called ***parameters***, e.g.\n",
    "$$\n",
    "f(x_1, \\ldots, x_D) = w_1\\sin(x_1) + \\ldots + w_D \\sin(x_D)\n",
    "$$\n",
    "3. We find the values of these parameters so to best explain the observed data (e.g. our model should explain what we've seen).\n",
    "4. To evaluate our model, we use our function to make predictions $\\widehat{y}$ and compare them to the true response $y$ in our training data. \n",
    "\n",
    "For classification, we will be using ***accuracy***, i.e. the percentage of points correctly labeled by our model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Testing**\n",
    "\n",
    "**Question:** Is it enough to check that we do well on the training data?\n",
    "\n",
    "**Hint:** remember that you optimized your model to explain the training data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# MODELS FOR CLASSIFICATION\n",
    "\n",
    "  - Logistic regression\n",
    "  - SVM\n",
    "  - Decision tree\n",
    "  - Ensemble methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "# Logistic Regression: The Idea"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Suppose our data has a small number of predictors, by visualizing the data we can intuitively check how easy it is to separate the classes.\n",
    "<img src=\"images/decision_boundary0.png\" alt=\"\" style=\"height: 300px;\"/>\n",
    "Ideally, the classes are easily separated by a curve (or surface) in the input space, this curve (or surface) is called the ***decision boundary***."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "We can try to model the **probability** of a data point being from a particular class by\n",
    "1. which side of the decision boundary it's on\n",
    "2. how far it is away from the boundary. Intuitively, the farther a data point is from the decision boundary, the more 'certain' we should be of it's classification.\n",
    "\n",
    "<img src=\"images/decision_boundary0.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Logistic Regression: Linear Decision Boundary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "When the decision boundary is linear, it is defined by the equation\n",
    "$$\n",
    "\\mathbf{w}^\\top \\mathbf{x} = w_0x_0 + w_1x_1 + \\ldots + w_D x_D = 0\n",
    "$$\n",
    "where $x_0 = 1$.\n",
    "\n",
    "The vector $\\mathbf{w}$ allow us to gauge the 'distance' of a point from the decision boundary\n",
    "<img src=\"images/decision_boundary1.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "To model the probability of labeling a point a certain class, we have to convert distance, $\\mathbf{w}^\\top\\mathbf{x}$ (which is unbounded) into a number between 0 and 1, using the ***sigmoid function***:\n",
    "$$\n",
    "\\text{Prob}[y = 1 | \\mathbf{x}] = \\text{sigm}(\\mathbf{w}^\\top\\mathbf{x})\n",
    "$$\n",
    "If $\\text{Prob}[y = 1 | \\mathbf{x}] \\geq 0.5$ we label $\\mathbf{x}$ class 1, otherwise we label it class 0.\n",
    "\n",
    "\n",
    "To *fit* our model, we need to learn the parameters of $\\mathbf{w}$ that maximizes the likelihood of our training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "skip"
    }
   },
   "outputs": [],
   "source": [
    "# read training toy data set\n",
    "linear_toy_data= pd.read_csv('toy_data_train.txt', delimiter=',')\n",
    "X_train = linear_toy_data.values[:, :-1]\n",
    "y_train = linear_toy_data.values[:, -1]\n",
    "# read testing toy data set\n",
    "linear_toy_data = pd.read_csv('toy_data_test.txt', delimiter=',')\n",
    "X_test = linear_toy_data.values[:, :-1]\n",
    "y_test = linear_toy_data.values[:, -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a logistic regression model with linear boundary\n",
    "logreg = linear_model.LogisticRegression(C=1.)\n",
    "# fit our logistic regression model\n",
    "logreg.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAFgCAYAAACmDI9oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzs3Xt8XVWd///XJ72QpGmaNr1Ykrah\nFJRSJJbG4ncGRFQKqJQOjl8woAhSGQSvU+1MHX+MilZxhrn8mHGKOlwMcnNicQQ7oNxGRVOgZbgM\nlEtbTmkhDaRJm4Resr5/rH3andOT5Jzk7HN9Px+PPPbZ6+zsvfblrPM5a6+1tjnnEBERERERryzX\nGRARERERyScKkEVEREREQhQgi4iIiIiEKEAWEREREQlRgCwiIiIiEqIAWUREREQkRAFyisxsjJnt\nNrPZmVxWciM4R7eZWaeZ/SZL29xsZk3Z2NZomFmVmTkze1tE619uZq2h+feb2UvBZ+YDZvawmZ0X\nwXZvMbMvZXq9UnrMrNnM/muE//u0mZ2W4SzlPTO718w+GdG6zzCzn4fmnZnNC17/wMz+Jortpiv4\nDvhAHuTjajP7SUTr/pyZrY5i3dlmxToOspntDs1WAm8BB4L5zzjnWrKfq9Exsx8C5wez4wHD7xfA\nA865j2RwWzvxx60f2Af8D3Aj8O8uhYvGzBYA651z5aPIw8eAvwEagD7gCeCTzrnto922mZ0F/BNw\ngnOub6R5TNjm/wB7gqTdwE+Bv3TOHRj0H3Mk+PF2DXAGUAHEgBbg+8A4oBuY6ZzbkYW8/AFY45z7\nUQbXeSXwYefcmZlapxQmM9sMfNo5d38Otn0jEHPOfW2U62kAXuZQ+bIT+IFzrigCkdEys/XAlc65\nR4N5BxzjnHshtzkbKJfXYkI+rgbmOecujGDd5cALwELn3OuZXn82FW0NsnOuKv4HbAU+Eko7LDg2\ns7HZz2V6nHOfDu3T94CW0D5lLDgOOT3Y1lx8MPm3wTRyZvZO4N+Ay4FqYB7wIyBTv+jmAC+OJDge\n4lp5K3R+FgJnAZ8aRR5Hm5/Bln8b8CiwF1jknKsGPgTMBuozn8NhzQGezsF2RQpNTVC+fBT4GzP7\nYKY3UAjfhWHBXblJ8eA4H5hXlPGVmY0Z6v3gO/Ve4BPZyVF0ivIEpsLMvmVmt5vZT82sG7jQzN5j\nZo8Gt923m9k/mdm4YPmxwW2bhmD+J8H795pZt5n93syOSnfZ4P2zzOx5M9tlZv9sZr81s4tHuF8f\nM7NnzexNM7vPzI4O0v/WzG5KWPbfzexbw63TOfemc+4u4CLgs2Y2N/j/88zsyWCftpjZV0P/9jBw\nRHDbfLeZnWBm84Pb52+Y2evB9qsG2exC4Gnn3G+dt8s5d3u8RjM4xleb2ctmtjO4fV492LYT9vtz\nwD8CHwje/0o83fyt/p1mdpeZTQ/S400OLjezF4GNKRyzV4HfAPND291pZicHr79vZjcH12C3mW0M\n5zM4X5uD9/4nqPGOv3dlcG5/YGZvAiuD5eaGlpljZntCxyRsJfCKc+5S59wrQX5fds59xjn3YuLC\nQ53n4NjcEZzTN4PPz6TgvcuD5bvN7EUz+7NQ/n8VvN4BzAB+Y2adQdp6Mzs/eG1mdpWZPRes50kz\nmz/UMTL/hfl3ofMbC9LvMrOVobwPd74/Hbz/hpl9f7hzLoXHzC4zsxeCc3y3mR0Zeu+M4LrbZWb/\nYmYPmdmng/cuNrP/Dl6bmV0XlGm7gmt0gZktB5qBrwTX4S+C5Q/eZjff1Ouvg89Ht5k9Zmazhsu3\nc249/kdlYyi/R5rZz8ys3Xy5+LnQexVmdlPwGX3WzL4S/1yE8vRVM3sS2GO+fB1qfe8OPqddZvaa\nmf19kF5u/vuuw/z3aJuZzQjeezB0/MrM7GtB+fC6+bIwXm40BJ+/T5rZ1uDzuWqIw3EW8NBgb5rZ\njRZ8z5nZaWYWM7MvB9vdbmafCi17hPmyeWuwXz8ws4rgvclm9p/B8XgzeF0f+t8HzewaM/st0IOv\nWEqmycyeCdbx7+ZrXOPrSHo9ho7J2NCy4eN5sZn9d5D3N4PzFf7OOCq4frvN7D5gasIxutPMdgTX\n78NmdnzC8ftXM7vHzPYAXwqOTTgv55nZhtAqH8RXuhQ251zR/wGbgQ8kpH0LX4P2EfwPhQqgCVgM\njMVf3M/jb9sQpDmgIZj/Cf421yL8LenbgZ+MYNnp+NvZS4P3voRv0nDxMPv0LeDGhLTGYF2n4ptg\n/C3wFDAGOBroAiqDZSuAXcA7Bln/TuDkJOlvABcFrz8IHBccv0XAm/HjDCwA+hL+dz5wWrCfM4E/\nAt8aZPvH45uPfBd4bzzfofe/hv8Qvi3Yl1uAGwbbdpL1Xwn8KjR/DrA9+N8KfG31vcF7VcH5vBuY\nBFQkWd+AbQKzgOeAjyU7pvimDHuA04Pz88/A/aFlzw/2bQy+FroLmBLK+37gkuD9CuBm4G9C/78K\n+Okg+/4UsGKIYxPf37elcJ6/jL+ey/HX/buD/EwLrpW5wXJ18WstybEfcK0B64Hzg9efAl4CTsQ3\nKXoHUJfiMfpVwn7dBaxM43zfCUzk0GfnT3Ndlukv/T+SlP9B+unBtbcQOCL4DD4cvDc1OOd/FlzX\nn8eXy58O3r8Y+O/g9RLgMaAmuEaPwzdPAt8s7VuD5QdYgW+a9fbgf08EapPktSG4JscG8yfjg7Bl\nwXxZkIev48v+ucHnZknw/mp8EDkZf5foSXzTj3CeNuDLrYoU1vd7Dn0PVHGoXPsM8At887wxwElA\ndfDeg6Hjdwn+Nvzc4P//A7glYV9vCPJyIv674LhBzu+dJJRnwf/PSzwH+O+f/cA38N9DZwfHcXLw\n/j/gy/kp+M/+L4DvBO/VAucF+zYx2O7PQ9t8EH+3+nj8NTNukGvxqeA4TwF+G8rbUNfjgPOf5Hhe\njL8+LwuO+18Ar3KoGe3vgb8P1nsqPk74SWhdlwT7dERwDDaE3rsRHyv8Cf66KAeeAc4KLdMKfDk0\nvxB4I9ef/VGXHbnOQFZ2cvAA+TfD/N9fAncGr5MFvT8ILXsO8NQIlr0EeCT0nuG/uC8eJm/JAuTv\nAj8OzY/FBymLgvlHgAuC1+cDfxxi/YMFyE8Bnx/kf34IfDN4nUqQemF435O8fyq+4OwAevFNLsqD\n914BFoeWPQboTmPbiUHa7cDXQ/O1wTmcyqGA6d1DrG9BsExnUJg44NeEAnsOD5DDheu7gZ1DrP8F\n4P2hvD+T8P77gf8NzT8LnD3IurYDFw6xrQEB8jDn+XMENeUJy9TiA+mPAEcMc+yHCpB/C1w61Lkc\n4hgNFSCncr4bQ+/fQ/BjWX+F9cfgAfKPgO+F5qvwQUYD/vbw70PvWVDmJAuQT8dXppwMlCVs40aG\nDpCfA5amsA8NofKlN3j9fQ4FQIuBrQn/81f4PiMQCm6D+U9zeIB8SWh+uPU9jK+AmZqwzCXA74B3\nJtmHB0PH79fAFaH33h4c+7Ghfa0Pvf/HeJmQZL33AZcnpA0VIPcyMNB8PTh3hq+0ODr03nuAlwfZ\nbiPwZsL+fSOFa/Hy0PzZ+KZ+MPT1GD8mQwXIL4TeqwyWfxu+6dx+YELo/VsJBcgJeawJ/ndS6Pjd\nnLDMV/FNPMEH+j0EPwqDtGOAA1F8nrP5V7JNLAKvhGfM7B1m9svgVkMX/lfm1OT/CkC4A1MP/oJO\nd9kjw/lw/uqKMTJHAltC69qP/xVZFyTdhA9KCaa3jGAbdfigGzM7Nbgds9PMdgXrHPR4mVm9+VvZ\nrwbH9wdDLe+ce9g592fBMu8HPgx82XwbqDrgv4LbeJ1AGzDOzGpGsE9w+LHrwBeWdaFlXkn8pwRv\nOedqnHOT8Of3f4G1Qyw/6PVjfqSH/wntXwMDj1ViXh4AKs2sycwW4QutwXrZd+Br8FMyzHleg/9C\nbDWzV8w3XSoLjt8ngS8Cr5nZWgua+6RpFnBYs48gX8Mdo6Gkcr7T+XxL4Um8BnbjPxt1pFEuO+d+\nA/z/wPX4a32NJW/alMyg1/cg4j/g/pJDd+PAt+M/Mv5ZCD4Pf41vvkTi/pC8LAunDbe+S4Fjgf8N\nmlF8OEi/BVgH3BaU89+zoJliggHHPng9NrR+SP3z9ya+9jNVHcF3Y+K6p+EDy8dC+/yrIB0zqzSz\nfwuahXThfyTU2MA2ucN9RyQuswV/LGDo6zEVB4+Xc64neFkVrPdN59ye0LIHt2O+mc9q8818uvBB\nPAz9ffMT4CPmm0h+DF/RFe48PxFfUVTQSj1Adgnz/4avIZ3nfMelr+N/VUZpO6GOUWZmpP6BSPQq\nvmCLr2ss/sOxLUi6AzjFfBvO04Hb0lm5+aGJJuFr9eLr+wn+lvek4HX8eCUeW/DtQnfhaxur8R3w\nhj2+zvsd/nbXAudHhdgOnBoEpPG/cudc5yDbHk7isZsCTODQsRtsnwbL8x58s4cPhNuYpSI4P3+P\n/xKa4pyrwRda4WM1IC/OuX78l9OF+LbiP034Egi7H3/rOFWDnmfnXJ9z7mvOubcD7wM+Dvx58N7d\nzrnT8dfzq/ggIl2v4Js4DJDCMRruXKVyvqW4JV4DE/B3EraRvFwetAOrc+6fnHMn4W+vH4tvOgHD\nX4dJr++hOOcOOOf+Dj+yzxWh9bycUB5OdM6dHbw/YH/wgflhq07I16Drc85tcs5dgG8i+F3gLjOb\n4Jzb55z7W+fcfOD/4Cs1knXWGnDsOVTL+VoahyLuSfwxH62d+Nrl40P7PMn5TpHgm5O9HX/nshp/\nhxOGKJcHET72s/HHAoa+HuPBbWXof1MdhnM7MDlYX3i7cR/HN/H8AP77vSGehdAyid832/DNNpbh\nv28SK9uOI4W+Ovmu1APkRPFfPXvM7Dh8e6qo/Sew0Mw+EgS0nyf4xToCtwEfNbM/CX61r8J/OJ4A\ncM51Ab8MlrvfOdeeykrNrMbMluEDvh84514y30O3Cv8Ld6+Z/Sm+fVbc6/iOcuFgfyK+7VOX+Q6M\nXxxim+8POh5MDeZPwHfGiPdU/gHw3fj6zWxGqBYj2baH81PgM2Z2vPlOGd8F1jnndqaxjnD+y/HB\n6ksu/ZEyqvDD67UDZWb2WQ4VWkO5Gfi/+OYzNw+x3GpgTlDTVR/kd7b5jkgDvqyHO89m9kEzOy5Y\nrgv/JXfAzGaZ2dnBsezDF/AjGe7uh8Bfm9k7zXtHcF6HO0avAbNt8B75GT3fkvfGme9AFv8bi7/N\n/CkzazSzI4BvA39wzm3Gl5MnmNm5wbKfZZCAJLhrszgoc/fgr/f4tf4ag3fWgqC5kpkdE1zf7zSz\n2hT3aTW+A2A5vglCl/mOdhVBreACOzTu+h3AX5nvaFaHb4I0lCHXZ2YXmtm04Id5Z/A/B8zsfeY7\nZI/Blwf7SP65/ynwRfOdx6rwx/72IX7UD+UefD+VUQn25QbgOjvUYbfOzJYEi0zEB9CdwQ/q/2+E\nm/qs+bupU/C18rcH6YNej8F39Tb8YAJjzOwSUvxh5Zzbgm+29rdmNj4ow8OjXk3Et/HuwAfg305x\nP24GvgKcgG+DHPZe/EgWBU0B8kBfxt8W7sbXJt8+9OKj55x7DR/U/D3+Aj0aH9C+NdT/DbKuJ/BB\n/Y/wgcOpwLlu4Di8N+Ev6FSaV/zG/HjSm/HH5pv4L4p4YXI5fjSIXfjOhXeF8tKOrzHeaP521Qn4\njnXvwxecd+E7OQymE39cngnysDbI+z8H738Hf4vroeC20H8D7xpi20Nyzv0c36bvP/G3Umvx10I6\nDo6cgf9hMh//Czstzrk/4s/hExxqIrNhyH/y//cc/lztdM49PsRyO/Bt7iqAx4Pj9yt8rVEsYdkh\nzzO+NuQX+M/MRvx5+g/87dJV+ABhJ7693ueH24ckbsSf85/hr5vb8Z1+hjtG9+C/UNrNLHwrN75f\nmTjfUjjuwQc38b+rnXO/xo+z/jP85/VognHmgx9Kf44fTrMD/1leT/JyuRofWL2Jv3Xdgb+2wF+j\n84Ny6OdJ/vfv8cHrf+Gv7x/hP5ep+GWwzcuCMv4j+M/Zy/jP3A/xNYLgmwvGgvfux3+GB/2OSWF9\nZwJPB2XdP+LbB/fhf0TcFezLs/iOgckeSPFj/HfQw8H6+4CrUtzvxLw+Duwys8Uj+f8EX8X3ZXg0\nKBfvx9cag++8VoE/Fo/iy8yRuBV/vl8K/r4FMNT1GLgMf2eiA3+n4ndpbPPj+Hblb+AD+3AFys34\n63YbvvNdqsPlteJrvFvDzTeCH2xn47+vC1rRPiikUAW/vF8FPuqceySC9c/Hf7DeNoKaTclzZnYH\n8LjTAwREMia4QxIDmp1zD+Q6P6NlZn+BD2pHXfOaD8zsDHynv3NznZdSYn7Y08+40INPzOwqYJZz\n7iu5y1lmFNSA4MXKzM7Et+fpw/cU3o+/xZXp7YzBN2u4WcFx8TGzY/E1O58bblkRGVpwa/0P+Brn\nFfg2mXnzMIp0mNlMfFOP3+NHGPgyI+sTkJecc//F4J2SJQJmdh6+bfJvwunOuX9O/h+FRwFyfvhT\n/GN+x+MHfz/XOZd2E4uhBG2qXgY24cftlCJifqD+y/BDl0X+eGiREvAe/O3w8fhbz+c653pzm6UR\nG49vNngUvvnabcC/5DRHUrDM7EF8s6OLgmZ4RUlNLEREREREQtRJT0REREQkpOCaWEydOtU1NDTk\nOhsiIkN67LHHdjrnRjpkY8GZUlvrZs2aM/yCIlJ0+vYdYOzYMsrHjsGifnrEKKVaNhdcgNzQ0MD6\n9etznQ0RkSElG16umM2aNYd7738o19kQkRzYtKObyZMncOz0KsaNia5xwpiy0UffqZbNamIhIiIi\nIhJScDXIIiIiIpIf2rv66OjZy6SaCcMvPBrbt8Mf/wDt7TBtGjQ1Qf2gT4AfNdUgi4iIiEja2rv6\niHX2MvfIycyZUhFd84rt22HdOujpgRkz/HTtWojFhv/fEYqsBtnMfgx8GHjdObcgyfuGf0Tl2UAP\ncPFQj8cdyr59+4jFYvT16dkXicrLy6mvr2fcuHG5zoqIiIgUiXhwXDd9EnOmVFBdHmGcsWEDVFfD\nEeP9fHW1n7a1RVaLHGUTixvxT+q5eZD3z8I/0ecY/DPC/zWYpi0WizFx4kQaGhqwfO8+mUXOOTo6\nOojFYhx11FG5zo6IiIgUgXhwXF5RHn1wDPBGB0ydBvv3HUqrqoLXXotsk5E1sXDOPQy8McQiS/GP\nPHbOuUeBmuBxmGnr6+ujtrZWwXECM6O2tlY16yIikndaW2HxYl8BuHixn5f8Fw6OF86piT44BphS\nC3v2DEzbvdu3RY5ILtsg1wGvhOZjQdphzGy5ma03s/Xt7e1JV6bgODkdFxERyTetrbBihW9C6pyf\nrliRJEjevh1+9StoafHT7dszmgcF6OmJB8eVlVkMjgEaG6Gry//19/tpZ6fvqBeRXAbIySK3pM+9\nds6tcc4tcs4tmhbhrwURERGJ3urV0Ns7MK2316cfFO+Y1dsLU6f66bp1GQmSUw7Q5aBwcNw4O4vB\nMcDMmbBkCVRW+mYVlZWwdGnRjmIRA2aF5uuBV3OUl1HbsWMH559/PkcffTTz58/n7LPP5vnnn2fz\n5s0sWHBYH8WMePjhh1m4cCFjx47lrrvuimQbIiIimbZtWwrpGzfCpEm+rWlZmZ9OmuTTRymlAF0O\nCgfHc6dXZTc4jps5E5Ytg+XL/TTC4BhyGyDfDXzCvJOBXc65zN07ySLnHMuWLeO0007jxRdf5Jln\nnuHb3/42r0XYeBxg9uzZ3HjjjXz84x+PdDsiIiKZVJe0QWVCekeHrykMq6z06aOUUoAuwOHBcX1N\nRa6zlBWRBchm9lPg98DbzSxmZpea2eVmdnmwyD3AS8ALwA3AFVHlJVFLCzQ0+B+kDQ1+fjQeeOAB\nxo0bx+WXX34wrbGxkVNOOWXAcps3b+aUU05h4cKFLFy4kN/97ncAbN++nVNPPZXGxkYWLFjAI488\nwoEDB7j44otZsGABJ5xwAtddd91h221oaOCd73wnZWUazlpERArHypVQkRBnVVT49INqa/14t2E9\nPT59lFIK0OWgqTUTSio4hgiHeXPOXTDM+w74bFTbH0xLi6+dj3/mtmzx8wDNzSNb51NPPcVJJ500\n7HLTp0/nvvvuo7y8nE2bNnHBBRewfv16br31VpYsWcKqVas4cOAAPT09bNiwgW3btvHUU08B0NnZ\nObLMiUh6YjE/tmaWntYkUoqWLfPT1at9rW1dnQ+O4+kAnHiib3MMvua4pwd27YKTTx719leu9G2O\nw80sDgvQpaSV3KOmV61K/oN01aqRB8ip2rdvH1deeSUbNmxgzJgxPP/88wA0NTVxySWXsG/fPs49\n91waGxuZO3cuL730EldddRUf+tCHOOOMM6LNnIj44HjtWqip8U9r2r3bz0fcGUSkFC1blhAQJ4p3\nzNq4EXbu9DXHJ5/s0zOwbRgmQJeSVnIB8tat6aWn4vjjj0+pk9x1113HjBkz2LhxI/39/ZSXlwNw\n6qmn8vDDD/PLX/6Siy66iBUrVvCJT3yCjRs3sm7dOq6//nruuOMOfvzjH488kyIyvLY2HxzHn9KU\nhac1icgQZs7MSECczLABugxof1xdXlohY8k1Xp09O730VJx++um89dZb3HDDDQfT2traeOihhwYs\nt2vXLmbOnElZWRm33HILBw4cAGDLli1Mnz6dyy67jEsvvZTHH3+cnTt30t/fz3nnncc3v/lNHn98\nRE/hFpF0tLf7nvJhVVU+XUSkxOR85IocKrkA+ZprkneKveaaka/TzGhtbeW+++7j6KOP5vjjj+fq\nq6/myCOPHLDcFVdcwU033cTJJ5/M888/z4QJEwB48MEHaWxs5F3vehc/+9nP+PznP8+2bds47bTT\naGxs5OKLL+Y73/nOYdtta2ujvr6eO++8k8985jMcf/zxI9+JEpHpDppSZKZN880qwiJ+WpOISD56\nYuubJTdyRZj5vnKFY9GiRW79+vUD0p599lmOO+64lNfR0uLbHG/d6muOr7km+vbHuZTu8SlWiR00\nwf84WrMm4fxH1Emr1K67ghRug1xV5YPjzs4RtUE2s8ecc4siymneObFxobv3/oeGX1BE8l44OJ4x\n8QjGjcmP+tQxZaN/OnCqZXN+7HGWNTfD5s3+aYWbNytIKRVDddA8KB4g9fT4Tlo9PX4+FhvVtuPB\n+ZYt/qlN8dFTVIOdZ+rrfTCcxac1iYjkk3wNjrOttFpcS0lLqYNmRJ20cjl6iqSpvl4BsYiUJAXH\nh5TunkvJSamDZkSdtKIYPUVERCTT5s2YWPLBMShAlhKSUgfNiDppRTF6ioiIiERDAbKUjOZm3yFv\nzhww89PDOug1NflOWV1dvpF6V5efb2oa1bajGD1FREREoqEAWUrKsB00I+qklVJwLiIikiPx9sf9\nBTa6WVTUSS9DduzYwRe+8AXa2to44ogjaGho4B/+4R8YP348H/7wh3nqqacyvs233nqLT3ziEzz2\n2GPU1tZy++2309DQkPHtlJyIOmk1NysgFhGR/LNpR7c65yXQEcgA5xzLli3jtNNO48UXX+SZZ57h\n29/+Nq+99lqk2/3Rj37E5MmTeeGFF/jiF7/IV7/61Ui3JyIiIsVl045u+seOVXCcoDSPQiwGra3+\nHndr66jHuH3ggQcYN24cl19++cG0xsZGTjnllAHLbd68mVNOOYWFCxeycOFCfve73wGwfft2Tj31\nVBobG1mwYAGPPPIIBw4c4OKLL2bBggWccMIJXHfddYdtd+3atXzyk58E4KMf/Si//vWvKbQHv4iI\niEhubNrRTfvuvQqOkyi9JhbhJ2XNmOFHKFi7dlTtTJ966ilOOumkYZebPn069913H+Xl5WzatIkL\nLriA9evXc+utt7JkyRJWrVrFgQMH6OnpYcOGDWzbtu1g04zOzs7D1rdt2zZmzZoFwNixY5k0aRId\nHR1MnTp1RPshIiIipSEeHDcdO13BcRKlFyBH9CCIVOzbt48rr7ySDRs2MGbMGJ5//nkAmpqauOSS\nS9i3bx/nnnsujY2NzJ07l5deeomrrrqKD33oQ5xxxhmHrS9ZbbHZ6B/DKCIiIsVLwfHwSu+IRPAg\niOOPP57HHnts2OWuu+46ZsyYwcaNG1m/fj179+4F4NRTT+Xhhx+mrq6Oiy66iJtvvpnJkyezceNG\nTjvtNK6//no+/elPH7a++vp6XnnlFQD279/Prl27mDJlyoj3Q0RERIrbph3ddPQoOB5O6R2VCB4E\ncfrpp/PWW29xww03HExra2vjoYceGrDcrl27mDlzJmVlZdxyyy0cOHAAgC1btjB9+nQuu+wyLr30\nUh5//HF27txJf38/5513Ht/85jd5/PHHD9vuOeecw0033QTAXXfdxemnn64aZBEREUkqHhzPPXKy\nguNhlF4Ti6Ym3+YYfM3x7t3+QRDvfe+IV2lmtLa28oUvfIHVq1dTXl5+cJi3sCuuuILzzjuPO++8\nk/e9731MmDABgAcffJBrr72WcePGUVVVxc0338y2bdv41Kc+RX9/PwDf+c53DtvupZdeykUXXcS8\nefOYMmUKt91224j3QURERIpXODh+x4wqBcfDsEIb9WDRokVu/fr1A9KeffZZjjvuuNRXEov5Nsft\n7b7muKkp8vbHuZT28RGRUTOzx5xzi3Kdj2w5sXGhu/f+h4ZfsIC1tsLq1bBtG9TVwcqVsGxZrnMl\nkppNO7qZWjuRIycdQXX5uFxnZ0TGlI3+LnmqZXPp1SBDZA+CEBGR4tTaCitWQG+vn4/F/DwoSBYp\nRqpfFxERGcbq1YeC47jeXp8uIsWnaALkQmsqki06LiIio7dtW3rpIvkk3v5434F+KsaNyXV2CkJR\nBMjl5eV0dHQoGEzgnKOjo4Py8vJcZ0VEpKDV1aWXLpIv1DlvZIqiDXJ9fT2xWIz2UYxlXKzKy8up\nV3trEZFRWblyYBtkgIoKny6SrxQcj1xRBMjjxo3jqKOOynU2RESkSMU74mkUCykUCo5HpygCZBER\nkagtW6aAWApDODieM6VCwfEI6IiJiIiIFInE4LhQxzzONQXIBaClBRoaoKzMT1tacp0jERERyTft\nXX0KjjNETSzyXEsLLF8OPT0Q9prqAAAgAElEQVR+fssWPw/Q3Jy7fImIiEj+aO/q48WdexQcZ4hq\nkPPcqlWHguO4nh6fLiIiIgIQ6+xlXp2C40xRgJzntm5NL11ERERKS3tXH1NrJjC5crweBJIhCpDz\n3OzZ6aWLiIiIyOgoQM5z11wDlZUD0yorfbqIiIgIQN/+fvr1ROGMUYCc55qbYc0amDMHzPx0zRp1\n0BMREZFDnfMmVY6nunysxjzOEI1iUQCamxUQi4iIyEAauSI6+pkhIiIiUmAUHEdLAbKIiIhIAWnv\n6iPW2avgOEIKkEVERKSgtbbC4sVQX++nra25zlF04sFx3fRJCo4jpABZRKTEmNmZZvacmb1gZiuT\nvD/bzB4wsyfM7EkzOzsX+RRJRWsrrFgBsRg456crVhRvkNzZs4+qqgoFxxFTgCwiUkLMbAxwPXAW\nMB+4wMzmJyz2NeAO59y7gPOBf8luLkVSt3o19PYOTOvt9enFaurE8lxnoegpQBYRKS3vBl5wzr3k\nnNsL3AYsTVjGAdXB60nAq1nMn0hatm1LL10kFQqQRURKSx3wSmg+FqSFXQ1caGYx4B7gqmQrMrPl\nZrbezNZ3dOyMIq8iw6pLvHqHSS9k7V197N67n30H+nOdlaKnAFlEpLRYkrTEx29dANzonKsHzgZu\nMbPDvi+cc2ucc4ucc4tqa6dGkFWR4a1cCRUVA9MqKnx6MVHnvOxSgCwiUlpiwKzQfD2HN6G4FLgD\nwDn3e6AcUAQseWnZMrj2Wj+ChZmfXnutTy8W8eC4vKJcwXGW6El6IiKlpQ04xsyOArbhO+F9PGGZ\nrcD7gRvN7Dh8gNye1VyKpGHZsuIKiMPCwfHCOTUKjrNENcgiI9DSAg0NUFbmpy0tuc6RSGqcc/uB\nK4F1wLP40SqeNrNvmNk5wWJfBi4zs43AT4GLnXOJzTBEJGLx4LiyUsFxtkVag2xmZwL/CIwBfuic\nW53w/mzgJqAmWGalc+6eKPMkMlotLbB8OfT0+PktW/w8QHNz7vIlkqqgnL0nIe3rodfPAH+S7XyJ\nyCHh4LhxtoLjbIusBlljbUqxWrXqUHAc19Pj00VEREYrHBzPnV6l4DgHomxiobE2pSht3ZpeuoiI\nSKoSg+P6morh/0kyLsoAOZKxNtvb1U9Ecmv27PTSRUREUqXgOD9EGSBHMtbmtGnTIsiqSOquuQYq\nKwemVVb6dBERkZFq7+pjas0EBcd5IMoAWWNtSlFqboY1a2DOHD/m5pw5fl4d9ERERIpDlKNYaKxN\nKVrNzQqIRUREilVkNcgaa1NEREQkNfHOeT17D+Q6K0LE4yBrrE0RERGR4alzXn7Rk/REREREcuiJ\nrW8qOM4zCpBFREREciQcHM+YeESusyMBBcgiIiIiOZAYHI8bo7AsX+hMiIiIiGTZph3dCo7zmM6G\niIiISJbt3rufeTMmKjjOUzojIiIiIiIhCpBFREREREIUIIuIiIhkUbxzXr+ejZa3FCCLiIiIZIk6\n5xUGnRURERGRLNi0o5v23XsVHBcAnRkRERGRiMWD46Zjpys4LgA6OyIiIiIRUnBceHSGRERERCKy\naUc3HT0KjguNzpKIiIhIBOLB8UnzFBwXGp0pERERkQyLB8dzj5ys4LgA6WyJiIiIZFB7V9/B4Pgd\nM6oUHBegsbnOgIiIiEgx6ezZx9tn1XLkJNUcFyqdNRERERGREAXIIiIiIiIhCpBFREREMiTeOW/f\ngX4qxo3JdXZkhBQgi4iIiGRAeOQKdc4rbDpzIiIiIqOk4Li46OyJiIiIjIKC4+KjMygiIiIyQuHg\neM6UCgXHRUJnUURERGQEEoPj6vJxuc6SZIgCZBEREZE0hZ+Wp+C4+ChAFhEREUlDe1cfL+7co+C4\niOlR0yIiIiJpiHX2Mq9uMrMmKzguVqpBFhEREUnT5MrxehBIEVOALCIiIiISogBZREREJEXtXX28\ntb+ffudynRWJkAJkERERkRSEO+dVl4/VmMdFTGdWREREZBgauaK0KEAWERERGUJ7Vx+xzl4FxyVE\nAbKIiIjIIOLBcd30SQqOS4gCZBEREZEk4sFxeUW5guMSoweFiIiIiCTR2bOPqqoK3lk/ScFxiVEN\nsoiIyDBaW2HxYqiv99PW1lznSLJl6sTyXGdBckA1yCIiIkNobYUVK6C318/HYn4eYNmy3OVLRKKj\nGmQREZEhrF59KDiO6+316VK82rv62L13P/sO9Oc6K5IDCpBFRESGsG1beulS+NQ5TxQgi4iIDKGu\nLr10KWzh4HjhnBoFxyVKAXKea2mBhgYoK/PTlpZc50hECp2ZnWlmz5nZC2a2cpBlPmZmz5jZ02Z2\na7bzmE9WroSKioFpFRU+XYpLPDiurFRwXOrUSS+PtbTA8uXQ0+Pnt2zx8wDNzbnLl4gULjMbA1wP\nfBCIAW1mdrdz7pnQMscAfwX8iXPuTTObnpvc5od4R7zVq32ziro6Hxyrg15xCQfHjbMVHJc6Bch5\nbNWqQ8FxXE+PT1eALCIj9G7gBefcSwBmdhuwFHgmtMxlwPXOuTcBnHOvZz2XeWbZMgXExSwcHM+d\nXqXgWNTEIp9t3ZpeuohICuqAV0LzsSAt7FjgWDP7rZk9amZnJluRmS03s/Vmtr6jY2dE2RWJVmJw\nXF9TMfw/SdFTgJzHZs9OL11EJAWWJM0lzI8FjgFOAy4AfmhmNYf9k3NrnHOLnHOLamunZjyjItmg\n4FiSiTRAVkeQ0bnmGqisHJhWWenTRURGKAbMCs3XA68mWWatc26fc+5l4Dl8wCxSVJ7Y+qaCY0kq\nsgA51BHkLGA+cIGZzU9YJtwR5HjgC1HlpxA1N8OaNTBnDpj56Zo1an8sIqPSBhxjZkeZ2XjgfODu\nhGV+DrwPwMym4ptcvJTVXIpErL2rj6k1ExQcS1JRdtJTR5AMaG5WQCwimeOc229mVwLrgDHAj51z\nT5vZN4D1zrm7g/fOMLNngAPACudcR+5yLSKSXVEGyMk6gixOWOZYADP7Lb6gvto596vEFZnZcmA5\nwGw1wBURGRXn3D3APQlpXw+9dsCXgj8RkZITZRvkSDqCTJs2LeMZFRERkdIS6+ylZ++BXGdD8lSU\nAbI6goiIiEjeUec8GU6UAbI6goiIiEheCQfHMyYekevsSJ6KLEB2zu0H4h1BngXuiHcEMbNzgsXW\nAR1BR5AHUEcQERERiUhicDxujB4HIckNemWY2QnBE5ReMbM1ZjY59N4fU1m5c+4e59yxzrmjnXPX\nBGlfD3pJ47wvOefmO+dOcM7dNtodEhEpZpkom0VK0aYd3QqOJWVDXR3/ClwNnAA8D/y3mR0dvKeH\nlIuI5IbKZpE0bdrRTf/YsQqOJWVDXSFVzrlfOec6nXPfxzeX+JWZnczho1GIiEh2qGyWQbW2wuLF\nUF/vp62tuc5R7m3a0U377r0KjiUtQ42DbGY2yTm3C8A594CZnQf8DJiSldyJiEgilc2SVGsrrFgB\nvb1+Phbz8wDLluUuX7kUD46bjp2u4FjSMtSV8l3guHCCc+5J4P3Af0SZKRERGZTKZklq9epDwXFc\nb69PL0WbdnTT0aPgWEZm0Bpk59ytg6RvxT8iWkREskxlswxm27b00kvB22fVKjiWEdEVI6PW0gIN\nDVBW5qctLbnOkYhI6amrSy9dRAanAFlGpaUFli+HLVvAOT9dvlxBsohItq1cCRUJD4WrqPDppepA\nf3+usyAFSgGyjMqqVdDTMzCtp8eni4hI9ixbBtde60ewMPPTa68tzQ568c55kyo08qGMzFCjWABg\nZsfix92c4ZxbYGbvBM5xzn0r8txJ3tu6Nb10EckMlc2SzLJlpRkQh6lznmRCKlfNDcBfAfvgYG/p\n86PMlBSO2bPTSxeRjFHZLJIgHhyfNE/BsYxOKldOpXMu8fGl+6PIjBSea66BysqBaZWVPl1EIqWy\nWSQkHhzPPXKygmMZtVSunp3BY0wdgJl9FNgeaa6kYDQ3w5o1MGeOb/M2Z46fb27Odc5Eip7KZpFA\nODh+x4wqBccyasO2QQY+C6wB3mFm24CXAYU/clBzswJikRxQ2SwCtHf1KTiWjBsyQDazMmCRc+4D\nZjYBKHPOdWcnayIikozKZpGBamsmMGdKhYJjyZghryTnXD9wZfB6jwpgEZHcU9ksIhKtVH5q3Wdm\nf2lms8xsSvwv8pyJiMhQVDaLAJ09+3KdBSlCqbRBviSYfjaU5oC5mc+OiIikSGWzlLyDnfNqJlAx\nbkyusyNFZNgA2Tl3VDYyIiIiqVPZLKVOI1dIlFJ5kt4nkqU7527OfHZERCQVKpullCk4lqil0sSi\nKfS6HHg/8DigQlhEJHdUNktJCgfHGrlCopJKE4urwvNmNgm4JbIciYjIsFQ2SylKDI6ry8flOktS\npEbys6sHOCbTGRERkVFR2SxFLfxAEAXHErVU2iD/guBRpviAej5wZ5SZEhGRoalsllLS3tXHizv3\nKDiWrEmlDfL3Q6/3A1ucc7GI8iMiIqlR2SwlQcGx5EIqTSzOds49FPz91jkXM7PvRp4zEREZispm\nKQmxzl7m1Sk4luxKJUD+YJK0szKdERERSYvKZikZkyvH60EgklWDNrEws78ArgDmmtmTobcmAr+N\nOmMiInI4lc0iUnK2b4cNG2BbDKZNg6YmqK+PdJNDtUG+FbgX+A6wMpTe7Zx7I9JciYjIYFQ2S8lo\n7+rjrf399Ds3/MJSnLZvh3XroLoaZsyA3bth7VpYujTSIHnQANk5twvYBVwAYGbT8YPRV5lZlXNu\na2S5EhGRpFQ2S6kId86rLh+rB4KUqg0bfHA8cSLsfcu/BmhrizRAHvZqM7OPmNkm4GXgIWAzvvZC\nRERyRGWzFDONXCEHvdEBEyYMTKuqgvb2SDebyjBv3wJOBu53zr3LzN5HUHMhIlJ0YjFfM9HenrW2\nbiOkslmKUntXH7HOXgXH4k2phT17fA1y3O7dvnyOUCr3K/Y55zqAMjMrc849ADRGmisRkVyIxXzb\ntp4e39atp8fPx/JyeGGVzVJ04sFx3fRJCo7Fa2yEri7o7ob+fv+6s9NXXkQolRrkTjOrAh4BWszs\ndfyg9CIixaWtDWpqDrVxy1JbtxFS2SxFJR4cl1eUKziWQ2bOhCVLBo5i8d735nQUi7ilQC/wBaAZ\nmAR8I8pMiYjkRHu7rzkOq6qC117LTX6GprJZikY4OF44p0bBsQw0c6b/K7OsbXLYANk5t8fM5gDH\nOOduMrNKQKN1i0jxmTbNt22L1xxDVtq6jYTKZikW8eC4srKcxtkKjiU/pDKKxWXAXcC/BUl1wM+j\nzJSISE40Nfm2bV1dWW3rNhIqm6UYKDiWfJVKJ73PAn8CdAE45zYB06PMlIhITtTX+8HnKyt9s4rK\nysgHox8Flc1S8Dp79lFVVcHc6VUKjiWvpNIG+S3n3F4z3+7DzMYCeqSNiBSn+vp8DYgTqWyWojB1\nYjnV5amEIyLZk0oN8kNm9tdAhZl9ELgT+EW02RIRkWGobBYRiUgqAfJKoB34H+AzwD3A16LMlIiI\nDEtlsxS09q4+du/dz4H+/lxnReQwg97TMLPZzrmtzrl+4IbgT0REckhlsxSDcOe8SRXj1P5Y8s5Q\nNcgHe0Ob2c+ykBcRERmeymYpaOHgeO70KuprKnKdJZHDDBUgh0djnht1RkREJCUqm6VgKTiWQjFU\ngOwGeS0iIrmjslkKloJjKRRDjatyopl14WsrKoLXBPPOOVc9+L+KiEhEVDZLQXpi65sKjqVgDFqD\n7Jwb45yrds5NdM6NDV7H51UAi4jkQCbKZjM708yeM7MXzGzlEMt91MycmS3K3B5IKWrv6mNqzQTm\nzZjIjIlH5Do7IsNKZZg3EREpEmY2BrgeOAuYD1xgZvOTLDcR+Bzwh0xuv7UVFi/2z2JZvNjPS2kZ\nN0ahh+S/SK9S1VKIiOSddwMvOOdecs7tBW4DliZZ7pvA94C+TG24tRVWrIBYDJzz0xUrFCSLSP6J\nLEDOdS2FiIgkVQe8EpqPBWkHmdm7gFnOuf8cakVmttzM1pvZ+o6OncNuePVq6O0dmNbb69PzmWq9\nRy/W2UvP3gP0O/UrlcIQZQ1yzmopBtPSAg0NUFbmpy0tUW9RRCTvWJK0g1GLmZUB1wFfHm5Fzrk1\nzrlFzrlFtbVTh93wtm3ppecD1XqPXrhzntofS6GIMkCOpJaivb19RJlpaYHly2HLFl/Ibdni5xUk\ni0iJiQGzQvP1wKuh+YnAAuBBM9sMnAzcnYkmcHV16aXng0Kt9c4XicGx2h9LoYjySo2klmLatGkj\nysyqVdDTMzCtp8en5yvVeItIBNqAY8zsKDMbD5wP3B1/0zm3yzk31TnX4JxrAB4FznHOrR/thleu\nhIqE0b0qKnx6virEWu98sWlHt4JjKVhRXq05q6VIZuvW9NJzTTXeIhIF59x+4EpgHfAscIdz7mkz\n+4aZnRPltpctg2uv9W15zfz02mt9er4qxFrvfLBpRzf9Y8cqOJaCZS6iBvNmNhZ4Hng/sA1fa/Fx\n59zTgyz/IPCXw9VSLFq0yK1fn35FRkODDzITzZkDmzenvbrIFVp+RWQgM3vMOVcyI/Oc2LjQ3Xv/\nQ7nORsbF2yCHm1lUVOR/YJ9Lm3Z00757L03HTldwLBk1pixZ44T0pFo2R3bV5rKWIplrroHKyoFp\nlZU+PR8VWo23iEgxKsRa71xScCzFYqhHTY+ac+4e4J6EtK8PsuxpUealudlPV63yQebs2T44jqfn\nm9mzk9cgz56d/byIiJSyZcsUEKdCwbEUk5K6epubffOE/n4/zdfgGAqvxltERErXph3ddPQoOJbi\noSs4TzU3w5o1vs2xmZ+uWZPfQb2IiJSut8+qpbp8rIJjKQqRNrGQ0WluVkAsIiIikm36mSciIiKj\ndqC/P9dZEMkYBcgiIiIyYvH2x5MqxlExbkyusyOSEQqQRUSkqLS2wuLFfki2xYv9vEQjHhyfNE+d\n86S46ErOMD0eWkQkd+IP9ojF/FNIYzE/ryA58+LB8dwjJys4lqKjqzmD9HhoEZHcWr164FPvwM+v\nXp2b/BSrcHD8jhlVCo6l6OiKzqBVq6CnZ2BaT49PFxGR6G3bll66pE/BsZQCXdUZpMdDi4jkVl1d\neumSnvauPgXHUhJ0ZWfQYI+B1uOhRUSyY+VKqKgYmFZR4dMlM2prJjBnSoWCYylqurozSI+HFhHJ\nrWXL4Npr/QgWZn567bU+XUQkVXqSXgbFn3q3apVvVjF7tg+O9TQ8EZHsWbZMAXFUOnv2wfhxuc6G\nSOQUIGeYHg8tIiLF6GDnvJoJeiCIFD01sRAREZEhaeQKKTW6wkVERGRQ4eBYnfOkVOgqFxERkaQS\ng+PqcrU/ltKgAFlEREQOEx7zWMGxlBoFyCIiIjJAe1cfL+7co+BYSpYCZBERETlIwbGIAmQREREJ\nKDgW8TQOsoiIiAAQ6+xlXt1kZk1WcCylTTXIIiIictDkyvF6EIiUPAXIIiIiIiIhCpBFRESE9q4+\n3trfT79zuc6KSM4pQBYRESlx4c551eVj9bQ8KXn6BIiIiJSw9q4+Yp29GrlCJEQBsoiISImKB8d1\n0ycpOBYJUYAsIiJSguLBcXlFuYJjkQQKkEVEREpMODheOKdGwbFIAj0oREREpITEg+PKynIaZys4\nFklGNcgiIiIlQsGxSGoUIIuIiJSAcHA8d3qVgmORIaiJhYiISImoqqqgYeoE6msqcp0VkbymGmQR\nEZESUl2uujGR4ShAFhEREREJUYAsIiJS5OLtj/v7Xa6zIlIQFCCLiIgUMXXOE0mfAmQREZEilRgc\nq3OeSGoUIIuIiBQpBcciI6MAWUREpAg9sfVNBcciI6QAWUREpMiEg+MZE4/IdXZECo4GQxQRESki\n7V19TK2ZwJGTK5lWNZ5xY1QXJpIuBcgiUnxiMWhrg/Z2mDYNmpqgvj7XuRIRKW3bt8OGDfBGB0yp\nhcZGmDkz17lKSj8rRaS4xGKwdi309MCMGX66dq1PFxGR3Ni+Hdatg95emDrNT9et8+l5SDXIIulS\n7WR+a2uDmhqorvbz8Wlbm86TlIT4yBX9zql5heSPDRt8eTxxop+PTzdsyMtaZH1yRNKh2sn8194O\nVVUD06qqfLpIkSu5znnbt8O990LLT/w0T2sjBd+sYsKEgWkTJvj0PBRpgGxmZ5rZc2b2gpmtTPL+\nl8zsGTN70sx+bWZzosyPyKiFayfLyvy0psanS36YNg127x6Ytnu3TxcpYonBcdHXHhfYLfuSN6UW\n9uwZmLZnj0/PQ5F9esxsDHA9cBYwH7jAzOYnLPYEsMg5907gLuB7UeVHJCPyvXYyFoPWVlizxk9L\nsWa7qQk6O6GrC/r7/bSz06cLoMqLYrRpR3dpBccw8JZ9WZmfVlf79HyhGu5DGht9edzd7cvm7m4/\n39iY65wlFeUn6N3AC865l5xze4HbgKXhBZxzDzjneoLZRwE1EJT8ls+1k2r+4dXXw9KlUFkJr73m\np0uXqv1xQJUXxWfTjm7ad+8treAY8v+WvWq4B5o5E5YsgYoK2Nnup0uW5GX7Y4i2k14d8EpoPgYs\nHmL5S4F7k71hZsuB5QCzZ8/OVP5E0tfU5INO8DXHu3f72sn3vje3+QJ1Tgurry+9fU7dwcoLADOL\nV148E1/AOfdAaPlHgQuzmkNJWTw4bjp2emkFx3Doln28sxfk1y37AuuUlhUzZxbMvkf5SbIkaS7p\ngmYXAouAa5O975xb45xb5JxbNC0fauqkdOVz7WS+N/+QfJGs8qJuiOWHrLwws/Vmtr6jY2cGsyip\nKOngGPL/ln2+13DLkKKsQY4Bs0Lz9cCriQuZ2QeAVcB7nXNvRZgfkczI19rJePOPeM0x5E/zD8kn\nI6m8SHqLxDm3BlgDcGLjwqTrkGhs2tFNR08JB8dw6Jb9hg3+lv2UWnjPe/KnhjLfa7hlSFEGyG3A\nMWZ2FLANOB/4eHgBM3sX8G/Amc651yPMi0jxy+fmH5JPVHlR4OLB8UnzSjg4jsvnW/aNjb7NMfia\n4z17fA33e96T23xJSiL7VDnn9gNXAuuAZ4E7nHNPm9k3zOycYLFrgSrgTjPbYGZ3R5UfkaKXz80/\nJJ8crLwws/H4yosBZW+o8uIcVV7kp7fPqqW6fGxpB8f5rsA6pclAkT5Jzzl3D3BPQtrXQ68/EOX2\nRUpOvjb/kLzhnNtvZvHKizHAj+OVF8B659zdDKy8ANjqnDtn0JWKSHL5XMMtQ9KjpkUkWno0d95R\n5UXhO9Dfn+ssiBQ13ZsRkehobGaRjIq3P55UMY6KcWNynR0pVHqAybAUIItIdPRobpGMiQfHc4+c\nrM55MnJ6gElK9OkSkehobGaRjAgHx++YUaXgWEauEB7RnQf0CROR6OTzo7lFCoSCY8koPcAkJfqU\niUh0mpr8WMxdXf5JV11dfr6pKdc5EykICo4l4+IPMAnTA0wOo0+aiERHYzOLjJiCY4lEvj+iO09o\nmDcRiZbGZhZJWzg4njOlQsGxZE6+P6I7T+gTJ3mppQUaGnz/gYYGPy8iUgoSg+Pq8nG5zpIUm5kz\n4ayzoPlCP1VwfBgFyJJ3Wlpg+XLYsgWc89PlyxUki0jxa+/qY/fe/XkVHLf+Byxe7G8ELV7s50WK\nnQJkyTurVvnnSYT19Ph0EZFiN7VmApMrx+VNcLziK/7ZPs756YqvKEiW4qcAWfLO1q3ppYuISDRW\nf9c/RyKst9enixQzBciSd2bPTi9dRKRYdPbso29/f66zcdC2bemlixQLBciSd665xo8GFlZZ6dNF\nRIpVe1cfHT17mVQ5nury/Bhkqq4uvXSRYqEAWfJOczOsWQNz5oCZn65Z49NFRIpRe1cfL+7ck1ed\n8wBWfhUqKgamVVT4dJFilh8/UUUSNDcrIBaR0pCvwTHAsj/z09Xf9c0q6up8cBxPFylWCpBFRERy\nJJ+D47hlf6aAWEqPmliIiIjkQCEExyKlSjXIIiIiOdDZs495dZOZNVnBsUi+UQ2yiIhIDlWMG5Pr\nLIhIAgXIIiIiIiIhCpBFRESyLD7mcb/LdU5EJBkFyCIiIlnU3tVHrLP3YOe8cWP0VSySb9RJLx2x\nGLS1QXs7TJsGTU1QX5/rXImISIGIB8d10ydp5AqRPKaframKxWDtWujpgRkz/HTtWp8uIiIyjHhw\nXF5RruBYJM+pBjlVbW1QUwPV1X4+Pm1rUy2yiEg2bd8OGzdCRwfU1sKJJ8LMmbnO1ZDCwfHCOTUK\njkXynGqQU9XeDlVVA9Oqqny6iIhkx/btsG4d9PbC1Kl+um6dT89T8eC4slLBsUihUICcqmnTYPfu\ngWm7d/t0ERHJjo0bYdIkX0FRVuankyb59DwUDo4bZys4FikUCpBT1dQEnZ3Q1QX9/X7a2enTRUQk\nOzo6oLJyYFplpU/PM+HgeO70KgXHIgVEAXKq6uth6VJfEL/2mp8uXar2xyIi2VRb6ztJh/X0+PQ8\nkhgc19dU5DpLIpIGddJLR329AmIRkVw68UTf5hh8RUVPD+zaBSefnNt8JVFVVUHD1AkKjkUKkGqQ\nRUSkcMycCUuWQEUF7Nzpp0uW5O0oFtXlqocSKUT65IqISGGZOTNvA2IRKQ6qQRYREcmgePvj/n6X\n66yIyAgpQBYREckQjVwhUhzUxEIkLBbzT0dsb/djXDc1qWOmiKRMI1eIFAfVIIvExWKwdq3vFT9j\nhp+uXevTRUSG8cTWNxUcixQJBcgicW1tUFMD1dX+CV3V1X6+rS3XORORPBcOjmdMPCLX2RGRUVIT\nC5G49nZfcxxWVeUfDCMiMojE4HjcGNU9ZcT27bBhA7zRAVNqobFRo5dI1uhTLBI3bRrs3j0wbfdu\nny4ikkR7Vx9TayYwb8ZEBceZtH27fyBMby9Mnean69b5dJEs0CdZJK6pCTo7oasL+vv9tLPTp4uI\nSPZs2OCbuU2c6Ju8TQ32yaMAAAl4SURBVJzo5zdsyHXOpEQoQBaJq6+HpUv942tfe81Ply7VKBYi\nItn2RgdMmDAwbcIEny6SBWqDLBJWX6+AWERSFh/Wrd/poSAZNaUW9uzxNcdxe/b4dJEsUA2yiIjI\nCKhzXoQaG30zt+5u3+Stu9vPNzbmOmdSIvRpFhERSdOmHd0KjqM0cyYsWQIVFbCz3U+XLNEoFpI1\namIhIiKShk07uukfO1bBcdRmzlRALDmjT7WIiEiKNu3opn33XgXHIkUu0k+2mZ1pZs+Z2QtmtjLJ\n+0eY2e3B+38ws4Yo8yMiIjJS8eC46djpCo5Filxkn24zGwNcD5wFzAcuMLP5CYtdCrzpnJsHXAd8\nN6r8iIiIjNSmHd109Cg4FikVUX7C3w284Jx7yTm3F7gNWJqwzFLgpuD1XcD7zcwizJOIiEha4sHx\nSfMUHIuUiig/5XXAK6H5WJCWdBnn3H5gF3DYIIdmttzM1pvZ+vb29oiyKyIiMlA8OJ575GQFxyIl\nJMpPerKa4MSR1FNZBufcGufcIufcomnTpmUkcyIipUr9Q9Lz9lm1zJlSoeBYpIRE+WmPAbNC8/XA\nq4MtY2ZjgUnAGxHmSUSkpKl/iIjI8KIMkNuAY8zsKDMbD5wP3J2wzN3AJ4PXHwV+45ye1ykiEiH1\nDxERGUZkDwpxzu03syuBdcAY4MfOuafN7BvAeufc3cCPgFvM7AV8zfH5w633scce22lmWxKSpwI7\nM7sHeUX7V9i0f4VtpPs3J9MZyZBk/UMWD7ZMUJbH+4cMOA5mthxYHsy+VTet+qlIcpw/iv1aB+1j\nsSj2fRzN/qVUNkf6JD3n3D3APQlpXw+97gP+PM11HtYI2czWO+cWjTSf+U77V9i0f4WtCPcvo/1D\ngDVQlMfpMNrH4qB9LHzZ2D/1OBARKS3qHyIiMgwFyCIipUX9Q0REhhFpE4ssWpPrDERM+1fYtH+F\nraj2L6r+IRTZcRqE9rE4aB8LX+T7Z6oUEBERERE5RE0sRERERERCFCCLiIiIiIQUVIBc7I9HTWH/\nvmRmz5jZk2b2azPL13FWkxpu/0LLfdTMnJkV1BA1qeyfmX0sOIdPm9mt2c7jaKRwfc42swfM7Ing\nGj07F/kcCTP7sZm9bmZJx/E175+CfX/SzBZmO4/5otjLYSj+shhUHgfLFGx5DMVdJsfltGx2zhXE\nH74zyYvAXGA8sBGYn7DMFcAPgtfnA7fnOt8Z3r/3AZXB678otv0LlpsIPAw8CizKdb4zfP6OAZ4A\nJgfz03Od7wzv3xrgL4LX84HNuc53Gvt3KrAQeGqQ988G7sWPD3wy8Idc5zmPr4OCLYfT2MeCLYtT\n3cdgOZXHefpX7GVyaB9yVjYXUg1ysT8eddj9c8494JzrCWYfxY9fWihSOX8A3wS+B/RlM3MZkMr+\nXQZc75x7E8A593qW8zgaqeyfA6qD15M4fGzdvOWce5ihx/ldCtzsvEeBGjObmZ3c5ZViL4eh+Mti\nUHkMhV0eQ5GXyXG5LJsLKUBO9njUusGWcc7tB+KPRy0Eqexf2KX4X02FYtj9M7N3AbOcc/+ZzYxl\nSCrn71jgWDP7rZk9amZnZi13o5fK/l0NXGhmMfwTNK/KTtayIt3PZ7Eq9nIYir8sBpXHUNjlMahM\njousbC6kcZAz9njUPJVy3s3sQmAR8N5Ic5RZQ+6fmZUB1wEXZytDGZbK+RuLv613Gr7G6REzW+Cc\n64w4b5mQyv5dANzonPs7M3sPfhzdBc65/uizF7lCLlsyqdjLYSj+shhUHkNhl8egMjkusvKmkGqQ\ni/3xqKnsH2b2AWAVcI5z7q0s5S0Thtu/icAC4EEz24xvS3R3AXUMSfX6XOuc2+ecexl4Dl9AF4JU\n9u9S+H/t3U+IVWUcxvHvk4lGli3sD7RoSAoTqSCiiAhLcFkJCUEQQYssCFoYBEWLFhEYrfpD0DYq\n3YQJZSQEJUaaTRoRGWm1kJQoragw/bU4Z+I4ic6MM3M91+8HDnPnznvOeV/uvc/8OOe897AeoKq2\nAfOBRbPSu5k3oc/nWWDYcxiGP4vBPB5r09c8BjN5zIxlc58K5GG/Peopx9ee8nqVJpD7dr3UScdX\nVYeqalFVjVTVCM11fXdW1Y7BdHfSJvL+fJtmcg9JFtGc4vtuVns5dRMZ3w/ACoAk19CE8cFZ7eXM\n2Qjc386Yvhk4VFX7B92pARj2HIbhz2Iwj6HfeQxm8piZy+bZnI14ugvNbMVvaGZuPtk+9wzNBxea\nF38D8C3wKXDloPs8zeP7APgJGG2XjYPu83SOb1zbD+nRrOkJvn4BXgC+AnYD9w66z9M8vqXAVprZ\n1KPAykH3eRJjewPYDxyhOSLxILAGWNN57V5qx767b+/NWX4f9DqHJzjGXmfxRMY4rq15fAYuw5zJ\nnTEOLJu91bQkSZLU0adLLCRJkqQZZ4EsSZIkdVggS5IkSR0WyJIkSVKHBbIkSZLUYYGs3khyNMlo\nZxmZwjYuSvLI9Pfuv+0vSbItyd9J1s7UfiTpTGE2axj5NW/qjSS/V9WC09zGCLCpqpZNcr05VXV0\nAu0uAa4A7gZ+qarnp9JPSeoLs1nDyCPI6rUkc5KsS7I9ya4kD7XPL0iyJcnOJLuT3NWu8hywuD3K\nsS7J8iSbOtt7MckD7eN9SZ5O8jGwOsniJO8l+SzJR0mWjO9PVR2oqu00X2ouSWcls1l9d+6gOyBN\nwnlJRtvHe6tqFc1ddQ5V1Y1J5gFbk7wP/AisqqrD7W1EP0myEXgCWFZV1wMkWX6Kff5VVbe2bbfQ\n3L1nT5KbgJeBO6Z7kJLUM2azho4Fsvrkz7Hw7FgJXJvknvb3hcBVNLekfDbJbcAx4HLg0ins8y1o\njnoAtwAbkoz9bd4UtidJw8Zs1tCxQFbfBXi0qjYf92RzKu5i4IaqOpJkHzD/BOv/w/GXGo1v80f7\n8xzg1xP8E5Ak/Z/ZrF7zGmT13Wbg4SRzAZJcneR8mqMVB9oAvp1mcgbAb8AFnfW/B5YmmZdkIbDi\nRDupqsPA3iSr2/0kyXUzMyRJ6j2zWb3mEWT13WvACLAzzfm1gzSzlF8H3kmyAxgFvgaoqp+TbE3y\nJfBuVT2eZD2wC9gDfH6Sfd0HvJLkKWAu8CbwRbdBksuAHcCFwLEkjwFL2xCXpLOF2axe82veJEmS\npA4vsZAkSZI6LJAlSZKkDgtkSZIkqcMCWZIkSeqwQJYkSZI6LJAlSZKkDgtkSZIkqeNfKsGllUWy\npJoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a2a24f860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot data and decision boundary of logistic regression\n",
    "fig, ax = plt.subplots(1, 2, figsize=(10, 5))\n",
    "# plot class 1\n",
    "ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], c='blue', label='Class 1')\n",
    "# plot class 0\n",
    "ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], c='red', label='Class 0', alpha=0.3)\n",
    "# label axes\n",
    "ax[0].set_xlabel('Feature 1')\n",
    "ax[0].set_ylabel('Feature 2')\n",
    "# label title\n",
    "ax[0].set_title('Training Toy Data Set for Binary Classification')\n",
    "# add legend\n",
    "ax[0].legend()\n",
    "\n",
    "# plot decision boundary\n",
    "ax[1] = plot_decision_boundary(X_train, y_train, logreg, \n",
    "                               'Logistic Regression (linear boundary)', \n",
    "                               ax[1])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>logistic regression</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train score</th>\n",
       "      <td>0.842105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test score</th>\n",
       "      <td>0.835443</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             logistic regression\n",
       "train score             0.842105\n",
       "test score              0.835443"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate model\n",
    "scores_df = pd.DataFrame(data={'logistic regression': [logreg.score(X_train, y_train), logreg.score(X_test, y_test)]}, \n",
    "                         index=['train score', 'test score'])\n",
    "scores_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Questions:** Why does the model do worse on testing data rather than training data?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Logistic Regression: Non-Linear Decision Boundary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWAAAAFgCAYAAACFYaNMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XmcXFWZ//HP00lDCE2IQAhCkzSb\nsgSMkCjjQhDRQVRiAMclMIOgkUEcdPQ3jsZR5qdRHPyJzuAWxRFIFAWNrbhAUBDFUbtBUDSIigE6\nbCFMSEKAbM/vj3MrqW5qudVdt869Vd/369Wv7qq6XXVqe+65z3nOuebuiIhI63XFboCISKdSABYR\niUQBWEQkEgVgEZFIFIBFRCJRABYRiUQBOAUzG2dmG8xsWjO3lTiS9+gqM1trZj9p0WOuNLPZrXis\nsTCzHjNzM9sno/tfYGbLyi6/3MzuSb4zJ5rZzWZ2WgaPe6WZ/XOz73fM3L3tfoANZT/bgCfLLs+P\n3b5RPqcvlz2HTcDmssvfa/JjPQpsTO77f4GbgbMBS/n/M4CnxtiGvwN+B6wHVgPXA89uxmMDrwL+\nBExo0us1A/Cy9+Mh4BJgXOzPTZX2TgOuBB4G1gF/ABYCOwM9yXPZp0Vt+RVwTpPv83zgR7Ff5zQ/\nbdkDdvee0g9wH/DasuuWjtzezMa3vpWNcfe3lj2n/wCWlj2n12bwkCckj3Ug8J/Avye/M2dmRwFf\nBM4FJgEHA5cRAkMzTAf+4u5PjaJt1T4rT5e9P0cTgvxbxtDGsban2vb7AL8k7MRnufsk4NWEoNzb\n/BbWNR34fYTHzYfYe4Csf4CVwIkjrvso8A3g64Qe1lnA3xA+mGuBBwnBpjvZfjzhy9+XXF6S3P7D\n5P//Bzig0W2T218F3A08DvwXcAtwVp3n9FHgqxWu/ztgBaHXuhw4KLn+34HLR2z738BHq9z/o8Cx\nI647nnA0cWBy+TTgt8lzuhd4X9m2jzG8R3gkcDihJ/0Y8Ejy+D1VHv8s4Oc1nv944ELgr0lbrwQm\nVXvsEf/7T8DTwJbk9n8pu/6e5P6uAfZOri/1CM8F/gL8vkJ7ntHrBj4HfKrSawp8EriC8BlcD9xR\n3s7k/VqZ3PY74FVlt52fvLdfSN7nDybbHVi2zXTgidJrMqJdnwZ+VeO1HdYDrvM+9wDfTF7z/yV8\nf3ZPbjs32X598rqdWtb+HyV/P5Q81kZgbXLdIPDG5G8D3gn8Mbmf3wKH13qNgNkj3t+h5PprgH8d\n8Tmo9X6/Nbn9MeCTmcWnrO44Lz9UD8CbgNcS8uC7JG/cCwlf7gMJQfH8si/8yKD6KDAL6E6+SEtG\nse3eyQdobnLbPxNSC2fVeU7PCMDAzOS+jgN2Sj6gdwLjgIMIh5oTk213IQT8Q6vc/zMCcHL9Y8CZ\nyd+vAA5LXr9ZhC/gicltlQLS4YQg3g08G/g11XcARyRfok8Ac0rtLrv9g8BNwD7Jc7kS+FK1x65w\n/8MOUYFTCDvdGcn9XQb8MLmt9IX8LrA7sEuF+xv2mMD+hKDxd5VeU0IAfgI4IXl//gu4oWzbNybP\nbRyhF70O2KOs7VsIKaFxSXuvAP6t7P8XAl+v8tzvBP5PjddmZACu9T6/h/B5nkD43L8gac+U5LNS\n2lnvV/qsVXjth33WGB6A30IIgs8jBONDgf1SvkY/GvG8tgfglO/31cBu7PjuvCSL+NSWKYiUfu7u\n33P3be7+pLsPuPuv3H2Lu98DLCZ8+au5xt0H3X0zsJQQABvd9jXA7e7en9x2CeEDORpvAq5295vd\nfRPwEWBf4Pnu/hdCL2tusu1c4I/ufleDj/EAsAeAuy939xXJ6zcIfIsar5e7/8Hdb3L3ze5eOsKo\nuL27/57wxT8E+Dawxsy+aGYTkk3eTuiJPeTuTwL/l/CFHK35wOfd/c7k/v4FOMnM9irb5qPu/nhy\neyU7J4N6jxPSXkPAtTUec7m7/8TdtxJ2INs/P+5+VfLctrr7fxOOGJ5f9r93u/tXktufBC5PnkPJ\nGcl9VrInIfikUud93kwItgcm35tfJ+3ZRgiYR5jZzu6+ahSfNQi90EXufocHd7n7qqRd9V6jWtK8\n34vcfX3y3fk5tb/fo9bJAfj+8gtmdqiZfd/MHjKzdYQv9V6V/xUIh08lGwl7zka33be8HR52wUMp\n2l7JvoRDvtJ9bSEEzP2Sqy4nfDGh9he0lv0IPRvM7LhkxPrRJOicQY3Xy8x6zewaM3sgeX2/UGv7\nZEdyarLNywk7q/eY2bikHdcnAW8tMAB0m9nkUTwneOZrt4bQQ92vbJv7R/7TCE+7+2R3353w/t4F\n9NfYvurnJ6kU+F3Z8+tj+Gs1si03AhPNbLaZzSLsJK+v8rhrCEcgqdR5nxcDvwCWmdn9ZvZRM+tK\nXr9/AN4NPGxm/WZ2UNrHLLM/IX1RqV31XqNa0rzfjXy/R62TA/DIAZ0vEg7PDvYwMPEhwl48Sw9S\nNvBhZsbwD0EjHiDk/kr3NZ7wQVuVXPVN4KVmdjjh0PeqRu7czI4nHILfUnZ/SwiHhLsnf5der0qD\nZf+PkPY4PHl9zyXF65v0fH4BfA+YkfQYHwSOSwJe6WeCu6+t8tj1jHzt9gB2ZcdrV+05VWvzE4S0\nwIllvfZUkvfnU8A5hEPqyYQ0WvlrNawt7r6NsEM9AziTkH7YUuUhbgBObaBJVd9nd3/K3T/o7s8F\nXga8GXh9ctt33f0Ewuf5AeDSBh6z5H5CCmCYFK9RvfcqzfvdEp0cgEfajRAgnjCzwwiHuVm7Fjja\nzF6bBMwLCId0o3EVcLqZvdjMugl5wAeB3wC4+zrg+8l2N7j76jR3amaTzWweIaB8wd3vMbMuQo9g\nDbDJzF5CGKwpeYRwSF6+M9mNkKNeZ2Z9hN5Rtcd8uZmdVTokNLMjCYOVv0w2+QLwidL9m9lUM3tN\njceu5+vA283sCDPbhZB7vs7dR5UOSoLuGcA93nilRQ/hEH410GVm7yD07uq5AngDIRVzRY3tLgKm\nm9liM+tN2jvNzD43spda7302s1eY2WHJdusIuemtZra/mZ2cvJZPEXqXW1M8h5G+DHzAzI6y4NDk\nfa33Gj0MTKtRIdLU93ssFIB3eA/hsGk9oTf8jawf0N0fJnxpPkX4kB9ECJhPj+K+fkPYaVxG+GAe\nB7wu6TGWXE6oSEiTfviJmW0g9CzeQ8gpvyN5rG2EHuxnCDutfyYMcpTasprQ470jOUQ8kjBw9jLC\nF/UawiBHNWsJr8sfkjb0J23/r+T2jxMqKn6apDN+TpL/q/LYNbn7dwgDY9cSUkB7Ej4Ljdg5mUyw\ngbDjOxyY1+B94O6/JryHv2FHCun2FP/3R8J79ai731Zju4eAYwmDT7clr9+PCL3NoRHb1nyfCSmC\n77GjkqOfkLMfT+gAPEwY05hJ6Fw06quE9/xbhM/NNwiVHfVeox8QerOrzexeRmjS+90UFtKOkgdJ\nfvMB4HR3/1kG9384IWe3zyh6ZpJzZvZN4DZ3vyh2WyQd9YAjM7OTzGx3M9sZ+DfCYdyvM3iccYTD\n/isUfNuPmT0HOInQa5SCiB6ALczL/42Z1SrZaWcvYUdB+EmEtEHDKYhazGxvwiHcbGBRM+9b4jOz\nTwG3Ah9OUgxSENFTEBYWyJhFyO28pt72IiLtImoPOBmFfTVhtFNEpKPEXoTm04RZKLtV28DMFgAL\nAHbddddjDj300BY1TURq2rQJ1q6F8eOhqwu2bYMtW2DyZNhpp9iti+rWW2991N3rlpRGC8BJ3eYj\n7n5rUuRfkbsvJsy4YdasWT44ONiiFopITcuWwcaNMGnSjuvWrYOJE2FewxV4baVS+VslMVMQLwZO\nMbOVhMkBJ5jZkojtEZFGrF4NPSNm6Pb0hOsllWgB2N3f7+697t5HmL3zE3c/o86/iUheTJkCGzYM\nv27DhnC9pBK9DE1ECmr27JADXrcu5H/XrQuXZ+f+zEu5EXsQDgB3v4mwvquI5MzmzZsZGhriqacq\nzN857rgwGLd1K4wbFwbf1q+HFSta39AIJkyYQG9vL93d3aP6/1wEYBHJr6GhIXbbbTf6+voIC/YJ\nhJNZrFmzhqGhIQ444IBR3YdSECJS01NPPcWee+6p4DuCmbHnnntWPjJISQFYROpS8K1srK+LArCI\nSCQKwCKSew899BBvfOMbOeiggzj88MM5+eSTufvuu1m5ciUzZszI5DFvvvlmjj76aMaPH88111xT\n/x9GQQFYRHLN3Zk3bx7HH388f/nLX/jDH/7Axz72MR5++OFMH3fatGl89atf5c1vfnNmj6EALCJN\ntXQp9PWF5SH6+sLlsbjxxhvp7u7m3HPP3X7dzJkzeelLXzpsu5UrV/LSl76Uo48+mqOPPppf/OIX\nADz44IMcd9xxzJw5kxkzZvCzn/2MrVu3ctZZZzFjxgyOPPJILrnkkmc8bl9fH0cddRRdXdmFSZWh\niUjTLF0KCxaEJSIA7r03XAaYP39093nnnXdyzDHH1N1u7733Zvny5UyYMIE//elPvOlNb2JwcJCv\nfe1r/O3f/i0LFy5k69atbNy4kdtvv51Vq1Zx5513ArB27drRNW6MFIBFpGkWLtwRfEs2bgzXjzYA\np7V582bOP/98br/9dsaNG8fdd98NwOzZszn77LPZvHkzr3vd65g5cyYHHngg99xzD+985zt59atf\nzStf+cpsG1eFUhAi0jT33dfY9WkcccQR3HrrrXW3u+SSS5g6dSp33HEHg4ODbNq0CYDjjjuOm2++\nmf32248zzzyTK664gmc961nccccdHH/88Xz2s5/lrW996+gbOAYKwCLSNNOmNXZ9GieccAJPP/00\nX/rSl7ZfNzAwwE9/+tNh2z3++OM8+9nPpquriyuvvJKtW8MJwe+991723ntv3va2t3HOOedw2223\n8eijj7Jt2zZOO+00PvKRj3DbbVVPJJ0pBWARaZpFi8JywOUmTgzXj5aZsWzZMpYvX85BBx3EEUcc\nwYUXXsi+++47bLvzzjuPyy+/nGOPPZa7776bXXfdFYCbbrqJmTNn8vznP59vfetbXHDBBaxatYrj\njz+emTNnctZZZ/Hxj3/8GY87MDBAb28vV199NW9/+9s54ogjRv8kqj232OeEa0RhF2QfGoKBgbBO\n6pQpYbWo3t7YrRJJZcWKFRx22GGpt1+6NOR877sv9HwXLco+/xtTpdfHzG5191n1/lc94KwNDUF/\nfxiJmDo1/O7vD9eLtKH582HlyrBC5cqV7R18x0pVEFkbGAjnyCqdtqX0e2Agu16wetwihaAecNZa\nfdoW9bhFCkMBOGutPm1LeY+7qyv8njw5XC8iuaIAnLVWn7ZFJ0oUKQwF4Kz19sLcuaEW5+GHw++5\nc7PLyepEiSKFoQDcCr29MG9emBQ/b162A2I6UaK0oRjLUT799NO84Q1v4OCDD+aFL3whK1eubPpj\nKAC3m1b3uEUyFms5yssuu4xnPetZ/PnPf+bd734373vf+5r+GCpDa0e9vQq4Ek+TyyCrLUcJDOuV\nrly5kjPPPJMnnngCgEsvvZQXvehFPPjgg7zhDW9g3bp1bNmyhc9//vO86EUv4pxzzmFwcBAz4+yz\nz+bd7373sMft7+/nwgsvBOD000/n/PPPx92benomBWARGW5kAD344Mb+t78/VN5MnRrGH/r7x3QU\nFms5ylWrVrH//vsDMH78eHbffXfWrFnDXnvtNarnUYkCsIjsUCmAPvkkbNoEO+1U//9jTDxKNHs5\nykrLNDT75KTKAYvIDpXqyLu6IDmsryuDMshYy1H29vZy//33A7BlyxYef/xx9thjj1E/j0oUgEVk\nh0oB1Ay2bEn3/xmUQcZajvKUU07h8ssvB+Caa67hhBNOaHoPWCkIEdmhFEBLqQMAdxifMlTMnh1S\nGBAC+YYNoQxyzpxRN6m0HOW73vUuLrroIiZMmEBfXx+f/vSnh2133nnncdppp3H11Vfzspe9bNhy\nlBdffDHd3d309PRwxRVXsGrVKt7ylrewbds2gIrLUZ5zzjmceeaZHHzwweyxxx5cddVVo34OVZ+b\nlqMUke3Kc8BJAF1xyCEcNnNmuhxw6T46aDGosSxHGa0HbGYTgJuBnZN2XOPuH47VHhFhRx35wECo\nI58yBXbZJX3wLd1HGwfcZoqZgngaOMHdN5hZN/BzM/uhu/8yYptEZGQAXbEiXlvaXLRBOA9K2fru\n5Kc4+RCRDlKkVGUrjfV1iVoFYWbjzOx24BFgubv/qsI2C8xs0MwGV2tFL5GWmzBhAmvWrFEQHsHd\nWbNmDRMmTBj1feRiEM7MJgPLgHe6+53VttMgnEjrbd68maGhIZ566qnYTcmdCRMm0NvbS3d397Dr\ncz8IV87d15rZTcBJQNUALCKt193dzQEHHBC7GW0pWgrCzKYkPV/MbBfgROCuWO2RYli6FPr6wuSs\nvr5wWaSoYvaAnw1cbmbjCDuCb7r7tRHbIzm3dGlYUnnjxnD53nvDZdCZd6WYcpEDTks54M7W1xeC\n7kjTp4fTn7e1DpvcUHSFygGLpHHffY1d3zYyWOKx7RVkh6XFeKQwpk2rfP0ee7R5Xlhnum5MaYe1\ncWPYYW3cGC4PDcVu2TMoAEthLFoUzrBUrrsb1q8PqQn3HXnhtgrCOtN1Ywq0w1IAlsKYPx8WLw45\nX7Pwe9KksFZ4uY0bYeHC9Peb+8oKnem6MQXaYSkAS6HMnx8G3LZtC78fe6zydmnzwqXKilz3oHWm\n68YUaIelACyFVi0vXO36kRYu3FHWVtJoDzpzOtN1Ywq0w1IVhBTaokXDa4MhxKdFi9L9f2EqK7TE\nY3qVltScMyeXr58CsBRaaQLGwoUhaE6bFoJv2okZ06ZVri1O24OWnCrIDkspCCm8kXnhRmbFVaqs\naKQHLTIWCsDS0SpVVixerKnN0hpKQUjHmz9fAVfiUA9YOlbu63+l7akHLB1JK6tJHqgHLB2pEPW/\n0vYUgKUjFab+V9qaUhBZKMhSeJ1M9b+SB+oBN1uBlsLrZKr/lTxQAG62Ai2F18lU/yt5oBRELaNJ\nJaxeHXq+5Xp6wpx0yRXV/0ps6gFXM9pUQoGWwpPGqXZYmkkBuJrRphIKtBSeNKYQawdLoSgAVzPa\nVfW1dmvbUu2wNJtywNWUUgmTJu24Lm0qoSBL4UljVDsszaYecDVKJcgIYz37hshICsDVKJUgI6h2\nWJpNKYhaWplK0Oy53Bvr2TdERjJ3j92G1GbNmuWDg4Oxm9F8pZK3yZPDQN+GDSHdoR63SCGZ2a3u\nPqvedtFSEGa2v5ndaGYrzOz3ZnZBrLZEp9lzIh0pZg54C/Aedz8MOBZ4h5kdHrE98Yy25E1ECi1a\nAHb3B939tuTv9cAKYL9Y7YlKs+dEOlIuqiDMrA94PvCrCrctMLNBMxtc3a49QpW8tZymFEseRA/A\nZtYDfAt4l7uvG3m7uy9291nuPmtKu/YIVfLWUppSLHkRtQrCzLqBa4Hr3P1T9bZv2yoIaam+vsqL\nsU+fDitXtro10o6KUAVhwGXAijTBV7LTaYfjmlIseRFzIsaLgTOB35nZ7cl1H3D3H0RsU8dp2tmB\nCzSRRKcjkryIWQXxc3c3dz/K3WcmPwq+LdaUFb5ydhqmej16TSmWvIg+CCdxNeVwPEcTSdIMsOl0\nRM3XaWmsZlEA7nBNWeErRxNJFi6EyRuHmMsy3sZi5rKMyRuHntGjnz8/DLht2xZ+Nyv4NhKI2iVo\nqapk9BSAO1xTDsdzNJFky71DzKWfXdnIQ0xlVzYyl3623Jt9OqSRQNROQataGuuCC9pjB5MlBeAO\n15TD8RxNJDl5rwEeZzLrmITTxTom8TiTOXmv7NMhjeTT2+nsGtXSVWvWtMcOJksKwDL2w/EcTSQ5\n55TVbOoeng7Z1N3DOadknw5pJJ/eTqVwadNVRd3BZEnrAUtz5OQ0TC98zRTetfMGvnLNpO0VcWef\nvoEXviL7dEgj5W3tVAq3aNHwUsZairiDyZIC8EgFqmeVCmbP5sUP9PPiFzN8beXZczJ/6EqBqFo+\nvZFt867SQvUbNoQUxEhF3MFkSSmIcjmrZ5VRiJgOaSSf3m6lcCPTWJ/5jGqt09AZMcotWxaCbvmZ\nkNetC5+cefOye1zJ1NKlOo1QDJ38uqddC0IpiHKrV4eeb7mentCTkkJq2lRradj8+XqN61EKolyO\n6lmlOdqp3EvajwJwuRzVs0pztFO5l7QfBeByOapnleZoylRrkYwoBzxSTupZpTnaqdxL2o8CsLS1\nSjWqqUbjVQ8uLaAUhLS9hqdaN7kevF1WPZPmUwCW3Gt5AGvi+sbttOqZNJ8CsORalADWxPWNVQYn\ntSgAS65FCWBNrAdXGZzUokE4ybUoAWz27JDzheEL+sxpfEGfdlr1rFAKMoiqHnBWhobC2hKLF4ff\nWtBnVKLU8TaxHlwnAI2gQItqqQechdIHYPLk8AHYsCFc1qSOhkWr421SPfioy+Bk9MoHUWHH74GB\n3H3/FICzUKAPQN61QwDTojQtVqBFtRSAs1CgD0ARKIBJQ0qDqOXLyuZ0US3lgLNQwFXVNFlA2kaB\nFtVSAM5CgT4AoMkCeaYd4ygUaFEtnREjKwUpg4Hwxa5UKjV9epi6K3GMXEweQiwp8qmLOkXaM2JE\nDcBm9hXgNcAj7j6j3vaFCsA15O1ULV1doec7klnowEsc2jEWV1FOSfRV4FLgisjtaJk8niJHkwXy\nKeosugIdwRVZ1Bywu98MPBazDa2Wx7UBNFkgn6ItJl+giQxFp0G4Fsvj2gDtdor0dhFtx9jE1eCk\nttwHYDNbYGaDZja4ehSrUeVNXk+R0/CauZK5aDvGJq4GJ7XlPgC7+2J3n+Xus6bkuI42raId7p93\nHowfHwLA+PHhsrROlB1jAevYiyr3AbjdFOlw/7zz4POfh61bw+WtW8NlBeE2V7A69iKLXYb2deB4\nYC/gYeDD7n5Zte3bpQytKMaP3xF8y40bB1u2tL490kKqghiTQpShufubYj6+1FYp+Na6XtqIzg7e\nEkpBSFXjxjV2vYg0RgFYqipNEEl7fTmtYSBSX+yZcJJjn/tc+L14cUg7jBsXgm/p+mryONtPJI+0\nGI80ndYwkE6XdhBOKQjZrllpgzzO9hPJo2IF4E2bYregrqLmPpu5JnBeZ/uJ5E2xAvDatbleEKTI\nC5s3c5Ggos32E4mlWAF4/PhcLwiSx5XO0mpm2iCr2X5FPboQqaZYAbirK9cLgsTOfY4lQDU7bdDs\nNQyKfHQhUk2xAvC2bbleECRm7nOsASrvaYMiH12IVFM1AJvZkWb2SzO738wWm9mzym77dWuaN8KW\nLbleECRmEBtrgMr7IkGxjy7aldI6kbl7xR/g58BJwGTgvcDvgYOS235T7f+y/DnmyCM975YscZ8+\n3d0s/F6ypDWPa+Ye+r7Df8xa8/hZmz698vPbc8/YLSuuJUvcJ04c/npOnNi6z2w7AwY9RUyrlYLo\ncfcfuftad/8kcD7wIzM7Fogze2OnnaI8bCNiLWze7qVfixZBd/czr1+/Xr220VJaJ75aAdjMbPfS\nBXe/ETgNuBKYnnXDpDF5z+GO1fz54cw4I23apIAxWkrrxFcrAH8COKz8Cnf/LfBy4NtZNkoal/cc\nbjM8VuX0rQoYo9PuR01FoLUgmkGLV7eE1phorpGLJkE4amq3HXcMWguiVXQK75Zp9zRLq3XCUVPe\naTnKsSo/hTfs+D0woF5wk5UCw8KFIe0wbVoIvgoYozd/fkFfvzY56lQPeKx0Cu+WilVlkomhIVi2\nLHQ7ly3bftSk2tw62uios24ANrPnmNmPzezO5PJRZvbB7JtWEDqFt4xGlSDy7f8c0pTresqPOru6\nwu/Jk3O9Tkw1aXrAXwLeD2yG7ZUQb8yyUYWiU3jnSmF6j1WCyI8+MqDa3Hra6KgzTQCe6O4jpx7r\npOQlvb0wd24YDXr44fB77txC5qOKrlAL9lQLIo9WDiIqtSvTRkedaQbhHjWzg0hmv5nZ6cCDmbaq\naHQK71yoNbMrd7niUhApn12yYQPsNQUefebmqs0tM3t2SN9A2Glt2BCOOufMiduuUUjTA34H8EXg\nUDNbBbwLODfTVomMQqFmdlVJXZ30b7NValdPGx111uwBm1kXMMvdTzSzXYEud1/fmqaJNGbatMoT\nNXLZeywFkYGBEESmTIE5czi1t5cn91SpXV1tctRZdyacmd3s7se1qD015XYmnOSCZnZJXjRzJtxy\nM3uvme1vZnuUfprQRpGm0swuKZo0PeC/Vrja3f3AbJpUnXrAIlIEaXvAdasg3P2A5jTpmczsJOAz\nwDjgy+5+UVaPJSKSN3UDsJn9faXr3f2KsTywmY0DPgu8AhgCBszsu+7+h7Hcr4hIUaTJAc8u+3kp\ncCFwShMe+wXAn939HnffBFwFzG3C/cpYVVmjQKQwMw0Lom4Advd3lv28DXg+0IxzA+0H3F92eSi5\nbhgzW2Bmg2Y2uLqAUw0Lp40WOilR0GiOQs00LIjRrIa2ETikCY9tFa57xoiguy9291nuPmtKAaca\nFk4bLXQCChrQvB2QziHXfGlywN9jR2DsAg4Hrm7CYw8B+5dd7gUeaML9ylisXh16vuV6esJkgQIq\n1PTkDIysjS7tgKDx51+omYYFkWYtiE+W/b0FuNfdm3E8OgAcYmYHAKsIK6y9uQn3K2NRbY2Cgh59\ndHrQaOYOqFAzDQsiTQriZHf/afJzi7sPmdknxvrA7r6FcKr764AVwDfd/fdjvV8ZozZbXrPTTzzZ\nzB1QLk4J1WYDxGkC8CsqXPeqZjy4u//A3Z/j7ge5e+GWG2nLwZ02WugEchI0ImrmDij6TMM2HCCu\nOhPOzP4ROA84EPhL2U27Abe4+xnZN2+4PM2E07oDxbF06fDFbU4+GX7wg85Y7KatPqfLloUnUp4e\nW7cuPKF58+K1q4JmrAXxNeC1wHeT36WfY2IE37zJ04hwW/bEm6j8PHKLFsHll3dOVUT0XmsztdGZ\nMErqrgWxfUOzvYEJpcvu3vJhjDz1gLu6whd4JLPwRW+VturhtEBfX+WBpOnTQ5CWHOuwHnDpjl5r\nZn8C/gr8FFgJ/HDMLSy4vAxMe0quAAAUCklEQVTu5KknXgSdXhVRaG02QAzpBuE+ChwL3J0szPNy\n4JZMW1UAeRncUUBpTF52nDIKbTZADOnqgDe7+xoz6zKzLne/sRllaEVXOryPfeYC1WY2ZtGiyimb\nTqmKKLw2ORNGSZoe8Foz6wF+Biw1s8+gsyIDwwd3Vq6Mk3MdbU+8Uwfu2mpQSgovzYLsuwJPEoL1\nfGB3YKm7r8m+ecPlaRAuT0aWWdXriWvgTiRbaQfhUlVBmNl04BB3v8HMJgLjYpycc3sAHhoKi8Os\nXh2myM6e3VaHJVlTJcAOje68RNJoZhXE24BrCKemh7Bk5HfG1rwxaMPZMK2mgbtAK6VJbGlywO8A\nXgysA3D3PwF7Z9momtpsucQYVAkQqIRPYksTgJ9OzlgBgJmNp8K6vS3ThrNhWi0vJXTNMJbBxFwc\nCbTZ4jLSmDQB+Kdm9gFgFzN7BWEt4O9l26waSssllivwcokxtEslwFhTCNGPBJJ02i3LN/LWhVOZ\nf+pGPnhUP9/+TwXhTpGmCqILOAd4JeEsFtcRzmDc8l7wrFmzfPA73wk538mTQ893w4YwG6bgBdnS\nuLEOJkavBlm2jFuWb+SSyybxdHKMOYl1bN1pIq/9yrzC7RBlhzEPwpnZNAB33+buX3L317v76cnf\n8VIQbTgbRkZnrCmE6EcCq1fz31f3bA++AOvpYdKm1cpDd4haM+G+AxwNYGbfcvfTWtOkFNpsNkw1\nKpGqrRmzAOfPj/iaTpnCk49uAHYsLrMbG3iEKR1XkZKpHJet1soBl58088CsGyLDqUSqvsIPJs6e\nzQHPWssk1mFsYxLr2J21DDA7PxUpRR8kzHnZaq0A7FX+lhZQiVR90VMIY9Xby9EXzmXrThPZh4d5\ngon0M5e1E3vzsRPJefBKJedlq7VSEM8zs3WEnvAuyd8kl93dJ1X/VxmrXJRIFUDUFEITnPpPvTy5\nZ++wVNPivKSayoMX7Pg9MJCbQ/i6cn6W76oB2N3HtbIhMpxWOescud2J5Dx4pZLzs3ynqQOWLFXJ\nsVXKb+60U/jsdNoKZhJJ0Wvuh4ZgzRq49lr48Y/hkUdyt4i7AnBMNXJsI/Obe+4ZBuPWrNGgXCWd\nurxmXWMZRCvyGShK361ddoETTwzX/fjH4TuWo7LV1OeEy4O2W46ygXNc5W4FsxyV9kSfUJFXpSA0\nlklLOXqfGxL5/HFpJ2KkOSOGZKWBHFuuBuXKvtg33TWVZR/YwJY1/QzuO5d/+o/elge9WhUjHR2A\nmzGIVtSa+4Lkr5WCiKmBHFv0dQvKJV/sm26bxKWf7eKeNZNYy2Se/cBAlLRIrnZOedLJC1cVJH+t\nABxTAzm2XE06SL7YV17J9mm06+lhb1ZHqVXO1c4pTwoShDJRkPy1UhBpZZELK61rMTAQDo2mTIE5\ncyreb15OAgps/2KvXv3MKbTQ+p6nTrSZGPkZ7e2FX/863FaeA54zJ247W6GB71ZMUQbhzOz1wIXA\nYcAL3D3VyFq0QbhmDGa0k+T1uODDk/nrmh52YwO7s5Z+5vIAvVEGBjt+3Yxqn9EXvCDcVrRBtIJr\n2imJMnIncCpwc6THb0wOpzNGLbtKehennjGRad07ptA+QG+0nmcezlAdVbXP6NBQGPVfsCD8VvDN\nlSgpCHdfAWBm9TbNh5yNqI4suyrVBEMLA09vL3M+3cvQ7NDzfPA+mN6JPc+8yNlnVNJRDjiNnE1n\nzFPZVW6n0cYSq242Z59RSSezFISZ3WBmd1b4mdvg/Swws0EzG1wdq3wmZyOqKrvKqZirh+XsMyrp\nZBaA3f1Ed59R4ae/wftZ7O6z3H3WlFh785ydhSNm2ZWm/NYQc6yg1Z/Roq8TnBNKQaSVwYyg0Y7c\nxyq7ykXuOZY0qYXYedhWzVorr7iYOjWkOvr7O7cqaAyiVEGY2TwzGwL+Bvi+mV0Xox0xjeWMF7EW\nIu/YReLTphY6ZeJDDquCikqL8USSu8V1UujqCjuLkcxC2rFtpV3YpVPqxRcvDjuirrL+27Ztoadf\nOiTqcHmvA+54RRxIyzL3nOvccto1FXI2VpCZTunpt4BywJEU8YwXWeWec59bbqTEq6irhzVi9uzQ\n04fOm+LcZOoBR5KrxXVSyir3HDu3XLf3rRKv4Tqlp98K7l6Yn2OOOcbbyZIl7tOnu5uF30uWxG5R\nHGbuIbs8/Mcs+8dessR94sThjztxYoX34v773b/9bfcvfjH8vv/+7BtXpb36zOQfMOgpYpoG4SS6\nmAOSRRoM1Zk/ikODcFIYMdMxRRoMjZ2qkeZTAJaoSpNRNm6EcePCddVyy1lUShRpMfci7SwkHQVg\niaZ8MgrA1q07er6Vgu9oJ67U0ored7N2HEXaWUhKaRLFeflpt0G42GIP6EyfXnnwbfr0sW3bqCxf\nh9SDfC2+L8kWGoSTWvIwoNPIzLqizsJr9iBfx5/5oyDSDsIpAHeoPIz+N9KGPLR3NIq645CxURWE\n1JSHAZ1G8q9FnLgCyttKbQrAHSoPgaGRmXWxVoAbq6LuOKQ1lILoUHnIAXcK5W07j1IQUlOrepS5\nXuWsRTr+jM1SlQJwB8s6MGRVu9toGzp9ByD5pQAsmYk9dXY0OwAF7BbQ+eS2Uw5YMhO7BKvR0rWO\nzounOeddsx6nA84aohywRBe70qLRUrvYPfZo0p7zrhl0PrlhFIAlM7FLsBrdAeShNjqKVgbFtKd3\n6hAKwJKZ2LW7je4AYvfYo2llUNT55IZRAJZMxSzBanQHELvHnlbTBwpbGRR1eqdhFIClrTWyA4jd\nY08jk9K+VgZFnU9uGFVBiBTIqBclqlfl0KoqiA6hKgiRMchrPfCoBgrTVDn09sK8eaE7PW+egm+L\nKACLjJCHGXzVjGqgUKVfuaUALDJCnuuBRzVQqNKv3FIAFhkhz/XAoxooVOlXbkUJwGZ2sZndZWa/\nNbNlZjY5RjtEKsl7PXDDpX0q/cqtWD3g5cAMdz8KuBt4f6R2iDxDUeqBU1PpV26Nj/Gg7n592cVf\nAqfHaIdIJaUeZVstot7bq4CbQ9HrgM3se8A33H1JldsXAAsApk2bdsy9lYogRURyJG0dcGY9YDO7\nAdinwk0L3b0/2WYhsAWoWuDj7ouBxRAmYmTQVBGRKDILwO5+Yq3bzewfgNcAL/fY3XARkQii5IDN\n7CTgfcAcd99Yb3sRySFNXx6zKAEYuBTYGVhuZgC/dPdzI7VFRBpVfmaLqVNDXXF/f/tWV2S0s4lV\nBXFwjMcVkSYpn94MO34PDBQjADcSUDPc2WgmnIg0rsjTmxs9BVOGa2koAItI44o8vbnRgJrhzkYB\nWKSAoi+XWeTpzY0G1Ax3NgrAIgWTi+Uyizy9udGAmuHOJvpMuEbojBgiYzgrhgTlg2o9PSH4rl1b\newfSYBVE9JlwIpKNPC+XWQil3vvAQOi9T5kCc+bU7r1ntJaGArBIwUybVrkHnJflMgshJ4sTKQcs\nUjBtt1xmB1MPWKRgcr9cpqYop6ZBOBFpntEMcGXVjog7AZ2WXkRaLw9nYG50pltECsAi0jx5mKKc\nh51ASgrAkkr0mVdSDHmYopyHnUBKCsBSVy5mXkkx5GGKch52AikpAEtdCxeGNFq5jRvD9SLD5GGK\nch52AimpDE3q0swraUjsSQ6jmekWiQKw1KWZV1I4sXcCKSkFIXVp5pVINhSApa7582Hx4rDalln4\nvXhxjmZeiRSUUhCSyvz5CrgizaYesIhIJArAIiKRKAXRDrT6lEghqQdcdAVaeEREhlMPuOjKFx6B\nHb8HBtQLls5VkKNC9YCLrkALj4i0RIGOChWAi65AC4+ItISWo6zNzD5iZr81s9vN7Hoz2zdGO9pC\ngRYeEWmJAh0VxuoBX+zuR7n7TOBa4EOR2lF8eVh9SiRPCnRUGGUQzt3XlV3cFSjOienyqCALj4i0\nxOzZIecLw89LN2dO3HZVEK0KwswWAX8PPA68rMZ2C4AFANO0/FZjCjISLNJUBVqOMrOzIpvZDcA+\nFW5a6O79Zdu9H5jg7h+ud586K3ID8nJ22k6kHV/HS3tW5Mx6wO5+YspNvwZ8H6gbgKUBqg+Oo3zH\nN3Vq2PH192vHJxXFqoI4pOziKcBdMdrR1go0EtxWClQCJfHFygFfZGbPBbYB9wLnRmpH+yqNBJd6\nvpDbkeC2snp16PmW6+kJuUh5pg5P10TpAbv7ae4+IylFe627r4rRjram+uA4ClQCFV2BZqxlRTPh\n2pXqg+PQji89pWu0GE9b6+T64FiHtgUqgYpO6RoFYGlDsSsROnnH1wiNUygFIW1Ih7bFoHSNArC0\nIZXgFYPGKZSCkDakQ9vi6PB0jQKwFFe1gbYCLcYinU0pCCmmWjWkOrSVglAPWIqp3loXHX5oK8Wg\nACw7FGlaqGpIpQ0oBSFB0aaFasqvtAEFYAmKVjurGlJpA0pBFFEWqYKiHdK345TfIqWApCkUgIsm\nq2m2RaydbaeBttjTpyUKpSCKJqtUgQ7p4ypaCkiaQgG4aLKaZqva2bg0fbojKQVRNFmmCtrpkL5o\nipgCkjFTD7holCpoT3pfO5ICcNEoVdCe9L52JKUgikipgvak97XjqAcsIhKJArCISCQKwCIikSgA\ni4hEokE4kU6nNSiiUQ9YpJMVbRnSNqMALNLJtAZFVArAIp1Ma1BEFTUHbGbvBS4Gprj7ozHbIgWk\n3OXYaQ2KqKL1gM1sf+AVwH2x2iAFptxlc2gNiqhipiAuAf4F8IhtkKJS7rI5tAZFVFFSEGZ2CrDK\n3e8ws3rbLgAWJBefNrM7s25fg/YC8pg+yWO7mtamqbDXZtg88vpu6H648cdo69eqifLYJshnu56b\nZiNzz6YDamY3APtUuGkh8AHgle7+uJmtBGalyQGb2aC7z2puS8cmj22CfLYrj22CfLZLbUovj+1K\n26bMesDufmKl683sSOAAoNT77QVuM7MXuPtDWbVHRCRvWp6CcPffAXuXLjfSAxYRaSdFqwNeHLsB\nFeSxTZDPduWxTZDPdqlN6eWxXanalFkOWEREaitaD1hEpG0oAIuIRFLYAGxm7zUzN7O9ctCWj5jZ\nb83sdjO73sz2zUGbLjazu5J2LTOzybHbBGBmrzez35vZNjOLWjpkZieZ2R/N7M9m9q8x21JiZl8x\ns0fyVO9uZvub2Y1mtiJ57y7IQZsmmNmvzeyOpE3/HrtNJWY2zsx+Y2bX1tu2kAE4h9OYL3b3o9x9\nJnAt8KHYDQKWAzPc/SjgbuD9kdtTcidwKnBzzEaY2Tjgs8CrgMOBN5nZ4THblPgqcFLsRoywBXiP\nux8GHAu8Iwev1dPACe7+PGAmcJKZHRu5TSUXACvSbFjIAEzOpjG7+7qyi7uSg3a5+/XuviW5+EtC\nvXV07r7C3f8Yux3AC4A/u/s97r4JuAqYG7lNuPvNwGOx21HO3R9099uSv9cTgst+kdvk7r4hudid\n/ET/3plZL/Bq4Mtpti9cAC6fxhy7LeXMbJGZ3Q/MJx894HJnAz+M3Yic2Q+4v+zyEJGDShGYWR/w\nfOBXcVuy/VD/duARYLm7R28T8GlC53Bbmo1zeUqiNNOYW9ui2m1y9353XwgsNLP3A+cDH47dpmSb\nhYRDyKVZt6eRduVApUVIoveg8szMeoBvAe8acdQXhbtvBWYm4xvLzGyGu0fLnZvZa4BH3P1WMzs+\nzf/kMgDncRpztTZV8DXg+7QgANdrk5n9A/Aa4OXewoLvBl6rmIaA/csu9wIPRGpL7plZNyH4LnX3\nb8duTzl3X2tmNxFy5zEHL18MnGJmJwMTgElmtsTdz6j2D4VKQbj779x9b3fvc/c+wpfo6NhrSJjZ\nIWUXTwHuitWWEjM7CXgfcIq7b4zdnhwaAA4xswPMbCfgjcB3I7cplyz0di4DVrj7p2K3B8DMppQq\ne8xsF+BEIn/v3P397t6bxKY3Aj+pFXyhYAE4xy4yszvN7LeE9Ej0Mh3gUmA3YHlSHveF2A0CMLN5\nZjYE/A3wfTO7LkY7kgHK84HrCINK33T338doSzkz+zrwP8BzzWzIzM6J3SZCz+5M4ITks3R70suL\n6dnAjcl3boCQA65b9pU3moosIhKJesAiIpEoAIuIRKIALCISiQKwiEgkCsAiIpEoAEshmNnWshKo\n25MpsY3ex2QzO6/5rdt+/4ea2f+Y2dNm9t6sHkfah8rQpBDMbIO794zxPvqAa919RoP/Ny6Z9lpv\nu72B6cDrgP9190+Opp3SOdQDlsJKFmO52MwGknWP355c32NmPzaz28zsd2ZWWuXsIuCgpAd9sZkd\nX75mq5ldamZnJX+vNLMPmdnPgdeb2UFm9iMzu9XMfmZmh45sj7s/4u4DwObMn7y0hVyuBSFSwS7J\nylcAf3X3ecA5wOPuPtvMdgZuMbPrCauczXP3dRYW7P+lmX0X+FfCGskzAVIsmPKUu78k2fbHwLnu\n/iczeyHwOeCEZj9J6SwKwFIUT5YCZ5lXAkeZ2enJ5d2BQwhrhHzMzI4jLAu4HzB1FI/5Ddi+CtiL\ngKuTRaAAdh7F/YkMowAsRWbAO9192HoSSRphCnCMu282s5WE1alG2sLwNNzIbZ5IfncBayvsAETG\nRDlgKbLrgH9MlkrEzJ5jZrsSesKPJMH3ZYSBMYD1hAWKSu4FDjeznc1sd+DllR4kWfv2r2b2+uRx\nzMyel81Tkk6iHrAU2ZeBPsKa0AasJlQgLAW+Z2aDwO0kyxS6+xozu8XCCS9/6O7/x8y+CfwW+BPw\nmxqPNR/4vJl9kHD6m6uAYWdlMbN9gEFgErDNzN4FHJ6Hxcsln1SGJiISiVIQIiKRKACLiESiACwi\nEokCsIhIJArAIiKRKACLiESiACwiEsn/BwWy69aLC9KJAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a2a24f0f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# generate more complicated toy data\n",
    "X_train, X_test, y_train, y_test = generate_toy_data(test_size, rand_state)\n",
    "\n",
    "# plot data and decision boundary of logistic regression\n",
    "fig, ax = plt.subplots(1, 1, figsize=(5, 5))\n",
    "# plot class 1\n",
    "ax.scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], c='blue', label='Class 1')\n",
    "# plot class 0\n",
    "ax.scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], c='red', label='Class 0', alpha=0.3)\n",
    "# label axes\n",
    "ax.set_xlabel('Feature 1')\n",
    "ax.set_ylabel('Feature 2')\n",
    "# label title\n",
    "ax.set_title('Training Toy Data Set for Binary Classification')\n",
    "# add legend\n",
    "ax.legend()\n",
    "# add x, y limits\n",
    "ax.set_xlim([-4, 4])\n",
    "ax.set_ylim([-4, 4])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "A simple non-linear boundary is a quadratic decision boundary. It is defined by the equation\n",
    "$$\n",
    "\\mathbf{w}^\\top \\mathbf{x}_{\\text{poly}} = w_0 + w_{11}x_1 + w_{12}x^2_1 + \\ldots + w_{D1}x_D + w_{D2} x^2_D = 0\n",
    "$$\n",
    "where $\\mathbf{x}_{\\text{poly}} = [1, x_1, x^2_1, \\ldots, x_D, x^2_D]$.\n",
    "\n",
    "What the above is saying is that a quadratic equation in $\\mathbf{x}$ is **secretly a linear equation** in the transformed vector $\\mathbf{x}_{\\text{poly}}$.\n",
    "\n",
    "Computing p-powers of the features:\n",
    "$$\n",
    "\\begin{aligned}\n",
    "\\mathbb{R}^D &\\to \\mathbb{R}^{D\\cdot p}\\\\\n",
    "[x_1, \\ldots, x_D] &\\rightarrow [x_1, x^2_1\\ldots, x^p_1, \\ldots, x_D, x^2_D\\ldots, x^p_D]\n",
    "\\end{aligned}\n",
    "$$\n",
    "\n",
    "<img src=\"images/poly_features.jpg\" alt=\"\" style=\"height: 80px;\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create polynomial features\n",
    "quad_features = preprocessing.PolynomialFeatures(degree=2)\n",
    "X_train_quad = quad_features.fit_transform(X_train)\n",
    "X_test_quad = quad_features.fit_transform(X_test)\n",
    "# fit our logistic regression model with non-linear decision boundary\n",
    "logreg.fit(X_train_quad, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsgAAAFgCAYAAACmDI9oAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzsnXt8FfWZ/z9PIEiuBELAmMNFxDtI\n1ESwVaG2FbVWjbjd2ujWesH+Wt3abVG2dHfbbWmxdmt32+4Fe/EWa1XMYi9CbSta1hYSNFGqFakl\ncCBiCOTOJZDv749nhkxOzmXOOTPznZnzvF+vvCbzPXNmvnM5z3zmmef7PKSUgiAIgiAIgiAITJ7u\nDgiCIAiCIAiCnxCBLAiCIAiCIAgWRCALgiAIgiAIggURyIIgCIIgCIJgQQSyIAiCIAiCIFgQgSwI\ngiAIgiAIFkQg24SIxhBRHxFNd3JZQQ/GOXqCiLqI6HcebXMHEdV6sa1sIKJiIlJEdKJL619KRI2W\n+Q8S0TvGb+ZDRPQSES1xYbuPEtE/OL1eIfgQUT0R/TrD7/6JiBY53CXfQ0TPEdEnXVr3ZUT0v26s\n28a2FRHNdnB9GV1bRPQQEX3dqX5kChEtIqKoS+s+h4hedmPdThBagWzcbM2/ISI6aJmvT3d9Sqlj\nSqlipdROJ5dNByL6oWUfjhDRoGX+5w5vax8RDRjrPmCIlluIiGx+fw4RHcqyDx8joteJqJeIOojo\n10RU6dC2LwNwPoATlVKXZtNPyzaV5Xy8S0QPENEYcxml1EylVFO223ICIppuCMa9RNRDRG8Q0Qoi\nOsHtbSulViul6ixN3wCw0vjN/EYpdYlSak022yCiO4loXcx2b1JKfSeb9Qr6MR40P+TkOpVSDUqp\ny2xse5RoUUqdrZTakM72iGhmjL3YQUTL0+y2VpRSVyilHnZp9d8AsMqldbuG5byONdvsXlu5iFLq\nNQBdRPRR3X2JR2gFsnGzLVZKFQPYCeCjlraG2OWtF7RfUUrdZtmnbwFosOyTGxfYpca2ZgH4DwBf\nNaauQ0TnAPgfAJ8GUApgNoAfAXCqss0MAH9RSqUt4pNcK4ct5+c8AFcA+FQWfcy2P4mWPxHAHwEc\nAVCjlCoF8BEA0wFEnO9hSmYA+JOG7QqCbsoMe3E9gH8iog87vYEg3NusGG/ZJiil/qi7L7FYHR5C\namxeew0A7nC7LxmhlAr9H4AdAD4U0/Z1AD8D8FMAvQBuBnAhWDh0AWgHi8F8Y/mxYHE205h/zPj8\nOeP7fwBwcrrLGp9fAWAbgG4A3wPwfwBuTrFPXwfwUJz2jwF4E8ABAM8DOMVo/yqAh2OW/QmArydY\n/z4AC2LaFgEYAjDLmF8C4DVjn9oA3GtZdr9xDPqMv7kAzgLwkvHZe8b2ixNs/2YAG5Ps/1gAXwHw\nV6OvjwIoTbTtmO/+PYDDAI4an99jaX/HWN/TAKYY7cXG+j4N4C8A/hSnP3MAHIpp+08A34l3TAF8\nG8Aj4GuwF0CrtZ/G+dphfPY6gCssn91pnNv/Ns7zl43lZlmWmQGg3zwmMf36LoBNSY6tub8n2jjP\nxQCeNI75AfDvZ4Lx2aeN5XuN43adpf/rjP/fNbY1AKDLaGsG8HHjfwJwF4C3jPW8BuCsZMcIQG3M\n+Y0a7U8DWB5zHSQ737cZn+8H8G3ddkz+jp+3HYix55bPbgew3ThnzwI4yfLZZcZ11G38Nl8EcJvx\n2c0w7I1xzT0AtlHdxjU3B8BSAIPgB8s+AD+P7Q+AMQC+ZFzvvQC2AJgWp58zjWtsrKVtM4BllvmT\nAKwB0AG2c39v+awAwMPGb+5NAPeY17mlT/cafT8MtpfJ1neB8bvrAbAXht0CMB58/+oE3xebAEw1\nPttgOX55YDvUZhy3RzBsB8x9/STYWbUPwIok5/efAfwwpu3DAP5snI/vx5y7rwB4LNGxBTsp3jTO\nxzsA7ohZ9zLw/X4PgFuM7842PnsIwH8B+BXYnn4I7Ex41ThWuwB8xbKunRh577kQMfcyAGeD7fd+\n41h/KcFxeAhs4583+v4igBmWz99nnI9uY/q+RL8R6zFKdT7A19ZD4GvrDeP4WK+t5Ri+vt8AUGf5\n7GawfnnA2L9vGlPrvW0KgIMAKoz5KmP+BN22ZdQ50N0BT3YysUA+AuCj4B93AfjGOh9sTGaBReud\nxvLxRO8+ADUA8sFC57EMlp1iXGjXGJ/9A9gI35xin0YJZADVxrouATAOLCC2go32KeAfdKHlR9AN\n4IwE6x8lkI32/QBuMv7/MIAzjeNXY/ygzBtFPMF4Flhk5wOoBN8QEgn0s8GG/T4AC81+Wz7/MthA\nn2jsy6MAHky07TjrPy7SjPmrwUZyjrG+HwF4zvjMFEzPApgAoCDO+kZsE8A08M34Y/GOKVgg9wO4\n1Dg/3wPwG8uyHzf2bQzYwPcAmGTp+1GwMR9j9PcRAP9k+f4KAD9NsO9bYbkRx/k8ViAnO89fAF/P\n48HX/QVGfyqMa8V8mKoyr7U4x37EtYaRAvlT4JvaPLBwOQNAlc1jtC5mv44LZJvn+ykAJRj+7Vyk\n25bJX2KBbPyW9oHf3pxg/KZeMj6bbJzD64zr9HNgOxtPIC8GC9sy45o7E0Cl8dlDiLFZGCmQl4Ef\n1k43vjsPQHmcvs7ESBG3APyQWGfM5xl9+GewLZ9l/A4WG5+vAgumieC3Pq9htEBuAduhAhvr+wOG\n7Xoxhu3UHQB+DqDQ+J2dj2FHxAbL8bsF/GAyy/j+MwAejdnXB42+zAPb9jMTnN+nMPJBwTx314Pv\nHZ8H2z+7Avkj4N8wge8lAwDOMz67HCxS5wAoAvA4RgvkbgDvN47hePA9bK4xf47x/WvjbTvOtVUC\ntjtfMNZVAmB+guPwEIbv5ycA+HfLeiaB7fBN4Ov5BmO+PN5vBPEFctzzAb62fm9sYxr4fmG9tv4G\n/LCVB+BvwfexSsu+HgU7NcYa6/9PAPdZvv85GA+XlrYeAOfoti2xf6ENsbDJRqXUz5VSQ0qpg0qp\nJqXUJqXUUaXUOwBWg39QiXhaKdWslBoEvyaozmDZqwC0KKXWGp89ADbymXADgKeUUi8ppY4A+Br4\nQj5XKfUXsJfyGmPZawC8pZT6c5rb2AP+4UAp9bxS6k3j+DWDvRMJj5dS6g2l1Aal1KBSyvTQx11e\nKfUnsDA7FWxsO4nof4hovLHIHWBP5rtKqYMA/hUsmDKlHsB/KaW2Guu7B8DlRDTZsszXlVLdxufx\nOMEY9NcNfjKPAvhFkm0+r5T6nVLqGFjgH79+lFJPGPt2TCn1E7BX5lzLd7cppX5sfH4Q7E2yxtbf\naKwzHuVgI22LFOd5ECyGZxm/m81Gf4bAN6SziegEpdTuDK41gL24K5VSrYr5s1Jqt9GvVMcoGXbO\n90qlVK/x29mI5L9vQT/1AH6slHpFKXUYwD8CuJCIZgK4Evzm5xml1FGw7Xk3wXoGwcLlDABkXPt2\nfy+3AfiyUuot43ptVUp1Jll+HxEdBAvU/wRgDkyrBXvY/lUpdcS4Hz2IYRv3MQDfUEodUEpFET/0\n7T+UUruM6zvV+gYBzCaiyUqpPjUc3jAIthezjd/ZFqVUT5xt1YO9zu8opfrAx/7jMa/Yv2rcZ1vB\n96J5CY5JGVgYmlwJ4A2l1NPGPfK7SHzuRqGU+qVS6i/G+XgRwK8BXGx8/DEAPzHsQD9YSMayVin1\nf4b9O2Tcw1435l8Dv4VOphOsXAXgXaXUvxnr6lVKbUqy/C+N+/lhsNPjQiKaBhb9byulHjXs7k/B\nHvZ0Qi0TnY+PgW3ffqXULsRcW0qpp5RSe4z9/xmAt8GOEZM9SqnvGf0y702fICJTb96E0femXvB5\n9xW5LpB3WWeI6Awi+qUxwKoHLLomx/8qgJE/0gHwk3O6y55k7YdSSoGFVSacBH7FZa7rKFjQVhlN\nD4OFE5BcQCWjCuwZBBFdYgze22eIwhuR5HgRUYSIniaiPcbx/e9kyxuG4TpjmQ+CjcsXjDiwKgC/\nNgSp+eovn4gy/ZHFHrtO8JNxlWWZXbFfiuGwUqpMKTUBfH7/DGBtkuUTXj/EmR5et+zfTIw8VrF9\neQFAIRHVElEN+CEm0cjpTrAH3xYpzvNqAC8DaCSiXUT0dSLKM47fJ8Henr1EtJaITrG7TQvTwK/z\n4vUr1TFKhp3znc7vW9BP7DntA1/rVUjDziqlfgd+jf8D8LW7mohKbfYh4fWagMng6+qLGH67BnCI\n1EnmtW1c318CMNX4fMT+IL5tsralWt+tAE4D8GciaiKiq4z2RwGsB/CEYbe/RUT5GM2IY2/8P9ay\nfsD+7+kA+AHFuu7Yc5fKFh+HiK4goj8S0X5jv6/EsJ2IPY5to1YwWifMJ6IXjIHj3eBQMrt2J93r\nw7rffeB770kYfbxhzFfBPrY0Sex2iOjviKjFch3NQZJ7k/EA0A9gIRGdAR5P9GxMX0rAITy+ItcF\nsoqZ/x/w64TZigcu/TPYC+Ym7bAMjDKyRKRzkVvZAzaE5rrMuLPdRtOTAC4morPAryOfSGflxKmM\nJoBjjMz1PQZ+5T3B+N88XrHHFgD+Dfy66izj+H4aNo6v8eT/MvhV3xzFHtd2AJcYgtT8G6+U6kqw\n7VTEHrtJ4Fduuy3L2F6v4Y14BMCHLF5vWxjn5zvgm9YkpVQZ+JWZ9ViN6ItSagh8M7sR/IT+U+MB\nKR6/Ab9qtkvC82x4Qb6slDodwAcAfAL8Cg5KqWcVZwipAh/f76exTZNd4NejI7BxjFKdKzvnWwgW\nsee0COz93I34djbhgFSl1H8opc4Hh3qdBg6dAFJfV3Gv12QYntl/A3AIwGcs6/lrjH0rUUpdaXw+\nYn/AwmvUqmP6lXB9Sqm3lVI3gEP+7gPwNBEVKX7b91Wl1FngmNerAPxdnG2NOPbgAb9HweEH6fIa\n+JibtFv3zzh31v3tB4eAmJxoWfYE8Buvb4Njp8vA8cSmnRixbqPfscSe88fBAm+aYQ//G/btTrrX\nh3W/i8GOjz0YfbwB7rtpvxIeExskPCZENAP85uFOcDhHGVgzJbw3GZjOuZvAb9OPD44nopPAYT9v\npdFHT8h1gRxLCVjA9RPRmfBmZOUvAJxHRB81BO3nwK+sM+EJANcT0fuNp/wV4Iv9VQAwXo390lju\nN0qpDjsrJaIyIqoDC77/Vkq9Y7wuKQZ7aI4Q0UXgwVwm74FDDqxivwT8KqXHeO35+STb/CAR3Wy+\n8iaiueDBjOarv/8GcJ+5fiKaavF6xNt2Kn4K4A4iOpuICsA3ifVKqYzCXQxRfCOAd1T6mTKKwSEK\nHQDyiOizYO9oKh4Bx4R93Pg/EasAzDA8YxGjv9OJ6D9jvbypzjMRfZiIzjSW6wHfFI8R0TQiutI4\nlofABvuYjX2I5YcAvkScL5OMtzxVSH2M9gKYTolHUTt6vgXPySei8Za/sWDh8ikiqjaE0TfAg1F3\ngO3eXCK61lj2s0ggGoy3MPMNG9oPvn7Na3cvOM42ET8E8DUiOtW4Xs8honKb+7QKwD2G7dgMtpP3\nElEBcd72OTScR/1JAP9IRBON38OdKdaddH1EdCMRVRgP2qYn7xgRfYCI5hK/tesBh1zE+x3/FMDn\niehkQ8h9A8DPkjykJ+NXGBmy8EtwqNZ1xrn7e4w8dy0ALjFs2ARweIfJOHD8bgeAo0R0BXiwpsmT\nAG4morOIqBDAv9joXwmA/UqpQ0R0AdgpYNIBYyB7gu/+AsCJRHQ3EZ1ARCVEND/Jtq4koouIaBw4\nZHKTEfbwKwCnEdEniGgsEf0teIyPGdLXAg5xySd+o3i9jf0ysV5bEXA8sUkRWAB3AAARfQrsQU7F\nowDqwPfE2HvTIgC/M8JIfIUI5JF8AfxauBfsTf6Z2xtUSu0Fi5rvgEXIKWBBm/bFopR6FSzqfwS+\ngC8BDx6wGrSHwQMM7IRX/I6I+sCeuS+Af6CfNbY1BPYA/zv4oeIfwIOgzL50gD3GrcSvYuaCB9Z9\nAGxonwYPxkhEF/i4vGH0Ya3R9+8Zn38TnBHjReJwjY0w4k8TbDspSqn/BXsZfgF+9VoOvhbS4QQy\n8pqCH0zOAhuFtFBKbQafw1cxHCLTYuN7b4HP1T6l1CtJlnsXPCioAMArxvFbB/ZuRGOWTXqewZ6G\nn2M4E8dacMz4WPAD2l5wTH01+OEvXR4Cn/M14OvmZ+BBQqmO0a/A3pQOIhr12tSh8y3o41fgke/m\n31eUUr8F8E/ga6UdbEs/DgDGg8/fgNNjdoJ/m82Ib2dLwV6yA+DXy53gawXga+4sw67EK2TxHbDA\n+DX4ev0R+Hdmh18a27zdsNkfBf9u/gr+Df0Q/AYP4PC/qPHZb8C/yYT3DBvruxzAnwzb9e/gQbKH\nwEL0aWNf3gQPDHwsziZ+DL6nvGSs/xBGCivbGLar2xSOlnO3CnwuTsXwW0wopZ4H24XXwAMRf2H5\nrBcsqJ8EH9tPwPJ6Xyn1HDim+XfgQYZ2ikZ9BsC/ElEv+C3zk5b1DQBYCeD/jGtkQcy+9YLH1nwU\nHOLwNviemIjHwaJ9P3iAZL2xnk4YIYfGMbkHwFWWB/x/Al//B8CD9R+3sV8mXwVf938FX8fHtYJS\n6g3wvfUPYNs+F5ZzkQjFcfKvgMX172M+rgc7vHwHcTiP4BeMJ/U9AK5XSsVeSE6s/yxwzOiJGXg2\nBZ9DRE8CeEUpFbgk+4LgFcYbjyiAeqXUC7r7ky1E9P/AotbuYDFfQ0SXAfiMUuraBJ9vAGdl+KGn\nHRMyhoh+DB7A92VL21wAq5VSF+rrWWLEg+wDiOhyIppgvBb8J/Br6s0ubGcMOKzhERHH4YOITgN7\ngh7S3BVB8B1EtJg4XOwE8AA1wnDIVqAgokriULo8Ijod7ElsTPW9oKCU+nUicSwED+KQyuvAb1SO\nozgbiC/FMeADgWzEQr1KRMnSYYWdizBcsOBycFiEo/E4RDQF/JqsFvwKSAgRRPQd8OvFfzFCKAQh\nK0Jomy8EZxDYB37Ffa1KnLLR74wDhwH2gsMC1oLTxAmCryCir4EH8t2vlPqr7v6kg/YQCyL6B3Dx\ngVKl1FWplhcEQRDcR2yzIAi5jFYPsjFC8iPgwQKCIAiCDxDbLAhCrpMoBZJXfBc8+rIk0QJEtBTA\nUgAoKio6/4wzzvCoa4IgCJmzZcuWfUqpTFM26kZssyAIocSubdYmkIlz1r6nlNpCXIAiLkqp1eBq\nXaipqVHNzc0e9VAQBCFz4qWXCwKZ2uZNm5s86qEgCELmjB2TZ8s26wyxeD+Aq4loB7hwxaVEFC+/\noiAIguAdYpsFQch5tHmQlVL/CKPijeGl+KJS6kZd/REcJhoFmpqAjg6gogKorQUiCSu7CoLgE8Q2\nh5z2dqClBdjfCUwqB6qrgcpK3b0SBN+hOwZZ0I0bQjYaBdauBcrKgKlTgb4+nr/mGhHJgiAIdnBD\nyLa3A+vXA6WlwOQKoL+f5xcvFpEsCDH4QiArpTYA2KC5G7mHW0K2qYnXWVrK8+a0qUkEckAZHBxE\nNBrFoUNSXyaW8ePHIxKJID8/X3dXHEdssybcErItLbzOEmPspTltaRGBLAgx+EIgC5pwS8h2dLDg\ntlJcDOzdm/k6Ba1Eo1GUlJRg5syZICLd3fENSil0dnYiGo3i5JNP1t0dISy4JWT3d7LgtlJUBOzr\nyHydghBStFfSEzTS0cHC1UpxMbdnQ0UFe6Ot9PVxuxBIDh06hPLychHHMRARysvLxbMuOMv+Thau\nVoqKuD0bJpWzN9pKfz+3C4IwAhHIuYxbQra2FujqAnp6gKEhnnZ1cbsQWEQcx0eOi+A4bgnZ6mq2\nx729bJt7e3m+ujq79QpCCBGBnMu4JWQjEY5jLizksIrCQhmgJwiCYBe3hGxlJccxFxRwWEVBgQzQ\nE4QESAxyLmMK2aYmFrIVFcDChc4I2UhEBLHgKO+++y7uvvtuNDU14YQTTsDMmTPx3e9+F+PGjcNV\nV12FrVu3Or7Nl156CXfffTdee+01PPHEE7j++usd34YgjMIUsi0tLGQnlQMXXuiMkK2sFEEsCDYQ\ngZzr+FXISh5lwYJSCnV1dfjkJz+JJ554AgDQ0tKCvXv3Ytq0aa5td/r06XjooYfw7W9/27VtCEJc\n/CpkJY+ykCNIiIXgP8z0cwMDnA1jYIDno1HdPRNs0tAAzJwJ5OXxtKEhu/W98MILyM/Px6c//enj\nbdXV1bj44otHLLdjxw5cfPHFOO+883Deeefh5ZdfBgC0t7fjkksuQXV1NebMmYPf//73OHbsGG6+\n+WbMmTMHc+fOxQMPPDBquzNnzsQ555yDvDwxlYJwPP3cwYOcDePgQZ5vb9fdM0FwHPEgC/5D8igH\nmoYGYOlSfq4BgLY2ngeA+vrM1rl161acf/75KZebMmUKnn/+eYwfPx5vv/02brjhBjQ3N+Pxxx/H\n4sWLsWLFChw7dgwDAwNoaWnB7t27j4dmdHV1ZdY5QcgVJI+ykEOIQBb8h+RRDjQrVgyLY5OBAWDF\nlxTqr+wCjh4Fxo7ltFXjxjm67cHBQdx5551oaWnBmDFjsG3bNgBAbW0tbrnlFgwODuLaa69FdXU1\nZs2ahXfeeQd33XUXPvKRj+Cyyy5ztC+CEDokj3I4kbCZuMh7Q8F/SB7lQLNzZ4L2XeAR+fn5PO3q\nAo4csbXOs88+G1u2bEm53AMPPICpU6eitbUVzc3NOGKs/5JLLsFLL72Eqqoq3HTTTXjkkUcwceJE\ntLa2YtGiRfjBD36A2267ze4uCkJuInmUw4eEzSREBLLgPySPcqCZPj1Be9UQMGYMz4wZw3+xN9sE\nXHrppTh8+DAefPDB421NTU148cUXRyzX3d2NyspK5OXl4dFHH8WxY8cAAG1tbZgyZQpuv/123Hrr\nrXjllVewb98+DA0NYcmSJfja176GV155Jf2dFYRcQvIohw9r2ExeHk9LS7k9xxGBLPgPyaMcaFau\n5FNmpbBAYeU/x1SbGzOGwy1sQERobGzE888/j1NOOQVnn302vvKVr+Ckk04asdxnPvMZPPzww1iw\nYAG2bduGIqMa2YYNG1BdXY1zzz0Xa9aswec+9zns3r0bixYtQnV1NW6++WZ885vfHLXdpqYmRCIR\nPPXUU7jjjjtw9tln2z8QghA2JI9y+HCramMIIKWU7j7YpqamRjU3N+vuhiDkHG+++SbOPPNM28s3\nNHAs8s6d7FFe+aU+1C85POxBBoBjx9hjMXGiCz32lnjHh4i2KKVqNHXJU2pqatSmzU26uyEIQro8\n9xyHVZgDLgF+M1BQAFxxhb5+ucjYMXm2bLN4kAVBcJz6emDHDn4Lu2MHUH/zOBbERsjD8f9jPReC\nIAiCd0jYTEJEIAuC4D7jxnHqvrw8YHCQp2VljmexEARBENJAwmYSImneBEYq1wluM26cCGJBSBdJ\nwSW4jV+rNmpGBLLTBFFompXryso4/3BfH8/LwDhBEMJCEIWmmYKrtJRTcPX387x4+ATBdSTEwkmC\nWiLZWrkuL4+nZWXcLgiCEHSCmutVUnAJgjZEIDtJUIVmRwdXqrNSXMztgiAIQSeoQlNScAmCNkQg\nO0lQhaZUrhMCwLvvvouPf/zjOOWUU3DWWWfhyiuvxLZt27Bjxw7MmTPHlW0ePnwYf/u3f4vZs2dj\n/vz52LFjhyvbEVwmqEJTKtcJgjZEIDtJUIWmVK4TfI5SCnV1dVi0aBH+8pe/4I033sA3vvEN7N27\n19Xt/uhHP8LEiROxfft2fP7zn8e9997r6vYElwiq0JQUXIKgDRHIThJUoSmV6wSniUaBxkZg9Wqe\nZhmH/8ILLyA/Px+f/vSnj7dVV1fj4osvHrHcjh07cPHFF+O8887Deeedh5dffhkA0N7ejksuuQTV\n1dWYM2cOfv/73+PYsWO4+eabMWfOHMydOxcPPPDAqO2uXbsWn/zkJwEA119/PX77298iSMWVBIOg\nCk1JwSUI2pAsFk5iCs2mJhaaFRXAwoXBEJqRSDD6KfgfF7KibN26Feeff37K5aZMmYLnn38e48eP\nx9tvv40bbrgBzc3NePzxx7F48WKsWLECx44dw8DAAFpaWrB7925s3boVANDV1TVqfbt378a0adMA\nAGPHjsWECRPQ2dmJyZMnZ7QfgiZModnSwkJzUjlw4YXBEJqSgksQtCAC2WlEaAq5jnWwKjA8bWpy\n/bcxODiIO++8Ey0tLRgzZgy2bdsGAKitrcUtt9yCwcFBXHvttaiursasWbPwzjvv4K677sJHPvIR\nXHbZZaPWF89bTESu7oPgEiI0BUFIAwmxEATBWVwYrHr22Wdjy5YtKZd74IEHMHXqVLS2tqK5uRlH\njhwBAFxyySV46aWXUFVVhZtuugmPPPIIJk6ciNbWVixatAg/+MEPcNttt41aXyQSwa5duwAAR48e\nRXd3NyZNmpTxfgiCIAjBQASyIAjO4sJg1UsvvRSHDx/Ggw8+eLytqakJL7744ojluru7UVlZiby8\nPDz66KM4duwYAKCtrQ1TpkzB7bffjltvvRWvvPIK9u3bh6GhISxZsgRf+9rX8Morr4za7tVXX42H\nH34YAPD000/j0ksvFQ+yIAhCDiAhFoIgOEttLcccA+w57uvjwaoLF2a8SiJCY2Mj7r77bqxatQrj\nx4/HzJkz8d3vfnfEcp/5zGewZMkSPPXUU/jABz6AIiO114YNG3D//fcjPz8fxcXFeOSRR7B79258\n6lOfwtDQEADgm9/85qjt3nrrrbjpppswe/ZsTJo0CU888UTG+yAIgiAEBwrSiOyamhrV3NysuxuC\nkHO8+eabOPPMM+1/IYgl17Mg3vEhoi1KqRpNXfKUmpoatWmzzwsiCYIgABg7Js+WbdbmQSai8QBe\nAnCC0Y+nlVL/oqs/giA4iAxWDSximwVBEPSGWBwGcKlSqo+I8gFsJKLnlFJ/1NincJJj3jxBELJC\nbLNXtLdz6rn9nZx6rrpaMm0Igk/QNkhPMeZInnzjLzjxHkHBzEk7MMA5aQcGeD7Lwg1hpaEBmDkT\nyMvjaUOD7h75hyCFY3lJ2I6L2GaPaG8H1q8HDh4EJlfwdP16bhdG0fgMMH8++3bmz+d5QXATrVks\niGgMEbUAeA/A80qpTXGWWUpB+yDyAAAgAElEQVREzUTU3JFFmqicxZqTNi+Pp2Vl3C6MoKEBWLoU\naGsDlOLp0qUikgFg/Pjx6OzsDJ0YzBalFDo7OzF+/HjdXXEUsc0e0NLC9rikhG1zSQnPt7To7pnv\naHwGWHYP+3WU4umye0QkC+7ii0F6RFQGoBHAXUqprYmWk0F6GbB6NXuO8yzPQkNDXOlv6VJ9/fIh\nM2eyKI5lxgxgxw6ve+MvBgcHEY1GcejQId1d8R3jx49HJBJBfn7+iPYwDNJLxzbLIL00aXiMPcex\ntnlfB1B/o75++ZD58+O/9IxEgE2jHt0EITm+H6RnRSnVRUQbAFwOIKERFgzSiSk2c9Ka1cyArHPS\nhpWdO9NrzyXy8/Nx8skn6+6G4DFim9MknZjiSeVAfz97jk36+7ldGMHu3em1C4ITaAuxIKIKwzsB\nIioA8CEAf9bVn8CQbkxxbS3noO3pYe9ETw/P19Z62+8AMH16eu2CEEbENmdIujHF1dVsj3t72Tb3\n9vJ8dbW3/Q4AVVXptQuCE+iMQa4E8AIRvQagCRzn9guN/QkG6cYURyLANdcAhYUcVlFYyPOSxWIU\nK1fy4bFSWMjtgpBDiG3OhHRjiisrgcWLgYICDqsoKOB5yWIxiuX38uGxUlDA7YLgFtpCLJRSrwE4\nV9f2A0tHB3uOrRQXs/hNhOSktUV9PU9XrOCwiunTWRyb7YKQC4htzpD9new5tlJUxOI3EZWVIoht\nUHcdT1fdx2EVVVUsjs12QXADX8QgC2kgMcWuUl8vglgQhAyQmGJXqbtOBLHgLSKQg0ZtLcccA+w5\n7uvjmOKFC/X2SxAEIZepruaYY4A9x/39HFN84YV6++URPYcGAQDPv7bHtW18+JyTjv9fOj4/yZKC\nkD0ikIOGGVPc1MRhFRUVLI4lhEIQBEEfZkxxSwuHVUwqZ3EcshCKwWNDeHbLrlHtY40RTUXj8nDu\n9ImOb/fVnQfwwlYW30eHRn9+9fnTAAD5Y7SWdxBChAjkICIxxYIgCP4jhDHFazaPTA6fZ/y9b7a3\noSPJRPcf3+nEL7bsQqxuXnLBDHc7JYQaEciCIOgjnZzegiC4zut7erAtemBE20Uei+F0WTBrdP82\nbu8cIe5Pi0zE3JNKRy0nJCCdnN4hRd5FCL6koYEr2+Xl8VTKPYeQdHN6C4LgCj2HBrFmcxvWbG7D\ntugBTC0eh4tmlx//s9LYyJXtIhGeNjZq6nQKrP2fWjwO26IHju/j63t6dHfP36Sb0zukiAdZ8B0N\nDVwFe2CA59vahqtiS4aJEGHN6Q0MT5uaxIssCC5j9a7mwZ6XuLERWLaM9RLAz7LLlvH/dXUudNIh\nTj2xBKda5v/4TucIL7mEYsRgzekNDE9bWnLKiywCWfAdK1YMi2OTgQFuF4EcIjLJ6S0IQlZYhXG6\noROrVg2LY5ODB7ndzwI5FmtIxsuWUAwRygaZ5PQOISKQBd+xc2d67UJAkZzeguAJ1rjisXnxY3bt\nsHt3eu1BwBxs+Pa7vceFcs7HK0tObwAikAUfMn06h1XEaxdChOT0FgRXsXqLT59ShIrS8Vmtr6oq\n/hCBqqqsVusLrGEYG7cPh2Bcff603Esdl+M5vU1y7KwLQWDlSqCwcGRbYSG3CyHCzOldWMhhFYWF\nPC/xx4KQFeZgtLF5w4PVshXHALB8OVBQMLKtoIDbw4R5zMbmAc9u2TUq1V3oMXN6FxRwWEVBAc/n\nUPwxIB5kwYeYccYrVnBYxfTpLI4l/jiESE5vQXAMU8i5lafYjDNetYrDKqqqWBwHKf44HcxQFDNO\nuXxCIRadniMhYCHM6Z0upJTS3Qfb1NTUqObmZt3dEATBCwKeI5mItiilanT3wwtqamrUps1NuruR\ns5jCOJv4YiE1G7d3Hv8/pwf0BTxH8tgxebZss4RYCILgPyRHsiCkxAylADgsQMSxu1jzQudsPuUc\nypEsIRaCIPgPyZEsCAnZ8FYHOrsHtJR8FobT45mD+XLKm5xDOZJFIAuC4D8kR7IgjGLw2BCe3bIL\ngP/LP+cCF80ux6s7D+RWHuUcypEsAlkQBP8hOZIFYQSmCJtaPA6nnliSYmnBK86dPhEAe5PXbG7D\n/NOmIFJWkOJbASaHciRLDLIgCP6jtpZzIvf0AENDPO3q4nZByCF6Dg2OiDMWcexPzPjkTdveC3da\nuOpqtse9vWybe3t5vrpad88cRwSyIAj+Q3IkCwLWbG7D86/tOZ7PWPA/Zv7k0IrkHMqRLCEWgiD4\nE8mRLOQoEmscbBbMCnlsco7kSBYPsiAIgiD4hGjXQTy7ZRemFo8TcRxgzp0+cURKOCF4iEAWBEEQ\nBB+wZnMbNm17D6dPKZJY45CQ83mTA4wIZEHwMQ0NwMyZQF4eTxsa4rcJghBs1mxuQx5YUFWUjtfd\nHSEFjY3A/PkcBTZ/Ps8najcH8G2LHhCRHCAkBlkQfEpDA7B0KReRA4C2NuBTnwKIgCNHhtuWLuX/\n6+v19FMQhMyRoh/Bo7ERWLaMi8gBXOBz2TKuY/Tkk6PbAaCujoVyThYXCSjiQRYcRbybzrFixbA4\nNhkcHBbHJgMDvKwgCMFizeY2dHYP4PQpRa6L40QeTyF9Vq0aFsEmBw8Cjz0Wv33VquF5iUsODiKQ\nBccwPZ5tbYBSw95NEcmZsXOnO8sKgqAfa25jt0MqTI9nNMq22fRsikjOjN2747cfO2ZveRHJwUAE\nsuAY8Tye4t3MnOnT3VlWEAS9WMWxFyTyeFo9m4J9qqrit48ZY395Ecn+R5tAJqJpRPQCEb1JRH8i\nos/p6ovgDIm8mOLdzIyVK7k+hpX8fGDcuJFthYW8rCA4gdhmd/FaHAOJPZ6J2oXkLF/O9TGsFBQA\nN94Yv3358vjrsYrknkODLvRUyAadHuSjAL6glDoTwAIAnyWiszT2R8iSRF5M8W5mRn09sHo1MGMG\nD8ybMQP4yU+AH/94ZNvq1TJAT3AUsc0uoUMcA4k9nonaheTU1QH338/x3EQ8vf9+4BvfiN9eV5d4\nXWaGi+df24No18HECwqeQ0op3X0AABDRWgDfV0o9n2iZmpoa1dzc7GGvhHSIzboAsHczLQEXjfJQ\n4I4OoKICqK2VampCICGiLUqpGt39yBa7tnnT5iYPexU8dIljYHTWBYA9m6nE2wja24HWVqCzEygv\nB+bNy4lqal6ycXunZLfwgLFj8mzZZl/EIBPRTADnAtgU57OlRNRMRM0dHR1ed01Ig3gez7TF8dq1\nrLCnTuXp2rXcLgiC54htdgZrjmMdJPJ4piWO169nhT15Mk/Xr+d2wVEkJtk/aPcgE1ExgBcBrFRK\nPZNsWfEgh5zGRhbFpaXDbT097Ia2bckFwR8E3YOcrm0WD3J8THEc6BzH69axKC4uHm7r62M39OWX\n6+tXCHl5eyeGAPEku0ggPMhElA9gDYCGVAZYyAE6OkYaYIDnxTslCJ4ittkZ1mxuw9i8gItjgMMq\nYkcMFxZyu+Ao75tdjjyIJ9kP6MxiQQB+BOBNpdR3dPVD8BEVFeyVsNLXx+2CIHiC2GZnMMXxglkB\nF8cAxxzHy+FZHoJ98yHvm12OsXkiknWj04P8fgA3AbiUiFqMvys19icQhLpSXW0t0NXFYRVDQzzt\n6uJ2QRC8QmxzBjQ+M1yp7oM3t+FPW0MijgEekNfdzQ6LoSGedndzu+AK5rUjIlkfY3VtWCm1EQDp\n2n4Qic0SYVaqA0KS5isSAa65hrNY7N3LnuOFCyWLhSB4iNjm9Gl8Blh2D4fpnnRBG3q6gfu/WI4p\nKiTDJyorgcWLOYvFvn3sOV6wQLJYuMxFs8uxcXsn1mxuk5hkDWgfpJcOuT5Ib+ZMFsWxzJgB7Njh\ndW8EQUhG0AfppUOuD9KbP5+T7Zx0ARvoV9ex9y8SATaNyv8hCOmxcXsnyicUYtHpEm7oBIEYpBcI\nolHOrrB6NU81phyTSnW5S6hDawQhE9rbgeeeAxoe46nGlGO7dwMzFvBgYlMcm+1CuGlsHA6tmT+f\n553motnl6OweSL2g4CgikJPhs7y8UqkuNzFDa9raAKWGQ2tEJAs5y4i8vBXa8/JWVQGDQwMjxLHZ\nLoQXswBLNMq2ORrleTdEsgza8x4RyMloagLKyjgvb14eT8vKuF0DK1fGz7SzcqWW7ggesWJF/AHk\nK1bo6Y8gaKelhe1xSQnb5pISnm9p0dKdj9zRhuhfR95OCwqA5cu1dEfwiFWrRlYnBHh+1Srnt7Vg\nlqR/8xoRyMnwWV7erCvVCYFEQmsEIYb9nUBR0ci2oiJu95g1m9tw9hzgX+6amHmlOiGQJAqhcSu0\nxsynveEtqQ3gBdqyWAQCMy+vtbKb5ry89fUiiHON6dPjD86U0BohZ5lUDvT3s+fYpL+f2z3E9OZd\nNLscmC2CONeoqoofcelmaI2Z2UJwH/EgJ0Py8go+QEJrBCGG6mq2x729bJt7e3m+utqzLgweGwJg\niGMhJ1m+nENprHgRWiPxyN4gAjkZZl7ewkLOy1tYyPOSl1fwEAmtEYQYzLy8BQXAvg6eLl7saV7e\nZ7fsQtE4uYXmMnV1HErjdWiNFBHxBsmDLAiC4AKSBzm8jAitEARNbNzeifmnTUGkrCD1wsJxJA+y\nIAiCIDiMOUBKxLGgm6Jxedi07T3d3QgtIpCFnEOKbvgUHxXlEYREdHYPiDh2CS+KboSJc6dPBOBB\nVgsfFeXxEsliIeQUZtENM6+wWXQDkJherZhFecrKuChPXx/PX3MNf97UxOkVKyp4kKyMAxA0sGZz\nm3iVXMIsumHmFTaLbgCSHSQZZlaLwWNDyB/jwtVpFuUpLeWiPP39PL94MX/e0sLpFSeV8yBZD8cB\nuI3EIAs5xcyZ8VOmzZgB7NjhdW+8oaGBi4rs3Mmp4Vau1PwwEI2OFrxNTfzUYk2p2NPDd8vDh4Fj\nx4A9e/g7Y8YAt9/u+2wyEoMcLja81SHeYxeZPz/+S6NIBNi0yfv+eEFjIxcV2b2bU8MtX57Zw8DL\n2zsxBGDJBTOy61B7+2jB29LCdtiaUrG3Fzh0CDhyhG3z3r1AZyfb5vpPAPO8yyaTCRKD7BfktbGv\nyLWiG74rU52ofPu2bfGL8jQ3swF+4w1gcJC9E2PHAg8+KL8lITvSfG0s4thdvC66oRsny1SbBUSi\nXQdTLJmEROXb33knflGe11rZNr+9DTg6CEydwgK54fHQhGCIQHaTRGIgx27sfor5TVRcI6xFN3xX\npjpR+fb9+zmswkpfH+dO2rOHDXJhIc9PnAgcPaqt5LsQAhKJgQQ39rCm0/JTzG+i4hpuFt3QidNl\nqi+aXZ7dgL1E5du7ujiswkp/P9tiM/1tQQHPl5WxbdZU8t1pRCC7SSIxkEM3dr95MHOt6IbvPOaJ\nyrdPnBi/KM955/F3rNn4Dx3iB05NJd+FEJBIDMS5sYe1IIiTHkwn0FV0QxduecwzHrCXqHz7hAnx\ni/LMmcthFePHDy9/+BCHzWko+e4GIpDdJJEY0Hxj99Kj6zcPpt+Kbrh9LnznMTfLt1vp6wNOPz1+\nUZ4rruDXdvv3s3EeGGDvRWWl1pLvQsBJJAbi3Nif3bILYz26U3rp0XXag5ktuopuJMLtc+GGx/yi\n2eXo7B5IvWA8zPLtVvr7gVNOiV+U59JL2TYf6AKGFF88AwPAlCmel3x3C8li4SamGLAOPOrr03pj\n9zqLg+88mOD99EPGCi/OxcqVI7cBaPaY19ZymBHAD4t9fewpXriQ70TxslPcfjvHHLe3s+d4xgw2\nzD4fpCf4GFMMWAce9fePurG/vqcHwHDlMjfxOouDH2N+6+r8kbHCi3OxfPnIbQDOeczXbG5Lf8Be\ndTWHGQH8sNjfz57iCy9kh0S87BT1n+CY4717WdecGmHb7GHJdzeRLBZuYk1dZRUDGstVe53FIRez\nRtjFq2MTiCwWqX4PmXxHM5LFwsdYU1dZxUBMueo1m9s8C63wOotDLmaNsItXx8apLBaxbNzemVlG\ni3hZLFKlbcvkO5qxm8VCBLLb+OzGnpfH8WaxEPEbbKeJ9ZIC7MHUGdbgF7w+F4K3iED2OSlu7F6X\nk45EEtsDN8Z1x3pJAfZg6gxr8Atenwun2bidQ4WyTvsWUiTNm1+IRNjaLF3KU0Mc68rs4HVMqt9i\nfv2E7+KDHcJPWUsEISGVlRzjXn8jTw1x3PgMexDvWwV8ob7cs0FrXmdx8FvMr58IekaNRA915rV9\nPK76GY87FjBEIGtAZ2YHHVkc6us5ZGBoiKcijpkwZtTwW9YSQUiHxmeAZfcAQye1eZ7ZQUcWh7o6\nDhmIRnkq4pgJS0YNa0YL89oekbXkHhHJyRCBrAGdmR3Eo+sfwngu/Ja1RBDSYdV9wyEHr65jL5xX\nmR3Eo+sfwnAuYjNaWK9tk4MHuV2Ij8Qga0BiT4WwItf2MBKDHDwiEaCylmOPTYEMBCf2VBCsvLy9\nE7MjEzH3pNLAx1U7icQg+5iwxp4KglzbQpAxY0yt4tjaLghB4n2zy7EtegBA8OOqdSACWQNhjD1N\nFxnIFU7k2haCzA2f78DY/JFtQYw9zQY/lZ8WnCHadRDL700QV32vnj4FARHIGghj7Gk6yECu8JLr\n17YQbKbNGsAnF5cHOvY0G/xWflrInqJxedi07T3UXQfc/62YuOpvAXXX6e6hf9Eag0xEPwZwFYD3\nlFJzUi0flhjkXEeKhwi5QJBjkDOxzUGPQR48NoRnt+zyLO+xH5HiIeEk48IhIcVuDLLuUtMPAfg+\ngEc090PwED+Wn84JfFa0xjHCul96eQg5Zpuf3bILY3P8naofy0/nBO3tQGsr0NkJlJcD8+Y5Xo1u\nw1sdWHR6haPrTEkAq+xZ0WoOlFIvAdivsw+C98hALg2YZc8HBoCpU3m6dm3why+Hdb80k2u2OdrF\n+a8WzMpd7zEgA7m0YJY9P3gQmDyZp+vXc7tDxKZ884QR+1Xhyn65TY4/Lws6kIFcGmhqAsrKgNJS\nHhlZWsrzTcF+LR7a/RI8ZdO29zC1eJzubmgnLAUyAkVrKzBhAlBczDasuJjnW1sd39TgMQ9zbba0\nsD0uKeH9Kinh+ZYW7/qQJb4XyES0lIiaiai5o6Mj9RcE3yMDuTTQ0cGG10pxMbcHmbDuVwAIm20+\n9cQS3V3QThgKZASOzs74HqPOTkc3kwcOI/KM/Z1AUdHItqIibg8IumOQU6KUWg1gNcADQTR3R3CI\n+noRxJ5SUQH09fETvElfH7cHmbDuVwAIi222luMVWAyLIPaQ8nIODbM+6A8McLuDvG92OTZu91Cc\nTioH+vvZc2zS38/tAcH3HmRBEBygthbo6gJ6erikXU8Pz9fW6u5ZdoR1vwTP6OweyOnMFYJm5s0D\nurv5wX5oiKfd3dzuAj2HBl1Z7yiqq9ke9/byfvX28nx1tTfbdwCtApmIfgrgDwBOJ6IoEd2qsz+C\n4BjRKCcPXb2ap7oHjUUiwDXX8Ku7vXt5es01wc/2ENb90ozYZiG0tLcD69Zx4v116/QPGqusBBYv\n5mDvfft4unixK9keisbl4fnX9ji+3riM2K8OV/fLLbSGWCilbtC5fUFwBTOzQlkZZ1bo6+N53cIt\nEgmncAzrfmkkV2zz63t65DVqLmFmVpgwgTNGDAzwvG7hVlnpyfbPnT7R2zALj/bLLcQ2CILTSGYF\nQQgE26IH8D4Jr8gdPMwYIQQfEciCkISGBq78l5fHU1vlsCWzgiAIgqs0NnLlv0iEp7bKYXuUMcLv\nyMBUe/g+i4Ug6KKhAVi6lN/CAVwee+lS/j9pBg7JrCAIvuf1PT26uyBkSGMjsGwZ154AOKpt2TL+\nP2kGDo8yRviZ06cU4a33+nV3IxCIB1kQErBixbA4NhkY4PakSGYFQfA926IHpDhIQFm1algcmxw8\nyO1J8ThjhB+pKB2vuwuBQQSyICRg58702o8jmRUEIRBIcZBgsnt3eu3H8TBjhN/xtKpeQJEQC0FI\nwPTpHFYRrz0lkllBEATBFaqq4mfOrKqy8eWAZ1ZwArOq3pILZujuiq8RD7IgJGDlyvjjOVau1NMf\nQRCcYc3mNoyVu19gWb6cnb9WCgq4XUiNZG6xh5gIQUhAfT3X+ZgxAyDi6erVUiJbEMLAglkiEoJK\nXR1w//38ko6Ip/ffLyWyBWeREAtBSEJ9vQhiQRAEv1FXJ4JYcJfgeZD9VsJX8CUZ5S8WBCFz2tuB\n554DGh7jqe4SvgmQwUl6ySh/seAK0a6DqRfKYRIKZCKaS0R/JKJdRLSaiCZaPtvsTfdiOHKES/YO\nDHAJ34EBnheRLFgw8xe3tQFKDecvdlIkiwAXdOFL2zw4yCV7Dx4EJlfwdP16X4rkZ7fskvhjTZj5\ni6NRts1m/mInRbIIcHsUjcvDpm3v6e6Gr0lmJv4LwFcAzAWwDcBGIjrF+Czf5X7FZ2BASvgKKck4\nf7FNvBDggpAEf9rm0lKgpIRtc0kJz7e0aOlOKiT+WA8Z5y+2iRcCPCycO31i6oVynGQCuVgptU4p\n1aWU+jaAOwGsI6IFAJQ33YthcFBK+AopyTh/sU3cFuCCkAL/2eajR4GiopFtRUXA/twq4SskJ+P8\nxTZxW4ALuUWyQXpERBOUUt0AoJR6gYiWAFgDYJInvYslP19K+AopySp/sQ3cFuCCS0Sj/Lapo4Nt\nRm1tUHNV+882jx0L9Pez59ikvx+YJJ5aYZis8hfbwG0BLrhEezu/bdrfyTajutoXuaqTeZDvA3Cm\ntUEp9RqADwJ4xs1OJaSwUEr4CilJlb842/jhRELbKQEuuEA0GqbxC/60zT09QG8v2+beXp6vrtbS\nnUS8vqdHdxdymlT5i7ONH04ktJ0S4IILtLf7dvxCQoGslHpcKfXHOO07lVK3u9utBIwbJyV8hZQk\ny1/sRPywXwuIyMDBJDQ1hWb8gi9tc36+pYRvh29L+G6LHsDU4nG6u5GzJMtf7ET8sF8LiPh54OCG\ntzSHqLa0+Hb8AimlJ2QtE2pqalRzc7PuboygoYFjT3fuZA/iypWSN9fPzJwZP/xixgxgxw776/Hb\neTeFvzU2urBQCpscZ/Vq9hznWXwCQ0P8oL10qSubJKItSqkaV1buM2pqatSmzf562Gh8Blh1H79e\nr6oClt8LDEXacJFUEfMl8+fHf6ETiQCbNtlfT2MjxxwfP+/L9eZLNoW/NTa6oMAfhU06eg5h32GF\nRadrDFNteIw9x7G2eV8HUH+jK5scOybPlm0WgZwFIkqCR14eeydiIeLfZFBxSviHlsbG4UwLJj09\n/IN16S4lAlkfjc8Ay+4ZLUr+7kttuONaEch+JBJJbJuDGQnFOCX83cAXAvm55/iHah2/0NvLP9gr\nrnBlk3YFsmSDzALJZhA8who/LAMHU1BbK+MXcohV98XPZvDiS3r6I6QmrPHDfh44WFE6Hp3dA6kX\ndJPqat+OX0gpkInoNCL6LRFtNebPIaIvu981/yOiJHj4NX44W8Iq/B0jEgnd+AWxzYlJJD56ZYye\nb/Fr/HC2hFX4O0ZlpW/HL9jxID8I4B8BDALHR0t/3M1OBQURJcEj2QC+IBNW4e8okQiHUyxdytMA\ni2MDsc0JSCQ+Skrjtwv6STaAL8iEVfg7SmUlh1PU38hTH4hjwJ5ALlRKxZYvPepGZ4KGiJJgUl/P\ncblDQzwNujgGwiv8haSIbU7A8ntHi5Lp72/D+2slqtDP1NVxXG40ytOgi2MgvMI/F0hWKMRkn1HG\nVAEAEV0PQH+COh9gig8/ZTMQcpf6ern2cgyxzQmou46n1iwWV1wO3HGtlNcVvKeuzt+CePDYEPLH\nyMNjLHYE8mcBrAZwBhHtBvBXAP68DWuolCWiRBAETQTHNmuolFV33bBQBoA1sb52QRAkU0MSkh4b\nIsoDUKOU+hCACgBnKKUuUkrFSSilmUwrZUWjnAJq9WqeBjmfjCD4AClY4j6Bss3ZVMpqb+c0UA2P\n8dQH1bUEIaj4uWCJH0nqQVZKDRHRnQCeVEr1e9SnzLBWygKGp01Nib3IpqguK2NR3dfH8wEf3R44\nNHj+A0WAjk9sbnCzUiEgb1qcJFC22VopCxietrQk9yKbwrq0lIV1fz/P+2SEe07Q3g60tgKdnUB5\nOTBvnhx7KwE6PrEFS8xKhZ+/D7jqfL198yt2vOvPE9EXiWgaEU0y/1zvWbp0dADFxSPbiou5PREh\nKj8bWDL1/OcKATs+khvcU4Jhm/d3AkVFI9uKirg9GT4uQZsTjPD8T07P858LBOz4rFoVPzf4r3+t\npz9BwE4M8i3G9LOWNgVglvPdyYKKCvYAWytl9fVxeyI6Olh0WCku5jypgjdk4vnPJQJ2fAKfGzxA\n3noExTZPKmfvr7VSVn8/tydjfyd7jq0UFXGuVMF9WluBCROGHU/mtLXVt15STwnY8UmUG7y7y9t+\nZIyGcQwpPchKqZPj/DligInociJ6i4i2E1F2WQEzqZRlimorqUS14CyZeP5ziYAdn0DnBg+Ytz4w\ntjnTSlmmsLZiR1gLztDZGT+PaWcKz3+uELDjkyg3+IQyb/uREdmMY8gCO5X0/i7eX7YbJqIxAH4A\n4AoAZwG4gYjOyniFmVTKkvKz+pGHlOQE7PgEOjd4wEKuAmObM62U5WAJ2sFjQxl2PocpL48fL1Uu\nDygAAnd8EhUsuewyPf1JC03hVnZikGstfxcD+AqAqx3Y9gUAtiul3lFKHQHwBIBrslpjupWyQlh+\nNnDIQ0pykhwfP2aLCHTBkoB56xEk25xJpSyHS9BKOqs0mTcP6O7mB/KhIZ52d3O7kPT4+DFbRKKC\nJecG4XRmOo4hS1LGICul7rLOE9EEAI86sO0qALss81EA82MXIqKlAJYCwHQ33tNGIiKIdWI+pDQ1\n8UNKRQWwcKGcE5MEx6fhxYhvs0UENjd4JuMYNBJ62wywGPZhPGdOYD6gtLYC+/axZ3TBAjkfJgmO\nT+MfK+NmiwD0FwuJVzjOD5IAACAASURBVLDk5e16+pIWmY5jyBI7g/RiGQBwqgPbpjhtalSDUqvB\nyfBRU1Mz6nPBvzQ02KwyKA8pyYlzfJJliwikOPUDtbUccwyw57ivj731Cxfq7Zd9xDYLtmhs5KwG\nZpXB5csTiDd5QElOnOOTKFvEqlX6BXJgqa7mmGOAPcf9/fw29cILXd1sSoFMRD/HsHHMA8ekPeXA\ntqMAplnmIwD2OLBewQdIPlx3CXy2CD8SsLcZYpuFTEiUDxcQAecEibJFJGoXbGB661taONxqUjmL\nY5cf3ux4kL9t+f8ogDallBPDupsAnEpEJwPYDeDjAD7hwHoFHyAeTodIkHZs+nR+6IglENki/Eyw\n3maIbRbSRjycDpGgSEhVVfzEN4mySAg20fA2w864hSuVUi8af/+nlIoS0X3ZblgpdRTAnQDWA3gT\nXBHqT9muV/AH4uF0oORykrRjgc4WEWYspesnAW4nUBLbLKSNeDgdKLmcpEhIomwRy7NLlihki6Vs\nvV3bbEcgfzhO2xXp9Sw+SqlfKaVOU0qdopSSW3uICHQ+XBOL2EFjY1r5cM0Qk7Y2QKnhEJNYkZxU\nRCdJOxbobBFhJeaBJs/9xAlim4W0SeTJDJSHs70dWLeODea6dWnlwzVDTKJRts1miEmsSE4qoq1F\nQvLyeDphAtDamjBbhHjnNRKTR9mubU64EBH9PyJ6HcDpRPSa5e+vAF5zqt9COAm8hzOO93bzl9di\nflXUlkfYTsnllCI6Rdqx+npgxw7OMLRjh4hj7cQ80AwBx9zYjNhmIRsC7+FM4L197sfttrzCyUJM\nTFKK6BRFQurqgE2b+HubNok41k5MHmW7tjmZin4cwEcBPGtMzb/zlVI3Zt1hIdQE3sMZI3Y2vFKK\n/3q8DJV7mpJ6hE3shJikFNEBKxJiFz/mb3aEeA807iC2WciYwHs443hvm7ZNwLNfa03pFQbshZik\nFNEBKxKSjCEA+WNYCjY+E+M1f0Zv3xwjXh5lGyQUyEqpbqXUDqXUDUqpNgAHwSOmi4koSC/KhTRw\nUrwE2sMZI3YefRToHCzGFAwXjYj1CFuxE2KSUkSHsIiK3dCTQBLvgcYFxDbnJk4Wnwi0hzOO9/Zn\nPy9EyZGRRSNivcImdkJMUoroEBZRaXwGWHZPjNf8npCI5Hhl621gp9T0R4nobQB/BfAigB0Ankt7\nS4LviSdebrqJvQyh8vTZIUbsdHQAJejDexjpvU0kcu2EmKQU0SGs9Ggn9CSwxDzQ5AFj3Nyc2Ob0\nCHKx6Xiv/O+6i0WdXyq1eUYc7+2h/QPYh9He23hC106ISUoRPaLK476sqzz6gVX3JfCaZz3s1wfE\nlK23a5vtBCp/HcACANuUUicD+CCA/8uiq4JPiSdelJFlNVSePjvEiJ2Ty3swAV1owkjvbSKRayfE\nxFacdrrl031OqLObxDzQDLmvycQ228R8hRxU4r3yN21zsnCCUBLHezu9rButGO29jSd07YSY2IrT\nrqwELr+cjfrllwdaHAMhz24SU7berm22YzUGlVKdAPKIKE8p9QKA6qw6K/iSVCIlNJ4+O8SInetu\nLMSvx1+DPRgWqKkGHaYKMQl8nHYGhCK7STIsDzT7gS6Xtya2OUdIJVIShROEkjje27lfWIzugpEC\nNdnAw1QhJoGP07ZJR8+h4/+HIrtJMiorgSuuAOpvtG2b7QjkLiIqBvB7AA1E9O/gpPRCyLAjUkLh\n6bOLRews/G4dvvrDiC0xm04ctxtx2n4eBLdyJZCfP7ItPz9A2U38hdjmHMGOSAmFp88uMd7bK26p\ntC1o7cZyuxWn7WQsuROUT+DXmMvvjW+bl9+roVM+wU4lvWvAg0DuBlAPYAKAf3WzU4IeVq4cWR46\nHqHx9GVAfX1qAau7xLbu7duBKPm8YBuxzTnC8uUjy0PHIzSevgypq0stYnWX2da9/VSIbR5JSg+y\nUqofwDQAi5RSDwP4IYAjbndM8B7rK39g9I8jUHmMNaF7EFq62/fa27xiBXAkxnocOZJDoTsOIrY5\nd7C+8gdG2+ZA5THWiJ0cyH7Zvhee5rfeG87ssOq++LY5FIP0MsROFovbATwN4H+MpioA/+tmpwR9\nmK/8leLUZrkUH+sEugehpbN9HSnXdB+fMCG2ObcwX/nv3g1873vhj491A90D0exu3261PydYdHpF\nWn3LJezEIH8WwPsB9ACAUuptAFPc7JQwjM540kDnMdaE7kFo6Wxfh7db9/EJGWKbNaIzljTQeYw1\nonsgmt3t6/B06z42fsSOQD6slDrueCeiseCk9ILLhLqoQkjRXWI7ne3r8ObqPj4hQ2xzmmzc3pl6\nIRt46eETnEN3mW2729fhzV1+b4K+5fAgPTsC+UUi+hKAAiL6MICnAPzc3W4JgP54ViF9dKduS2f7\nOry5uo9PyBDbnAZLLpjh2Lp0x7IKmaE7fZvd7evw5tZdB9z/rZi+fYvbcxVSKrnDgYjyANwK4DIA\nBGA9gB+qVF90gZqaGtXc3Oz1ZrWRlzecDN4KEYc9COxNX7GCvZ7Tp7MnUsSWPWIzXgDszRXB6gxE\ntEUpVePi+n1lmzdtbvJ6s2mzZnMbLpo9uuJaukQiiW1zNJr16kNBYyM/MOzezcJu+XIJBbFLbLYL\ngL25Tor5V3cewJyZkxEpK0i9cMgYOybPlm1OmOaNiKYrpXYqpYYAPGj8CR4yfTqHVcRrF4KR0szP\nmMdIHjCChdhm/VRVxRfCuRyvacXv6cz8jnmM3HzAOHxUvGypSBZicXw0NBGt8aAvQgwSr5kcCUHJ\nHhmIGUjENmtGdyyr35EQlOxxeyDm0SHkpPc4HZIJZGumxVlud0QYjcRrJidsKcP8XAFP8BVim7Pg\n7Xd7s16H7lhWvxO2lGF+q34neEMygawS/C94SBg9fMmEoPWzyZP5L5FgDFPKMMlYIqSB2OYMOS0y\n0bF1hTHVWjIhaP1szhz+SyQYw5QyTDKW5C7JSk3PI6IesLeiwPgfxrxSSpW63jshdCSLGwZGftZp\nycgUL744XmnsoIagJAsXCcNDkeAoYpuzYG/fEZyquxM+JFncMDDyswMHhtvjxRfHK40d1BCUZOEi\nQX0oetmhdIdhJ2UWCz+Ra1kswsjMmfEHHprlreN9Frvcjh3D82HJYiEZS1wgGgWamoCODqCiAqit\nHa7V6wFuZ7HwE0HJYgE4l8kibMyfH3/gofmTSZWdIxJhT7pJWLJYhDFjycbtnbj6/GnIH2Mn068L\ntLcDLS3A/k5gUjlQXQ1UVnq2+ayzWAiCG2QbNxy7XH19MAVxLJKxxGGiUWDtWqCsDJg6Fejr4/lr\nrvFUJAtCUMg2bjh2ubq6YAriWMKasUSrOF6/HigtBSZXAP39PL94saci2Q6ajpCQqySLG7YjBsMg\nGOPFYIc9Y4nnAxCbmlgcl5byRktLeb4pGF5OQfCaZHHDdsRg0AUjED8GO2wZSzp6Do2Yb3wmZp+f\ncbkDLS1sj0tK2DaXlPB8S4vLG04fEciCpyQTgvE+i7dckEk0GA8Ib8YSLQMQOzqA4uKRbcXF3C7k\nNB8+5yT88R2JwYwlmRCM91m85YJMosF4QLgylrz1Xj/KJ/CNtvEZYNk9Mft8j8sieX8nUFQ0sq2o\niNt9hoRYCJ5ipziF+dmkSTy/f3+w44utJBuMF5YsJbFoGYBYUcFhFaWW8Wp9fdwu5DQF+WMgNRJG\nY6c4hflZWRnPd3UFO77YSrLBeGHJUmKy6HS2g6vuS7DP97lYYnpSOYdVlJQMt/X3c7vPkEF6guAh\nngzG0zw4LRYtAxCtMcjFxSyOu7o8jUGWQXr+RQbqCbF4MhivvR1obeUUTeXlwLx5nsfdbtzeiSUX\n8Kh4LQMQrTHIRUUsjnt6PI1BtjtIT0IsQkiQC04Eue92cD13sykMBwZ4cNrAAM9rHG6tJV91JMJi\nuLAQ2LuXpzJAT9BMkAtOBLnvdnA9d7MpDA8e5AT/Bw/yfHu7QxtIzas7D4yY15KvurKSxXBBAbCv\ng6c+HKAHSIhF6EiWZ9iR19kueidd77sPcD13s3VwGjA8bWrSJg615auOREQQC3Epn1CIjds7PfUi\nJ8sz7Mjrexe9k6733Qe4nru5tRWYMGF4bIQ5bW31TBz2HxnC/NOmHJ9ffi/HHI/a53td7khlpS8F\ncSxaPMhE9DdE9CciGiKinHgF6RXJ4j2zxmXvpKt9d4hsPdyulw/34eA0KZkeHHLFNpsxmF6SLMY1\na1z2Trrad4fI1sPtevnwzs74I9Q7vR2cFikbHm1Zdx1w/7di9vlbLsYfBwxdHuStAK4D8D+ath9a\nss0znBSXvZOu9t0BnPJwu5q72aeD08KSrzoHENvsEtnmGU6Ky95JV/vuAE55uF3N3VxezjcPqwNj\nYIDbPSBR9by660QQJ0KLB1kp9aZS6i0d2w47rsZ7uuyd1BKrmgZ+8nAn9GTX1vJgtJ4eHgHX08Pz\ntbXed1IIHLlkm0+LTPQ03Zur8Z4ueye1xKqmgZ883Ak92fPmAd3d7LAYGuJpdze3e8AQgKvPn+bJ\ntsKCDNILGVdemV57WpjeSSsOeif9XizDLx7upHmFZXCaINhi7kmlnqZ7++AH02tPC9M7acVB76Tf\ni2X4xcOdKJdyYyNiBqft0zI4TVv1vIDiWogFEf0GwIlxPlqhlFqbxnqWAlgKANP94kr0Mb/6VXrt\naVFbyzHHwMjUWQsXOrByezmSdeKXctAp8wrL4DQhCWKb9fDb36bXnhbz5nHMMcAPxQMD7J1csMCB\nldvLkawTv5SDTubJrquDtsFpG7d3Hi8OIthHax5kItoA4ItKKVvJjSUPcmpczznrsxy7XhIbgwzw\nvcjrAWda8goLaRPkPMiZ2OYg5UE2WbO5DWPzgAWz3I8DdT3nrA9y7OoiNgYZYAet1xXvtOQVtoE1\n97FgPw+ypHkLGa57OXPYO+kXD7dfPNmCEHSuPn8ant2yy5Ntue7lDEjqLDfwi4fbL55sK7G5jwX7\n6ErzVkdEUQAXAvglEa3X0Y8w4vc43qBTX88loYeGUpeGdqvoiZzjDIhG2c20ejVPdbpzfEyu2WYv\nYzL9HscbdOrquCR0NJq6NLRbRU/8eI5jcx/7jvZ24LnngIbHeOph4ZRU6Mpi0aiUiiilTlBKTVVK\nLdbRjzAiOWft4XbFvqQD6bJEznGa+LC6oF/JVdu8MUEKLCdxPc9uSHC7Yl/SgXRZ4tdzbM197CtG\n5O+u0FJdMBlaY5DTRWKQg0lDg/6whNj+uB1LPHNm/DCIGTPY8yx4SGMjn2xrbuieHj7pLt65ghyD\nnC5BjUE2WbO5zdOqen6hsVF/WEJsf9yOJZ4/P/6zcSTCnucw8fL2TgwB/o0/fu45PtklJcNtvb18\n0q+4wrXN2o1Blpwfgqu46UnNFC/yGfslJZwAX1YXFPzH2+/26u6Cp7jpSc0UL/IZ+yUlnBf4WhwD\nwP5OoKhoZFtREbf7ABHIgqv4qbiGiRfi1e9FT3IKl/N3C8Fn/mlTsLfviO5ueIqfimuYeCFe/V70\nxCk6eg7p7kJqJpUD/f0j2/r7ud0HiEAWXMWPnlQvxKsMpPMRUl1QSIFvYzRdxI+eVC/Eqx8H0rnB\nW+/148PnnKS7G8mprmZ73NvLtrm3l+erq3X3DIAIZMFl/OhJ9UK8BmUgXcrBimHI/iDVBQWbeDFY\nzy/40ZPqhXj160C6WFIOVmxvB9atY6O9bt2IgW1marfS8fke9jgDRlQX7NBSXTAZIpAFV8lUjLqZ\nZcIJ8Wqnf+mkhNNByvhwS/aHDX+eis/dPoDPTluL+VVRrTHkGRGJ8B1w6VKeijgWYvB1rKYLZCpG\n3cwy4YR4tdO/dFLC6SBlfLgl+8Mftk/GP3/xIFbUrMdV57ejsZFTu/nee2xSWckD8upv5KlPxDEg\nAllwmUzEqFMD+5KJ2GzEqx8HHmZCyvjwpiagrAwbXinF93+Qh3c6S9GFMlTuaQrk/gqCHXLFi5yJ\nGHVqYF8yEZuNePXjwMNMSBkf3toKTJiAP7xejJ88lIedB4rRjQmY8m4rvvVQJ/60NQDe4wAgAjmg\nuJ3H10nSFaNODOxzU8T6ceBhJqSMDzeyPzz6KHDYGL/Ui2JMQUcg91cQUuGEF9ntPL5Okq4YdWJg\nn5si1o8DDzMhZXx4ZydQWIinnwaOGLa5D4WYjE6UTwY2PjnNk36GHRHIASQsHsxEODGwz00R68eB\nh5mQMj48Lw944QWc27EOC/AHlGMfStCH98DZH4K2v4Jgl0y9yGHxYCbCiYF9bopYPw48zISU8eF5\nBLz8MuZ0voDzsAUTsR/FGEDHXA5PeGOrSDsnkKMYQMLiwUyEEwP73BSxfhx4mAlJ48OjUeDdd4Gu\nLpxQNA4n4BAWYgOmYweawNkfgra/gmCHbLzIYfFgJsKJgX1uilg/DjzMhKTx4e3twHsdQE8PTigc\nh3E4jAX4A6qwC4eqZmLP5hmB21+/IgI5gITFg5kIJ7JMuCliw5LCLWl8eFMTcPLJwMKFWHBpAYrH\nHMYBTMReTMEeRAK5v4KQDi9n4EUOiwczEU5kmXBTxIYlhVvS+PDWVr6RLViA894/HkVjDqMbE7Dz\n5GnoxQTe33t170E4EIEcQMLiwUyEE1km3BSxQUnhZoeE8eFm9bnJkzH39gtxxucuxxuTP4CxUIHe\nX0Gww5ILZmAog++FxYOZCCeyTLgpYoOSws0OCePDjfhjTJqEM+rPx+zbPoBtk96HSacXI2/PDNz/\nLaDuOq1dDw2klNLdB9vU1NSo5uZm3d3QjhmDbA2zKCwU0RJLQwOHnezcyQ8PK1fK8bFNYyNfYKWl\nw209PXyhBe1uE42yR7yjg6vn1dZ6kuaNiLYopWpc35APqKmpUZs2N+nuhqOs2dwGALhotv2qXmYM\nsjXMoqAguCLNLRobOexk925+eFi+XI6Pbdat4wusuPh408b+cUB+PpbcdJnGjmVAezvQ0sKlpSeV\nc4EQD9K8jR2TZ8s2565A1nTTdAoRf4KrmDmQy8rYEPf1cfW5oBXY0LgfIpAzRNNNMx5rNrdhavE4\nnHpiie3viPgTXMXMgTxhAlBYiLf3H8JeKsCSugt9lUM4JeZ+lJYCRUVcYrqnx5NCIXYFcm6GWFgK\nIGDqVJ6uXRuoKmF+KkIRpJRzgk3CUn3OyOWM0lK+QEtLeb4pXN7O0GApgIDJFTxdv35ElTAvmX/a\nFOztO5LWd/xUhCJIKecEm4yoPrcPe0srgieOAX4ILi0FSkrYNpeU8HxLi+6eHWes7g5owXrTBIan\nTU3BEwCaiQ33MFPOAeLRDjyRSPB/Dx0d/BBspbiYRb/gP6w3TWB42tKiRQBE/n97dxsc1XXeAfz/\nSGCjV8TLmggWRCbGpB7A64JwcDz1WxxMm6ASmpm0pE2aTGjappNkGsekdNzpZEhpyLT9kLYpk3b6\nIWrTdBwNqTMUO4k7KYkNwmZjnNgY6hgskNEiIesdDHr64eyiK3kX7ZXu3XPuvf/fDHN1Lyvtw4p9\n9tmz5zynqQZHYLbuvXPFgorf/2xMne5RaDkHcEQ78pqbgebmiXaEUSuOAfMJ0eLU5Gt1dWbLaUck\ncwS5sADJq77eXCdf4t5yzhUcpZ+hVMpMq/AaGjLXyT19veZF0quuzly3ZPvGFgxfmcmSPbvi3nLO\nFbZG6XMDYwAivEX6wkVmWoXX8LC57ohoFcj9/cFMg+CLZmDi3nLOBXHfGCZUra0mbwwMmPlIAwPm\nvLXVdmTx8uabwUyDcPhFM2pbUMe95ZwLbG4Mc7JnGFvXR3jHvEzG5OPBQZObBwfNeSZjO7LrolUg\nj48HM1eYL5qBiXvLORfEdpS+q8u8kuzfb45hrAGIy1xq142PBzNX2NEXzcIo3ak3Bq3G4UfcW865\nwNYofeHN2tzqkEq47m7g4EGg/VvmGMYagElzqXPmWIEFen5Eq0Curg5mgQ1fNANTiU0zkj69IJaj\n9JVcKJtOm0mXO3eaI5/nwauuDmaBjcMvmjNZsGdTJTbNSPoiQBuj9KFPrajkQtnmZmDLFmDHR83R\ngee5V/QW6QW1wCYOC5AcUFiIF1bLOS4CNI/pmTPFr0cWF8rGT1ALbPILkFxTWLB3+HSvr97IthQW\n4oXVco6LAM1jWuw9fZij9Cd7hvHQuqXh3YFjC2VtitYIMsC5wg4Ks+VcbKcX+BDUKL1TI/FcKBs/\njswVDlNh1K4wiue6MFvOcRFgcKP05Y7EF6ZWNM6bO4Noy+TgQllbolUgX7vGucIJU2zkFIj49AKf\ngtja2rmFflwoGy/XrjkxV7gStq5fjpM9w9PfMOZKzYZK0iLAILa2LnehX6E4Dr1rhcMLZSstWgVy\nVRXnCidIe7tJOsVEenrBDMx2lN65kXgulI2Xqipn5gqHrbAwKmpdLYLU0VE6NydtEeBsR+nLGYmv\naEs3RxfK2hCtArmpicVxguzebd5RTyUS7CLAJHBuoR8XysbL/PmJKI4LCoVKUovkvXtL5+YgFwEm\nQTkL/UKfd+zl8ELZSoveIj1KjFLFm2pyFugFxcmFflwoSxG2fWMLHj96BrmBMaQa59kOp6JKFXWq\nyVmgF5TpFvpVZN7xVI4ulK20aI0gUyyUu1isVPHWEtGNg2yqRDs+oqSJ23zkcheLlZpGwfe7/t1o\noV/F5h1TUSyQqaL8LBZjURecIBb6EdFkcZqP7GdXuEr0WE6KUgv9UmtZHNsmWmwiUdh3KrIPwAcB\nXAHwfwB+X1X7p/u+DRs26LFjx8IOj0K0cmXxj/pbWszis6na28PrsUwUJhF5TlU32I7Dj9nk5iNH\nZ7mBU4Q9ftQktSj0Ry7lrruKf9SfTpvFZ1N1dITXYznpcgNjONkzzOI4JHOqq8rKzbYK5PcD+JGq\nXhWRvwYAVX10uu9jgRx9VVWlF3eMj1c+HqKwRLRAnnFuTnKBDES/SE6nS+fmMDa4pOJOvTGIC0NX\nsHX98vC2kk64cgtkK4++qj6pqlfzp88C4MylhCg1rzhpbduIXMTcPHOF0b5TbwxajmRmSs0rTlrb\nNptYHLvFhd/AJwAcLPWXIrJTRI6JyLEcd9mKPM4rJooM5maftq5fjgtDVyJZJHNesV3Hz17ChaEr\n2L6xhcWxI0L7LYjID0TkxSJ/2jy32Q3gKoCS+3mp6n5V3aCqG1LcZSvyuFiMyC7m5vDMra7C9o0t\nuDB0BcfPXrIdji9B7ApHMzd8ZRxb1y+3HQZ5hNYHWVXfd6O/F5GPAfgAgAfVxkRosmbHDhbERLYw\nN4ev0CP5+NlLuHPFAtvhlG3bNhbENhS6oHDk2C1Wfhsi8jCARwFsVdWR6W5PREThY24OzvaNLRi+\nMo5nX41+CzgKD3sdu8vW25WvA2gA8JSIZEXkG5biICKiCczNAdq6fjmusjsPlXD4dC8Wza9lcewo\nK1tNq+qtNu6XiIhKY24O1tzqKiyaX4vDp3tRBeDuiLaAo2AVRo1vSy/A2qWNlqOhUqwUyEREkdDV\nBXR2ArkckEoBra3cT5d8uW+1WcD4+NEzOHy6N7J9kikYnFIRkO5uIJsF+nqBhYuATAZobg70Ljgj\nPKHa282udlVV5lhsq2eiROvqAg4cAEZGgCVLzPHAAe6aQDNSKIjK2Za6o8PsbJdOm2Ox7Z4pelgc\nB6S7Gzh0CBgdBRanzPHQIXM9QCyQE6i9Hdi502z5rGqOO3eySCaapLMTaGoCGhvNO8nGRnPemewd\n42jmyimSOzqARx4x78NUzfGRR1gkRx2L4wBlsyYfNzSY3NzQYM6z2UDvhgVyAu3ebQbDvEZGzHUi\nysvlgPr6ydfq6811ohmarkjeu9cMiHmNjprrFE0sjgPW1wvU1U2+VldnrgeIBXICnT3r7zpRIqVS\nwNDQ5GtDQ+Y60SzcqEg+d67495S6Tu569tVeFsdhWLgIGB6efG142FwPEAvkBFqxwt91okRqbQX6\n+4GBAWB83Bz7+811olnyFsneXsnLlhW/fanr5KZnX+3F1XHze2ZxHLBMxuTjwUGTmwcHzXkmE+jd\nsEBOoD17gNrayddqa811IspLp4G2NvPkuHDBHNva2MWCAlMonq6OAz/NjzTu2gXU1Ey+XU2NuU7R\ncPj0RHFMIWhuBjZvNk+Mizlz3Lw58C4WbPOWQIVtnnfvNtMqVqwwxTG3fyaaIp1mQUyhK2xNffh0\nL7ZtMx8T791rplUsW2aKY24B7b7cwBhO9piP/lkch6y5OfCCeCoWyAm1YwcLYiIiV2zf2IKBsbfw\n1AvnsWbTTThypMF2SORDYa7xQ+uWonHeXMvRUBA4xYKIiMgBjfPm4qF1S3Fh6EpZ/ZLJvqkL8Vgc\nxwcLZCIiIkc0zptbcgEfucU715hTKuKHBTIREZFjvAv4OJrsluNnL7F9WwJwDjIREZGjvHOTAeCe\nW4Pt9Url8y7C41zj+OMIMhERkcOmTrs4fvaS5YiS5/DpXpzsGcZt6QWca5wQHEEmIiKKgEKRXGgJ\nx9Hk8BWmUtx12y1IN9VMc2uKExbIREREEbJ9Ywu6+kdx+JUeAMCS+puw6h1sCxck77xvzjNOJhbI\nREREEZNuqkHaMz/5wulerL6lDqnGebZDiyzvHOPb0guwdmmj5YjIJhbINDNdXUBnJ5DLAakU0NrK\nHceIiCqsMD/5xPkBnOy6hJOnz2P12ZNIDfcBixYBd9wR+o5jcVAYMWZhTAUskMm/ri7gwAGgqQlY\nsgQYGjLnbW0skomILFi7tBFrZRhvPf0LfA8NOLlgPqqGR3D3oUPA5s0skov46elejOe/XjS/Fvet\nTlmNh9zCApn86+w0xXFj/l124djZyQKZiMiWbBZz592E7ambAACP99TjcPU7ged/ibq183DnigWW\nA7Tv+NlLGL4yfv186/rlmFvNhl70diyQyb9czowce9XXAxcu2ImHiIiAvl5g8cQo6PZbFBifg67h\nKzhyZfz6NIKkH226ogAAEHdJREFUzVX2zi0G2JGCysMCmfxLpcy0ikbPPK2hIXOdiIjsWLgIGB4G\nGjwdLYaHka4xC/oATMxVzheMcW0Vd+qNQVwYunL9nHOLyS8WyORfa6uZcwyYkeOhIaC/H7j3Xrtx\nERElWSYDHDpkvq6rM8XywACwadP1m6xd2ni9UDxxfmBSO7MqAHdHuGD2zikGOH2CZocFMvmXTpsF\neZ2dZlpFKmWKY84/JiKyp7nZLMjLZoGLOTOivGlTyQV63mL5rWvjuDB4+XpvZQCYUwXcPKfKybnL\nU+cSF7AopqCwQKaZSadZEBMRuaa5eUYdK+ZWV13vrVzQ1T+KI6/0TBplBoC6m8xtw57HnBsYu/61\ndw5xAecSU5hYIBN5tLcDu3cDZ88CK1YAe/YAO3bYjiqh2GubyKqpBXPBifMDuDh4GSenFM6lFJvn\nPLXoLmXR/FoAwJvnF+Cf/qoR584By5YBux4F0hvL+hEUtO5u8ylFX6/5lCKTiWUbQRbIRHnt7cDO\nncDIiDk/c8acAyySA1Nu0VvotX3tGnD+PHDkCHDwIPCpT5nvISJr/Cx2e/zomaLFsJ++wx3fBb7y\nRWB01Jx3dQGPfNF8ve1DZYdCN1Ju0dvdbea5X7tmplg+fxx4+mlgx+8Ad2QqH3eIRFUrf6ciXwbQ\nBmAcQA+Aj6vq+em+b8OGDXrs2LGww6OEWrnSFMVTtbQAr71W6WhiyLvBjHdxZ7ENZjo6gNdfB37x\nC7PYqKYG6OsDrl4FHnssEiPJIvKcqm6wHYcfs8nNR452hh0eJdRdd5n0MVU6bd470ywVit7GxsmL\nO4ttMHPwoBm0OPUKUFsLzJsHXOo3BfPnPx+JkeQ51VVl5WZbM9n3qeo6Vc0AeALAY5biILru7Fl/\n18kn7wYzVVXm2NRkrk+Vy5kkXFdnkrAIsGCBKZCL3Z6CwtxMzjl3zt918imbNfm4ocHk5oYGc57N\nvv22fb1m5Li21gxciJg8fvVq8dtHmJUCWVUHPKd1ACo/jE00xYoV/q6TT7mcGTn2qq8316dKpcz1\nGs8CnLExs0FNsdtTIJibyUXLlvm7Tj719ZrBCK+6OnN9qoWLgN5eM3JccHnM5Oxit48wa3OQRWQP\ngN8D8CaA+29wu50AdgLAClYq8eTIYqw9eybPQQbMm+Q9eyoeSvhsPOZ+NphpbTUf5fX1mZHjsTHz\nsV9LCzekCRlzM13nyGKsXY+aOceFOciAee+869GKhxI+G495iQ1msLBIT+xMxsw5vtRvRo4vj5kX\nzVXp4rePsNBGkEXkByLyYpE/bQCgqrtVdTmAdgCfKfVzVHW/qm5Q1Q0pvjDGT2Fe6siIGR0cGTHn\nxSachWzHDmD/flODiZjj/v0xXKBn6zFvbTVzjgcGgPFxc+zvL73o7tZbgRMngGeeMQXy7bcD1dVc\npDdLzM1UlsK81NFRs3316Kg57+6ueCjbPgTs+6p5Dy9ijvu+GsMFerYe80zG5OPBQZObBwfNeabE\noruVK4GXXwaeOwaMXQZW3WZyc6nbR5SVRXqTAhBpAfB9VV0z3W2dX6TnyEhopHR0mALNO6o4MGCG\nbrdtsxdXnNl8zMt5jngX842NmSK5pwe4/35gy5bIPKeiuEjPy29udnqRniMjoZFy8KAp0LyjioOD\nZuh2yxZ7ccWZzce8nOeIdzHf5cumSL54Ebj7buCBByLznCp3kZ6VKRYiskpVT+VPtwJ42UYcgfK+\nqC9ZYj46PnCg+Ap9mpDLmcfLq77eLAKgcNh8zMvZYMa7mK+xEXjwwYkCns+lUMUyN3tf1BenzEfH\nhw4VX6FPE/p6zePlVVdnduijcNh8zMvZYMa7mK+hAbjnnokCPobPJVtzkPeKyGqYVkJnAHzaUhzB\n8b6oAxPHzk6+qN+In3mpFAzXH3O+abIpfrnZ+6IOTByz2Vi+qAfGz7xUCobrj3nC3jTZ6mKxXVXX\n5NsJfVBVo9+sxc8KfZrgd14qzZ7rj3mhgPdyqYCPsVjmZj8r9GmC33mpNHuuP+aFAt7LpQI+YLb6\nIMcPX9RnJp0201Bqayd6K3JaSrhcf8xdL+ApWhL2oh6Y5mYzDaWmxowQ1tRwWkrYXH/MXS/gA8at\npoPS2mrmHAOTdwm79167cUVBOfNSKVguP+aFAr6z0xTwqZR5HrkaL7ktkzFzjoHJu4Rt2mQ3rigo\nZ14qBcvlx7xQwGezpoBfuMg8j1yNd5ZYIAeFL+pEwXG5gKdoSdiLOlGoXC7gA8YCOUh8UScick+C\nXtSJKBicg0xERERE5MECmYiIiIjIgwUyEREREZEHC2QiIiIiIg8u0rOlq8t0vMjlTMeL1lYu8CMi\nsq2723S86Os1HS8yGS7wI0ogjiDb0NVleiaPjJgtdUdGzHlXl+3IiIiSq7vb9EweHTVb6o6OmvPu\nbtuREVGFsUC2obMTaGoCGhuBqipzbGoy14mIyI5s1uTjhgaTmxsazHk2azsyIqowFsg25HJmtz2v\n+npznYiI7OjrNbvtedXVmetElCgskG1IpcxW1F5DQ+Y6ERHZsXCR2Yraa3jYXCeiRGGBbENrK9Df\nDwwMAOPj5tjfb64TEZEdmYzJx4ODJjcPDprzTMZ2ZERUYSyQbUingbY2oLYWuHDBHNva2MWCiMim\n5mZg82agpga4mDPHzZvZxYIogdjmzZZ0Ot4FMdvYEVEUNTfHuyBmGzuisnAEmYLHNnZERO5hGzui\nsrFApuCxjR0RkXvYxo6obCyQKXhsY0dE5B62sSMqGwtkCh7b2BERuYdt7IjKxgKZgsc2dkRE7mEb\nO6KysUCm4LGNHRGRe9jGjqhsbPMWRVFooRb3NnZERFNFoYVa3NvYEQWEI8hRwxZqRETuYQs1olhh\ngRw1bKFGROQetlAjihUWyFHDFmpERO5hCzWiWGGBHDVsoUZE5B62UCOKFRbIUcMWakRE7mELNaJY\nsVogi8gXRERFZLHNOCKFLdSIKGTMzTPAFmpEsWKtzZuILAfwEICztmKILLZQI6KQMDfPAluoEcWG\nzRHkvwXwRQBqMQYiIpqMuZmIEs/KCLKIbAVwTlV/JiLT3XYngJ3508si8mLY8fm0GMBF20EU4WJc\nLsYEMC4/XIwJcDOu1bYD8Gs2uXlOdZVruRlw8/+FizEBbsblYkyAm3G5GBPgZlxl5WZRDWeQQER+\nAOAdRf5qN4A/A/B+VX1TRF4DsEFVp30AReSYqm4INtLZcTEmwM24XIwJYFx+uBgT4GZcLsYEJCc3\nA27G5WJMgJtxuRgT4GZcLsYEuBlXuTGFNoKsqu8rdl1E1gJ4J4DCCEUawPMislFV3wgrHiIiYm4m\nIipHxadYqOoJALcUzv2MUhARUTiYm4mIJkStD/J+2wEU4WJMgJtxuRgTwLj8cDEmwM24XIwpLK7+\nW12My8WYADfjcjEmwM24XIwJcDOusmIKbQ4yEREREVEURW0EmYiIiIgoVCyQiYiIiIg8Ilsgu7QV\nqoh8WUReEJGsiDwpIkttxwQAIrJPRF7Ox9YhIk0OxPRhEfm5iIyLiNXWLyLysIicFJHTIrLLZiwF\nIvIvItLjWr9vEVkuIk+LyEv5399nHYhpnogcFZGf5WP6S9sxFYhItYgcF5EnbMdSaczN02NuLise\n5ucyMDf74yc3R7JAdnAr1H2quk5VMwCeAPCY7YDyngKwRlXXAXgFwJcsxwMALwL4EIAf2wxCRKoB\n/D2ALQBuB/DbInK7zZjy/hXAw7aDKOIqgD9V1V8B8B4Af+zA43UZwAOqegeADICHReQ9lmMq+CyA\nl2wHUWnMzWVjbr4B5mdfmJv9KTs3R7JAhmNboarqgOe0Du7E9aSqXs2fPgvT19QqVX1JVU/ajgPA\nRgCnVfVVVb0C4NsA2izHBFX9MYA+23FMpardqvp8/utBmASzzHJMqqpD+dO5+T/Wn3sikgbwGwC+\naTsWC5iby8DcPC3m5zIxN5fPb26OXIHs3QrVdixeIrJHRF4HsAPujFJ4fQLAQdtBOGQZgNc9512w\nnFSiQkRWArgTwBG7kVz/uCwLoAfAU6pqPSYAfwdTJI7bDqSSmJtnjLn57ZifZ4C5eVq+cnPFNwop\nRzlboVY2ohvHpKoHVHU3gN0i8iUAnwHwFy7Elb/NbpiPYdpdickBUuSa9Xe4rhORegCPA/jclNE5\nK1T1GoBMfg5nh4isUVVr8wNF5AMAelT1ORG5z1YcYWFuDi6u/G2Ym4tjfvaJufnGZpKbnSyQXdwK\ntVRMRfwbgO+jQkl4urhE5GMAPgDgQa1Q02sfj5VNXQCWe87TAM5biiUSRGQuTAJuV9Xv2o7HS1X7\nReR/YOYH2lxA814AW0Xk1wHMA9AoIt9S1Y9ajCkwzM3lY26eFeZnH5iby+I7N0dqioWqnlDVW1R1\npaquhHkS/WrYCXg6IrLKc7oVwMu2YvESkYcBPApgq6qO2I7HMZ0AVonIO0XkJgAfAfA9yzE5S0zV\n888AXlLVv7EdDwCISKqw+l9EagC8D5afe6r6JVVN5/PTRwD8KC7F8Y0wN/vD3Dwt5ucyMTeXZya5\nOVIFssP2isiLIvICzEeM1tus5H0dQAOAp/Jtjr5hOyAR2SYiXQA2Afi+iByyEUd+gcxnAByCWdTw\nHVX9uY1YvETk3wE8A2C1iHSJyCdtx5T3XgC/C+CB/P+lbP6duE3NAJ7OP+86Yea5Ja6tGt0Qc3OZ\nXMnNAPOzT8zNIeFW00REREREHhxBJiIiIiLyYIFMREREROTBApmIiIiIyIMFMhERERGRBwtkIiIi\nIiIPFsgUCSJyzdPCJpvfUtPvz2gSkT8KPrrrP//dIvKMiFwWkS+EdT9ERK5gbqa4Yps3igQRGVLV\n+ln+jJUAnlDVNT6/rzq/beZ0t7sFQAuA3wRwSVW/NpM4iYiigrmZ4oojyBRZIlItIvtEpFNEXhCR\nP8hfrxeRH4rI8yJyQkTa8t+yF8C78qMc+0TkPhF5wvPzvi4iH89//ZqIPCYihwF8WETeJSL/LSLP\nicj/isi7p8ajqj2q2gngrdD/8UREjmJupjiYYzsAojLViEg2//UvVXUbgE8CeFNVW0XkZgA/EZEn\nAbwOYJuqDojIYgDPisj3AOwCsEZVMwAgIvdNc59jqnpP/rY/BPBpVT0lIncB+AcADwT9jyQiihjm\nZoolFsgUFaOF5OnxfgDrROS38ufzAawC0AXgKyLyawDGASwDsGQG9/kfgBn1AHA3gP8UkcLf3TyD\nn0dEFDfMzRRLLJApygTAn6jqoUkXzUdxKQDrVfUtEXkNwLwi338Vk6cZTb3NcP5YBaC/yIsAERG9\nHXMzRR7nIFOUHQLwhyIyFwBE5DYRqYMZrejJJ+D7YRZnAMAggAbP958BcLuI3Cwi8wE8WOxOVHUA\nwC9F5MP5+xERuSOcfxIRUeQxN1PkcQSZouybAFYCeF7M52s5mFXK7QD+S0SOAcgCeBkAVLVXRH4i\nIi8COKiqj4jIdwC8AOAUgOM3uK8dAP5RRP4cwFwA3wbwM+8NROQdAI4BaAQwLiKfA3B7PokTESUF\nczNFHtu8ERERERF5cIoFEREREZEHC2QiIiIiIg8WyEREREREHiyQiYiIiIg8WCATEREREXmwQCYi\nIiIi8mCBTERERETk8f97gheFb7zDRwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1d3b72e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot data and decision boundary of logistic regression\n",
    "fig, ax = plt.subplots(1, 2, figsize=(10, 5))\n",
    "# plot class 1\n",
    "ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1], c='blue', label='Class 1')\n",
    "# plot class 0\n",
    "ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1], c='red', label='Class 0', alpha=0.3)\n",
    "# label axes\n",
    "ax[0].set_xlabel('Feature 1')\n",
    "ax[0].set_ylabel('Feature 2')\n",
    "# label title\n",
    "ax[0].set_title('Training Toy Data Set for Binary Classification')\n",
    "# add legend\n",
    "ax[0].legend()\n",
    "# set x, y limits\n",
    "ax[0].set_xlim([-4, 4])\n",
    "ax[0].set_ylim([-4, 4])\n",
    "\n",
    "# plot decision boundary\n",
    "ax[1] = plot_decision_boundary(X_train, y_train, logreg, \n",
    "                               'Logistic Regression (quadratic boundary)', \n",
    "                               ax[1], bounds=(-6, 7), poly_flag=True)\n",
    "# set x, y limits\n",
    "ax[1].set_xlim([-4, 4])\n",
    "ax[1].set_ylim([-4, 4])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>logistic regression</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train score</th>\n",
       "      <td>0.871429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test score</th>\n",
       "      <td>0.816667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             logistic regression\n",
       "train score             0.871429\n",
       "test score              0.816667"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# evaluate model\n",
    "scores_df = pd.DataFrame(data={'logistic regression': [logreg.score(X_train_quad, y_train), logreg.score(X_test_quad, y_test)]}, \n",
    "                         index=['train score', 'test score'])\n",
    "scores_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Support Vector Classifiers (SVC)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Another way to use a decision boundary for classification is to build a Support Vector Classifier. \n",
    "\n",
    "1. Logistic regression: \n",
    "  - uses the decision boundary to model the **probability** of labeling a point a certain class\n",
    "  - maximizes the likelihood of the training set\n",
    "\n",
    "2. Support vector classifier: \n",
    "  - classifies points on one side of boundary as one class and the other side of the boundary the other class\n",
    "  - maximizes the distance between the points and the boundary, while minimizing the number of misclassifications"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Recall from building logistic regression with quadratic boundaries: the key observation is that training a polynomial model is just training a linear model on data with transformed predictors. \n",
    "\n",
    "In our previous example, transforming the data to fit a 2nd degree polynomial model requires a map\n",
    "\\begin{aligned}\n",
    "&\\phi: \\mathbb{R} \\to \\mathbb{R}^3\\\\\n",
    "&\\phi(x) = (x^0, x^1, x^2)\n",
    "\\end{aligned}\n",
    "where $\\mathbb{R}$ called the ***input space***, $\\mathbb{R}^3$ is called the ***feature space***. \n",
    "\n",
    "While the data may not be linearly separable in the input space, it may be in the feature space.\n",
    "<img src=\"images/feature_space.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**The Kernel Trick:** since the feature space can extremely high dimensional, computing $\\phi$ explicitly can be costly. \n",
    "\n",
    "Instead, we note that computing $\\phi$ is unnecessary. For training a (support vector) classifier we just need to compute the inner products of vectors in feature space, this inner product is called ***the kernel function***.\n",
    "\n",
    "**For Practitioners:**\n",
    "\n",
    "A support vector classifier model, as implemented in `sklearn`, has several 'customizations' that affect model performance:\n",
    "\n",
    "1. a choice of the kernel function (i.e. a choice of the map $\\phi$ of the input into some feature space)\n",
    "2. for each kernel function, a number of constants (hyper-parameters)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Decision Trees"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Logistic regression models with linear boundaries are intuitive to interpret. We often examine the impact of each predictor on the log-odds of a positive classification:\n",
    "$$\n",
    "12 x_1 - 0.4 x_2 + -20x_3 = 0.\n",
    "$$\n",
    "\n",
    "\n",
    "But it is less straightforward to interpret nonlinear decision boundaries in the problem context:\n",
    "$$\n",
    "(x_3 + 2 x_2)^2 - x_1 + 10 = 0.\n",
    "$$\n",
    "\n",
    "It would be desirable to build models with complex decision boundaries that are also easy to interpret."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "But people in every walk of life have long been using ***decision trees*** (flow charts) for differentiating between classes of objects and phenomena:\n",
    "<img src=\"images/dt_partition.png\" alt=\"\" style=\"height: 300px;\"/>\n",
    "Every flow chart tree corresponds to a partition of the input space by axis aligned lines or (hyper) planes.\n",
    "\n",
    "Conversely, every such partition can be written as a flow chart tree."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Fitting a decision tree on training data:**\n",
    "\n",
    "1. Start with an empty decision tree (undivided feature space)\n",
    "2. Choose the 'optimal' predictor on which to split and choose the 'optimal' threshold value for splitting.\n",
    "3. Recurse on on each new node until some stopping condition is met\n",
    "\n",
    "Typically, we measure optimality by the purity (in terms of observed clases) of each region defined by the tree.\n",
    "\n",
    "We usually stop until a pre-defined maximum depth is reached."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**For Practitioners:**\n",
    "\n",
    "A decision tree model, as implemented in `sklearn`, has several hyper-parameters. Two important ones being:\n",
    "\n",
    "1. stopping condition (usually the max depth, but can also be minimum number of observations per region defined by tree)\n",
    "2. splittin condition (what criteria of purity, in terms of observed classes, used to determine an 'optimal' split)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Deep vs Shallow Trees**\n",
    "\n",
    "<img src=\"images/dt_bias.png\" alt=\"\" style=\"height: 500px;\"/>\n",
    "\n",
    "***Exercise:*** implement decision trees of various depth on the toy data set with non-linear boundary. What is the depth you need to get a 'good' fit?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**Deep vs Shallow Trees**\n",
    "If you randomly draw 50 sample from our toy data with non-linear boundary and fit a decision tree on each sample individually, their decision boundaries will look like the following:\n",
    "\n",
    "<img src=\"images/dt_var.png\" alt=\"\" style=\"height: 300px;\"/>\n",
    "\n",
    "***Exercise:*** plot the training score and the testing score as a function of the depth of the decision tree. What do you observe?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Ensemble Methods: Bagging (Random Forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "As we've seen:\n",
    "\n",
    "1. a shallow decision tree is often unable to capture complex decision boundaries (high bias)\n",
    "2. a deep decision tree tends to be overly sensitive to the noise the training data leading to overfitting (high variance)\n",
    "\n",
    "A compromise for reducing variance is to fit a large number of sensitive models (deep trees) on the training data and then average the results (reducing the variance).\n",
    "\n",
    "<img src=\"images/random_forest.jpg\" alt=\"\" style=\"height: 300px;\"/>\n",
    "\n",
    "A random forest is the 'averaged model' of collection of deep decision trees each learned on a subset of the training data (each branch in each tree is also trained on a randomized set of input dimensions to reduce correlation between trees)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**For Practitioners:**\n",
    "\n",
    "A random forest, as implemented in `sklearn`, has a number of hyper-parameters that effects model performance. Some important hyper-parameters are:\n",
    "\n",
    "1. number of trees in the forest\n",
    "2. max depth of trees\n",
    "3. hyper-parameters that determines the subsets of input dimensions to learn each branch and the subsets of training data to learn each tree"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Ensemble Methods: Boosting (AdaBoost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "fragment"
    }
   },
   "source": [
    "Instead of 'averaging' over a large number of complex models (to reduce variance or overfitting), we can aggregate a large number of simple models into a complex model (to overcome high bias).\n",
    "\n",
    "Each model $T_h$ might be a poor fit for the data, but a linear combination of the ensemble\n",
    "$$\n",
    "T = \\sum_h \\lambda_h T_h\n",
    "$$\n",
    "can be expressive."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "***Gradient boosting*** is a method for iteratively building a complex regression model $T$ by adding simple models. Each new simple model added to the ensemble compensates for the weaknesses of the current ensemble.\n",
    "\n",
    "1. We start by fitting a simple model $T^{(0)}$ on the training data, $\\{(x_1, y_1),\\ldots, (x_N, y_N)\\}$.\n",
    "2. Compute the ***residuals*** or errors $\\{r_1, \\ldots, r_N\\}$ for $T$\n",
    "3. Fit a simple model $T^{(i)}$ to models the errors of $T$\n",
    "4. Set $T\\leftarrow T + \\lambda T^{i}$\n",
    "\n",
    "\n",
    "Intuitively, with each addition of $T^{(i)}$, the error is reduced \n",
    "\n",
    "Note that gradient boosting has a hyper-parameter, $\\lambda$. This is called the ***learning rate***."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "Gradient boosting is in fact gradient descent, with the objective of minimizing model error:\n",
    "\n",
    "<img src=\"images/descent.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "**For Practitioners:**\n",
    "\n",
    "All of this formalism is to say that ***it is important to choose a good learning rate***!\n",
    "\n",
    "\n",
    "<img src=\"images/learning_rate.png\" alt=\"\" style=\"height: 300px;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# EXAMPLE: CLASSIFICATION WITH REAL DATA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "##  Predicting Hubway User Types\n",
    "### Using Two Covariates:\n",
    "- **Simple Covariates:** Trip duration, check-out hour\n",
    "- **Response:** 1 for subscriber, 0 for non-subscribe\n",
    "- **Models:** Logistic regression (linear boundary), SVM (RBF kernel), Random Forest, AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# define covariates: duration, checkout hour, gender, age, distance from start station to city center, \n",
    "# distance from end station to city center\n",
    "X = ride_data[['duration', 'check_out_hour']]\n",
    "X_expanded = ride_data[['duration', 'check_out_hour', 'start_dist_to_center', 'end_dist_to_center']].fillna(0)\n",
    "\n",
    "# define response variable as subscriber type: 1 for subscriber, 0 for non-subscriber\n",
    "y = ride_data['subsc_type']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# train test split\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X.values, y.values, test_size=test_size, random_state=rand_state)\n",
    "\n",
    "# standardize the features\n",
    "scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AdaBoost</th>\n",
       "      <th>RF</th>\n",
       "      <th>logistic</th>\n",
       "      <th>svm</th>\n",
       "      <th>tree</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train score</th>\n",
       "      <td>0.764286</td>\n",
       "      <td>0.997143</td>\n",
       "      <td>0.658571</td>\n",
       "      <td>0.662857</td>\n",
       "      <td>0.962857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test score</th>\n",
       "      <td>0.693333</td>\n",
       "      <td>0.653333</td>\n",
       "      <td>0.716667</td>\n",
       "      <td>0.680000</td>\n",
       "      <td>0.620000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             AdaBoost        RF  logistic       svm      tree\n",
       "train score  0.764286  0.997143  0.658571  0.662857  0.962857\n",
       "test score   0.693333  0.653333  0.716667  0.680000  0.620000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# list of model names\n",
    "methods = ['logistic', 'svm', 'tree', 'RF', 'AdaBoost']\n",
    "# list of model for classification\n",
    "models = [linear_model.LogisticRegression(C=10, random_state=rand_state), \n",
    "          svm.SVC(C=1, gamma=0.5, random_state=rand_state), \n",
    "          tree.DecisionTreeClassifier(max_depth = 20, random_state=rand_state), \n",
    "          ensemble.RandomForestClassifier(n_estimators=50, random_state=rand_state), \n",
    "          ensemble.AdaBoostClassifier(tree.DecisionTreeClassifier(max_depth=2), \n",
    "                                      n_estimators=50, random_state=rand_state, learning_rate=1e-1)]\n",
    "# apply models to our data\n",
    "scores = {}\n",
    "for i in range(len(methods)):\n",
    "    models[i].fit(X_train, y_train)\n",
    "    scores[methods[i]] = [models[i].score(X_train, y_train), models[i].score(X_test, y_test)]\n",
    "# compare results\n",
    "scores_df = pd.DataFrame(data=scores, index=['train score', 'test score'])\n",
    "scores_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "##  Predicting Hubway User Types\n",
    "### Using Four Covariates:\n",
    "- **Simple Covariates:** Trip duration, check-out hour, distance between start and end stations from city center\n",
    "- **Response:** 1 for subscriber, 0 for non-subscribe\n",
    "- **Models:** Logistic regression (linear boundary), SVM (RBF kernel), Random Forest, AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true,
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# train test split\n",
    "X_train, X_test, y_train, y_test = model_selection.train_test_split(X_expanded.values, y.values, test_size=test_size, random_state=rand_state)\n",
    "\n",
    "# standardize the features\n",
    "scaler = preprocessing.StandardScaler().fit(X_train)\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AdaBoost</th>\n",
       "      <th>RF</th>\n",
       "      <th>logistic</th>\n",
       "      <th>svm</th>\n",
       "      <th>tree</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>train score (expanded covariates)</th>\n",
       "      <td>0.767143</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.665714</td>\n",
       "      <td>0.697143</td>\n",
       "      <td>0.975714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test score(expanded covariates)</th>\n",
       "      <td>0.706667</td>\n",
       "      <td>0.673333</td>\n",
       "      <td>0.690000</td>\n",
       "      <td>0.653333</td>\n",
       "      <td>0.676667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   AdaBoost        RF  logistic       svm  \\\n",
       "train score (expanded covariates)  0.767143  1.000000  0.665714  0.697143   \n",
       "test score(expanded covariates)    0.706667  0.673333  0.690000  0.653333   \n",
       "\n",
       "                                       tree  \n",
       "train score (expanded covariates)  0.975714  \n",
       "test score(expanded covariates)    0.676667  "
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# apply models to our data\n",
    "scores = {}\n",
    "for i in range(len(methods)):\n",
    "    models[i].fit(X_train, y_train)\n",
    "    scores[methods[i]] = [models[i].score(X_train, y_train), models[i].score(X_test, y_test)]\n",
    "# compare results\n",
    "scores_df = pd.DataFrame(data=scores, index=['train score (expanded covariates)', 'test score(expanded covariates)'])\n",
    "scores_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# Lingering Questions:\n",
    "\n",
    "-  You will notice that the train/test score of the various models are affected by the values of the hyper-parameters as well as the train/test split.\n",
    "   - How do we determine the \"best\" hyper-parameters? Should we pick ones that maximizes the test score?\n",
    "   - How do we compare the performance of the various models in a fair way? Maybe the high performing models just got lucky with this particular test set?\n",
    "- Is accuracy (percent of data correctly classified) a good measure for evaluating the performance of a classifier? Can you think of a situation where accuracy can be misleading?\n",
    "\n",
    "We will explore answers to these questions in Part III: Model Evaluation."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python [conda env:py36]",
   "language": "python",
   "name": "conda-env-py36-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}