week03_model_free/seminar_qlearning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q-learning (3 points)\n",
    "\n",
    "This notebook will guide you through implementation of vanilla Q-learning algorithm.\n",
    "\n",
    "You need to implement QLearningAgent (follow instructions for each method) and use it on a number of tests below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys, os\n",
    "if 'google.colab' in sys.modules and not os.path.exists('.setup_complete'):\n",
    "    !wget -q https://raw.githubusercontent.com/yandexdataschool/Practical_RL/spring20/setup_colab.sh -O- | bash\n",
    "\n",
    "    !touch .setup_complete\n",
    "\n",
    "# This code creates a virtual display to draw game images on.\n",
    "# It will have no effect if your machine has a monitor.\n",
    "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\")) == 0:\n",
    "    !bash ../xvfb start\n",
    "    os.environ['DISPLAY'] = ':1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "import random\n",
    "import math\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "class QLearningAgent:\n",
    "    def __init__(self, alpha, epsilon, discount, get_legal_actions):\n",
    "        \"\"\"\n",
    "        Q-Learning Agent\n",
    "        based on https://inst.eecs.berkeley.edu/~cs188/sp19/projects.html\n",
    "        Instance variables you have access to\n",
    "          - self.epsilon (exploration prob)\n",
    "          - self.alpha (learning rate)\n",
    "          - self.discount (discount rate aka gamma)\n",
    "\n",
    "        Functions you should use\n",
    "          - self.get_legal_actions(state) {state, hashable -> list of actions, each is hashable}\n",
    "            which returns legal actions for a state\n",
    "          - self.get_qvalue(state,action)\n",
    "            which returns Q(state,action)\n",
    "          - self.set_qvalue(state,action,value)\n",
    "            which sets Q(state,action) := value\n",
    "        !!!Important!!!\n",
    "        Note: please avoid using self._qValues directly. \n",
    "            There's a special self.get_qvalue/set_qvalue for that.\n",
    "        \"\"\"\n",
    "\n",
    "        self.get_legal_actions = get_legal_actions\n",
    "        self._qvalues = defaultdict(lambda: defaultdict(lambda: 0))\n",
    "        self.alpha = alpha\n",
    "        self.epsilon = epsilon\n",
    "        self.discount = discount\n",
    "\n",
    "    def get_qvalue(self, state, action):\n",
    "        \"\"\" Returns Q(state,action) \"\"\"\n",
    "        return self._qvalues[state][action]\n",
    "\n",
    "    def set_qvalue(self, state, action, value):\n",
    "        \"\"\" Sets the Qvalue for [state,action] to the given value \"\"\"\n",
    "        self._qvalues[state][action] = value\n",
    "\n",
    "    #---------------------START OF YOUR CODE---------------------#\n",
    "\n",
    "    def get_value(self, state):\n",
    "        \"\"\"\n",
    "        Compute your agent's estimate of V(s) using current q-values\n",
    "        V(s) = max_over_action Q(state,action) over possible actions.\n",
    "        Note: please take into account that q-values can be negative.\n",
    "        \"\"\"\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "\n",
    "        # If there are no legal actions, return 0.0\n",
    "        if len(possible_actions) == 0:\n",
    "            return 0.0\n",
    "\n",
    "        <YOUR CODE>\n",
    "\n",
    "        return value\n",
    "\n",
    "    def update(self, state, action, reward, next_state):\n",
    "        \"\"\"\n",
    "        You should do your Q-Value update here:\n",
    "           Q(s,a) := (1 - alpha) * Q(s,a) + alpha * (r + gamma * V(s'))\n",
    "        \"\"\"\n",
    "\n",
    "        # agent parameters\n",
    "        gamma = self.discount\n",
    "        learning_rate = self.alpha\n",
    "\n",
    "        <YOUR CODE>\n",
    "\n",
    "        self.set_qvalue(state, action, <YOUR CODE: Q-value> )\n",
    "\n",
    "    def get_best_action(self, state):\n",
    "        \"\"\"\n",
    "        Compute the best action to take in a state (using current q-values). \n",
    "        \"\"\"\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "\n",
    "        # If there are no legal actions, return None\n",
    "        if len(possible_actions) == 0:\n",
    "            return None\n",
    "\n",
    "        <YOUR CODE>\n",
    "\n",
    "        return best_action\n",
    "\n",
    "    def get_action(self, state):\n",
    "        \"\"\"\n",
    "        Compute the action to take in the current state, including exploration.  \n",
    "        With probability self.epsilon, we should take a random action.\n",
    "            otherwise - the best policy action (self.get_best_action).\n",
    "\n",
    "        Note: To pick randomly from a list, use random.choice(list). \n",
    "              To pick True or False with a given probablity, generate uniform number in [0, 1]\n",
    "              and compare it with your probability\n",
    "        \"\"\"\n",
    "\n",
    "        # Pick Action\n",
    "        possible_actions = self.get_legal_actions(state)\n",
    "        action = None\n",
    "\n",
    "        # If there are no legal actions, return None\n",
    "        if len(possible_actions) == 0:\n",
    "            return None\n",
    "\n",
    "        # agent parameters:\n",
    "        epsilon = self.epsilon\n",
    "\n",
    "        <YOUR CODE>\n",
    "\n",
    "        return chosen_action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try it on taxi\n",
    "\n",
    "Here we use the qlearning agent on taxi env from openai gym.\n",
    "You will need to insert a few agent functions here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "env = gym.make(\"Taxi-v3\")\n",
    "\n",
    "n_actions = env.action_space.n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = QLearningAgent(\n",
    "    alpha=0.5, epsilon=0.25, discount=0.99,\n",
    "    get_legal_actions=lambda s: range(n_actions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_and_train(env, agent, t_max=10**4):\n",
    "    \"\"\"\n",
    "    This function should \n",
    "    - run a full game, actions given by agent's e-greedy policy\n",
    "    - train agent using agent.update(...) whenever it is possible\n",
    "    - return total reward\n",
    "    \"\"\"\n",
    "    total_reward = 0.0\n",
    "    s = env.reset()\n",
    "\n",
    "    for t in range(t_max):\n",
    "        # get agent to pick action given state s.\n",
    "        a = <YOUR CODE>\n",
    "\n",
    "        next_s, r, done, _ = env.step(a)\n",
    "\n",
    "        # train (update) agent for state s\n",
    "        <YOUR CODE>\n",
    "\n",
    "        s = next_s\n",
    "        total_reward += r\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "    return total_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEICAYAAAC3Y/QeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO2deXhU1fnHP28mG4FAwk4IEJYAAgJCFBFRcANX6lJFrWKrte4/W1vFfSsttbZW60rValstdRcVRVFxRzaVHY2ALLKHNYGQZM7vj3tncmcyM5k1y8z7eZ48mXvOufe8986d73nve849R4wxKIqiKKlFWmMboCiKojQ8Kv6KoigpiIq/oihKCqLiryiKkoKo+CuKoqQgKv6KoigpiIq/oihNChGZIyKXNbYdyY6KfwoiIveLyHcisldEVorIxSHKiojcKiLrRGSPiEwXkdaO/HNF5HMRqRCROQH2P11ElorIPrvcAEfeIBGZJSLbRaTOCyci0lZEXhWRchH5QUQuCNeuKK5JkYh8aJ/HShE5wZF3iYjU2Ofg+RsTbV1KwyMib/t9fwdFZEmI8sfb90GFfV/0aEh7GwIV/9SkHDgdaANMAh4UkaOClL0YuAgYBRQALYC/O/LLgL8BU/13FJFi4DngCiAPeAOYISLpdpEq4AXg0iB1PwIcBDoBFwKPicjAMO2KlP8CXwHtgFuBl0SkgyP/C2NMK8ffnBjqanI4vpOGrFNEpEE0yBhzsvP7Az4HXgxiV3vgFeB2oC2wAPhfQ9jZoBhj9C8Bf1iC9DKwDVgDXOfIuwt4CeuG2gssAoY48m8CNtp5q4DjE2zrDOCGIHkvAb9zbB8FHABy/MpdBszxS7sGeMuxnQbs9z8foI91K/qktcQS/r6OtH8DU8OxC6thewrYZF/L3wOuIOfYF6gEch1pnwBX2J8vAT6N8tpeAnwGPADsAlbbtl4CrAe2ApMc5bOA+4F1wBbgcaCFnZcPvGnfUzvtz4WOfecA99r17QXeBdoHsWsMsMG+1zbb1zYNmAx8D+zAapjb2uWf9dwjQFfAAFfb272xnIC0MG2cYtu43/7uTwRWAruBh4GPgMsSeL8XATVAUZD8y4HP/e7F/UD/RP4OG/pPPf8EYHszbwDfYP1QjgeuF5FxjmITsDyPtsDzwGsikiEi/bBE83BjTC4wDlgbpJ7JIrIr2F+YtrYADgeWhSrm9zkLKA7n+AH2FWBQGPv1BaqNMd860r4BBjq2Q9n1DFCNJS6HASdhNVCBGAisNsbsDVHXYXZ46lsRuT1CT3kEsBjrqeJ5YDrWNe8D/Ax4WERa2WWnYp37UDu/K3CHnZcG/BPoAXTHEqSH/eq6APg50BHIBH4bwq7OWPdfDyzBuxb4CXAslvOyE+vpCyxBHmN/PharETvGsf2JMcYdpo0X2fXlYgn+K8BtQHushmdUMINF5IJQ97yIdA9xvh4utu1dGyR/INb3D4Axpty2a2CQ8s2Txm59kvEP68e+zi/tZuCf9ue7gLmOvDQsD3U01g9+K3ACkNEAtj4LvANIkPzLgG+xvKU2WE8JBhgZoNwcv7T+WCGmMVhCdDvgBm72KxfI8x8NbPZL+6WnjlB2YYWJKrE9Zrv8+cCHQc7xIuf3YadNAZ6xP/cCetrf06HAcv9zCHF9LwG+c2wfatvZyZG2A0vsxb5evR15I4E1QY49FNjp2J4D3ObYvgp4J8i+Y7CerLIdaStwPJUBXbBCc+lY3v1O+xo8DvwK2OC4h34TgY33OLYv9vstCNYTSSI9/1LgkhD5T2E/YTrSPgu1T3P8U88/MfQACvy88FuwRMnDes8HY3lMG4ACY0wpcD1WA7HV7sgsSISRIvJnLC/8XGPf4QF4GisePgfr6eBDO31Dfcc3xqzE6lN4GKtxa48lnPXuC+wD/DtwW2OFM+qzqweQAWxyXP8nsLxhRGSZo+NvdH11GWNWG2PWGGPcxpglwD3AOWGcg4ctjs/77WP6p7UCOgA5wEKH3e/Y6YhIjog8YXd+7wE+BvJExOU41mbH5wr7uMHYZow54NjuAbzqqHsFVnikkzHme6yGaShWw/wm8KP9pHos1pNBuDaud3wuwPe3YPzy44qIHI31xPNSiGL13XtJgYp/YliP5a3lOf5yjTGnOMp083yww0SFwI8AxpjnjTFHY/0YDfCnQJWIyC1+Ixh8/kIZKCJ3AycDJxlj9gQrZwvencaYImNMIZbQbrT/6sUY85IxZpAxph1wJ5anPj+MXb8F0u1OYw9D7Prrs2s9luff3nH9WxtjBtr7DjS1nX+f2Pv2EpHcQHUFOi18Q07xYjtWQzDQYXcbY3VQAtwA9ANGGGNaUxt2idYW/wZ/PXCy332bbYzxfNcfYTV6mXbaR1iNez7wdQQ2OuvdhO9vQZzb/ojIhaHu+TDCPpOAV4wxoX4fy7C+f0+dLbGefEKFRpsdKv6JYR6wV0RuEpEWIuISa1jj4Y4yw0XkLDt2fD2WWM0VkX4icpyIZGF1YO7HCpXUwRjzB+M7AsXnL5hxInIzVmz4BGPMjlAnItZwy972yIwBwF+xHtvddr5LRLKxQgNpIpItIhmO/YfbZToA04AZ9hOBZ7RHNlZICHvfLPvcyrFiwfeISEsRGYXVT/Lv+uwyxmzC6uz8i4i0FpE0u+yxQa7jt1jidadtw5nAYKwOe0TkZBHpZH/ujxW+et1xjnNE5K5Q1zEc7Gv6D+ABEfE8pXSV2r6iXKz7YZeItMVqTOPJ48AUsYc1ikgHEZngyP8Iqz/qY3t7jr39qTGmJkob3wIGOn4L12F55gExxjwX6p43xqwLtq9Y/VvnYvUHheJVYJCInG3fn3cAiz33bbKg4p8A7B/CaViPyGuwPLonsWLTHl4HzsOKo14EnGWMqcLqtJxq77MZK1Rxc5xN/ANWZ1ypw2O6xZPpCIeAFaqZifXI/zbwtDFmmuNYF2H92B/DCgfsxxIwDw9ijXJZZZ/rLx15PezyHo9qv13Ow1VYQzi3YoV4rjTGeMrWZ9fFWI3Kcrvel7Bi2MGYCJTYZacC5xhjttl5xwOLRaTcrvMVrGvooRtWTDge3IQVk55rh01mY3nSYA2pbYF1b8zFCgnFkwex+k7eFZG9dh0jHPkfYYm7R/w/xQpTfewoE5GNxpjtwE+xrvkOrA77eF1Lf36CdS9+6J9hhwIvtG3aBpyN1e+zE+saTEyQTY2GBA/1KonC9hL7GGN+1ti2KLEhIoXAC8aYYO9JKEqTpMFf7FCUZMIYswFr3L6iNCs07KMoipKCaNhHURQlBVHPX1EUJQVpFjH/9u3bm6KiosY2Q1EUpVmxcOHC7caYDoHymoX4FxUVsWDBgsY2Q1EUpVkhIj8Ey9Owj6IoSgqi4q8oipKCqPgriqKkICr+iqIoKYiKv6IoSgqi4q8oipKCqPgriqKkICr+TYwat2FXxcGAeQeqath7oMq7XVXj5oX563G7g0/RUVZ+0LMMHW63YdveypD1G2PYuvdAwLyqGjd7HPVHQln5QaprAi5LEBXGGLbv8z0X/+vjpKrGzaJ1O3lz8Y/etK17D/D2kk1R21Bd42b6vHUcrA59Xjv2VdIQ06gYY9ixL/T36+FgtZvd+8P/LsvKD4a8z5ys3V7Ohyu3hn3spsCa7eV89O22+gtGif+92hRQ8W8k7nx9KRc+OZf73lnJmY9a05d/Vrqd8574gqH3vBdQxM574gsOvetd7/YjH5Zy48uLecMhaB52VRykaPJbDLv3PabPt1bFu+31pRz5x/f5at3OoHZ9sXoHR0x5n//MrftuyJX/Wcjgu95l0bqdfLN+F2634Ycd5Rhj2LLnAP+bX3cdjd37q9iy5wDD7n2P66Z/FbRh8Wd9WQUvLQy82uPa7eXc8+ZySn4/m8+/305Z+UG276tk/N8+5rB73gu4z29e+IazHv2ca57/ytsIXfmfRVz53CK276vkhx3l/G/+Ol5csJ6qGjf/+Hg176/Ywp4DVT6CumTDbt5esonNuw8wc+lmJr+yhKueW+QV9zXby3nmszV8/O027p+1ir/N/pbhv5/NeU/M5X/z19UR0B937ad061427669Li8v3MDiDbtYX1bBxl37uemlxVzyz3lcP/0rLn1mvreu8spqzn38C95abDVgt722lOG/n82m3fvZV1nNj7v2M29NGWc9+hlzV+9g0+79HKiqoeJgNedN+4Ihd79LeWU1a7aX89pXG9lzoIpzn/iCZT/uZsmG3fywo5x3lm7inaWbGHbvezww+1t2V1Tx1Kdr+GrdTlZtDryq4Zj75/DzZ+YHbRTXl1X4XK+qGjfrdljnunt/FffPWsWUt5b77FNeWc3WPaHvne37Kus0aG63Yd2OCu/2roqD/OPj1dS4Dau31S7mNfb+OUx6eh77D9b4fBc1bsOTn6xmxaY9XPP8In76+OdM+/h7b94363fx5CereeazNcxcsokPV9U2egvWlvH61xv5z9wfKPn9bOatKQto94ertrLI8Ztct6OCNdvLqXEb1pdVUBVHp8lJs3jDNxl59gtLXD8rtRbSMsZw4ZNfevP3HqgmN9u7IBZl5Qf5ZsNuwPJYO+Zms3HnfsDyeP1xejEff7uN84/ozswlm6hxG5Zu3E27lllc+ux8fnlML578ZDU3je/P8Yd0YleF9eN5bM73nHlYV655fhGHdGnNjeP7M3uFdWOf9ejnAJxyaGdmLtnM1LMO5dWvNvLlmjJWbNpLhkv4Zv1u/nTOYE576BPKD1r2zVyymZlLNvPpTWMpzM8B4Icd5fy46wDfbtnLEx99T4tMFy9dcRSj77PW22iV5eKkAZ158tPVHNe/E3//4Dte/7q2sbvgH7XXzMO4Bz7mgfOGUuM2DOraGhHhjW9q91lXVkF5ZQ0Lf7B+cHe+voy3HE8Av3tpcZ1jHt2nPWu2l7Nx1/46ebNXbOHyfy/krjMGct87K3l76eY6ZeatLWPe2jJuenkJlxxVxKmDu/DLfy3wXu+8nAy+vuMkVm7eww0vflNnfyezlm3hsO55nPb3T9m2t5J5a8v477z2fFq6HYDrp3/Nl35CM3HaXO9nV5pQYzdCA++cVef4pz70acB6//5BKf+dtz5sL7bvbW9z4oBOHNmrHa9/vZEtew6Qk5nOmu3lPuVG9mrHF6vrLiiXl5PJiQM6ce4TX3ivE8AFI7pzXom10mOH3CxyMl08+P53/POztQwpbMMlo4po3yqLjrnZnPb3T6iqMfzqmF488fFq7zGe+nQNm/cc4LELh/l8p4fcUbv2zKmHdsFgmLnE9/ucv3Ynqzbv4+VFgZ2TId3y+Gb9rjrp5z7xBbedegiXHt2TZz5fy/R568nLyajzXXlo2zKTsvKDXHd8Mb85sW/AMrHQaLN6ish4rJWDXMCTxpipwcqWlJSYZJveoWjyWz7b3//hFHrfMtO7/fnk4yjIa+HdvmvGMp75fC0A71w/mv6dW3PN84t4c/EmHpw4lJ7tWwIwuDAPgF/+awHvLbfWCB8/sDOPXzScwXfNYs+Ban59Ql8emP0tADmZLioO1tAhN4tteyu58/QB3P2G5XUN657HonXWTXz3GQO5c0bgJUzPHlZI6bZ9dW74rPQ0KoN4f2unnkpVjZviW9+u91odXpTP/LXBn1ZCMapPO649rthH/BSlOXFs3w48+4sjotpXRBYaY0oC5TVK2EdEXMAjWAuIDwDOt9dhTVnc9TTC+TmZ3s8Hq91U1biZaXurWekuznj4M8542AofGWO8wg+wbV8lRZPfYs+BasDytj1U2F65py9gif10AXiFHwgq/AAvL9rA0o2766QHE36wGr9zn/giaL6TaIUfrCercIW/dXbtg/CQbnn06tAy6noBjihq6/38/g3HUtyxFYX5LchwxW/t96vG9ObSo3sGzHvgvCFMGFoQ9rFKeuRzWZBjtWuZ6bM9sKA1H/52DFPOHMQFI3zXTO/tuG6BzrVLm+yANnd1ODuh0uJBmsBffjqk/oLA0G55SD1f2SVHFQVMnzC0gA65WRFa50ui3PPGCvscAZQaY1YDiMh0rMW5l4fcq5mxr7KaVll1L3GgGJ6/+Pt/4V3aZHs/v7Vkk1foAa74z0Kfsje/ssRne11Zhc/2K19tDGqzKy0yYSrpkc+CH3Z6wwiR8NU63yeF4/p35ANHR+HAgtZs21vJVr9O6lZZ6bzwq5Gc8tAnEdX3syO70yorg8c/+j5g/sLbT/Q+ibx21VGICJf/awHvLt/CX88dwr/n/lDH5lA8MHEod76+jNkrttCuZSZvXWcti7x+ZwUPf1DK4UVtueXVJXX2G9S1Nf+4uISRf/wAgDOGFFBjDG8t3sQxfTvw8bfb+PM5gzl9SAHZGS6+XL2Dpz5dQ//Oucy8bjS/ffEbXvlqI4d2bcOZhxXyl58O4ZPS7fRq35KD1W5OfMBacreoXQ6vXjWKD1dtRQROG1xAhiuN347rR2WVmwmPfMraHRWMLm7Pvy8dwcIfyjj7MavB/s+lI8hvmel94jx5UGd27DvIkb3akd8yg8UbdtO5dTZd81rQy36ifXDiUP5v+td0bpPN7acN4PbTBjBq6gds3LWfrPQ0BhS0ZuOu/eTlZPDxjWNpmZmOK004UFXDwRo3rbMzuO+dlTw6x/r+vrzleHIyXcxbU8bhPduSne6i723W9/fZ5ONYsmE3FQer+c0LVhjt29+fjMFQXWPIyXQhIizZuJv3V27hT2cP9oYQ/3jWoXzy3TZaZaVzdHEHzhhSwOINuzjj4c/46fBCBhe2oWt+C/YeqGZ9WQUXH1VE6+wMxvTrQKusdLbvO8gbi3/k3gmDaGs3mnNX76Bb2xxGTf3Ae/36dc7l8CmzAXj96lG8uHA9k08+hNXb9rF2RwVnDCng3jeXex20eNNY4t8VWO/Y3oDvQtGIyOXA5QDdu/t6Fs2BN775kWv/+xVvXns0g7q28ckLFKO/K4RnDWAczcHrX9Xt4PVQ4zbeDl4PGREIenoEXukJh3TiD2cO4oYXv+GT77aHvV8wrh7bh7tOH8gxf7bi/c/+4ghq3Ibxf/uY/JxMVtux4m5tcxhQ0Npn3wtGdGfh2p2s3VHO6OIOzF6xpc7xM1xp3DS+H6cP6cLSjbu56WVLeIf3yKddy0wyXGk8dP5hVFW7EdvVG1zYhneXb6FXh1b8blw/LvjHl1wwojs/GdrV++TSOjuddq2y6sSy83MyeHDiUJZv2kOe48mtd4dWPHDeUADeWbaZj/1GmeRmZdDS4TQ8dP5hADxygdWBmeb3febbApObnU5amjD17MGcOawrfTrmApDuSmNsv46A7733whUjyW+ZyVnDCn2Ol53hIjvDxezfHMsf317JpJFF9vnU1pPv9yQwuth31uDDHU891jml076V5QF3yq11ZDwe9eM/G07fzrkUtMnm1lMHkJleG5Tw2ANw5ZjevL9iK2P7d6RTa+s4xx/SyVt26d3jWLFpD13zWnifGob3yKfabbzHdPpjd55uNUJOB+n8I7pz/hG+mjO4MI/nLxtBSVFbH9ucjLGvMcD4QZ198o7s1Q6At647mg6tsujYOtsnf0i3PIZ0y/PW5Qnf3n5a4gIiTbbD1xgzDZgGVsy/kc3xYc32crq3zQnpJXs6XJdv2sOgrm146tM1zFq2mRd+NZL9AcT/v/PW10lz4nSsA+0fKu9APUMRnUTi+T85yQolBhv1EYgzD+vKqwGePG44sS/De+T7jKzJa5FBuiuNr+44iQfe+5YH3/8OwNv5deWY3ryzdDPP/PxwerSzPFBjDJ+V7ggo/oIgIgwsaIPbcUlevrJ2Cd4zhviGSa4c04fRxR28P8y1U0+tc9xFt59IuivN24/z4MShbN1TSU6m9fPyF0IngdrazPQ0WmYG/mn6Cz9AccdW/ObEvpw9vNC7v78Ye/CI6JDCNnTMzQ5YxkO6K81HfDwNWEGbyEIxH/1uDC2z0snPyeSasX24ZFSRN89zv+Vmp9M1rwV3TxgU8li52RnM+vUxQfNbZaXXud6eeyMQIoJLoF2rzKBlPBzVp329ZepjYIGvI/i384bGHBaKlsYS/41AN8d2oZ3WZFlfVkF2houKg9WMvX8OV43pzY3j+4e9/71v1ka0DhyMfOiWMyxUWR1c/KcFCGnsj+Cx0VVfcDMAbVtm1gnNBGNAl9YBxb+VHW93erzprloPq7394zxneCEnDrA8vZvG9+cmv+9ARMJ6eilqn8Ogrq25ZmyfkOVcaeIV/mA47QTrBz5haKt6bbCOX9eLbJHhiqgRFhGuO7447PKfTz6ONi0y6i/oR9uWmdz7k0Ec179j/YUdOMX3t+P6+eR57rdW2Y3rh+YGCM82BD85rGuj1AuNN85/PlAsIj1FJBOYCMxoJFvCYvR9H3L4lNlekQs2ZtdDqP7bUJ57MJye/4Gq4I3HQx+UxlTf1wGGqNXHoxcOC7tsIM8VakU/K8gj9ZnDChnTr0PQjjUnzk7Ghy84zPvZGTrLzc7gzWtHM35Ql3DMDkgwW1tkusI+hqfdGF1c61UWta8VS2d6vCjIa+HTyEbCRUf2iGsnrOd+SIvC6YgnIsI5wwt5cOLQRrWjIWkU8TfGVAPXALOAFcALxpjQQe8mQqQjYwPd0p63ZAvaBH/s9uxnjOFAVU2DvCEKeN8liISiEI/Vd/jFLP21/7DullftSZYgItAqK51nfn5Enf6TQKQ7vOnTBhdw26mH1LtPNLx/w7E8/8sRddJzMsIX/z4drSeEq8b0Yd6txzN+YGeuOLYXACvuGc/TlxweH2ObKI9cMIyzhnWlV/vYRlbFg/t/OoQJQxvPE29oGu1ZyxgzE5hZb8EmhkeE63NUnF7mWr+OwB/tl0qK2rfkx92h31p89vO13PXG8piHHcaLqWcdymS/0UTBvHmo6wX7hzNG9mrHV+t2sa+yOm42RtJpHQuF+Tnel9WcROL5X39CX0p6tGVkb6tD8PGLhkd1nOZKv865/PXc1PG2mxJNtsO3qSOI91XyUPFTEWHM/XN80jbYb+b2bN+Sz7+v+2YjwB/fXklxx1bejsvV28oDlouFKWcO4tZXl0a0T++O4cWyPaT7ib3z8f5v5w3lhAGdOFDl5tyS2i6g+84e7POCW6Rk+MXge3ewbD6kS+tAxeNGiwwX+6tqgoaDApHhSmNshDF0RYkHKv4R4gy+DLnbmmfHMwLk2c/Xclz/jnRrW9cbdLKr4iDZGWneMcCB8ExH0L9zbti2Fea38DYs4RAqXBOMnAi9UX8hdop/ZnoarbLSueN039DQuYd3Ixb8G5yx/Tsy87rRHNIl/GsZDW9cO4ov15QFDV0pSlNCJ3aLEE/o3f/3va+ymjtnLKv3bVK32+A2Vlw6lPh7iGRSp6H1jErxJ9IXuoCAL60FwuP9+odgnG2Bf8MQLwIdd0BB64SLcp+OuVw4okdC61CUeKHiHyGeWL6/jniGYtY3TW6NMbiNQQTvSy+hqKoJv6M32MsnwfD3kMMh3FEiHqvT/YYyip/nnwgaKuavKM0ZFf9ICaLFztE5oahxG4yxwh/hvFgSieeflR5ZSMaVJsz+zbERdSZnBxnJ0ql1FoML2/Der4/hghHd6ZZvxez953ZxvkcQzzlunPg3OIqi1EV/JRHikXbxG8Tp8WgNULp1H68sCvzOWrXb8vzThLiHfSLpaARLJPt0bMXzlx0ZdsgoM0io5stbTmDGNUdT3CmXP5x5qFeA/V+AcoaaIrU3XBLVqChKMqHiHyXBwsfGwN8/+K62nF9+jVf8JSxPPdTMmP5EGkbxCHHnNtncfcbAsPYJV1g91ycjTZh/6wkMKWzjk24dK1FhH72tFaU+9FcSIcGiOp6Yv8GEfFuxxu7wFZGwPN+9B3zHv4cK00fs+TuEPNzO30g7TdNdaXTIzaKrHQZy1pMw8Y+iL0NRUg0V/wgJ1uHraRSM8c3zL2fF/K0O32g6PEOJdLCQTDCcIhlM028+Ofz5i5x4Ggn/zldnuCxRHb6JalQUJZnQX0mEeId6+gd0TO2/OnkOatwGt9vy4KOJeYfqT86IIubvIVij4pzuN9iiIYHwHC2UFx5pYxUu0QxhVZRUQ1/yihBvh28d7a9V5VDaU+12e0ND0Xi+ocYSRdqYuBxeebBQldNGz/S+d54+oN4XxMKJDiXK81cUpX5U/OthfVkFhfn1TzXgnXXThJ6h0O22yqaJROX5hhpKGo6YpqcJ1baxTq88mM2BGpSfj6r/CcBzvJBPKhqeUZRGQ8U/BJ5l66aedag3LZj4GkeHbyivt9rt9r7kFc0bp6E8/3Aak7Q08bZUztqDPa1EG5rxnFpIexPo+d93zmAODWMGUEVJVVT8Q7B8k7VC1WLH4uS1YR9ftXSKXChRd5val7yiIZQnHY6Yemo9rn9Hn6XkgsXJQ83YGU49oZ5UEjke3zlRnKIoddHn7hActMfY+6yB6+3w9cU71NP4etH+Gu98ySveBGtQzi0pZNJIa84Zz8RsPzvSd43SeC+m8cezBnNs3w511tp1kqgOX0VR6kc9/xB4xN+51F6woZ4+o31ChX1qjDfmH2+Cjay54/SBtMx0ccuph3DsfXOAqgBvKMfXlgEFrXn2F0eELKOzXypK46GuVwg8Uys4x6qbAJ6/2228yzsa4/uSl7/Iuh0Tu8WbYCGaNPG8VOaqrdevqA6PVJTUQsU/BLWef11hdHqt1//va077+6eA5fk7xf+A3/q51fZLXp4yZzvG0cdKsMXXAz1l+KfV9yQSyboCiqI0fTTsE4LKaku4neGUQP2XM+yFVwLl/232dz7bbu9LXtYxJwwt4OVFG+JirytIB2rAxstvO5T4f3HzcbTODr5amaIozQ8V/xB4PH/nm7Cejt1QfvLc1bVLM27e47tGr6fD16O18Yr9d8jNCur5B0r3TwoV9enSJvolFRVFaZqo+IfgYICYv+dlrlALjq/cvDdonhXzrxX9eIXa/37+YdS4Aw+rDNQX4N/oNGTMf85vx7B9X2WD1acoSl1U/EPgmU7ZKZQez//LNWVRHfP215by/bZyBtpDIOM14iVNhAM1vv0L4wd25rLRgd/G9a+1IUfeFLVvSVH7yNcPVou46bsAABlNSURBVBQlfsTU4SsiPxWRZSLiFpESv7ybRaRURFaJyDhH+ng7rVREJsdSf6LxLKFY466dU98d6i2rMPh+WzkQf8/flQZb9/h603k5GZQUtfVJkzofPPvraB9FSSViHe2zFDgL+NiZKCIDgInAQGA88KiIuETEBTwCnAwMAM63yzZJPG+nOtfRDRJZiRiP1kb7Bq0/IsLovu3DfnGq7mgf3/xrj+tDSY/8uNimKErTI6awjzFmBQQMGUwAphtjKoE1IlIKeN74KTXGrLb3m26XXR6LHYnCo/POWHp9a/SGS7zDLILVMbvsnnEU3/p2WOWd+DcGN5zUL37GAdceV8yyH/dwdJ/2cT2uoijRkahx/l2B9Y7tDXZasPQ6iMjlIrJARBZs27YtQWaGxuv5xzHs48HjaQfrpI38eOLzH0K/tev/xJGIN46dHNKlNR/9bixtcnTIqKI0Ber1/EVkNtA5QNatxpjX42+ShTFmGjANoKSkJE7BlkhtsP4vWLvTm+YOf0ndkHjE1h138Q+vfF3PPy5mKIrSTKhX/I0xJ0Rx3I2Ac1rFQjuNEOlNDo+Xv/CHWvGviZvnL2Efr1VWesihpVDr5TvDSaEO7e/oa4evoqQWiQr7zAAmikiWiPQEioF5wHygWER6ikgmVqfwjATZEDOBnPL4xfyt/+GEfTq3ya63TKRRG/8+BxFh7dRTIzuIoijNlliHep4pIhuAkcBbIjILwBizDHgBqyP3HeBqY0yNMaYauAaYBawAXrDLNkkC6Xz8RvvYYZ8oG5NHLxwW8Hjhon6+oqQ2sY72eRV4NUjeFGBKgPSZwMxY6m046gpz3Dp87Wa3Jso+BP/pm8MVf4/Hr9MpK0pqo7N6+rHwhzJ+3LUfCOzlx9vzDyfsEyjUlO7yD9tEWn9k5RVFSS5U/P04+7EvGH3fh0BgLz/e4/wPLQxvndmPfzeWd399jHfbOdkcBBbzUA2C/zoDiqKkFjq3TwA83nggnY/fuHzrf9e8FrjSpN7jdm+X47Nd1/OvK+aRjPZRFCW1UM8/BIG0M15hn3VlFTHtX9fzj7DDV8VfUVIaFf8QBArxRBP2GdGzbZ20NdvLo7LJg/+4/Ei1XMM+ipLaqPiHIFDMP5rRPmcPr7tUYySHCVQ0Kz02zz9Nv3lFSWlUAkIQr3H+LTNDd61E44P7i3/EL3mp568oKY2KfwgCi3/k6u8v1PEgK93lsx35G75xNEZRlGaHin8IAg/1jPw4Mc+bE6DOzFjDPir+ipLSqPiHIJD4RzPUMxFedjjif8moohBHUPVXlFRGx/mHIJDQRxP2iXWu/PA6fOuW6d+5dZ202tk/A9f1zvWj2bHvYIQWKorS3FDxD0Fg8Y/8OPWJfzRtg7/nH6kjH8ymQA2GoijJh4p/CALNtR/NOP86Qh0HQk3s9ua1R9O2ZWbI/TXooyipjYp/CALNuBlN2CdW8Q/U4PhP5+AU/0Fd658vSEf7KEpqox2+IagJsGZjNGGfRAz19Eff8FUUJRJU/EMQMOYfhfoH8vyHdMsLe/9wakz0AuyKoiQXGvYJQeD5/GN/yWvp3ePIdNWmWV54bDPGiTbjiqJEgIp/CAJ5/pXVkS+95e/5t8qK/2VXz19RlEhQfzEEgbz8f33xQ8TH8Z+KwZ/fjusbMj/Yw8bkk/t7P4cr/dpGKIoCKv4hidOiXfV2+F5+TG8K81uEdaxcx1PDFcf29n5Wz19RlEjQsE8D4IzvByPchuazm4+jsqpu6Em1X1GUSFDxD0G81utNC2MWtVB1GUdncOvsDMiuWyZS8TcxdjAritK80bBPCBpSHmOtS8M+iqJEQkziLyJ/FpGVIrJYRF4VkTxH3s0iUioiq0RknCN9vJ1WKiKTY6k/0cQr5g9w4/h+EZV3TgMdjh0q/oqiREKsnv97wCBjzGDgW+BmABEZAEwEBgLjgUdFxCUiLuAR4GRgAHC+XbZJEs/QyFVj+oSuy6+qSOfbD7e8vtmrKArEKP7GmHeNMdX25lzAs1jtBGC6MabSGLMGKAWOsP9KjTGrjTEHgel22SZJPD3/+vAfVhqpSPvP9ROMDrlZAKSH0QmtKEryEs8O318A/7M/d8VqDDxssNMA1vuljwh0MBG5HLgcoHv37nE0M3waNebv0PJ4NkKP/2w4H67cSte88IaWKoqSnNQr/iIyG+gcIOtWY8zrdplbgWrguXgZZoyZBkwDKCkpaZShKYnw/I/r3zGsuhIVnOmQm8W5h3dL0NEVRWku1Cv+xpgTQuWLyCXAacDxpna84kbAqTCFdhoh0psczuGXWelpUU3t4OS7KSfjChqe8Qv7aGheUZQEEuton/HAjcAZxpgKR9YMYKKIZIlIT6AYmAfMB4pFpKeIZGJ1Cs+IxYZ48e6yzazdXu6T5pTjo/u0j7mODFda0DH/dT1/VX9FURJHrDH/h4Es4D27w3GuMeYKY8wyEXkBWI4VDrraGFMDICLXALMAF/C0MWZZjDbEhcv/vbCOt+30/MN5USsWQkWY8ltmJLRuRVFSj5jE3xgTdPyiMWYKMCVA+kxgZiz1Jgp/79u5GTxcE6+6fSt3tjVPTTo8oXUripJ66Hi/EDj1OC3BV8rf8/cM3ezVviWdWgeYz0FRFCUGVPxD4BP2Sbjn77utEX9FURKJin8IfMI+iY75+6n/xUf1SGh9iqKkNir+oXDoccJj/n7bZw0rDFhOURQlHqj4h8ApyIke7aMzLCuK0pDofP4hcIZiGtrzL8xvQf/Oudx+WpOd905RlGaMin8IGtLz79upFYvW7fJuZ6W7eOf6YxJap6IoqUvKh32qatwcqKqpk75p934qDtamJ3oSzKcvOZznLgs4x52iKErcSXnP/6xHP2fJxt110k958BOf7fQwB/o/OHEo/zf964jtyMvJZFSf9jw4cShZ6a6I91cURYmElBf/QMIPsLOiymc7PYywT35OBicO6BSTPROGdq2/kIOrxvTmveVbYqpTUZTUI+XFP1xcrvBi/g29nOKN4/tz4/j+DVqnoijNn5SP+YdLRqLnd1AURWlAVNHCJD0Mz19EdB5+RVGaBSr+YRJOzF9o+LCPoihKNKj4h4lzwfMXrxjJlDMHBSyn0q8oSnNAO3zDxOn5F+S1ICcz8HBM9fwVRWkOqOcfJuIQ9WARoG5tczTmryhKs0DFP0yc/b3OeX4K2tQutPLUpBKfRkJRFKWpouIfJs75/EXEu8B66xa16+u2a5XV4HYpiqJEg4p/mDgndkv07M6KoiiJRsU/TJyhnkSv6qUoipJoVPzDJM0v7KMoitKciUn8ReReEVksIl+LyLsiUmCni4g8JCKldv4wxz6TROQ7+29SrCfQUDg9fxF0VI+iKM2aWD3/PxtjBhtjhgJvAnfY6ScDxfbf5cBjACLSFrgTGAEcAdwpIvkx2tAg6NQ+iqIkEzFJmjFmj2OzJbWLX00A/mUs5gJ5ItIFGAe8Z4wpM8bsBN4DxsdiQ7T86t8L+HDV1rDLi9+7u564f1a6tgqKojQ/Yn7DV0SmABcDu4GxdnJXYL2j2AY7LVh6oONejvXUQPfu3WM10wdjDLOWbWHWsvrnwT+uf0dG9mrnE+ZpnZ1BblY61x1fzLklhRz9pw/jap+iKEqiqddtFZHZIrI0wN8EAGPMrcaYbsBzwDXxMswYM80YU2KMKenQoUO8DmsfO/yy/Trn8stjenk7eU8b3AWwOn1/c2JfCvNz4mqboihKQ1Cv52+MOSHMYz0HzMSK6W8EujnyCu20jcAYv/Q5YR4/bkSg/V48jn8kDYeiKEpTJdbRPsWOzQnASvvzDOBie9TPkcBuY8wmYBZwkojk2x29J9lpDYqJQME9RT1hHxOg6XjruqN57epR3u33fn0ML14xMiYbFUVREkmsMf+pItIPcAM/AFfY6TOBU4BSoAL4OYAxpkxE7gXm2+XuMcaUxWhDveyrrCYnw+Udqx+N8+6ZrdPtrps3sKCNz3Zxp9woalAURWk4YhJ/Y8zZQdINcHWQvKeBp2OpNxJ2769iyN3vcvXY3vxuXH/bhvD393j6aSE8f0VRlOZG0o9T3FVxEIA3vtnkTYtOwG3PP4Jd27fK5LZTD4miLkVRlMSSkou5RNRp6x/zj2DfBbedGEFFiqIoDUfSe/7xGp1TO8xfwz6KojR/kl78PThf0oqmQfB0+OpQT0VRkoGkF/9AWh1JzN9T0tN4uFX9FUVJApJe/D04Z+aJRr8HF+YBcMmonvExSFEUpRFJzQ7fSMraLUWH3CzWTj01MQYpiqI0MCnj+TuJ5A1fRVGUZCTpxT+Q0Efm+cfPFkVRlKZC0ou/B+fSiyroiqKkOikj/gBLN+7m5Ac/obyyurFNURRFaVRSSvynvr2SFZv2sPCHnY1tiqIoSqOSUuLvIYrZHRRFUZKKpBf/gC95adBfUZQUJ+nF34PUX0RRFCVlSBnxdxLRfP76kKAoShKSmuKvkXxFUVKcpBf/QJ57NCt5KYqiJBNJL/5eYpzSWVEUJZlIAfHX6R0URVH8SXrx96y56zulsyq6oiipTdKLf8CYf8OboSiK0qRIfvEPJPWq/oqipDhxEX8RuUFEjIi0t7dFRB4SkVIRWSwiwxxlJ4nId/bfpHjUH4rAnn8EyzhqiEhRlCQk5pW8RKQbcBKwzpF8MlBs/40AHgNGiEhb4E6gBMv/XigiM4wxCZtpLdahnoqiKMlIPDz/B4Ab8Q2mTAD+ZSzmAnki0gUYB7xnjCmzBf89YHwcbAiKZ8F153z+VW5Vf0VRUpuYxF9EJgAbjTHf+GV1BdY7tjfYacHSAx37chFZICILtm3bFouZdbj9taVxPZ6iKEpzo96wj4jMBjoHyLoVuAUr5BN3jDHTgGkAJSUlUbvqGuJRFEWpS73ib4w5IVC6iBwK9AS+sUMqhcAiETkC2Ah0cxQvtNM2AmP80udEYXfYxDo9g7YdiqIkI1GHfYwxS4wxHY0xRcaYIqwQzjBjzGZgBnCxPernSGC3MWYTMAs4SUTyRSQf66lhVuynEcrORB5dURSleRLzaJ8gzAROAUqBCuDnAMaYMhG5F5hvl7vHGFOWIBuAWs892vn8tfFQFCUZiZv4296/57MBrg5S7mng6XjVWx+1o30aqkZFUZSmT/K/4Ruj565TOiuKkowkvfhrl62iKEpdkl78o/H8Ref+VxQlyUl+8Y9inzTtIFAUJclJevF321M5SATjfVwO8T9hQKe426QoitLYJL34R+P5e7T/pvH9GduvY1ztURRFaQokv/hHof6uNEv9a9zuOFujKIrSNEh+8Xf4/uGG8j1hn2qd/VNRlCQl6cXfGfcJ9ykgzfb83Sr+iqIkKYma3qHJ4NHvjbv2s2rL3rD28YR91PNXFCVZSXrP3xP22VdZHfY+tvZTo4P8FUVJUpJf/KPQb884fw37KIqSrCS/+EexT262FQ1rkeGKrzGKoihNhKSP+ZswXX+R2qeEc4Z3I01g0lFFiTNMURSlEUl+8Y9inzSBXx3bO+62KIqiNBWSP+wTRdBfI/2KoiQ7KSD+4ZVzvv+lg3wURUl2VPwD7aO+v6IoSU7yi3+Y5USncVYUJYVIfvEPd7SPzz6JsUVRFKWpkPTiH+57Wur4K4qSSiS9+EczdieaEUKKoijNiZjEX0TuEpGNIvK1/XeKI+9mESkVkVUiMs6RPt5OKxWRybHUHw6q44qiKHWJx0teDxhj7ncmiMgAYCIwECgAZotIXzv7EeBEYAMwX0RmGGOWx8GOgETW4WuV1gZDUZRkJ1Fv+E4AphtjKoE1IlIKHGHnlRpjVgOIyHS7bOLEP5px/gmxRFEUpekQj5j/NSKyWESeFpF8O60rsN5RZoOdFiy9DiJyuYgsEJEF27Zti9q4cMfsOzt81fNXFCXZqVf8RWS2iCwN8DcBeAzoDQwFNgF/iZdhxphpxpgSY0xJhw4doj5OuKN9VPAVRUkl6g37GGNOCOdAIvIP4E17cyPQzZFdaKcRIj0hRDe3j7YEiqIkN7GO9uni2DwTWGp/ngFMFJEsEekJFAPzgPlAsYj0FJFMrE7hGbHYEC/SHHEffQpQFCXZibXD9z4RGYrVR7oW+BWAMWaZiLyA1ZFbDVxtjKkBEJFrgFmAC3jaGLMsRhtCEq6Qt2mRwf6qGqB2MRdFUZRkJSaVM8ZcFCJvCjAlQPpMYGYs9UZCuCGcy0b39H7WRVwURUl2kt7FdbvDK5eeJlwyqmf9BRVFUZKApJ/eQcP3iqIodUl+8dfeW0VRlDokv/iHWU7n81cUJZVIevHXuI+iKEpdkl789YUtRVGUuiS9+K/YtDescto3oChKKpH04v/M52sb2wRFUZQmR9KLv6IoilKXpBb/g9VhvuGFjvZRFCW1SGrx37X/YGOboCiK0iRJavHvmJvN2qmnNrYZiqIoTY6kFn9FURQlMCr+iqIoKYiKv4329yqKkkqo+CuKoqQgST+fv5NJI3vgSkujc5ssNu+u5OnP1njz9AVfRVFSidQS/6OK6NWhFQB7DlT5iL+iKEoqkVJhH1dabWDfpUF+RVFSmJQS/zSH4DsbAkVRlFQjtcTfIfhpfp6/PggoipJKpJT4O0M9/o7/wII2DWyNoihK4xGz+IvItSKyUkSWich9jvSbRaRURFaJyDhH+ng7rVREJsdafyS0yHB5PzvDPt/ccRLDe+Q3pCmKoiiNSkyjfURkLDABGGKMqRSRjnb6AGAiMBAoAGaLSF97t0eAE4ENwHwRmWGMWR6LHeGSnVnb1jln8WyTk9EQ1SuKojQZYh3qeSUw1RhTCWCM2WqnTwCm2+lrRKQUOMLOKzXGrAYQkel22QYR/0xX3QedG07sG6CkoihKchOr+PcFRovIFOAA8FtjzHygKzDXUW6DnQaw3i99RIw2hI3/nP0646eiKKlKveIvIrOBzgGybrX3bwscCRwOvCAiveJhmIhcDlwO0L1793gcUlEURbGpV/yNMScEyxORK4FXjLX6+TwRcQPtgY1AN0fRQjuNEOn+9U4DpgGUlJTo5AuKoihxJNbRPq8BYwHsDt1MYDswA5goIlki0hMoBuYB84FiEekpIplYncIzYrRBURRFiZBYY/5PA0+LyFLgIDDJfgpYJiIvYHXkVgNXG2NqAETkGmAW4AKeNsYsi9EGRVEUJULENIPpLEtKSsyCBQui3v/9FVuoqnEzflCXOFqlKIrStBGRhcaYkkB5KTGr5/GHdGpsExRFUZoUKTW9g6IoimKh4q8oipKCqPgriqKkICr+iqIoKYiKv6IoSgqi4q8oipKCqPgriqKkICr+iqIoKUizeMNXRLYBP8RwiPZYcw4pei380evhi16PWpLhWvQwxnQIlNEsxD9WRGRBsFecUw29Fr7o9fBFr0ctyX4tNOyjKIqSgqj4K4qipCCpIv7TGtuAJoReC1/0evii16OWpL4WKRHzVxRFUXxJFc9fURRFcaDiryiKkoIktfiLyHgRWSUipSIyubHtaQhEpJuIfCgiy0VkmYj8n53eVkTeE5Hv7P/5drqIyEP2NVosIsMa9wzij4i4ROQrEXnT3u4pIl/a5/w/ez1p7DWn/2enfykiRY1pdyIQkTwReUlEVorIChEZmar3hoj82v6NLBWR/4pIdirdG0kr/iLiAh4BTgYGAOeLyIDGtapBqAZuMMYMAI4ErrbPezLwvjGmGHjf3gbr+hTbf5cDjzW8yQnn/4AVju0/AQ8YY/oAO4FL7fRLgZ12+gN2uWTjQeAdY0x/YAjWdUm5e0NEugLXASXGmEFYa4pPJJXuDWNMUv4BI4FZju2bgZsb265GuA6vAycCq4AudloXYJX9+QngfEd5b7lk+AMKsQTtOOBNQLDe2kz3v0+AWcBI+3O6XU4a+xzieC3aAGv8zykV7w2gK7AeaGt/128C41Lp3khaz5/aL9fDBjstZbAfTQ8DvgQ6GWM22VmbAc/Cxsl+nf4G3Ai47e12wC5jTLW97Txf77Ww83fb5ZOFnsA24J92GOxJEWlJCt4bxpiNwP3AOmAT1ne9kBS6N5JZ/FMaEWkFvAxcb4zZ48wzlvuS9GN8ReQ0YKsxZmFj29JESAeGAY8ZYw4DyqkN8QApdW/kAxOwGsQCoCUwvlGNamCSWfw3At0c24V2WtIjIhlYwv+cMeYVO3mLiHSx87sAW+30ZL5Oo4AzRGQtMB0r9PMgkCci6XYZ5/l6r4Wd3wbY0ZAGJ5gNwAZjzJf29ktYjUEq3hsnAGuMMduMMVXAK1j3S8rcG8ks/vOBYrv3PhOrM2dGI9uUcEREgKeAFcaYvzqyZgCT7M+TsPoCPOkX2yM7jgR2O0IAzRpjzM3GmEJjTBHW9/+BMeZC4EPgHLuY/7XwXKNz7PJJ4wUbYzYD60Wkn510PLCcFLw3sMI9R4pIjv2b8VyL1Lk3GrvTIZF/wCnAt8D3wK2NbU8DnfPRWI/ti4Gv7b9TsOKT7wPfAbOBtnZ5wRoV9T2wBGv0Q6OfRwKuyxjgTftzL2AeUAq8CGTZ6dn2dqmd36ux7U7AdRgKLLDvj9eA/FS9N4C7gZXAUuDfQFYq3Rs6vYOiKEoKksxhH0VRFCUIKv6KoigpiIq/oihKCqLiryiKkoKo+CuKoqQgKv6KoigpiIq/oihKCvL/cUAkcW6+f1kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import clear_output\n",
    "\n",
    "rewards = []\n",
    "for i in range(1000):\n",
    "    rewards.append(play_and_train(env, agent))\n",
    "    agent.epsilon *= 0.99\n",
    "\n",
    "    if i % 100 == 0:\n",
    "        clear_output(True)\n",
    "        plt.title('eps = {:e}, mean reward = {:.1f}'.format(agent.epsilon, np.mean(rewards[-10:])))\n",
    "        plt.plot(rewards)\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binarized state spaces\n",
    "\n",
    "Use agent to train efficiently on `CartPole-v0`. This environment has a continuous set of possible states, so you will have to group them into bins somehow.\n",
    "\n",
    "The simplest way is to use `round(x, n_digits)` (or `np.round`) to round a real number to a given amount of digits. The tricky part is to get the `n_digits` right for each state to train effectively.\n",
    "\n",
    "Note that you don't need to convert state to integers, but to __tuples__ of any kind of values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_env():\n",
    "    return gym.make('CartPole-v0').env  # .env unwraps the TimeLimit wrapper\n",
    "\n",
    "env = make_env()\n",
    "n_actions = env.action_space.n\n",
    "\n",
    "print(\"first state: %s\" % (env.reset()))\n",
    "plt.imshow(env.render('rgb_array'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Play a few games\n",
    "\n",
    "We need to estimate observation distributions. To do so, we'll play a few games and record all states."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def visualize_cartpole_observation_distribution(seen_observations):\n",
    "    seen_observations = np.array(seen_observations)\n",
    "    \n",
    "    # The meaning of the observations is documented in\n",
    "    # https://github.com/openai/gym/blob/master/gym/envs/classic_control/cartpole.py\n",
    "\n",
    "    f, axarr = plt.subplots(2, 2, figsize=(16, 9), sharey=True)\n",
    "    for i, title in enumerate(['Cart Position', 'Cart Velocity', 'Pole Angle', 'Pole Velocity At Tip']):\n",
    "        ax = axarr[i // 2, i % 2]\n",
    "        ax.hist(seen_observations[:, i], bins=20)\n",
    "        ax.set_title(title)\n",
    "        xmin, xmax = ax.get_xlim()\n",
    "        ax.set_xlim(min(xmin, -xmax), max(-xmin, xmax))\n",
    "        ax.grid()\n",
    "    f.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHgAAAKACAYAAADn488NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzde7hlV1kn6t8n4RIpSILBEpJIoURsIC0NZYgHj1aMHQKIiTYgdIREI2mOoNimWwOi0Fw02tAIrcCJIU0QmkAjSCRgiEhJ04cACbdwkU4RgqmQECEhUIBo4Xf+WLN0987eVfu29l6z9vs+z35qrTHHHHOsMWftGvVb81LdHQAAAADG69s2ugMAAAAArI6ABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMwT1Xtqarv2c/yT1TVjnXsEgCwiVXVq6vqBats41lVdcFa9QmYPQIeYE1V1b+tqiuHkOTGqnpHVf3wKtrrqrrffpafWVXfGrb3lar6SFX9xEq3lyTdvaW7rx3av92Eqrsf2N07V7MNAGBjreecpapOqKqvVdWWBZZ9uKqevtLtLlV3/3Z3/8KwzW1Dfw+Z9naB9SPgAdZMVf1qkt9P8ttJtib57iQvT3LqCtpazoTjfd29JcnhSV6V5I1VdcRytwkAbA7rPWfp7iuS7E7y2HnrPijJA5K8frnbBZhPwAOsiao6LMnzkjytu9/c3V/r7n/o7j/r7v841Dm+qt5XVV8evin7g6q605w2uqqeVlXXJLmmqt4zLPro8O3az+yvD939j0kuTHJoku+tqsOq6jVV9bdV9bmqenZVfduwrftV1V9V1W1V9cWqesO8ftyvqs5OcnqSXxu2/2fD8uuq6seH13euqt+vqs8PP79fVXcelu2oqt1VdU5V3Tx85p9bkwEHAFZkA+csFyV58ryyJyd5e3d/qaq+v6our6pbqurTVfX4/XyGp1TVrqHuJVV17znLHjinnS9U1bOG8udW1WuHavv6++Whvz861D9uTjvfWVVfr6p7LmVcgY0n4AHWyg8luUuSt+ynzreS/PskRw71T0ryi/PqnJbkYUke0N0/MpT9wHDZ1BuyH8M3aL+QZE+Sa5L81ySHJfmeJD+aySRqX8Dy/CTvTHJEkqOHuv+H7j4/yeuS/N6w/ccssNnfSHJCkgcn+YEkxyd59pzl3zX04agkZyX5Q2cXAcCG2qg5yx8n+ZGqOiZJhi+d/m2Si6rqrkkuT/Lfk3xnkickeXlVPWB+I1X1Y0l+J8njk9wryeeSXDwsu1uSv0jy50nuneR+Sd61QF/29ffwob9/NbTxs3PqPDHJu7r7bxdYH5hBAh5grXxHki92997FKnT3Vd19RXfv7e7rkvy/mQQvc/1Od9/S3d9YxrZPqKovJ7kpk8nIT2US8jwhyTO7+6vD9l6c5EnDOv+Q5D5J7t3df9fd713G9uY6PcnzuvvmYQL0n+ZsY992njd8M/j2oV/3X+G2AIDV25A5S3dfn2Rn/nmecFKSOye5NMlPJLmuu//bsM0PJ/mTJI9boKnTk1zY3R/q7m8meWaSH6qqbUM7N3X3i4f5zVe7+/1L6V8mZxg9sapqeP+kTEIpYCQEPMBa+VKSI/d3HXpVfV9Vva2qbqqqr2Ry3fuR86pdv4JtX9Hdh3f3kd19Qnf/xdDuHTP5Vmufz2VyJk2S/FqSSvKBmjwV6+dXsN1k8u3Y/G3ce877L82bQH49ye1usAgArJuNnLNclH8OeJ6U5OLu3vel08OGS8K+PHxxdXomZwLP93/MPbp7z/CZjkpyTJLPrKBfGYKgryfZUVXfn8nZP5espC1gYwh4gLXyviTfzOR05cW8IslfJzm2u++e5FmZhCxz9Rr154v557N09vnuJDckSXff1N1P6e57J/l3mZwGvdCTLw7Un88vsI3Pr7jXAMC0beSc5c1Jjq6qE5P8dCaBTzIJi/5q+MJq38+W7v5/Fmjj/5h7DJd3fUcmc5zrM7k0/UAW6/tFmVym9aQkb+ruv1vKhwJmg4AHWBPdfVuS38rkHjOnVdW3V9Udq+qRVfV7Q7W7JflKkj3DN0MLTVrm+0KWNlGZ359vJXljkhdW1d2q6j5JfjXJa5Okqh5XVUcP1W/NZKLzjyvY/uuTPLuq7llVR2YyBq/dT30AYANt5Jylu7+W5E1J/luSz3X3lcOityX5vqp60tCXO1bVD1bVv1igmdcn+bmqevDwYIffTvL+4VKytyW5V1X9yvAgiLtV1cMWaONvM5n3zO/vazO51P1nk7xmCZ8ZmCECHmDNdPeLMwlRnp3JxOH6JE9P8qdDlf+Qyc0Ev5rkj5Ls96bJg+dmcvPBL+/vaRKL+KUkX0tybZL3ZnLjwguHZT+Y5P1VtSeT04+f0d3XLtDGq5I8YNj+ny6w/AVJrkzysSRXJ/nQUAYAzKgNnrNclMkZOP8UoHT3V5OcnMn9Az+fyX0FfzeTe/TM7/tfJPnNTO7Rc2OS7x3W29fOv07ymKGNa5KcuEAbX0/ywiT/a+jvCUP59ZnMZTrJ/1zCZwZmSHWv1dUQAAAAjFlVXZjk89397ANWBmbKojcWAwAAYPMYnsT100n+1cb2BFgJl2gBAABsclX1/CQfT/Kfu/uzG90fYPlcogUAAAAwcs7gAQAAABi50d6D58gjj+x73vOeuetd77rRXTmofe1rXzPGU2aMp8v4Tp8xnj5jPF37xveqq676Ynffcy3bPvLII3vbtm1r2eRBwTE9O+yL2WFfzA77YnbYFwtbbM4y2oBn27ZtedGLXpQdO3ZsdFcOajt37jTGU2aMp8v4Tp8xnj5jPF37xreqPrfWbW/bti1XXnnlWjc7eo7p2WFfzA77YnbYF7PDvljYYnMWl2gBAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjd8hGdwBgObade+my6p9z3N6ceYB1rjvv0avpEgAAwIZzBg8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkTtgwFNVF1bVzVX18QWWnVNVXVVHDu+rql5WVbuq6mNV9ZA5dc+oqmuGnzPmlD+0qq4e1nlZVdVafTgAAACAzWApZ/C8Oskp8wur6pgkJyf5mznFj0xy7PBzdpJXDHXvkeQ5SR6W5Pgkz6mqI4Z1XpHkKXPWu922AAAAAFjcAQOe7n5PklsWWPSSJL+WpOeUnZrkNT1xRZLDq+peSR6R5PLuvqW7b01yeZJThmV37+4ruruTvCbJaav7SAAAAACby4ruwVNVpya5obs/Om/RUUmun/N+91C2v/LdC5QDAAAAsESHLHeFqvr2JM/K5PKsdVVVZ2dy6Ve2bt2aPXv2ZOfOnevdjU3FGE+fMV6ec47bu6z6Ww898DrGf3Ucw9NnjKdrrcd3/nzFvrs9x/TssC9mh30xO+yL2WFfLM+yA54k35vkvkk+OtwP+egkH6qq45PckOSYOXWPHspuSLJjXvnOofzoBeovqLvPT3J+kmzfvr23bNmSHTt2LFadNbBz505jPGXGeHnOPPfSZdU/57i9efHV+/9Vd93pO1bRIxzD02eMp2utx3f+fMW+uz3H9OywL2aHfTE77IvZYV8sz7Iv0eruq7v7O7t7W3dvy+Syqod0901JLkny5OFpWickua27b0xyWZKTq+qI4ebKJye5bFj2lao6YXh61pOTvHWNPhsAAADAprCUx6S/Psn7kty/qnZX1Vn7qf72JNcm2ZXkj5L8YpJ09y1Jnp/kg8PP84ayDHUuGNb5TJJ3rOyjAAAAAGxOB7xEq7ufeIDl2+a87iRPW6TehUkuXKD8yiQPOlA/AAAAAFjYip6iBQAAAMDsEPAAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkDhjwVNWFVXVzVX18Ttl/rqq/rqqPVdVbqurwOcueWVW7qurTVfWIOeWnDGW7qurcOeX3rar3D+VvqKo7reUHBAAAADjYLeUMnlcnOWVe2eVJHtTd/zLJ/07yzCSpqgckeUKSBw7rvLyq7lBVd0jyh0kemeQBSZ441E2S303yku6+X5Jbk5y1qk8EAAAAsMkcMODp7vckuWVe2Tu7e+/w9ookRw+vT01ycXd/s7s/m2RXkuOHn13dfW13/32Si5OcWlWV5MeSvGlY/6Ikp63yMwEAAABsKoesQRs/n+QNw+ujMgl89tk9lCXJ9fPKH5bkO5J8eU5YNLf+7VTV2UnOTpKtW7dmz5492blz52r7z34Y4+kzxstzznF7D1xpjq2HHngd4786juHpM8bTtdbjO3++Yt/dnmN6dtgXs8O+mB32xeywL5ZnVQFPVf1Gkr1JXrc23dm/7j4/yflJsn379t6yZUt27NixHpvetHbu3GmMp8wYL8+Z5166rPrnHLc3L756/7/qrjt9xyp6hGN4+ozxdK31+M6fr9h3t+eYnh32xeywL2aHfTE77IvlWXHAU1VnJvmJJCd1dw/FNyQ5Zk61o4eyLFL+pSSHV9Uhw1k8c+sDAAAAsAQrekx6VZ2S5NeS/GR3f33OokuSPKGq7lxV901ybJIPJPlgkmOHJ2bdKZMbMV8yBEPvTvLYYf0zkrx1ZR8FAAAAYHNaymPSX5/kfUnuX1W7q+qsJH+Q5G5JLq+qj1TVK5Okuz+R5I1JPpnkz5M8rbu/NZyd8/QklyX5VJI3DnWT5NeT/GpV7crknjyvWtNPCAAAAHCQO+AlWt39xAWKFw1huvuFSV64QPnbk7x9gfJrM3nKFgAAAAArsKJLtAAAAACYHQIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJE7YMBTVRdW1c1V9fE5Zfeoqsur6prhzyOG8qqql1XVrqr6WFU9ZM46Zwz1r6mqM+aUP7Sqrh7WeVlV1Vp/SAAAAICD2VLO4Hl1klPmlZ2b5F3dfWySdw3vk+SRSY4dfs5O8opkEggleU6ShyU5Pslz9oVCQ52nzFlv/rYAAAAA2I8DBjzd/Z4kt8wrPjXJRcPri5KcNqf8NT1xRZLDq+peSR6R5PLuvqW7b01yeZJThmV37+4ruruTvGZOWwAAAAAswSErXG9rd984vL4pydbh9VFJrp9Tb/dQtr/y3QuUL6iqzs7kzKBs3bo1e/bsyc6dO1f4EVgKYzx9xnh5zjlu77Lqbz30wOsY/9VxDE+fMZ6utR7f+fMV++72HNOzw76YHfbF7LAvZod9sTwrDXj+SXd3VfVadGYJ2zo/yflJsn379t6yZUt27NixHpvetHbu3GmMp8wYL8+Z5166rPrnHLc3L756/7/qrjt9xyp6hGN4+ozxdK31+M6fr9h3t+eYnh32xeywL2aHfTE77IvlWelTtL4wXF6V4c+bh/Ibkhwzp97RQ9n+yo9eoBwAAACAJVppwHNJkn1PwjojyVvnlD95eJrWCUluGy7luizJyVV1xHBz5ZOTXDYs+0pVnTA8PevJc9oCAAAAYAkOeIlWVb0+yY4kR1bV7kyehnVekjdW1VlJPpfk8UP1tyd5VJJdSb6e5OeSpLtvqarnJ/ngUO953b3vxs2/mMmTug5N8o7hBwAAAIAlOmDA091PXGTRSQvU7SRPW6SdC5NcuED5lUkedKB+AAAAALCwlV6iBQAAAMCMEPAAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjNyqAp6q+vdV9Ymq+nhVvb6q7lJV962q91fVrqp6Q1Xdaah75+H9rmH5tjntPHMo/3RVPWJ1HwkAAABgc1lxwFNVRyX55STbu/tBSe6Q5AlJfjfJS7r7fkluTXLWsMpZSW4dyl8y1EtVPWBY74FJTkny8qq6w0r7BQAAALDZrPYSrUOSHFpVhyT59iQ3JvmxJG8all+U5LTh9anD+wzLT6qqGsov7u5vdvdnk+xKcvwq+wUAAACwaVR3r3zlqmckeWGSbyR5Z5JnJLliOEsnVXVMknd094Oq6uNJTunu3cOyzyR5WJLnDuu8dih/1bDOmxbY3tlJzk6SrVu3PvSCCy7Ili1bVtx/DmzPnj3GeMqM8fJcfcNty6q/9dDkC9/Yf53jjjpsFT3CMTx9xni69o3viSeeeFV3b19te/PnKxdffPGq+3iwcUzPDvtidtgXs8O+mB32xcIWm7McstIGq+qITM6+uW+SLyf5H5lcYjU13X1+kvOTZPv27b1ly5bs2LFjmpvc9Hbu3GmMp8wYL8+Z5166rPrnHLc3L756/7/qrjt9xyp6hGN4+ozxdK31+M6fr9h3t+eYnh32xeywL2aHfTE77IvlWc0lWj+e5LPd/bfd/Q9J3pzk4UkOHy7ZSpKjk9wwvL4hyTFJMiw/LMmX5pYvsA4AAAAAB7CagOdvkpxQVd8+3EvnpCSfTPLuJI8d6pyR5K3D60uG9xmW/2VPrg+7JMkThqds3TfJsUk+sIp+AQAAAGwqK75Eq7vfX1VvSvKhJHuTfDiT05EvTXJxVb1gKHvVsMqrkvxxVe1KcksmT85Kd3+iqt6YSTi0N8nTuvtbK+0XAAAAwGaz4oAnSbr7OUmeM6/42izwFKzu/rskj1uknRdmcrNmAAAAAJZptY9JBwAAAGCDCXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARm5VAU9VHV5Vb6qqv66qT1XVD1XVParq8qq6ZvjziKFuVdXLqmpXVX2sqh4yp50zhvrXVNUZq/1QAAAAAJvJas/geWmSP+/u70/yA0k+leTcJO/q7mOTvGt4nySPTHLs8HN2klckSVXdI8lzkjwsyfFJnrMvFAIAAADgwFYc8FTVYUl+JMmrkqS7/767v5zk1CQXDdUuSnLa8PrUJK/piSuSHF5V90ryiCSXd/ct3X1rksuTnLLSfgEAAABsNtXdK1ux6sFJzk/yyUzO3rkqyTOS3NDdhw91Ksmt3X14Vb0tyXnd/d5h2buS/HqSHUnu0t0vGMp/M8k3uvtFC2zz7EzO/snWrVsfesEFF2TLli0r6j9Ls2fPHmM8ZcZ4ea6+4bZl1d96aPKFb+y/znFHHbaKHuEYnj5jPF37xvfEE0+8qru3r7a9+fOViy++eNV9PNg4pmeHfTE77IvZYV/MDvtiYYvNWQ5ZRZuHJHlIkl/q7vdX1Uvzz5djJUm6u6tqZQnSArr7/ExCpWzfvr23bNmSHTt2rFXzLGDnzp3GeMqM8fKcee6ly6p/znF78+Kr9/+r7rrTd6yiRziGp88YT9daj+/8+Yp9d3uO6dlhX8wO+2J22Bezw75YntXcg2d3kt3d/f7h/ZsyCXy+MFx6leHPm4flNyQ5Zs76Rw9li5UDAAAAsAQrDni6+6Yk11fV/YeikzK5XOuSJPuehHVGkrcOry9J8uThaVonJLmtu29MclmSk6vqiOHmyicPZQAAAAAswWou0UqSX0ryuqq6U5Jrk/xcJqHRG6vqrCSfS/L4oe7bkzwqya4kXx/qprtvqarnJ/ngUO953X3LKvsFAAAAsGmsKuDp7o8kWehmhCctULeTPG2Rdi5McuFq+gIAAACwWa3mHjwAAAAAzAABDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACN3yEZ3AAAAYNZtO/fSqbZ/znF7c+YUt3HdeY+eWtvAbHAGDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgdstEdANho2869dCrtXnfeo6fSLgAAwHzO4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkVh3wVNUdqurDVfW24f19q+r9VbWrqt5QVXcayu88vN81LN82p41nDuWfrqpHrLZPAAAAAJvJWpzB84wkn5rz/neTvKS775fk1iRnDeVnJbl1KH/JUC9V9YAkT0jywCSnJHl5Vd1hDfoFAAAAsCmsKuCpqqOTPDrJBcP7SvJjSd40VLkoyWnD61OH9xmWnzTUPzXJxd39ze7+bJJdSY5fTb8AAAAANpPq7pWvXPWmJL+T5G5J/kOSM5NcMZylk6o6Jsk7uvtBVfXxJKd09+5h2WeSPCzJc4d1XjuUv2pY503zNpeqOjvJ2UmydevWh15wwQXZsmXLivvPge3Zs8cYT5kxXp6rb7htWfW3Hpp84RtT6swBHHfUYRuz4XXmGJ4+Yzxd+8b3xBNPvKq7t6+2vfnzlYsvvnjVfTzYOKZnh32xdMudgyzXtOcsm2Veshb8vZgd9sXCFpuzHLLSBqvqJ5Lc3N1XVdWO1XRuqbr7/CTnJ8n27dt7y5Yt2bFjXTa9ae3cudMYT5kxXp4zz710WfXPOW5vXnz1in/Vrcp1p+/YkO2uN8fw9Bnj6Vrr8Z0/X7Hvbs8xPTvsi6Vb7hxkuaY9Z9ks85K14O/F7LAvlmc1v0EenuQnq+pRSe6S5O5JXprk8Ko6pLv3Jjk6yQ1D/RuSHJNkd1UdkuSwJF+aU77P3HUAAAAAOIAV34Onu5/Z3Ud397ZMbpL8l919epJ3J3nsUO2MJG8dXl8yvM+w/C97cn3YJUmeMDxl675Jjk3ygZX2CwAAAGCzmcY5gL+e5OKqekGSDyd51VD+qiR/XFW7ktySSSiU7v5EVb0xySeT7E3ytO7+1hT6BQAAAHBQWpOAp7t3Jtk5vL42CzwFq7v/LsnjFln/hUleuBZ9AQAAANhsVvWYdAAAAAA2noAHAAAAYOQEPAAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABG7pCN7gBwcNp27qUb3QUAAIBNwxk8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGzk2WAQAADnLr8QCM68579NS3ASzOGTwAAAAAIyfgAQAAABg5l2gBAACjth6XHwHMOmfwAAAAAIycgAcAAABg5AQ8AAAAACO34oCnqo6pqndX1Ser6hNV9Yyh/B5VdXlVXTP8ecRQXlX1sqraVVUfq6qHzGnrjKH+NVV1xuo/FgAAAMDmsZozePYmOae7H5DkhCRPq6oHJDk3ybu6+9gk7xreJ8kjkxw7/Jyd5BXJJBBK8pwkD0tyfJLn7AuFAAAAADiwFQc83X1jd39oeP3VJJ9KclSSU5NcNFS7KMlpw+tTk7ymJ65IcnhV3SvJI5Jc3t23dPetSS5PcspK+wUAAACw2VR3r76Rqm1J3pPkQUn+prsPH8orya3dfXhVvS3Jed393mHZu5L8epIdSe7S3S8Yyn8zyTe6+0ULbOfsTM7+ydatWx96wQUXZMuWLavuP4vbs2ePMZ6yg3WMr77hto3uQpJk66HJF76xMds+7qjDNmbD6+xgPYZniTGern3je+KJJ17V3dtX2978+crFF1+86j4ebBzTs+Ng2RezMu9YjY2cs6yVg2Xuc7D8vTgY2BcLW2zOcshqG66qLUn+JMmvdPdXJpnORHd3Va0+Qfrn9s5Pcn6SbN++vbds2ZIdO3asVfMsYOfOncZ4yg7WMT7z3Es3ugtJknOO25sXX73qX3Urct3pOzZku+vtYD2GZ4kxnq61Ht/58xX77vYc07PjYNkXszLvWI2NnLOslYNl7nOw/L04GNgXy7Oqp2hV1R0zCXde191vHoq/MFx6leHPm4fyG5IcM2f1o4eyxcoBAAAAWIIVR8TD5VevSvKp7v4vcxZdkuSMJOcNf751TvnTq+riTG6ofFt331hVlyX57Tk3Vj45yTNX2i8AAADW37Ypn0l13XmPnmr7MHarOQfw4UmelOTqqvrIUPasTIKdN1bVWUk+l+Txw7K3J3lUkl1Jvp7k55Kku2+pqucn+eBQ73ndfcsq+gUAAACwqaw44BlullyLLD5pgfqd5GmLtHVhkgtX2hcAAGB2TfvMDgBWeQ8eAAAAADaegAcAAABg5AQ8AAAAACMn4AEAAAAYudU8RQsAAADWhceww/4JeAAAYJPzlCuA8XOJFgAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAA8oKzcAACAASURBVIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjNwhG90BAABgcdvOvXRqbZ9z3N6cOcX2YUz2/V2b5t+L68579FTahcQZPAAAAACjJ+ABAAAAGDmXaAFMyTROqXdaLwAAsBABD2xy07yuHwAAgPXhEi0AAACAkRPwAAAAAIzczFyiVVWnJHlpkjskuaC7z9vgLgEAwAG53BlYqmn/vnC/xs1tJgKeqrpDkj9M8q+T7E7ywaq6pLs/ubE9g9liAsm0jgGTAQCA8RMgbW4zEfAkOT7Jru6+Nkmq6uIkpyYR8DBKy/nFes5xe3Om4IYNtprJwGLHsAkAMCt8QQLAZlDdvdF9SFU9Nskp3f0Lw/snJXlYdz99Xr2zk5w9vL1/ki8l+eJ69nUTOjLGeNqM8XQZ3+kzxtNnjKdr3/jep7vvudrGFpivfHq1bR6EHNOzw76YHfbF7LAvZod9sbAF5yyzcgbPknT3+UnO3/e+qq7s7u0b2KWDnjGePmM8XcZ3+ozx9Bnj6Vrr8Z0/X+H2HNOzw76YHfbF7LAvZod9sTyz8hStG5IcM+f90UMZAAAAAAcwKwHPB5McW1X3rao7JXlCkks2uE8AAAAAozATl2h1996qenqSyzJ5TPqF3f2JJazq9OfpM8bTZ4yny/hOnzGePmM8XcZ3/Rnz2WFfzA77YnbYF7PDvliGmbjJMgAAAAArNyuXaAEAAACwQgIeAAAAgJEbVcBTVY+rqk9U1T9W1aKPSquq66rq6qr6SFVduZ59HLtljPEpVfXpqtpVVeeuZx/HrqruUVWXV9U1w59HLFLvW8Mx/JGqctPxAzjQMVlVd66qNwzL319V29a/l+O2hDE+s6r+ds5x+wsb0c+xqqoLq+rmqvr4Isurql42jP/Hquoh693HsVvCGO+oqtvmHMO/td593Eyq6j9X1V8Px/Nbqurwje7TZrXU+R/TYV49Ow707wTro6qOqap3V9Unh99Nz9joPo3FqAKeJB9P8tNJ3rOEuid294O72z9Sy3PAMa6qOyT5wySPTPKAJE+sqgesT/cOCucmeVd3H5vkXcP7hXxjOIYf3N0/uX7dG58lHpNnJbm1u++X5CVJfnd9ezluy/h7/4Y5x+0F69rJ8Xt1klP2s/yRSY4dfs5O8op16NPB5tXZ/xgnyf+ccww/bx36tJldnuRB3f0vk/zvJM/c4P5sZsuZY7OGzKtnzqtz4H8nmL69Sc7p7gckOSHJ0/y9WJpRBTzd/anu/vRG9+NgtsQxPj7Jru6+trv/PsnFSU6dfu8OGqcmuWh4fVGS0zawLweLpRyTc8f9TUlOqqpaxz6Onb/3U9bd70lyy36qnJrkNT1xRZLDq+pe69O7g8MSxph11N3v7O69w9srkhy9kf3ZzMyxN5R/X2eIfydmQ3ff2N0fGl5/Ncmnkhy1sb0ah1EFPMvQSd5ZVVdV1dkb3ZmD0FFJrp/zfnf8hVuOrd194/D6piRbF6l3l6q6sqquqCoh0P4t5Zj8pzrDfyhuS/Id69K7g8NS/97/m+FyizdV1THr07VNw+/e9fFDVfXRqnpHVT1wozuzifx8kndsdCdgA/jdDvsx3FbhXyV5/8b2ZBwO2egOzFdVf5HkuxZY9Bvd/dYlNvPD3X1DVX1nksur6q+HNJas2RizH/sb47lvururqhdp5j7Dcfw9Sf6yqq7u7s+sdV9hDf1Zktd39zer6t9lcsbUj21wn2A5PpTJ7949VfWoJH+aySVxrNBS5hxV9RuZnI7/uvXs22Zj/geMTVVtSfInSX6lu7+y0f0Zg5kLeLr7x9egjRuGP2+uqrdkcuqjgGewBmN8Q5K538wfPZQx2N8YV9UXqupe3X3jcHnFzYu0se84vraqdmaSXAt4FraUY3Jfnd1VdUiSw5J8aX26d1A44Bh399zxvCDJ761DvzYTv3unbO7ksbvfXlUvr6oju/uLG9mvMTvQnKOqzkzyE0lO6u7FvvBgDazFHJup8LsdFlBVd8wk3Hldd795o/szFgfdJVpVddequtu+10lOzuTGcaydDyY5tqruW1V3SvKEJJ7ytHSXJDljeH1Gktt9a1ZVR1TVnYfXRyZ5eJJPrlsPx2cpx+TccX9skr/0n4llOeAYz7sfzE9mcr00a+eSJE8enqZ1QpLb5lzuyRqoqu/ad2+uqjo+k3mSIHhKquqUJL+W5Ce7++sb3R/YIObVMM/wb/Grknyqu//LRvdnTEYV8FTVT1XV7iQ/lOTSqrpsKL93Vb19qLY1yXur6qNJPpDk0u7+843p8fgsZYyH+5c8PcllmfwH7o3d/YmN6vMInZfkX1fVNUl+fHifqtpeVfueOvQvklw5HMfvTnJedwt4FrHYMVlVz6uqfU8ge1WS76iqXUl+NYs/vYwFLHGMf3l4lOVHk/xykjM3prfjVFWvT/K+JPevqt1VdVZVPbWqnjpUeXuSa5PsSvJHSX5xg7o6WksY48cm+fhwDL8syRMEwVP1B0nulsnl9B+pqldudIc2q8Xmf0yfefVsWejfiY3u0yb18CRPSvJjw78PHxkuneYAyrwFAAAAYNxGdQYPAAAAALcn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAPMtKq6rqp+fKP7MV9V7ayqX9jofgAA+7cRc4mqem5VvXaVbZxeVe9cqz5NU1W9sqp+c6P7AZudgAdYF8Pk6htVtaeqvlBVr66qLeuw3ftW1T9W1SumvS0AYHrWcy5RVUdV1d6q+t4Flr2lql40je3O1d2v6+6T52y3q+p+q2lzsXlRVe2oqt37We8dw7jvqap/qKq/n/P+ld391O5+/mr6BqyegAdYT4/p7i1JHpJke5Jnr8M2n5zk1iQ/U1V3XoftAQDTsy5zie6+Icm7kjxpbnlV3SPJo5JcNI3troMVzYu6+5HdvWUY+9cl+b1977v7qdPqLLA8Ah5g3Q2TpnckeVCSVNVPVtUnqurLw6VP/2Kh9arq26rq3Kr6TFV9qareOEy0FlRVlclE5tlJ/iHJY+Yt76p6alVdM2z7D4d1UlV3qKoXV9UXq+qzVfX0of4hi2zr56vqU1V1a1VdVlX3WcnYAAAHtk5ziYsyL+BJ8oQkn+zuq6vq3lX1J1X1t8Nc4ZcX6+/++ldVx1TVm4d2vlRVfzCUn1lV7x1ev2eo/tHhrJmfqaqPV9Vj5rRzx2He8q8W6cOC86Kquuswlveec1bOvRf7LIu0/eqqesHwekdV7a6qZw39ua6qTl9Oe8DKCHiAdVdVx2Ty7deHq+r7krw+ya8kuWeStyf5s6q60wKr/lKS05L8aJJ7Z/IN1B/uZ1M/nOToJBcneWOSMxao8xNJfjDJv0zy+CSPGMqfkuSRSR6cybeEp+3n85ya5FlJfnr4DP9z+EwAwBSs01ziLUmOrKofnlP2pCQXVdW3JfmzJB9NclSSk5L8SlU9Yn4j++tfVd0hyduSfC7JtqGti+e30d0/Mrz8geGsmTckeU2Sn51T7VFJbuzuDy/yeRacF3X31zKZ83x+zlk5n1+kjaX6riRHDp/njCTnV9X9V9kmcAACHmA9/WlVfTnJe5P8VZLfTvIzSS7t7su7+x+SvCjJoUn+rwXWf2qS3+ju3d39zSTPTfLYxc6qyWRC8Y7uvjXJf09ySlV957w653X3l7v7b5K8O5NAJ5mEPS8dtnVrkvP287memuR3uvtT3b13+FwPdhYPAKy5dZtLdPc3kvyPTM56SVUdm+ShmcwpfjDJPbv7ed399919bZI/yuQMn/n217/jMwma/mN3f627/66737vEsXhtkkdV1d2H909K8sf7qb+UedFa+s3u/mZ3/1WSSzOZWwFTJOAB1tNp3X14d9+nu39xmDjdO5NvrZIk3f2PSa7P5Buf+e6T5C3D6c1fTvKpJN9KsnV+xao6NMnjMrlOPN39viR/k+Tfzqt605zXX0+y72aN9x76sc/c1wv166Vz+nVLklrkMwAAK7duc4nBRUkeV1V3ySRAuay7bx7aufe+doa2nrVIO/vr3zFJPjd8QbQsw1k2/yvJv6mqwzM5C+d1C9Vdxrxordw6nBm0z+cyGQdgigQ8wEb7fCaTpCT/dH34MUluWKDu9UkeOUzs9v3cZbgOf76fSnL3JC+vqpuq6qb882nCS3FjJqcx73PMfupen+TfzevXod39/y1xWwDAyk1rLpFMzhS6JcmpmVwOddGcdj47r527dfejltm/65N8937ORj6Qi4Z+PS7J+/bzOQ40L+oVbn8xRwz39tnnuzMZB2CKBDzARntjkkdX1UlVdcck5yT5ZpKFwpFXJnnhvkufquqew/1vFnJGkguTHJfJZVcPTvLwJD9QVcctsV/PqMljUg9P8uv7qfvKJM+sqgcO/Tqsqh63hG0AAKs3rblEurszudfN7yY5PJP77iTJB5J8tap+vaoOrcnDGR5UVT+4zP59IJMvlc6rqrtW1V2q6uGLdOcLSb5nXtmfZnKvwGcM/VzMgeZFX0jyHVV12H7aWK7/NNxn6P/O5J6H/2MN2wYWIOABNlR3fzqTb57+a5IvZvJEh8d0998vUP2lSS5J8s6q+mqSK5I8bH6lqtp3s8Pf7+6b5vxcleTPs7SzeP4oyTuTfCzJhzO5IeLeTE7jnv8Z3pLJxO/iqvpKko9ncpo0ADBl05hLzPOaTM5AecNw355097cyCS0enOSzw3YvSHK7gGR//RvaeUyS+2VyydTuTO7Zs5DnZnKD5y9X1eOHtr+R5E+S3DfJmxdaaSnzou7+60xuBH3t0P5qL6e6KZMbWH8+k8vCnjpsA5iimoTSAOxPVT0yySu7242TAYCZUVW/leT7uvtnD1h5HVTVjiSv7e6jD1QXWFvO4AFYwHC69aOq6pDhm6/nZPK4VACAmVBV90hyVpLzN7ovwMYT8AAsrJL8p0xOL/5wJk/Z+K0N7REAwKCqnpLJTZrf0d3v2ej+ABvPJVoAAAAAI+cMHgAAAICRO2SjO7BSRx55ZG/btm2ju3FQ+drXvpa73vWuG92NTcFYrx9jvb6M9/ox1mvvqquu+mJ333Mt2xzrfMXxtTTGaWmM09IYp6UxTktjnJZmrOO02JxltAHPtm3bcuWVV250Nw4qO3fuzI4dOza6G5uCsV4/xnp9Ge/1Y6zXXlV9bq3bHOt8xfG1NMZpaYzT0hinpTFOS2Oclmas47TYnMUlWgAAAAAjJ+ABAAAAGDkBDwAAAMDICXgAAAAARk7AAwAAADByAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQOGPBU1YVVdXNVfXyBZedUVVfVkcP7qqqXVdWuqvpYVT1kTt0zquqa4eeMOeUPraqrh3VeVlW1Vh8OAAAAYDNYyhk8r05yyvzCqjomyclJ/mZO8SOTHDv8nJ3kFUPdeyR5TpKHJTk+yXOq6ohhnVckecqc9W63LQAAAAAWd8CAp7vfk+SWBRa9JMmvJek5ZacmeU1PXJHk8Kq6V5JHJLm8u2/p7luTXJ7klGHZ3bv7iu7uJK9JctrqPhIAAADA5nLISlaqqlOT3NDdH513RdVRSa6f8373ULa/8t0LlC+23bMzOTMoW7duzc6dO1fSfRaxZ88eY7pOjPX6Mdbry3ivH2M9uw6G+Yrja2mM09IYp6UxTktjnJbGOC3NwTZOyw54qurbkzwrk8uz1lV3n5/k/CTZvn1779ixY727cFDbuXNnjOn6MNbrx1ivL+O9foz17DoY5iuOr6UxTktjnJbGOC2NcVoa47Q0B9s4reQpWt+b5L5JPlpV1yU5OsmHquq7ktyQ5Jg5dY8eyvZXfvQC5QAAAAAs0bIDnu6+uru/s7u3dfe2TC6rekh335TkkiRPHp6mdUKS27r7xiSXJTm5qo4Ybq58cpLLhmVfqaoThqdnPTnJW9foswEAAABsCkt5TPrrk7wvyf2randVnbWf6m9Pcm2SXUn+KMkvJkl335Lk+Uk+OPw8byjLUOeCYZ3PJHnHyj4KAAAAwOZ0wHvwdPcTD7B825zXneRpi9S7MMmFC5RfmeRBB+oHAAAAAAtbyT14AAAAAJghAh4AAACAkRPwAAAAAIycgAcAAABg5AQ8AAAAACMn4AEAAAAYOQEPAAAAwMgJeAAAAABGTsADAAAAMHICHgAAAICRE/AAAAAAjJyABwAAAGDkBDwAAAAAIyfgAQAAABg5AQ8AAADAyAl4AAAAAEZOwAMAAAAwcgIeAAAAgJET8AAAAACMnIAHAAAAYOQEPAAAAAAjJ+Dh/2/v/oM1q+v7gL8/A0FNaARNZmuBBmZkzCAYE7dIx06ySCKrZoR2jMWxcUlImUwxNQ0zEeKktCozWJMYbZO0TCDB1LpSEgdGSZAgd0xmAhKVqoCGDf6CoqSCmh2N6ZpP/7iHeLve3T3cH89zz3Nfr5k79znf8+N+7mfPvc/Z9z0/AAAAgIkT8AAAAABMnIAHAAAAYOIEPAAAAAATJ+ABAAAAmLgjBjxVdW1VPVJVn1gx9paq+mRVfayq3lNVx62Yd3lV7auqT1XVuSvGdw9j+6rqshXjp1TVncP4u6vqmI38BgEAAAAW3ZgzeH43ye6Dxm5Ncnp3PyfJXyS5PEmq6rQkFyR59rDOb1bVUVV1VJLfSPLiJKcleeWwbJK8Oclbu/uZSR5LctG6viMAAACAbeaIAU93fzDJoweNvb+7DwyTdyQ5cXh9XpK93f2N7v50kn1Jzhw+9nX3A939t0n2JjmvqirJC5PcMKx/XZLz1/k9AQAAAGwrR2/ANn46ybuH1ydkOfB53IPDWJJ8/qDx5yd5epIvrwiLVi7/barq4iQXJ8mOHTuytLS03tpZYf/+/Xo6I3o9O3o9W/o9O3q9dS3C8Yr9axx9GkefxtGncfRpHH0aZ9H6tK6Ap6pen+RAknduTDmH191XJ7k6SXbu3Nm7du2axZfdNpaWlqKns6HXs6PXs6Xfs6PXW9ciHK/Yv8bRp3H0aRx9GkefxtGncRatT2sOeKrqwiQ/nuSc7u5h+KEkJ61Y7MRhLIcY/1KS46rq6OEsnpXLAwAAADDCmh6TXlW7k/xikpd199dWzLopyQVV9aSqOiXJqUk+lOSuJKcOT8w6Jss3Yr5pCIZuT/LyYf09SW5c27cCAAAAsD2NeUz6u5L8WZJnVdWDVXVRkv+S5B8kubWq7q6q/5ok3X1PkuuT3Jvkj5Jc0t3fHM7OeU2SW5Lcl+T6YdkkeV2SX6iqfVm+J881G/odAgAAACy4I16i1d2vXGX4kCFMd1+Z5MpVxm9OcvMq4w9k+SlbAAAAAKzBmi7RAgAAAGDrEPAAAAAATJyABwAAAGDiBDwAAAAAEyfgAQAAAJg4AQ8AAADAxAl4AAAAACZOwAMAAAAwcQIeAAAAgIkT8AAAAABMnIAHAAAAYOIEPAAAAAATJ+ABAAAAmDgBDwAAAMDECXgAAAAAJk7AAwAAADBxAh4AAACAiRPwAAAAAEycgAcAAABg4gQ8AAAAABMn4AEAAACYOAEPAAAAwMQJeAAAAAAmTsADAAAAMHECHgAAAICJE/AAAAAATNwRA56quraqHqmqT6wYe1pV3VpV9w+fjx/Gq6reXlX7qupjVfVDK9bZMyx/f1XtWTH+vKr6+LDO26uqNvqbBAAAAFhkY87g+d0kuw8auyzJbd19apLbhukkeXGSU4ePi5P8VrIcCCW5Isnzk5yZ5IrHQ6FhmX+9Yr2DvxYAAAAAh3HEgKe7P5jk0YOGz0ty3fD6uiTnrxh/Ry+7I8lxVfWMJOcmubW7H+3ux5LcmmT3MO+7u/uO7u4k71ixLQAAAABGOHqN6+3o7oeH119IsmN4fUKSz69Y7sFh7HDjD64yvqqqujjLZwZlx44dWVpaWmP5rGb//v16OiN6PTt6PVv6PTt6vXUtwvGK/WscfRpHn8bRp3H0aRx9GmfR+rTWgOfvdXdXVW9EMSO+1tVJrk6SnTt39q5du2bxZbeNpaWl6Ols6PXs6PVs6ffs6PXWtQjHK/avcfRpHH0aR5/G0adx9GmcRevTWp+i9cXh8qoMnx8Zxh9KctKK5U4cxg43fuIq4wAAAACMtNaA56Ykjz8Ja0+SG1eMv3p4mtZZSb4yXMp1S5IXVdXxw82VX5TklmHeV6vqrOHpWa9esS0AAAAARjjiJVpV9a4ku5J8T1U9mOWnYV2V5PqquijJZ5O8Ylj85iQvSbIvydeS/FSSdPejVfXGJHcNy72hux+/cfO/yfKTup6S5A+HDwAAAABGOmLA092vPMSsc1ZZtpNccojtXJvk2lXG/zzJ6UeqAwAAAIDVrfUSLQAAAAC2CAEPAAAAwMQJeAAAAAAmTsADAAAAMHECHgAAAICJE/AAAAAATJyABwAAAGDiBDwAAAAAEyfgAQAAAJg4AQ8AAADAxAl4AAAAACZOwAMAAAAwcQIeAAAAgIkT8AAAAABMnIAHAAAAYOIEPAAAAAATJ+ABAAAAmDgBDwAAAMDECXgAAAAAJk7AAwAAADBxAh4AAACAiRPwAAAAAEycgAcAAABg4gQ8AAAAABMn4AEAAACYOAEPAAAAwMStK+Cpqn9XVfdU1Seq6l1V9eSqOqWq7qyqfVX17qo6Zlj2ScP0vmH+ySu2c/kw/qmqOnd93xIAAADA9rLmgKeqTkjyb5Ps7O7TkxyV5IIkb07y1u5+ZpLHklw0rHJRkseG8bcOy6WqThvWe3aS3Ul+s6qOWmtdAAAAANvNei/ROjrJU6rq6CTfmeThJC9McsMw/7ok5w+vzxumM8w/p6pqGN/b3d/o7k8n2ZfkzHXWBQAAALBtVHevfeWq1ya5MsnXk7w/yWuT3DGcpZOqOinJH3b36VX1iSS7u/vBYd5fJnl+kv8wrPPfh/FrhnVuWOXrXZzk4iTZsWPH8/bu3bvm2vl2+/fvz7HHHjvvMrYFvZ4dvZ4t/Z4dvd54Z5999oe7e+d6t7MIxyv2r3H0aRx9GkefxtGncfRpnKn26VDHLEevdYNVdXyWz745JcmXk/zPLF9itWm6++okVyfJzp07e9euXZv55badpaWl6Ols6PXs6PVs6ffs6PXWtQjHK/avcfRpHH0aR5/G0adx9GmcRevTei7R+tEkn+7uv+ru/5vkD5K8IMlxwyVbSXJikoeG1w8lOSlJhvlPTfKlleOrrAMAAADAEawn4PlckrOq6juHe+mck+TeJLcnefmwzJ4kNw6vbxqmM8z/QC9fH3ZTkguGp2ydkuTUJB9aR10AAAAA28qaL9Hq7jur6oYkH0lyIMlHs3w68vuS7K2qNw1j1wyrXJPk96pqX5JHs/zkrHT3PVV1fZbDoQNJLunub661LgAAAIDtZs0BT5J09xVJrjho+IGs8hSs7v6bJD9xiO1cmeWbNQMAAADwBK33MekAAAAAzJmABwAAAGDiBDwAAAAAEyfgAQAAAJg4AQ8AAADAxAl4AAAAACZOwAMAAAAwcQIeAAAAgIkT8AAAAABMnIAHAAAAYOIEPAAAAAATJ+ABAAAAmDgBDwAAAMDECXgAAAAAJk7AAwAAADBxAh4AAACAiRPwAAAAAEycgAcAAABg4gQ8AAAAABMn4AEAAACYOAEPAAAAwMQJeAAAAAAmTsADAAAAMHECHgAAAICJE/AAAAAATJyABwAAAGDi1hXwVNVxVXVDVX2yqu6rqn9aVU+rqlur6v7h8/HDslVVb6+qfVX1sar6oRXb2TMsf39V7VnvNwUAAACwnaz3DJ63Jfmj7v7+JD+Q5L4klyW5rbtPTXLbMJ0kL05y6vBxcZLfSpKqelqSK5I8P8mZSa54PBQCAAAA4MjWHPBU1VOT/HCSa5Kku/+2u7+c5Lwk1w2LXZfk/OH1eUne0cvuSHJcVT0jyblJbu3uR7v7sSS3Jtm91roAAAAAtpv1nMFzSpK/SvI7VfXRqvrtqvquJDu6++FhmS8k2TG8PiHJ51es/+AwdqhxAAAAAEao7l7bilU7k9yR5AXdfWdVvS3JV5P8XHcft2K5x7r7+Kp6b5KruvtPh/Hbkrwuya4kT+7uNw3jv5zk6939K6t8zYuzfHlXduzY8by9e/euqXZWt3///hx77LHzLmNb0OvZ0evZ0u/Z0euNd/bZZ3+4u3eudzuLcLxi/xpHn8bRp3H0aRx9Gkefxplqnw51zHL0Orb5YJIHu/vOYfqGLN9v54tV9Yzufni4BOuRYf5DSU5asf6Jw9hDWQ55Vo4vrfYFu/vqJFcnyc6dO3vXrl2rLcYaLS0tRU9nQ69nR69nS79nR6+3rkU4XrF/jaNP4+jTOPo0jj6No0/jLFqf1nyJVnd/Icnnq+pZw9A5Se5NclOSx5+EtSfJjcPrm5K8enia1llJvjJcynVLkhdV1fHDzZVfNIwBAAAAMMJ6zuBJkp9L8s6qOibJA0l+Ksuh0fVVdVGSzyZ5xbDszUlekmRfkq8Ny6a7H62qNya5a1juDd396DrrAgAAANg21hXwdPfdSVa7Vv2cVZbtJJccYjvXJrl2PbUAAAAAbFfreYoWAAAAAFuAgAcAAABg4gQ8AAAAABMn4AEAAACYOAEPAAAAwMQJeAAAAAAmTsADAAAAMHECHgAAAICJE/AAAAAATJyABwAAAGDiBDwAAAAAEyfgAQAAAJg4AQ8AAADAxAl4AAAAACZOwAMAXQQFsQAADzFJREFUAAAwcQIeAAAAgIkT8AAAAABMnIAHAAAAYOIEPAAAAAATJ+ABAAAAmDgBDwAAAMDECXgAAAAAJk7AAwAAADBxAh4AAACAiRPwAAAAAEzc0fMuAAAAYBGcfNn71rzupWccyIWHWf8zV710zdsGtod1n8FTVUdV1Uer6r3D9ClVdWdV7auqd1fVMcP4k4bpfcP8k1ds4/Jh/FNVde56awIAAADYTjbiEq3XJrlvxfSbk7y1u5+Z5LEkFw3jFyV5bBh/67Bcquq0JBckeXaS3Ul+s6qO2oC6AAAAALaFdQU8VXVikpcm+e1hupK8MMkNwyLXJTl/eH3eMJ1h/jnD8ucl2dvd3+juTyfZl+TM9dQFAAAAsJ2s9wyeX0/yi0n+bph+epIvd/eBYfrBJCcMr09I8vkkGeZ/ZVj+78dXWQcAAACAI1jzTZar6seTPNLdH66qXRtX0mG/5sVJLk6SHTt2ZGlpaRZfdtvYv3+/ns6IXs+OXs+Wfs+OXm9di3C8Yv8aR5/G2U59uvSMA0de6BB2POXw62+XHh7Jdtqf1kOfxlm0Pq3nKVovSPKyqnpJkicn+e4kb0tyXFUdPZylc2KSh4blH0pyUpIHq+roJE9N8qUV449buc7/p7uvTnJ1kuzcubN37dq1jvI52NLSUvR0NvR6dvR6tvR7dvR661qE4xX71zj6NM526tPhnoJ1JJeecSC/+vFD//fsM6/ateZtL5LttD+thz6Ns2h9WvMlWt19eXef2N0nZ/kmyR/o7lcluT3Jy4fF9iS5cXh90zCdYf4HuruH8QuGp2ydkuTUJB9aa10AAAAA2816zuA5lNcl2VtVb0ry0STXDOPXJPm9qtqX5NEsh0Lp7nuq6vok9yY5kOSS7v7mJtQFAAAAsJA2JODp7qUkS8PrB7LKU7C6+2+S/MQh1r8yyZUbUQsAAADAdrPep2gBAAAAMGcCHgAAAICJE/AAAAAATJyABwAAAGDiBDwAAAAAE7cZj0kHAADYck6+7H3zLgFg0ziDBwAAAGDiBDwAAAAAE+cSLQAAgC1usy8v+8xVL93U7QObzxk8AAAAABMn4AEAAACYOAEPAAAAwMS5Bw+wsE6+7H259IwDuXATr1l3vToAALAVCHiAudnsmwUCAABsFy7RAgAAAJg4AQ8AAADAxAl4AAAAACbOPXgA1mGz7yPkJs4AAMAYzuABAAAAmDhn8ACH5ClXAAAA0yDgAQAAtgR/XAJYO5doAQAAAEycM3gAAAC2uc08e8pDI2A2nMEDAAAAMHECHgAAAICJE/AAAAAATJyABwAAAGDi1hzwVNVJVXV7Vd1bVfdU1WuH8adV1a1Vdf/w+fhhvKrq7VW1r6o+VlU/tGJbe4bl76+qPev/tgAAAAC2j/WcwXMgyaXdfVqSs5JcUlWnJbksyW3dfWqS24bpJHlxklOHj4uT/FayHAgluSLJ85OcmeSKx0MhAAAAAI5szQFPdz/c3R8ZXv91kvuSnJDkvCTXDYtdl+T84fV5Sd7Ry+5IclxVPSPJuUlu7e5Hu/uxJLcm2b3WugAAAAC2m+ru9W+k6uQkH0xyepLPdfdxw3gleay7j6uq9ya5qrv/dJh3W5LXJdmV5Mnd/aZh/JeTfL27f2WVr3Nxls/+yY4dO563d+/eddfOt+zfvz/HHnvsvMvYFqbS648/9JV5l7BuO56SfPHr865i7c444anzLuEJmcq+vQj0euOdffbZH+7unevdziIcr9i/xtGncZ5Inxbh2GOtpn7McjgbeTzj524cfRpnqn061DHL0evdcFUdm+T3k/x8d391OdNZ1t1dVetPkL61vauTXJ0kO3fu7F27dm3UpkmytLQUPZ2NqfT6wsveN+8S1u3SMw7kVz++7l91c/OZV+2adwlPyFT27UWg11vXIhyv2L/G0adxnkifFuHYY62mfsxyOBt5POPnbhx9GmfR+rSup2hV1XdkOdx5Z3f/wTD8xeHSqwyfHxnGH0py0orVTxzGDjUOAAAAwAjreYpWJbkmyX3d/WsrZt2U5PEnYe1JcuOK8VcPT9M6K8lXuvvhJLckeVFVHT/cXPlFwxgAAAAAI6znHMAXJPnJJB+vqruHsV9KclWS66vqoiSfTfKKYd7NSV6SZF+SryX5qSTp7ker6o1J7hqWe0N3P7qOugAAAAC2lTUHPMPNkusQs89ZZflOcskhtnVtkmvXWgsAAADAdraYd/ECAAA2xclP8EbIl55xYFvfPJknvs8czmr702eueumGbR+mbF03WQYAAABg/pzBAxO2kX8NAQAAYLoEPABb2GaHeE5pBgCAxeASLQAAAICJE/AAAAAATJyABwAAAGDiBDwAAAAAE+cmy7BJDndz3EvPOJALPQELAACADSLgAdjGNvopXQeHl57SBQBsNk8dhWUCHratzX4jAAAAgFkR8AAAwALxRyyA7clNlgEAAAAmzhk8AGyaWfwV2XXxAADgDB4AAACAyXMGD1uW68eBMTw5AwAAnMEDAAAAMHnO4GHNnGEDbAfOEAKA7W0zjwUcB7CRBDwLai2/hC4940AuFNoAAADA5Ah4AGCO1vpXwbGhvL8MwtbkTGgANpp78AAAAABMnIAHAAAAYOJcojUnTssFYBZm8X7jMjAAgPkT8AAAAMAcbNYfYh6/V58/wmwvAh4AADjImP90eQIpAFuJe/AAAAAATNyWOYOnqnYneVuSo5L8dndfNeeSAAAAYLI28158Lv/aerZEwFNVRyX5jSQ/luTBJHdV1U3dfe98KwMAYKvy0AoA+JYtEfAkOTPJvu5+IEmqam+S85LMLeBxwAAA42z2e6a/EE6X4ymAxeX9f+up7p53DamqlyfZ3d0/M0z/ZJLnd/drDlru4iQXD5PPSvKpmRa6+L4nyf+ZdxHbhF7Pjl7Pln7Pjl5vvO/r7u9d70YW5HjF/jWOPo2jT+Po0zj6NI4+jTPVPq16zLJVzuAZpbuvTnL1vOtYVFX15929c951bAd6PTt6PVv6PTt6vXUtwvGK/WscfRpHn8bRp3H0aRx9GmfR+rRVnqL1UJKTVkyfOIwBAAAAcARbJeC5K8mpVXVKVR2T5IIkN825JgAAAIBJ2BKXaHX3gap6TZJbsvyY9Gu7+545l7UdTfp08onR69nR69nS79nRazaT/WscfRpHn8bRp3H0aRx9Gmeh+rQlbrIMAAAAwNptlUu0AAAAAFgjAQ8AAADAxAl4trGqelpV3VpV9w+fj19lmedW1Z9V1T1V9bGq+pfzqHXqxvR6WO6PqurLVfXeWdc4dVW1u6o+VVX7quqyVeY/qarePcy/s6pOnn2Vi2FEr3+4qj5SVQeq6uXzqHGRjOj3L1TVvcPv6Nuq6vvmUSeLp6reOOxXd1fV+6vqH827pq2oqt5SVZ8cevWeqjpu3jVtRVX1E8Px5N9V1cI8kngjHOn3PMuq6tqqeqSqPjHvWraqqjqpqm4fjgvuqarXzrumraiqnlxVH6qq/zX06T/Ou6aNIuDZ3i5Lclt3n5rktmH6YF9L8urufnaS3Ul+3YHLmozpdZK8JclPzqyqBVFVRyX5jSQvTnJakldW1WkHLXZRkse6+5lJ3prkzbOtcjGM7PXnklyY5H/MtrrFM7LfH02ys7ufk+SGJP9ptlWywN7S3c/p7ucmeW+Sfz/vgraoW5OcPvwM/kWSy+dcz1b1iST/IskH513IVjLy9zzLfjfL/x/h0A4kubS7T0tyVpJL7E+r+kaSF3b3DyR5bpLdVXXWnGvaEAKe7e28JNcNr69Lcv7BC3T3X3T3/cPr/53kkSTfO7MKF8cRe50k3X1bkr+eVVEL5Mwk+7r7ge7+2yR7s9zzlVb+G9yQ5JyqqhnWuCiO2Ovu/kx3fyzJ382jwAUzpt+3d/fXhsk7kpw44xpZUN391RWT35XEkzlW0d3v7+4Dw6SfwUPo7vu6+1PzrmMLGnMMQ5Lu/mCSR+ddx1bW3Q9390eG13+d5L4kJ8y3qq2nl+0fJr9j+FiI9zgBz/a2o7sfHl5/IcmOwy1cVWcmOSbJX252YQvoCfWaJ+yEJJ9fMf1gvv3N7O+XGQ7Ev5Lk6TOpbrGM6TUb54n2+6Ikf7ipFbGtVNWVVfX5JK+KM3jG+On4GeSJ8b7KphhuR/CDSe6cbyVbU1UdVVV3Z/kEhlu7eyH6dPS8C2BzVdUfJ/mHq8x6/cqJ7u6qOmRqWVXPSPJ7SfZ0t7/Kr2Kjeg2wFlX1r5LsTPIj866F6Tjce1d339jdr0/y+qq6PMlrklwx0wK3iCP1aVjm9Vm+POKds6xtKxnTJ2DzVdWxSX4/yc8fdDYmg+7+ZpLnDrcfeU9Vnd7dk7+/k4BnwXX3jx5qXlV9saqe0d0PDwHOI4dY7ruTvC/Lb853bFKpk7cRvWbNHkpy0orpE4ex1ZZ5sKqOTvLUJF+aTXkLZUyv2Tij+l1VP5rlMPlHuvsbM6qNBXC4966DvDPJzdmmAc+R+lRVFyb58STndPe2/SPOE9if+Bbvq2yoqvqOLIc77+zuP5h3PVtdd3+5qm7P8v2dJh/wuERre7spyZ7h9Z4k3/aXlao6Jsl7kryju2+YYW2L5oi9Zl3uSnJqVZ0y7LMXZLnnK638N3h5kg9s54PwdRjTazbOEftdVT+Y5L8leVl3C4/ZMFV16orJ85J8cl61bGVVtTvJL2b5Z/BrR1oeDuJ9lQ0z3F/ymiT3dfevzbueraqqvvfxBwdV1VOS/FgW5D2u/P9m+6qqpye5Psk/TvLZJK/o7keHR1f+bHf/zHDK/+8kuWfFqhd2992zr3i6xvR6WO5Pknx/kmOzfHbJRd19y5zKnpSqekmSX09yVJJru/vKqnpDkj/v7puq6slZvszwB7N8g74LuvuB+VU8XSN6/U+yHAwfn+RvknxheBIfazCi33+c5Iwkj9/n63Pd/bI5lcsCqarfT/KsLN8w/bNZfr9yZsFBqmpfkiflW2eF3tHdPzvHkrakqvrnSf5zlh/W8eUkd3f3ufOtamtY7ff8nEvakqrqXUl2JfmeJF9MckV3XzPXoraYqvpnSf4kycfzrYdd/FJ33zy/qraeqnpOlh++clSWT3q5vrvfMN+qNoaABwAAAGDiXKIFAAAAMHECHgAAAICJE/AAAAAATJyABwAAAGDiBDwAAAAAEyfgAQAAAJg4AQ8AAADAxP0/v9BVMKnnwScAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x648 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "seen_observations = []\n",
    "for _ in range(1000):\n",
    "    seen_observations.append(env.reset())\n",
    "    done = False\n",
    "    while not done:\n",
    "        s, r, done, _ = env.step(env.action_space.sample())\n",
    "        seen_observations.append(s)\n",
    "\n",
    "visualize_cartpole_observation_distribution(seen_observations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Binarize environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gym.core import ObservationWrapper\n",
    "\n",
    "\n",
    "class Binarizer(ObservationWrapper):\n",
    "    def observation(self, state):\n",
    "        # Hint: you can do that with round(x, n_digits).\n",
    "        # You may pick a different n_digits for each dimension.\n",
    "        state = <YOUR CODE: round state to some amount digits>\n",
    "\n",
    "        return tuple(state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = Binarizer(make_env())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seen_observations = []\n",
    "for _ in range(1000):\n",
    "    seen_observations.append(env.reset())\n",
    "    done = False\n",
    "    while not done:\n",
    "        s, r, done, _ = env.step(env.action_space.sample())\n",
    "        seen_observations.append(s)\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "visualize_cartpole_observation_distribution(seen_observations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learn binarized policy\n",
    "\n",
    "Now let's train a policy that uses binarized state space.\n",
    "\n",
    "__Tips:__\n",
    "\n",
    "* Note that increasing the number of digits for one dimension of the observations increases your state space by a factor of $10$.\n",
    "* If your binarization is too fine-grained, your agent will take much longer than 10000 steps to converge. You can either increase the number of iterations and reduce epsilon decay or change binarization. In practice we found that this kind of mistake is rather frequent.\n",
    "* If your binarization is too coarse, your agent may fail to find the optimal policy. In practice we found that on this particular environment this kind of mistake is rare.\n",
    "* **Start with a coarse binarization** and make it more fine-grained if that seems necessary.\n",
    "* Having $10^3$–$10^4$ distinct states is recommended (`len(agent._qvalues)`), but not required.\n",
    "* If things don't work without annealing $\\varepsilon$, consider adding that, but make sure that it doesn't go to zero too quickly.\n",
    "\n",
    "A reasonable agent should attain an average reward of at least 50."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def moving_average(x, span=100):\n",
    "    return pd.DataFrame({'x': np.asarray(x)}).x.ewm(span=span).mean().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = QLearningAgent(\n",
    "    alpha=0.5, epsilon=0.25, discount=0.99,\n",
    "    get_legal_actions=lambda s: range(n_actions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rewards = []\n",
    "epsilons = []\n",
    "\n",
    "for i in range(10000):\n",
    "    reward = play_and_train(env, agent)\n",
    "    rewards.append(reward)\n",
    "    epsilons.append(agent.epsilon)\n",
    "    \n",
    "    # OPTIONAL: <YOUR CODE: adjust epsilon>\n",
    "\n",
    "    if i % 100 == 0:\n",
    "        rewards_ewma = moving_average(rewards)\n",
    "        \n",
    "        clear_output(True)\n",
    "        plt.plot(rewards, label='rewards')\n",
    "        plt.plot(rewards_ewma, label='rewards ewma@100')\n",
    "        plt.legend()\n",
    "        plt.grid()\n",
    "        plt.title('eps = {:e}, rewards ewma@100 = {:.1f}'.format(agent.epsilon, rewards_ewma[-1]))\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('Your agent has learned {} Q-values.'.format(len(agent._qvalues)))"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python",
   "pygments_lexer": "ipython3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}