4.acer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import random\n",
    "\n",
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import clear_output\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Use CUDA</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "use_cuda = torch.cuda.is_available()\n",
    "device   = torch.device(\"cuda\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Episodic Replay Buffer</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import deque\n",
    "\n",
    "class EpisodicReplayMemory(object):\n",
    "    def __init__(self, capacity, max_episode_length):\n",
    "        self.num_episodes = capacity // max_episode_length\n",
    "        self.buffer = deque(maxlen=self.num_episodes)\n",
    "        self.buffer.append([])\n",
    "        self.position = 0\n",
    "        \n",
    "    def push(self, state, action, reward, policy, mask, done):\n",
    "        self.buffer[self.position].append((state, action, reward, policy, mask))\n",
    "        if done:\n",
    "            self.buffer.append([])\n",
    "            self.position = min(self.position + 1, self.num_episodes - 1)\n",
    "            \n",
    "    def sample(self, batch_size, max_len=None):\n",
    "        min_len = 0\n",
    "        while min_len == 0:\n",
    "            rand_episodes = random.sample(self.buffer, batch_size)\n",
    "            min_len = min(len(episode) for episode in rand_episodes)\n",
    "            \n",
    "        if max_len:\n",
    "            max_len = min(max_len, min_len)\n",
    "        else:\n",
    "            max_len = min_len\n",
    "            \n",
    "        episodes = []\n",
    "        for episode in rand_episodes:\n",
    "            if len(episode) > max_len:\n",
    "                rand_idx = random.randint(0, len(episode) - max_len)\n",
    "            else:\n",
    "                rand_idx = 0\n",
    "\n",
    "            episodes.append(episode[rand_idx:rand_idx+max_len])\n",
    "            \n",
    "        return list(map(list, zip(*episodes)))\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.buffer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h2>Neural Network</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorCritic(nn.Module):\n",
    "    def __init__(self, num_inputs, num_actions, hidden_size=256):\n",
    "        super(ActorCritic, self).__init__()\n",
    "        \n",
    "        self.actor = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(hidden_size, num_actions),\n",
    "            nn.Softmax(dim=1)\n",
    "        )\n",
    "        \n",
    "        self.critic = nn.Sequential(\n",
    "            nn.Linear(num_inputs, hidden_size),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(hidden_size, num_actions)\n",
    "        )\n",
    "        \n",
    "        \n",
    "    def forward(self, x):\n",
    "        policy  = self.actor(x).clamp(max=1-1e-20)\n",
    "        q_value = self.critic(x)\n",
    "        value   = (policy * q_value).sum(-1, keepdim=True)\n",
    "        return policy, q_value, value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_env(render=False):\n",
    "    state = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    if render: \n",
    "        env.render()\n",
    "    while not done:\n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        policy, _, _ = model(state)\n",
    "        action = policy.multinomial(1)\n",
    "        next_state, reward, done, _ = env.step(action.item())\n",
    "        state = next_state\n",
    "        total_reward += reward\n",
    "        if render: \n",
    "            env.render()\n",
    "    return total_reward\n",
    "\n",
    "def plot(frame_idx, rewards):\n",
    "    clear_output(True)\n",
    "    plt.figure(figsize=(20,5))\n",
    "    plt.subplot(131)\n",
    "    plt.title('frame %s. reward: %s' % (frame_idx, rewards[-1]))\n",
    "    plt.plot(rewards)\n",
    "    plt.subplot(132)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1>Sample Efficient Actor-Critic with Experience Replay</h1>\n",
    "<h2><a href=\"https://arxiv.org/abs/1611.01224\">Arxiv</a></h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_acer_loss(policies, q_values, values, actions, rewards, retrace, masks, behavior_policies, gamma=0.99, truncation_clip=10, entropy_weight=0.0001):\n",
    "    loss = 0\n",
    "    \n",
    "    for step in reversed(range(len(rewards))):\n",
    "        importance_weight = policies[step].detach() / behavior_policies[step].detach()\n",
    "\n",
    "        retrace = rewards[step] + gamma * retrace * masks[step]\n",
    "        advantage = retrace - values[step]\n",
    "\n",
    "        log_policy_action = policies[step].gather(1, actions[step]).log()\n",
    "        truncated_importance_weight = importance_weight.gather(1, actions[step]).clamp(max=truncation_clip)\n",
    "        actor_loss = -(truncated_importance_weight * log_policy_action * advantage.detach()).mean(0)\n",
    "\n",
    "        correction_weight = (1 - truncation_clip / importance_weight).clamp(min=0)\n",
    "        actor_loss -= (correction_weight * policies[step].log() * (q_values[step] - values[step]).detach()).sum(1).mean(0)\n",
    "        \n",
    "        entropy = entropy_weight * -(policies[step].log() * policies[step]).sum(1).mean(0)\n",
    "\n",
    "        q_value = q_values[step].gather(1, actions[step])\n",
    "        critic_loss = ((retrace - q_value) ** 2 / 2).mean(0)\n",
    "\n",
    "        truncated_rho = importance_weight.gather(1, actions[step]).clamp(max=1)\n",
    "        retrace = truncated_rho * (retrace - q_value.detach()) + values[step].detach()\n",
    "        \n",
    "        loss += actor_loss + critic_loss - entropy\n",
    "        \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def off_policy_update(batch_size, replay_ratio=4):\n",
    "    if batch_size > len(replay_buffer) + 1:\n",
    "        return\n",
    "    \n",
    "    for _ in range(np.random.poisson(replay_ratio)):\n",
    "        trajs = replay_buffer.sample(batch_size)\n",
    "        state, action, reward, old_policy, mask = map(torch.stack, zip(*(map(torch.cat, zip(*traj)) for traj in trajs)))\n",
    "\n",
    "        q_values = []\n",
    "        values   = []\n",
    "        policies = []\n",
    "\n",
    "        for step in range(state.size(0)):\n",
    "            policy, q_value, value = model(state[step])\n",
    "            q_values.append(q_value)\n",
    "            policies.append(policy)\n",
    "            values.append(value)\n",
    "\n",
    "        _, _, retrace = model(state[-1])\n",
    "        retrace = retrace.detach()\n",
    "        compute_acer_loss(policies, q_values, values, action, reward, retrace, mask, old_policy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"CartPole-v0\")\n",
    "model = ActorCritic(env.observation_space.shape[0], env.action_space.n).to(device)\n",
    "\n",
    "optimizer = optim.Adam(model.parameters())\n",
    "\n",
    "capacity = 1000000\n",
    "max_episode_length = 200\n",
    "replay_buffer = EpisodicReplayMemory(capacity, max_episode_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAwIAAAE/CAYAAAD1x3TiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XmcZHdd7//Xp7au6r17pmdPMplkkjgkEGBEFNkuLgmIoN6LRH+KyjWi4NUrV0W9V7ioqPc+0J88RCBe+YHKIoJolChwQUCQAAHCZCeTCcns3TO913Zq+f7+OOdUV1XX1t3VXb28n4/HPKb7VHX16Upm5nzOZzPnHCIiIiIisrNEen0CIiIiIiKy8RQIiIiIiIjsQAoERERERER2IAUCIiIiIiI7kAIBEREREZEdSIGAiIiIiMgOpEBgmzKz683sXjNbMLP/0uvzkbUxsxeY2Zlen4eIiIhsHwoEtq9fA/7VOTfknHtbr0+mnpm91MzuN7NFM/t3MztW9Vifmf2xmZ0zsxkz+zMzi1c9Pm5mHzWztJk9YWY/VvfaPxYcT5vZ35vZ+Eb+bL1mZnvM7APB+zdnZl8ws++oe07T96jd+1v3OmZmf2hml4Nff2hmtp4/n4iIiHSHAoHt6yrggWYPmll0A8+l/nsfBd4HvAYYBf4RuNPMYsFT3gAcB24ErgOeAfz3qpd4O+ABe4EfB95hZk8JXvspwLuAnwgezwB/tsrzjLV/Vvd14fsOAl8BngmMA+8FPmZmg8Hrt3uPmr6/DdwOvBx4GvBU4KXAz63x/EVERGQDKBDYhszs08ALgT8N7rhfZ2bvMbN3mNldZpYGXmhmLzGzr5vZvJmdNrM3Vb3GYTNzZvbTwWMzZvYaM/t2MzthZrNm9qd13/dnzOyh4LkfN7Ormpzi9wP/5pz7vHOuCPwhcBB4fvD4S4G3OeemnXNTwNuAnwm+xwDwI8D/cM4tOuc+D9yJf1EL/oXrPzrnPuecWwT+B/DDZjbUwfv2AjM7Y2a/bmYXgP8vOP4DQZnVbJC9eGpw/KfN7B+rvv5RM/vbqs9Pm9nNwcd/Enw+b2ZfNbPnVj3vTWb2YTP7azObB37KzFLBf7MZM3sQ+PZ25x9yzp1yzv2Rc+68c67knLsDSADXt3uPOnh/670KeKtz7oxz7izwVuCnOj1XERER6R0FAtuQc+4/AP8GvM45N+ic+2bw0I8BvwcMAZ8H0sBP4t+Vfwnw82b28rqX+w7gKPCjwP8L/BbwPcBTgFeY2fMBzOxlwG8CPwxMBN//Ay1O0+o+NvwMQLPHD5nZCH6GoFj1MwF8Izgfgt+/UfVePIZ/d/u6FudSbR/+XfSrgNvN7OnAu/Hvcu/Cv5N+p5n1AZ8FnmtmETM7gH+x/Z0AZnYE/878ieB1vwLcHLz2+4G/NbNk1fd9GfBh/P8W7wPeCFwT/Pp+/AvupTfEL5fqKNMRBCMJ4GRwqNV71O79rVfzWm2eKyIiIpuIAoGd5R+cc19wzpWdcznn3Gecc/cFn5/Av3B/ft3X/E7w3E/gBw4fcM5NBnd//w14evC81wC/75x7KLjL/xbg5iZZgf8LPD+4A5/ADyASQH/w+L8Av2RmE2a2DwibnfvxL67n615vDj+4IXh8rsXj7ZSBNzrn8s65LH7py7ucc18K7q6/F8gDz3bOnQIW8C/wnwd8HDhnZjfgv4//5pwrAzjn/to5d9k5V3TOvRXoY+kOPcAXnXN/H/y3yAKvAH4vyIqcxs+KVDjnfsE59wvtfhgzGwb+CvifzrnwfWn1HrV7f+vVv9YcMKg+ARERkc1PgcDOcrr6EzP7DjP7VzObMrM5/Iv53XVfc7Hq42yDzweDj68C/iQon5kFpvHv5B+sPwnn3MP4d7j/FDgffM8HgXAqzu8BXwfuBf4d+HugEHzvRWC47iWH8S/I6eDxdqacc7mqz68CXh/+XMHPdgVwIHj8s8AL8AOBzwKfwQ8Cnh98DoCZ/begbGoueI0Rat/rmv82wetXH3uiw/OvMLMUfv/F3c653696qNV7tNL3r/75w8Cic86t9HxFRERkYykQ2FnqL87ej1//fYVzbgR4J7UlOStxGvg559xo1a+Uc+7fG56Icx92zt3onNuFXwZzGL98Budc1jn3OufcQefcEeAy8NXg7vo3gVjQcBx6GkuN0Q8EnwOVEp2+4Os6Uf8enca/M1/9c/U758KypzAQeG7w8WepCwSCfoBfw7/LP+acG8W/c179Xtd/3/P4AUfoyg7Pn+B79uEHUGdY3rzb6j1q9/7Wq3mtNs8VERGRTUSBwM42BEw753Jm9iz8HoLVeifwG1XTe0bM7D81e7KZPdPMomY2AdwB3BlkCjCzg2Z2wHzPxm9mfSOAcy4N/B3wZjMbMLPn4NfX/1Xw0u8DXmpmzw0aX98M/J1zrtOMQL0/B14TZE8s+J4vqWo+/ix+Y3bKOXcGv1zqFvx+gq8HzxkCisAU/kX2b7P8rnu9D+G/n2Nmdgj4xU5P2PxRqx/Gz9i8KixPqtL0Perg/a33l8CvhP/NgNcD7+n0XEVERKR3FAjsbL+Af8G3APw2/sXnqjjnPoo//eeDweSb+4FbW3zJnwCzwCPADPCzVY9dg18SlMYfffmGoEeh+rxTwCR+X8PPO+ceCM7jAfwSp/cFjw8FzwfAzP7ZzH5zBT/XPcG5/WlwniepmooTNNUu4gcAOOfmgVPAF5xzpeBpH8fve/gmfolPjuWlQPX+Z/Dcx4FPUHchbmbvNLN3Nvna7wJ+APg+YNb8yVGL4aSidu8RLd7fIHhYrHruu/DLj+7D/2/+seCYiIiIbHKmUl4RERERkZ1HGQERERERkR1IgYCIiHSNmb3bzCbN7P4mj5uZvc3MTpq/nPAZG32OIiLiUyAgIiLd9B78hvlmbsVfUngUf0/HOzbgnEREpAEFAiIi0jXOuc/h7xFp5mXAXzrf3cCome3fmLMTEZFqCgRERGQjHaR2atYZGiweFBGR9Rfr9QkA7N692x0+fLjXpyEisil99atfveScm+j1eWw0M7sdv3yIgYGBZ95www09PiMRkc1nLf9GbIpA4PDhw9xzzz29Pg0RkU3JzJ7o9Tl00Vlqt2YfCo4t45y7A3/hIMePH3f6d0JEZLm1/Buh0iAREdlIdwI/WbU5fM45d77XJyUishNtioyAiIhsD2b2AeAFwG4zOwO8EYgDOOfeCdwFvBh/S3cG+OnenKmIiCgQEBGRrnHO3dbmcQe8doNOR0REWlBpkIiIiIjIDqRAQERERERkB1IgICIiIiKyAykQEBERERHZgdoGAmZ2hZn9q5k9aGYPmNkvBcfHzeyTZvZo8PtYcNzM7G1mdtLMTpjZM9b7hxARERERkZXpJCNQBF7vnDsGPBt4rZkdA94AfMo5dxT4VPA5wK3A0eDX7cA7un7WIiIiIiKyJm3HhwaLXs4HHy+Y2UPAQeBl+LOiAd4LfAb49eD4XwYj4u42s1Ez26+FMSK9N7WQ59MPX6TsWj/vqvF+vuva3U0fn8sW+PgDFyi1e6EVGutPcMuN+2qOFUtlvnjqMs89uvLt6cVSmY/dd56MV1rV+STjEV5y0wESsaV7Js45/uX+C8xmCyt6rW8/PM61ewZXdR4iIiLrYUV7BMzsMPB04EvA3qqL+wvA3uDjg8Dpqi87ExyrCQTM7Hb8jAFXXnnlCk9bRFbjXZ99jP/z+cfbPi8WMR548/fTF4s2fPyDX36S3//nh7t9egB8/Jefx/X7hiqf3/mNc/zKh77Bp17/fK6ZWNmF9Je/Nc0vffDeNZ3PYF+c7z22t/L5yclFfv59X1vx6/zBD9+kQEBERDaVjgMBMxsEPgL8snNu3swqjznnnJmt6Nagc+4O4A6A48ePd/e2oog0dH4ux5Xj/Xzo576z6XP+6cQ5fvdjD3FxLs+Vu/obPudy2qMvFuGzv/rCrp3bE5fT/Ogdd3P/2bmaQODEmTn/ey56XLPCpMB8tgjAX/7Ms7hu71CbZ9eaWsjz0j/9PFML+Zrjk8Hn7/jxZ/D0K8c6fr3hlPY3iojI5tLRv0xmFscPAt7nnPu74PDFsOTHzPYDk8Hxs8AVVV9+KDgmIj02tZBn30iSfSPJps8JL8LPzWWbBgJzmQKj/fGWr7NSE0N9JOMRHjg3z488c+n4g+fmAZhfYSkOQK7glwQdGkut+FxH++MAzGS8muPTaf/za/cMdvXnFxER2WidTA0y4C+Ah5xzf1T10J3Aq4KPXwX8Q9XxnwymBz0bmFN/gMjmMLmQY89QX8vn7B9JAXB+Ltv0OXPZAiOpeFfPLRoxbtg3zIPn5yrHymXHg+eDQCC38kAgGwQCqUTjEqdWkvEoA4lo5cI/FAYGYwOJFb+miIjIZtJJRuA5wE8A95lZWGz7m8AfAB8ys1cDTwCvCB67C3gxcBLIAD/d1TMWkVWbWsgz0SYQODDq3+U+N5tr+pz1CAQAjh0Y5p++cQ7nHGbG6ZkMi3m/vGc1GYGwSTgVX3kgAP7F/ky6cUZgdB1+fhERkY3UydSgzwPW5OEXNXi+A167xvMSkQ588MtP8uXHp/mjH7257XPT+SJpr8SeodblLP2JGCOpeNuMQBgwdNNTDgzz/i89yZmZLFeM9/NAUBYEMJ8rrvj1cmvICACMDySYrisNmkl7jKTixKLaxygiIlub/iUT2cL+7dFLfPKhix09N2x6bZcRANg/kuR8m4zA8HpkBPYPA1QCgAfPzRONGIlYZFUZgaxXImKQWOVF+1h/Yllp0OW0x7jKgkREZBtQICCyhV1O51nMFyl3MM8/nHbTrkcA4MBoinNzzQOB+XUqDbph3zARgwfP+X0CD5yb4+ieQcb7E6vuEUjFo1RPOVuJ8YHlgcBMxmOsX2VBIiKy9SkQENnCZtIFnIOFfPuymRVnBJqUBhVLZRbyxXUJBFKJKNdMDFYahB88P8+xA8MMp2KVUaArkfFKqy4LAj8QWN4jUGB8oP17KCIistkpEBDZwi4HF6mdlM1MLvh3+DvNCMxmCmQbbOQNa/XXIxAAv2H4gXPzXFrMc3E+z7H9wwwn48ytcnzoWgOBtFeq9BqA3yMwPqCMgIiIbH0KBES2KOdcZZRlJ2UzUwt5YhFjrL99ffv+YD7+uQZZgfCCfL0CgaccGOb8XI4vnLwUfD7CcCq+utIgr7TqiUFA5b2azfjf2znHdMbT6FAREdkWFAiIbFHz2SKloDegk7KZyYU8uwf7iETa18tXdgk0aBhe70Dg2P4RAD781TPB58MMJ2OrCgQyhbUFAuGd/7BPIOOV8IplxjsIpkRERDY7BQIiW1T1WMtOMwKd9AdA1S6BHmQEjh3wJwd9/uQlDo2lGOmPM5KKr6pHILfGHoEwIxAGAuHvygiIiMh2oEBAZIuaTucrH3fWI5DvqD8AYF9QGtSLjMD4QIL9I0mc88uEAIZTcRZyhY6mI1XLrjkjEAQCmdpAQBkBERHZDhQIiGxR0+mli/+FDpZtrSQj0BeLsnsw0XBy0HoHArAUAIRlQsPJOGUHaW9lWYFsF5qFgcrkoDAgGB9UICAiIlufAgGRLaomI9CmNKhYKnM53XlGAPw+gUa7BMLsw3osFAsdO+AHAEsZAX8J+kq3C2e9Esk1ZARGUnHMljIBM8oIiIjINqJAQGSLCkeHJqKRtvXz02kP5zrbIRDytws3zgj0xSJrusBu54XXT3DleD/PuGoM8DMC0FkJVLVsoUT/GjICsWiEkVS8Mp1JPQIiIrKdKBAQ2aJm0h6peJSJob62GYHJyjKxZMevf2A0xbnZLM7V1uXPZdZnq3C1p185xud+7YWV0pww+7DiQGCN40PBv/tfyQhkPKIRYzgZW9NrioiIbAYKBES2qMtpj/GBBEPJWNsL5JVsFQ4dGE2S9krLynHmsusfCNQLMwIrWSrmnFtzszD4d/+XMgIFxvoTmLUfwSoiIrLZKRAQ2aJmgkBgONl+2dZKtgqHKrsE6hqGexIIrKJHIFcoA5BKrO3u/Vh/gsuLYSCQ11ZhERHZNhQIiGxR02l/w+1wKta2R2C1GQFYPkK0lxmBlZQGZQslAFLxtf01t6sqIzCTLlTKlURERLY6BQIiW9R0xmNXxxmBPMPJ2IoafMOMQP1SsV4EAkPJMCOwikBgDc3CEJQGpQs455jOeAoERERk21DHm8gWNb3oMdafoOxcRz0CK8kGgF9GFLHlGYH5bGFdR4c2EotGGOxrn/molg12Dqx1utH4QByvVCbtlZhJe5VtwyIiIludAgGRLShXKJH2SuwaTJAvlFjIFymXHZFI4yZWf6tw5xODwL/43jucrMkIlMqOhXxxwzMCAMPJ2MoyAp7fI9DfhR4BgMuLeWaUERARkW1EpUEiW1BYsz4+kGA4Fcc5WGyxdXc1GQEIdwksZQTmN2CrcDPDqfgqewTWmhHwL/y/dTlD2aGMgIiIbBsKBES2oHCKzVh/om0jrXOOyYXciiYGhfaPpmqmBs31MhDooBei2lKPwNr+mguXh52cXARQRkBERLYNBQIiW1CYEdg1mFgardmkfn4xXyRXKK8qI3BgJMm5uVxlqVhPA4EOpiNVC3sEUvG1lQbtUiAgIiLblAIBkS0o3HRbkxFocrc8HB26Z3jlgcCVuwbwimXOz/nlQZVAoH8rZQTWPjUI4LEpBQIiIrK9KBAQ2YLCQGBX0CMAzUuDJsMdAoMraxYGuHZiEIBHg7vhvc0IxFe0WThsFl5rj8BQX4xYxDgVBAJjCgRERGSbaBsImNm7zWzSzO6vOvY3ZnZv8OtbZnZvcPywmWWrHnvnep68yE41nfaImH9BHmYEFpps3V1LRuDaPX4gcHIzBALJGIvBdKR6xVKZj504XylhAshUSoPWFgiYGWMDCS4FfRnjahYWEZFtopOMwHuAW6oPOOd+1Dl3s3PuZuAjwN9VPfxY+Jhz7jXdO1URCU0H8+wjEVvqEWhSNrOUEVh5ILB7MMFof3xzBALBdKSF/PKA59MPT/La93+NE2fmKsdyXSoNgqWL/2Q80pXXExER2QzaBgLOuc8B040eMzMDXgF8oMvnJSItTKeX5tkP9rVuFp5ayBOPGqOrqOs3M66dGOSxIBCYzxZIxCJrXtK1Gq1KoM7M+JONwpIp8HsEohEjHm28W2Elxgb8761sgIiIbCdr7RF4LnDROfdo1bGrzezrZvZZM3vuGl9fRBq4nPYqteqVrbtNMgIzQdDgx+0rd+2eQU5OLWUEepENAFo2RV+Y95uZZ7NVgYBXJhWPrvrnrrZrwM+mjA8qEBARke1jrYHAbdRmA84DVzrnng78CvB+Mxtu9IVmdruZ3WNm90xNTa3xNER2lpm0VxlrCTCUjDVtFp7JeIymVn8Be+2eQabTHtNpr7eBQIsxqZWpRpml9yBbKHatjCfMCGiZmIiIbCerDgTMLAb8MPA34THnXN45dzn4+KvAY8B1jb7eOXeHc+64c+74xMTEak9DZEeqLg2C1qM1Z7OFNY37vKaqYXizZgTOz/qlQXNVQULWK625UTgUlgRpdKiIiGwna8kIfA/wsHPuTHjAzCbMLBp8fAQ4Cpxa2ymKSLVy2TGTqQsEWizbmssUGFtDIHB0kwQCIy16BOr3HIDfI9CtQCAsw1JGQEREtpNOxod+APgicL2ZnTGzVwcPvZLlTcLPA04E40Q/DLzGOdew0VhEVmcuW6DsWEFGYG2lQQdGUqTiUR6dXNgkGYHagKdcdlycXx4IZLwSyS6VBoXvtTICIiKyncTaPcE5d1uT4z/V4NhH8MeJisg6mc4E8+xrMgJxvjm50PD5s5nCqiYGhSIR45o9Az3PCAwm/b+u6peKXUrnKQa7BaofyxVK9HcrIxBkArRMTEREthNtFhbZYsIRmbUZgcalQVmvRL5YXlOPAPgbhh+9uMhCrlgZ47nRohFjqG95U/SFoCwIasuGsoVS15qF94/4W5n3Da98O7OIiMhmpUBAZIu5HGy4ra5XH07FWcgVlm3dDcdprqU0CPzJQeGIzl5lBMD/OetLoM7N+ud1cDRV2yPQxWbho3uHeP/Pfgf/4YY9XXm97c7MbjGzR8zspJm9ocHjV5rZvwajpk+Y2Yt7cZ4iIjudAgGRLWYmKA3aNVjbI1B2kPZqswKzwTjNtZQGAVy7Z6jycc8DgbrMx4U5f2LQDfuG6vYIdC8jAPBd1+wmGln7ToLtLhgY8XbgVuAYcJuZHat72n8HPhSMmn4l8Gcbe5YiIgIKBES2nLA0qDYjEMzYz61XIDBY+bingUBy+eK08/M5EtEIh3cPrNvUIFmRZwEnnXOnnHMe8EHgZXXPcUC4Y2YEOLeB5yciIgEFAiJbzHTaYyARJVl1kVuZqFNXPz/XpdKgq3b1Ewvuhvc+I7C8R2DfSJLRVJxcoUy+WAK62yMgK3IQOF31+ZngWLU3Af+PmZ0B7gJ+sdELafGkiMj6UiAgssVMp71l02uGm8zY71ZGIB7ccYdeZwTiLNRlPc7PBoFA8DPOZf1eiVyhXBMsyaZyG/Ae59wh4MXAX5nZsn+PtHhSRGR9KRAQ2WKm0x676gOBICNQf5E806VAAPzJQdDrjMDyqUHn57PsH0nWBEO5ICvQr4xAL5wFrqj6/FBwrNqrgQ8BOOe+CCSB3RtydiIiUqFAQGSLmck0ygiEPQJ1GYGsRyIa6Uqt/PX7hohFrCtBxWoNJ+Ms5IuUgulI5bLj4lyefSPJSoAyly2Q9fxAQD0CPfEV4KiZXW1mCfxm4DvrnvMk8CIAM/s2/EBAtT8iIhus7UIxEdlcLi96Nc270KJHIFgmZrb2aTevfu7VPOfa3T0ttwnv+i/mioz0x5nOeHilMgdGUrWBQEGBQK8454pm9jrg40AUeLdz7gEzezNwj3PuTuD1wJ+b2X/Fbxz+Keeca/6qIiKyHhQIiGwxMxmP8f7ajMBQsvnUoG7dwR9OxnnW1eNdea3Vn8PSduGR/nhlmVjTjIBKg3rCOXcXfhNw9bHfrvr4QeA5G31eIiJSS6VBIltIrlAi45WWlQbFohEGEtHlzcJZb80TgzaTA6MpAB6dXADg3Ky/Q2B/dSCQUUZARESkEwoERLaQyjKxgeUX90PJ5Vt3ZzP+nfPt4vjhMYb6Ynz8gQsAlW3H+6qahWeVERAREemIAgGRLeTyYrBMrEEg4E/UaVAa1MMpP93WF4vyom/bwycfvEixVOb8XI541Ng90Ec8GmGwL1bTI6DxoSIiIs0pEBDZQsKMwHijQKBRRiC7fMLQVnfLjfuYyRT48uPTXJjLsXc4SaRq2Vl1j4DGh4qIiDSnQEBkC5lOBxmB/kYZgdpAIFcokSuUezr3fz0877oJkvEI//LABc7N+jsEQuHmYfUIiIiItKdAQGQLCQOBRj0Cw8na0qC5bPeWiW0m/YkYL7huDx9/4ALn5rLsG0lVHhtJ1ZYGqUdARESkOQUCIlvITNojYkvz9KuND/RxaTFPOI49LCPaTlODQrfcuI+L83lOT2c5UJURqC8NUiAgIiLSnAIBkS1kOuMx2p8gGlm+IOzq3f1kvBIX5/OA3ygM2y8jAPDCG/YQj/rvwb5WgYBKg0RERJpSICCyhUynvYaNwgBHJvxtw6cuLQLbOxAYScV5zrW7AWp6BCqBQKFELGLEo/orTkREpBn9KymyhUynl28VDl29ewCAU1NpAOayQWlQk+dvdbfeuA+AQ2P9lWMjqTi5QpnZbEFlQSIiIm3Een0CItK5mXSBw7v7Gz62bzhJKh7l8Ut+IFDJCGyzqUGh//jMKzgwmuIpB4Yrx0aCoOfiXE5lQSIiIm0oIyCyhVxuURoUiRiHdw9wasovDZrJFIhHbdvO0o9GjOcencBsqV8iHJV6fi6njICIiEgbCgREtgjnHDOZ5oEAwJGJgUpGYC7rMZJK1Fwob3dhIHBxXhkBERGRdhQIiGwR87kipbJruEwsdGT3AKdnsnjFMrOZAmPbsFG4lTAQuJz2lBEQERFpo20gYGbvNrNJM7u/6tibzOysmd0b/Hpx1WO/YWYnzewRM/v+9TpxkZ1mJlgm1i4jUCo7npzOMJspbMuJQa1Ub1FWRkBERKS1TjIC7wFuaXD8j51zNwe/7gIws2PAK4GnBF/zZ2amf41FuuByB4HA1buDEaJTi8xmC4xsw2VirSgQEBER6VzbQMA59zlgusPXexnwQedc3jn3OHASeNYazk+k5177/q/xWx+9r9en0VFGIBwh+vilNHMZb8dlBIaTS4PQVBokIiLS2lp6BF5nZieC0qGx4NhB4HTVc84Ex5Yxs9vN7B4zu2dqamoNpyGyvh6fSvPNiwu9Pg2mM34g0KpHYCQVZ/dgglNTaWYyhW07OrSZWDTCUJ8fDCgjICIi0tpqA4F3ANcANwPngbeu9AWcc3c45447545PTEys8jRE1l+hVGYhV+z1aTDdQUYA4MjuQR6+ME+2UGKszXO3o+Eg+FFGQEREpLVVBQLOuYvOuZJzrgz8OUvlP2eBK6qeeig4JrJlbZZAYCbt0ReLtN0LcPXuAR48Pw/U1szvFOHPrIyAiIhIa6sKBMxsf9WnPwSEE4XuBF5pZn1mdjVwFPjy2k5RpLcKJcd8ttDr02A6WCbWbi/AkYkBCiUHsON6BKAqEFBGQEREpKVYuyeY2QeAFwC7zewM8EbgBWZ2M+CAbwE/B+Cce8DMPgQ8CBSB1zrnSutz6iIbwyuVWfSKlMuOSKR3y7mm017L/oBQ2DAMMLrDpgaBMgIiIiKdahsIOOdua3D4L1o8//eA31vLSYlsJoVSGedg0SsynOzdHfbpjMeuwfYX9kcmBisfKyMgIiIizWizsEgbhWIZoOd9AjMdZgSuHO8nGmQudmQg0K+MgIiISCcUCIi0EdbbL+R62ydwOegRaCcRi3DFWAqA0Q4Ch+1GGQEREZHOKBAQacE5h1fqfUYgnFzUSUYA/D6BWMQY2IEXw+oREBER6UzbHgGRnaxYdpWPe5kRmAmWiY130CMA8F3X7GY2W2g7YWg7UiAgIiLSGQUCIi0UgmwA9DYjMJP2g5DxDjMCP/u8I/zs846s5yltWkf3DvrlUeP9vT4VERGRTU2BgEi2bbrYAAAgAElEQVQLheJSRqCXuwQup/MAjA3svObflbph3zDf/N1be30aIiIim556BERa8KoyAvObICOwa6CvZ+cgIiIi24sCAZEWNktp0HTQI6CMgIiIiHSLAgGRFmoDgd6VBk0vBoHADhwHKiIiIutDgYBIC5slIzCT8RhKxohH9UdWREREukNXFSIteMXNMT50usNlYiIiIiKdUiAg0oK3iTICCgRERESkmxQIiLQQlgYN9sV62yyc9jreISAiIiLSCQUCIi0Uin4gMD6QYL6Xm4XTHmPKCIiIiEgXKRAQaSEsDdo1mOj5+FCVBomIiEg3KRAQaaFQ8puFdw0kWMwXKZVdm6/ovqxXIlcoa3SoiIiIdJUCAZEWwh6B8G78Yn7jswLhMrFxLRMTERGRLlIgINLCUiDQB/RmhOhMWsvEREREpPsUCIi04AXNwruCjEAv+gSm02FGQIGAiIiIdI8CAZEWwh6B8R4GAjNBaZCmBomIiEg3KRAQaaFSGjQYBgIbXxpUyQioNEhERES6SIGASAthIBCWBvVil8BM2iNiMJxSs7CIiIh0jwIBkRaW9giEzcK9mRo02p8gGrEN/94iq2Fmt5jZI2Z20sze0OQ5rzCzB83sATN7/0afo4iIdBAImNm7zWzSzO6vOva/zexhMzthZh81s9Hg+GEzy5rZvcGvd67nyYust0JxaY8A9KhHIF1grF/ZANkazCwKvB24FTgG3GZmx+qecxT4DeA5zrmnAL+84ScqIiIdZQTeA9xSd+yTwI3OuacC38T/Cz30mHPu5uDXa7pzmiLd8+mHL3LizGxHzy2UykQjRjIeJRGL9KQ0aDqtrcKypTwLOOmcO+Wc84APAi+re87PAm93zs0AOOcmN/gcRUSEDgIB59zngOm6Y59wzoW3Ru8GDq3DuYmsi9/9p4d41+dOdfTcQqlMPOqX5AwnYz2bGqQdArKFHAROV31+JjhW7TrgOjP7gpndbWb1N5tERGQDdKNH4GeAf676/Goz+7qZfdbMntuF1xfpqlyhVNkP0I5XKhOP+n9MhpLxnu0RUEZAtpkYcBR4AXAb8OdhiWk1M7vdzO4xs3umpqY2+BRFRLa/NQUCZvZbQBF4X3DoPHClc+7pwK8A7zez4SZfq7/gpSe8UrkyDaidQqlMohIIxDZ8fKhzzs8IKBCQreMscEXV54eCY9XOAHc65wrOucfxS0yP1r+Qc+4O59xx59zxiYmJdTthEZGdatWBgJn9FPADwI875xyAcy7vnLscfPxV4DH8FPAy+gteesUrriAQKLqqjMDGlwYt5osUSk47BGQr+Qpw1MyuNrME8Ergzrrn/D1+NgAz243/70Rn9XoiItI1qwoEgnrOXwN+0DmXqTo+EUyMwMyO4N/h0V/usql4pXJlGlA7hVKZeMzvERjqizOf3diMwEza/37KCMhWEfSPvQ74OPAQ8CHn3ANm9mYz+8HgaR8HLpvZg8C/Ar8a3kQSEZGNE2v3BDP7AP6dm91mdgZ4I/6UoD7gk2YGcHcwIeh5wJvNrACUgdc456YbvrBIj3jFcmU/QNvnVvUIDKc2PiMwnQm2Cg9ofKhsHc65u4C76o79dtXHDr989Fc2+NRERKRK20DAOXdbg8N/0eS5HwE+staTElkvpbKj7Oi8WbhY3SMQr+kRmFrIYwa7g2Vj62Em7QcCmhokIiIi3abNwrKjhAHAipqFY0s9AmmvRKnslxX9/F9/lTd85MT6nGhgOh1mBBQIiIiISHcpEJAdZeWBgKsZHwqwmCuykCvwtSdnuLTorc+JBmYyCgRERERkfbQtDRLZTvKlEuBf4HfCq1ooNpT0/7jM5wqcnFqk7PydBOtpOu0RjxqDffqjKiIiIt2ljIDsKGEA0GmzcKG6WTgIBBZyRb50yu+BX+9AINwqHDTli4iIiHSNAgHZUVbVI1BXGrSQK/Clx/1Jh7lCZ6+zWtoqLCIiIutFgYDsKJVAoMOpQdULxYaDQODCfI4TZ+YAyG5AaZAmBomIiMh6UCAgO0qYCei0R8BfKLY0NQjgs49MUSo7vm3/8IYEAsoIiIiIyHpQB6LsKPkgE+CVyjjn2tbeN2oW/vQjk0Qjxndfu4uHzs9TLjsikfY1/CfOzPIv91/g8qLH5XSeZx/ZxX9+7pGWXzOTKTCmZWIiIiKyDhQIyI5SvUisUHIkYq0v4Bv1CMxmCjztilF2BYvEcsUS/Yn2f5R+7cMneHRykV0DCfLFMifOzLUMBEplx2zGY1ylQSIiIrIOVBokO0p1k3AnDcPVewQSsQh9QZnQs68eJxl83EnD8GNTizx8YYHfevG38eXf+h5e8/xrmFzIM1+1qbjefLZA2cGYSoNERERkHSgQkB2lNiPQQSBQXBofCktZge84Mk4qEQU6axi+68R5AG69aR8A10wMAPDY5GLTr5nWMjERERFZRwoEZEep3h/QyS4Br1QmXlU+NJyMETE4fnicZNwPBDrZJXDX/Rd45lVj7B9JAXDtnkEATrYIBGbSfiCgqUEiIiKyHhQIyI5S3yPQTnWPAPhlOk85MMJwMl4JBLJe60Dg8UtpHjo/z4tv2l85duV4P/GocXKqRUYgrYyAiIiIrB81C8uOUp0FaLdLoFR2lB01pUG/87IbiQVThMJAIF9sHQjcdV9QFnTjvsqxWDTC4V0DPDaZbvp1M0FpkHoEREREZD0oEJAdpToj0K40KOwhqA4Ejh0YrnycqmQEWr/Ox06c5+lXjnJgNFVz/No9gzx8YaHp102n/UZiTQ0SERGR9aDSINlRagKBNhkBrxIINB4xmoyHU4OaZwS+dSnNg+fneUlVWVDo2j2DPDmdaZpRmMl4JOORSlOyiIiISDcpEJAdZSXjQ8PSoUSs8R+TSkagRSBw1/1+WdAtVWVBoWsmBimVHU9czjT82suLnhqFRUREZN0oEJAdZSXNwuHj1aVB1TqZGvSlU9PcsG+IQ2P9yx4LJwc1GyF64swsR/cOtTxHERERkdVSICA7ireSjECDHoFqnQQCi/kiu4MNxPWOBLsEGo0QvbSY59HJRb7zyK6W5ygiIiKyWgoEZEdZyR6BfLF1j0BYu99qs3A6X2xa49+fiHFwNMVjDUaI3n3qMgDPPjLe8hxFREREVkuBgOwoNaVBbZqFw4xAX5MegWRwvFWPQMYrMdCi2feaPYMNdwncfeoyA4koNx0caXmOIiIiIqulQEA2nf/83nu48xvn1uW11zo+tFosGiEetbaBQH9f8ym910z4uwTK5dp+hS8+dplvv3qcWJPvLSIiIrJWusqQTaVcdnzq4Yt8KSiN6bbaZuG1BQIAyVi0ZY9Axiu2zAhcu2eQbKHE+flc5djkQo7HptLqDxAREZF1pUBANpW0V8Q5mMsW1uX1C6UysYhf818otp4a5BVbTw0CSCaaBwLlsiPjlUglWmUE/MlB1Q3DXzo1DcCzFQiIiIjIOuooEDCzd5vZpJndX3Vs3Mw+aWaPBr+PBcfNzN5mZifN7ISZPWO9Tl62n4VcEYD54Pdu80plBoJSnU5LgxKxxs3C4C8Va9YsnAsWhbXLCEDtCNEvnrrMUF+Mp1RtMRYRERHptk4zAu8Bbqk79gbgU865o8Cngs8BbgWOBr9uB96x9tOUnSIMBNYrI+AVy5UL826UBqXiUbJe44xAOu8fb9UjsGsgwWh/vKZh+O5T6g8QERGR9dfRlYZz7nPAdN3hlwHvDT5+L/DyquN/6Xx3A6Nmtr8bJyvb32LeDwAW1isQKDkGk/6FeVd6BOLRyp3/ehnPD2r6480zAmbGNROD3POtaWbSHhfnc5xSf4CIiIhsgLXcctzrnDsffHwB2Bt8fBA4XfW8M8Exkbbm1z0jUKqUBrXbLOy12SwMfiDQLiMw0Nc8EAD40W+/gsem0rzwrZ/hLXc9BKg/QERERNZfV2oPnHMOaH1VVcfMbjeze8zsnqmpqW6chmwDSz0CBfz/rbrLK5bpD0qDvHZ7BILHE20zAo1fJ1sIMgItmoUBXnH8Cu76L8/lur1D/MO95xhKxjim/gARERFZZ62vUFq7aGb7nXPng9KfyeD4WeCKqucdCo7VcM7dAdwBcPz48e5f8cmWtBgEAoWSI1sotb2IXqlCyTESjZCIRjrfI9CiWTgVj3Bxbm0ZAYDr9w3xN7c/m386cZ541IhGmn9PERERkW5YS0bgTuBVwcevAv6h6vhPBtODng3MVZUQibS0kFsqCZrPdn9ykFcsk4j5i8A63Sy81h6BVLyzYMbMeOnTDnDLjWqpERERkfXX6fjQDwBfBK43szNm9mrgD4DvNbNHge8JPge4CzgFnAT+HPiFrp+1bFsLVWND16NPwCuVScSixGORts3CnfQItJoalPE6zwiIiIiIbLSOblU6525r8tCLGjzXAa9dy0nJzrWYX+dAoFgmHjXi0UjlQr+Zyh6BdhmBJgvF0kEg0O3yJhEREZFu0KBy2VTma0qD1icj0BfzewTajg8thqVBrRaKRZsuFMvkw2ZhZQRERERk81EgIBvqjs89xiMXFpo+vpArkgrm7q9XRiARDXoEOmgWNqNl424qHsUrlSmVl2cXwoxAqsUeAREREZFeUSAgG8YrlnnLXQ/zN1853fQ5i7kiB8dSQG12oJvnEI9GSHTYIxCPRjBrlRHw/wg1Kg/KekX6E1EimgAkIiIim5ACAdkw6aBU5uxspulzFvIFDoz6gcB6ZAQKpXBqUKT9HoFSuWV/AEAqKPvJNggE0l73x5+KiIiIdIsCAdkw6WCc5pmZbNPnLOSKjPXHGeyLdX18aLnsKJbdUiDQQbNwq/4AgGTMDwQaZQQy+aL6A0RERGTTUiAgGyZcsHV2tnkgsJgrMtgXYzgZ63pGwKvaC5CIRjraI9BqdChAMtEiEPBKCgRERERk01IgIBsmHA06mynUjAmttpArMpSMM5yKd71HIAwE+mIR4rH2zcL5YgeBQCzsEVj+WhmvxECfSoNERERkc1IgIBsmXXXxf7ZBeVCuUMIrlRlKxhhJxbufEQgyAGFpUPupQX4ZUSutewRUGiQiIiKblwIB2TDVgcCZmeUNw2GWYCgZ8zMC6xUIRDvsESi2bxZOxlv1CKg0SERERDYvBQKyYarLgRr1CSzklgKBkXUMBDodH1oolYnHWjcLhzsCsl6DQKBQZEBTg0RERGSTUiAgG6Y2I9AoEPAv/If64gwnu18aFF74J4LNwu3Gh3qdNAuHewQavFYmX6qUDomIiIhsNgoEZMOEm3YPjqYa9ggsBhmBwSAjkPZKFNvctV+JfE2PQGebhdsHAkFpUIOMQNorqllYdiQzu8XMHjGzk2b2hhbP+xEzc2Z2fCPPT0REfAoEZMMs5ovEIsaRiYGGPQLzueoegVjNsW4IpwaFPQIdNQt3GggUawOBUtmRK5TVIyA7jplFgbcDtwLHgNvM7FiD5w0BvwR8aWPPUEREQgoEZMNk8v4d8kNjqSY9An4p0HAyzkgqDtDVPoFC3dSgTjYLt1so1qxHIJwipB4B2YGeBZx0zp1yznnAB4GXNXje7wB/COQ28uRERGSJAoEeOjOT4eVv/wJTC/len8qGWMyXGOyLcWisn0uL3rKL57CZ2F8o5gcC3ewT8Kp7BGIRCm2mBnmd7BGoTA2qDSoywc+iHgHZgQ4Cp6s+PxMcqzCzZwBXOOc+tpEnJiIitRQI9NC9p2e59/Qs952d7fWpbIh0vshAX5SDoykAzs7WlgctVPcI9AcZgS4uFaueGtRxj0CbPQLRiJGIRpbtEQj7IQb6FAiIVDOzCPBHwOs7eO7tZnaPmd0zNTW1/icnIrLDKBDooZm0B8CFuZ2REQibZw+N+YFA/eSghVyBVDxKPBpZn4xA3R6BYtlRLjfPCnTSIwD+5KD6PQIZzw9q+lUaJDvPWeCKqs8PBcdCQ8CNwGfM7FvAs4E7GzUMO+fucM4dd84dn5iYWMdTFhHZmRQI9NBMxr/IvTC3vF5+O1rMFxnsi3GwSSCwmC8ymPQvnJd6BNahWTgoDQIolJtnBTrpEQC/PGh5IOB/rmZh2YG+Ahw1s6vNLAG8ErgzfNA5N+ec2+2cO+ycOwzcDfygc+6e3pyuiMjOpUCgh6bDjMD8zuiVS+f9BVt7hpLEo7asYXg+V2SoLhBYj4xAX7BHoPpYI52MDwW/D2BZaVBeGQHZmZxzReB1wMeBh4APOeceMLM3m9kP9vbsRESkmq5Semgm4wcC5+d2SiBQor8vSjRiHBhNNSgNKjIUlAQl434d/3o0C8eD0iCgZcNwJ83CAMnY8oxAVj0CsoM55+4C7qo79ttNnvuCjTgnERFZThmBHgozAhd3SEYgLA2CcKlYfbNwgeEgI2BmjKTiXW0Wrh8fCrRsGC6UXKWEqJVkIkq2bmpQpVlYGQERERHZpBQI9FCYEbiwAzICzrlgapB/YXxobHlGYDG3FCiAv09gvcaHhrX/7UuDOugRiDVvFtb4UBEREdmsFAj00Ezav8idzxUrF47blVcqUyy7qoxAP5MLefJVG3kXqnoEAIZT8a4uFFsaH2pLzcJNMgLlsqNYdh33CNQHAum8MgIiIiKyua06EDCz683s3qpf82b2y2b2JjM7W3X8xd084e1kOu2xe7AP2P5ZgaULY/8OeThC9Nzs0s+9kCtUegRgHQKBoB8g0UGPQDhNaPU9AkXM/F4HERERkc1o1VcpzrlHnHM3O+duBp4JZICPBg//cfhY0DQmdbJeiWyhxLftHwJ2QiDgZzzC0qBwhOjZoDyoVHakvVJNaZDfI9DF8aHFMoloBDOrTA1qlhEoVAUN7TScGuSV6I9HMWtfWiQiIiLSC926Xfki4DHn3BNder1tL+wPOLZ/GNj+I0QXg0BgsKpHAOBM0DAcPl5TGpSMdX18aFgSFG4MzjfpEShUlRG14y8Uq32djFekv09lQSIiIrJ5dSsQeCXwgarPX2dmJ8zs3WY21qXvsa2EE4O+LQgEtvsI0fqMwL7hJLGI8cS0HwgsBNOBhqtKg0aC0iDnmo/4XAmvVKpc2Ie/N88IBIFAJ1OD4lFy3vKFYgNqFBYREZFNbM2BQLA58geBvw0OvQO4BrgZOA+8tcnX3W5m95jZPVNTU2s9jS0nzAgcGE0xkopv+xGii3WBQCwa4bq9Qzxwbr7m8cG6ZuFi2VW29K5Vobg0DrRdaVC+uIIegXiUXHF5s7CWiYmIiMhm1o2MwK3A15xzFwGccxedcyXnXBn4c+BZjb7IOXeHc+64c+74xMREF05ja5nJ+HfAxwfi7BtO7oCMwPIFW089NMJ9Z2ZxzrGQW14aFG4X7tYuAa9UVRrUtkdgaQtxO6l4lELJUax6rYxXpF8ZAREREdnEuhEI3EZVWZCZ7a967IeA+7vwPbadmaA0aKw/wb6R5LbPCFRKg6rukt90aISZTIEzM9lKadBQXWkQ0LU+gepNweHvXrHJ1KCgWbizjID/nFxVv0HaK6lHQERERDa1NQUCZjYAfC/wd1WH/5eZ3WdmJ4AXAv91Ld9ju5pOe5j5F7v7hpPbf2qQV9ssDPDUg6MA3Hd2rpIRqF8oBjCX6WJGILiwT8Q67BHoZGpQ3L/zn60qYcp6RfUIiIiIyKa2pluWzrk0sKvu2E+s6Yx2iJmMx0gqTiwaYd9IkqnFfLDJdnvOna9vFga4bt8giWiEE2fmKlOEhhuWBnVnhKhXLFdKfdqVBnmllUwN8i/4q3cJpPMlbRUWERGRTW17XnVuAdNpj/H+BAD7RpI4B1ML+R6f1fpZzJdIRCOVGn2AvliUG/YPcd/Z2aoegeqFYn5QsB6lQeF5eG3Gh3ayR6BRIJDxitoqLCIiIpuaAoEemcl4jPb7F737RpLA+owQzRe7M3FnrdL5Yk2jcOimgyOcODPHQq5ANGI1m3grGYEuBQKFFTULBz0CHTYLAzW7BDJeif4GP6+IiIjIZqFAoEem0wXGB4KMwLAfCHS7YfjcbJab3vQJvnDyUldfdzX8QGD5HfKnHhphIVfk/nPzDCVjNZt4w+xA1zICDQIBr9SsWXhl40OBynbhYqlMvlimP66MgIiIiGxeCgR6ZCbtMdZfGwh0OyNw4swcXrHMZx6Z7OrrrsZivljTCBy6KWgY/srj0zWjQwGiEWPPUB//9ugUpfLal4p5xapm4S72CKQS/muFgUCmsHxUqoiIiMhmo0CgB5xzTGe8SkZgtD9OXyzS9YzAyckFAL76xExXX3c10k3m6h/dO0hfLEK2UGKwL77s8V+/5Qa+9uQs7/7842s+B69YrpT6VDYLN+sRKHXeI9AXq+0RyAQ7E7RQTERERDYzBQI9kPFKeMUyY0EgYGbsG+n+UrGTk4sA3H92vqaRtRcW86WGpUHxaIRjB4YBlmUEAH74GQf5vmN7+d+feIRvXlxY0zl4pTJ9wYV9NGKYdWl8aKIuEPDCCUnKCIiIiMjmpUCgB2Yy/jKxcGoQ+OVBF9sEAhmvyJvufKDjmvmTU4skYhG8Upn7z86t/oS7IN2kNAjgqQdHgNrRoSEz4y0/fBODfTFe/6FvNL1w74RXXOoRMDPi0UjzHoFi583C9VODMsE+gbCJWERERGQzUiDQAzNp/0I+zAiAPznoQpvSoC89Ps17/v1b/HsHzb/lsuOxyTS33rgP6H15UKZJszDATYf8PoHq0aHVdg/28ZYfupH7zs7xl198YtXn4NXtaeiLRrrTI1C3UKzRzgQRERGRzUaBQA9MhxmBgaUL3zAQcK55U+y52SzQ2XShc3NZsoUS33H1Lq7a1d/zQKBZszD4k4OApo8D3HLjfg6OptaU2ShUZQTAv9vfdI/ACnoEwpGnueC1wmbhRj0RIiIiIpuFAoEemEn7gcBYXWmQVywzk2le9nN2xg8ELsy3Xzz2aNAfcO2eQZ555Rhfe3KmZZCxnpxzpL1S05r5ayYG2TPUx1W7+lu+zp7hvjUtXaseHwr+3f7qjMD5uSyXF/3XX9H40FhtRkDNwiIiIrIVKBDogekGgcD+ylKxbNOvCzMCkx1kBB6rCgSecdUYlxY9npzOdHyOC7kCz37Lp/jcN6c6/ppm8sUypbJrWioTjRif+dUX8NPPubrl60wMrj4QKJcdhZKrubD3ewSWAoGf/+uv8esfuQ+oWijWQSAQiRiJWIRcsLwtHTQLKyMgIiIim5kCgR6YyXhEDIZTS6VBuwb7gKUgoZGzYWnQQvtA4OTkIrsGEowPJDh+eAxYWZ/ANy8ucGE+x4Pn5zv+mmYWg5r5VqU//YkY0UjrevyJoT6mFlcXCBTK/gV/X1VGIBGNVC74AS7M5fjKt6ZxzlVKhjrpEQC/TyBXyQioR0BEREQ2PwUCPTCd9hjtT9Rc+I4GQcFsB6VBFzsoDTo5ucg1ewYBOLpniKG+2IoCgcem0m3Pp1Nh8+xaS2UmhvqYTnurmhwUXtgn6jIC1XsEZrMec9kCj19KUyiViUetZtNxK8l4hFxBPQIiIiKydSgQ6IGZjMdYf+2EnJEgEGg2GrRYKlemCrUbM+qc49HJRa4NAoFoxLj5ytEVBQKngkBgLts8Q9GppYzA2i6MJ4b8rMmlVWQFKoFATbPwUo9ArlCqXMh//cnZIBDo/I9HKh5d2iycLxGx2uyDiIiIyGajK5UemE4vbRUODbcJBC7M5yg7uGpXPwv5YuUueyOXFv0729dODFaOPfOqMR65uMBCrrM7/Kem/B6D7mQE/AvktZbK7Bny+yhW0yfgNWj+re4RqH7fv356Zlk/QTvJeLSyRyDtFRlIxDrOJoiIiIj0ggKBHphJF2oahcG/kEzFo00DgXOzfhbg6Vf4M/cnW1wMn6xqFA4986oxnIN7T892dI6nLvkZgXD52Vqkve7UzIcZgdUEAuGCsERdj0CYKQjfdzM/I5AvriwjkKzKCGS9Ev3aKiwiIiKbnAKBHpjJLM8IgF8eNNvkwvvsrD/x5xlX+Y2/rXYJnJxaHghcv3cIgG8FF/itFEtlnrjc/R6BVs3CnVhLIOCV/Iv0mkAgtrRQLPw5b75ilIcvLDCfLayotOfgaIovPnaZ37/rIaYW8hodKiIiIpuerlY2mHPO7xFoEAiM9sebXngvZQTaBwKPTS4ykIhWRpKCP5UoGrG224sBTs9kKZQciWikaYZiJbq1aXf3oP+erSYQyFeahZfKdeJVU4PCAOyF1+/h60/O8rUnZ1YUCPzOy29koC/Kuz53CoCnHBhe8TmKiIiIbCRlBDbYYr5IoeQY718eCAyn4k0vvM/MZNk1kOCq3f7SrZYZgWBiUHWNejRi7B3q43ybRmNY6g+48eBwVzICi0GPwOAa75L3xaKMpOKrGiEaXvA3Wyg2G7zvL7h+AoDzc7kVlQaNDyT4X//xaXzk57+Lm68Y5elXjq74HEVEREQ2kgKBDTaT9i84G2YEWgQC52azHBhNMdQXIxWPthwh+ujkQk1ZUGjvSLJlABEKJwY948oxsoVSpQk29MXHLrdsVq63lBFYe938xNDqlootjQ9dOofqZuH54H0/vHuAw8GG45UEAqFnXjXG37/2Ofzuy29a8deKiIiIbCQFAhtsOhNuFY4ve2ykRSBwdjbLwdEUZsbe4b6mF/QX53NcnM83DAT2jyQ7ywhcWmSsP87h3QPA0kUy+BOPfuz/3M3f3nO67euE0vkifbEIsVVcWNdb7XbhRuND/YViSz0C0Ygx1Bfj6Vf65Vdxjf8UERGRbUxXOhtsJtgcvJIeAedcJSMAsHc4yWRdRuBf7j/Pq9/zFb77Dz8NwLcfHl/2OnuHk1yYy+GcW/ZYtcem0hyZGKxMNpqpOif/61tPLaq3mC92bcvuarcLF0rLNwX7C8WCHoGsx0gqjplVynoSHW4VFhEREdmKFAhssPAidvdA37LHRlJxsoUS+WJtKc5spkDGK3FwbCkQqG76vff0LK/566/x4Pl5fuq7DiM8ZtsAAB71SURBVPNPv/jdDQOB/SNJMl6JhTZlPaem0hzZPcBof7jteGmSUbjMa2YFvQPpfLErZUGw+tKgfKOMQGypNGg2U6hsdw4bsldTGiQiIiKyVWhq0AYLL6R3DzUYHxrcgZ/LFtgztHThfHY2C8DBUX8KUFga5JzDzPjK49MA3Pm6766M2Gxk77D/9Rfncgwnl5cmAcznClxazHNkYrCy7Xi2qjQovAgPMxudWMyXGOjSOM09Q31kvBKL+eKKxpGGF/x9sdqFYoWqPQLhUrcb9g/RF4soEBAREZFtbc1XOmb2LTO7z8zuNbN7gmPjZvZJM3s0+H1s7ae6PVxa8BhIRBvOmQ8vvOfr+gSWAgG/iXXvcJJ8scx81r+zf++ZWQ6OploGAQD7R/yMQnWfgHOOf/zGucrG4bBR+MjEUkZgrurufxjITK9g0VjGW9lFeyur3SUQ9gjUbBaOWc1m4fDnjUcjvOSp+7np4Eg3TllERERkU+rWLc8XOududs4dDz5/A/Ap59xR4FPB54JfGrS7yQV7WJpS3ydwLggEDgQZgT3hnf0F/4L+3idnubmDcZX7gq+rLis6ObnIL37g6/zuPz0ELI0OvWZigNEgQzGbXbroX01GIN3lHoHq8+hU2CPQqlk4fP8B/ugVN/Pfvv/6tZ6uiIiIyKa1XrUPLwPeG3z8XuDl6/R9tpxLC3l2DzYOBMKMQP3koLMzWZLxSGUbceWCfi7H1EKes7NZbj7UPhDYM+x/34tVGYFvXvQv/D/01dPcd2aOU1NpohHjyvEBBhJRYhGrCUxW0yOw0jKeVtaaEUhEa0uDyg5KZcdsxqsEPiIiIiI7QTeuzhzwCTNzwLucc3cAe51z54PHLwB7u/B9tgW//n6g4WNLzbl1GYE5f2JQuCBsb3hBP5+rNMF2khFIxqOMDyQ4X5cRMIPx/gRv+scH2DPUxxVjqcqd89H+RM1F/1QlEPAqPQrtpPOl7jULD4aBQPsxqNUqpUF1PQIAuYLfQD2catw3ISIiIrIddSMj8N3OuWcAtwKvNbPnVT/o/FmVy+ZVmtntZnaPmd0zNTXVhdPYGi4tts8IzDbICBwMRocC7BnyMwKTC3m+cXqWaMS48UBn9ex7h5M1GYGTU4scGkvxa7dcz1efmOH/PnSRIxNLOwhG++PMVZUGXVrwPy6VHfO5zpaKdbM0aKw/QTRiKx4hGvYC1GYE/CBmOu3hHDWlQSIiIiLb3ZoDAefc2eD3SeCjwLOAi2a2HyD4fbLB193hnDvunDs+MTGx1tPYEgqlMjOZQtOm3qFkHLMGpUGztYFAKhFlOBnj4nyOe0/PcsO+IVKJzu641y8VOzm5yLUTg/ynZ17BTQdHKJQcR3YvZSxGU7W7DaYW8/QH36uTPgHnHGmv2LWpQZGIsXsw0ZXSoDDrEe5EGG2w5E1ERERku1pTIGBmA2Y2FH4MfB9wP3An8Krgaa8C/mEt32e7mA4unJtlBMLNtnNVE3lyhRKXFr2aQAD8O/vnZnN84/QsN1/Rviyo+uvCrcSlsuPU1CLX7hkkEjHe+NJjmMGxA8OV51cvOfMDGY+jwdbimQ4mB52fy1F2sHck2fE5trOaXQJeqUw8akQiS6VMYVAwpUBAREREdqC13qbdC3w0qBOPAe93zv2LmX0F+JCZvRp4AnjFGr/PthBecDYLBMCvya/OCCxNDKoNBPaNJPnKt6ZZyBdXFAjsH0lyOe2RL5a4MOf3GFwbXNgfPzzO5371hTXfaySV4KHzC8BSCc3RvUN848xcR4HAIxf9r71+71DH59jOxODKtwt7xfKyvQDh5+Frjag0SERERHaQNQUCzrlTwNMaHL8MvGgtr70dhRecEw2WiYVGUvGaHoHKDoGx2kBgz1CSuewlAJ7eQaNwKJw4NDmf5+SkPzEoDAQArhjvr3m+nxHwL/jDQCa8qJ9Ot58c9GgQCFy3d7DNMzs3MdTHg+fnV/Q1hVK5ZnQoLDUOhz/XSEpTg0RERGTn0OrUDXSpo4xAvCYj8OR0BoAr6y7Qw8lBQ8kYR3Z3fpG9LyjROT+XWwoEJprfrR/rj5P2SnjFciWQORpc1HfSI/DIhUX2DPV1dTTnnqEklxY9yuVlPehNecVyTX8AQCJoFlZpkIiIiOxECgQ20KXF1j0C4GcEqjf5PjmdIRGNVO7kh/YGnz/t0GhN3Xs7YSBwYd4PBCaG+hhpcQE8UrVULAxkrt49QCxiHZUGffPiAtfv615ZEPgZgVLZrWi7ccvSoAWVBol0k5ndYmaPmNlJM1u2UNLMfsXMHjSzE2b2KTO7qhfnKSKy0ykQ2ECXgok7rUZpjqRqMwKnpzMcGk8tu9ivBAJXdDY2NFQJBOaynJzyJwa1Eo7UnMsUqkqb+hgbSLQNBMplx6OTCxzd0/1AAFa2VMwrlemrLw0KAoFLi3kGEtFlgYKIrJyZRYG344+UPgbcZmbH6p72deD/b+/eo+O86zuPv78aaXSzrpYj2ZZtJbEd2zgOcUwSEu5JuyFAWEi6S2ja9NRb2rNlC7v0cGhZWqCXpdwLh1JogVCgtCRQ6gaTLAnpthvWCXac+E4sO5avimTdL7Zu890/nmfGI1myRvZoZjTzeZ3j45lnZqTv83tG8zzf+f1+398Wd98EPAJ8MrNRiogIKBHIqEutIRBXWxHMEQiWXwh6BKYOCwISi5K9+pqGOcVQVVpMRTRCe18wRyB5fsBM8UCwtsHZgVEqoxEqosXUV0QTVZBmcqJnmPNjMa5rSt/8ALjMRGB8mjkCST0CWlVYJG1uBlrd/ai7jwL/QLDafIK7P+Xuw+HdHUBzhmMUERGUCGRUkAhc+oKzpryEiZgzNDoBwPGu6ROBtY1V/PsH38hr1swtETAzmmrK2Huql4Hz47MnAuEE2t6wR6AhvAivqyyhZ5bJwi++HMxBWJPGikGQvLrw3HoEpn7jH08MOgdHNCxIJH2WAyeS7p8Mt81kK/Dj6R4o1IUnRUQyRYlABp0dGJ29RyBx4T1K3/AY/efHp00E4OIKP6lqqi5j9/FegNR7BIaDOQLxi/C6itmHBr0YVgxaM8vvmKtEj8AcSohOVzUoPnl4dDymicIiWWBmDwBbgE9N93ghLjwpIpJJSgQyKPkb9ZlUl8cvvMcSFYMu94J/Jk01ZYyHFXdmu0iPXyD3nQt6BJYkegQuTgRGx2OJIU0Av2gfYHltOVVl6b3IrgyHN3X0z3Fo0NTJwsUX5l2oR0AkbU4BK5LuN4fbJjGzO4EPA/e4+9wWBhERkbRQIpAh4+GqvKnMEQDoPzc2Y+nQKxWvQFRVVpy4sJ/JotJiImGFoM6BC3Mc6iui9AyPJUp4nh+b4JY/f4Jv7WhLvPbFlwfSun5AspX1FRw9O5jy8y81RwBUOlQkjX4OrDGzq80sCryLYLX5BDO7EfgKQRLQkYUYRUQEJQIZE1+Vd7YL7/g3073nxmjrHgLmp0cAgmFB4arQMzIzastL6Ogfoe/c2KQegYmYM3B+HIDWjkF6hsf4yv85yvhEjLGJGEc7h1ib5tKhceuXVnMoXPE4FSPTlA9N7iHQYmIi6eHu48B7gceBg8D33H2/mX3czO4Jn/YpYBHwsJk9b2bbZvhxIiIyj65oZWFJXaL05iyThZOH4pzoHmZxZZRFlyg3ejniPQKzlQ6Nq6ko4Uhn8O17Q2KOQBBnz/DopMdP9Z7jiYMdrL6qktGJGGvTXDo0bl1TFf+0+xS9w6MpVfwZu0T5UFCPgEg6uft2YPuUbX+UdPvOjAclIiIXUY9AhqSymBgk9QiEcwTS3RsAk3sEUlFbXpJYhTi5RwBILOrV2jFIkcGymjIe+tlLiYpB6V5MLC7+cw+1p9Yr0HdujKqyyQlVSURzBERERKRwKRHIkPiqvLMlAuUlEaKRIvrCOQLpnh8AQenRt92wjP/wiqaUnl9bEaU/HAIUL39aH34L3zN0IRFYtbiSB29rYcfRbrY9fxozuDbFXoe5Wr+0GggmJM+mb3iMs4OjibUX4kqSeghqlQiIiIhIgVEikEYHTvfzjr96mt5pymqeDYcGzVY1yMyoLi+ha3CE073n5yURKCuJ8MX7b6SloXL2JzN52Ey8R6A+3iOQlAisvmoR//lVKygrKeKx/e2sqq+gPBpJc/SBq6pKqa0o4VB7/6zPPRJOKp6alEyaI6ChQSIiIlJglAik0XefPc7u47083dp10WOdAyOUlRRRmcKFcW1FCQfb+5mI+bwkAnNVmzSRNt6jcWF9gTHGJmK8dHaI1VctorYiyjtuDNYOSvdCYsnMjHVNVSkNDTrSMX0iMGmOgCYLi4iISIFRIpAmsZjz+P52AHa2dV/0eLCqcOmsVXogGK8eH/KycnEOJALhRX9VWTFlJUEis6i0mJKI0T08SlvXMOMxT0w+fvC2FgDWz9P8gLh1TdX8on0gUcJ0Jkc6h4hGimiuK5+0PVJkRIqC46EeARERESk0qhqUJrtP9NAxMEI0UsTOYz0XPX52cHTW0qFxteUljE0EF7c50SMQXiQvSZrfYGbB6sJDo4mJxPHJx+uaqvnW1pvZuKxmXuNa11TF8OgEJ3vOXTJhOtI5SEtDBcWRi/PekogxEXPNERAREZGCox6BNHlsXzslEePdt6zkwJl+hkbGJz0e7xFIRbyCTTRSRGNY6jOb4vFMnd9QXxmle2g0UTr02qQqRK9dsyRRWWi+rAsnDB+cZZ7Akc7BGSctl0SKKIkYFfM0l0FEREQkVykRSAN357H97bxmdQOvv24JEzHnhRO9k54zp0Qg/Aa+ua48MXQlm+J1+pdMib+2ooTe4TFaOwZZWlOW9vUOZrO2cRFml64cNDYR43jX8IyJQDRSRE15NKUhWyIiIiL5RIlAGuw/3c+J7nPctbGJzSvrMIOdbReGB03EnO6h0VkXE4uLT1ydjzUELkd82MzUoU31lVG6h0cTFYMyrSJazKr6iktWDorPX7j2qukrJJVEiqgp1wg5ERERKTxKBNLg8f3tFBncub6RmvISrmus4ufHLkwY7h4aJeazlw6Ni1+Y5sL8AIC6sEegYUoiU1cRpWtwhNaOmYfezLfrmqo4dGbmHoHEsKWZegSKi1JamVhEREQk3ygRSIPH9rVzy9WLWRwOnblpVR27j/cyEVaz6UxxMbG4+IVpriQCTTVl3L56Ma++tmHS9vrKKD3DY5wbm2BNY3YSgXVN1RzrGuLc6MS0j8cTgWtmnCNgWlVYRERECpISgSvU2jHI4Y5B7tp4YZXeV7XUMzgynhi7fqr3HHDx0JqZxC9Mc2VoULS4iO/8l1u5aVXdpO3J36SvzlKPwLqmKmIOhzum7xU40jFEU/XM8xd+942r+Y2w3KmIiIhIIdHg6Cv07EvBEKA3XndVYlv8gnlXWzfL68r50x8doLG6lHUp1tW/qaWO+29ewW2rF6c/4DSqr7zwTXo25gjAhcpBh9oH2NRce9HjrZ2DM84PAHjn5uZ5i01EREQkl112j4CZrTCzp8zsgJntN7P3hds/amanzOz58N/d6Qs397R1DREtnrxYVXNdOY3VpTx7rIfff/gFTvWc40vv3kxVWWpDUKrLSvhf79xEdYrPz5b43IG6ipLEsKhMW1lfQXlJhINnLp4w7O4czeL8BREREZFcdiU9AuPAB9z9OTOrAnaZ2U/Cxz7n7p++8vByX1vXMCvqyilKKvNpZmxZVc+P9pwm5vCRt25gS0t9FqOcH/XhOgHZ6g2AYHXg9Uur2H/q4kSgc2CEgZFxJQIiIiIi07jsHgF3P+Puz4W3B4CDwPJ0BbZQHOsaomXxxUNPtrTUEXN4y/VL+c3bWzIfWAbEewSymQgAbGquZd/pvsTk7LjWWSoGiYiIiBSytEwWNrMW4EbgmXDTe81sj5l93czqZnzhAufuHO8eZtU0icA9Nyzjv77hWj5x7/V5u1jVkqpSqsuKuWlVdns7rl9ew/DoBEfDC/+4I51DAJecIyAiIiJSqK44ETCzRcD3gfe7ez/wZeBa4JXAGeAzM7zuPWa208x2dnZ2XmkYWdE5OMLw6ASrFl9c3WfxolI+eNe6lOcFLERlJRF2/OEd3Ls5ux1Bm5prANhzsm/S9iMdg1REIzRVl2UjLBEREZGcdkWJgJmVECQB33H3HwC4+8vuPuHuMeBvgJune627f9Xdt7j7liVLllxJGFnT1jUMMG0iUCgqosVZ7/G4ZskiKqIR9p6akgh0BhOFsx2fiIiISC66kqpBBnwNOOjun03avjTpae8A9l1+eLnt2Nlg6Ml0cwQkcyJFxsZlNew52ZvYFos5B8/0sybL8xdEREREctWVVA26Hfg1YK+ZPR9u+0PgfjN7JeDAMeC3ryjCHHa8e5hIkbE8qXSoZMf1zTV8e0cb4xMxiiNF7Gzr4ezgKG9Yd9XsLxYREREpQJedCLj7/wWmG3Ox/fLDWViOdQ2zvLackogWaM62Tc01jIzHONwxyPql1Wzfe4bS4iLepERAREREZFoL+grW3TnRPZy139/WNVTQ8wNyyfXLgwnDe0/2EYs5P953htevXcKiUi2eLSIiIjKdBZ0IfOKxQ9z9hX9naGQ8K7//2Nnp1xCQzGtZXElVaTF7TvXy3PEeXu4f4S2bls7+QhEREZECtaATgV/e0MTA+XH+afepjP/u3uFR+s+Pq0cgRxQVGRuX17D3ZB/b97YT1bAgERERkUta0InA5pW1bGqu4aGfHcPdZ39BGh1LlA5Vj0Cu2NRcw8EzA2zfe4bXrVmS12s4iIiIiFypBZ0ImBkPvrqF1o5Bnm7tyujvbuuKlw5Vj0CuuL65htGJGO3953nLpqZshyMiIiKS0xZ0IgDw1huW0rAoykM/eymx7RtPv8Svf/1ZRsdj8/Z744uJrahXIpArNi2vBSAaKeKO9Y1ZjkZEREQkty34kiqlxRHeffNKvvhUK8e7hnnkuZN84cnDAPz0UAd3bZyfb4aPdQ2xtKaMspLIvPx8mbsV9eXUV0bZvLKWag0LEhEREbmkBd8jAPCrt64iYsYDX3uGLzx5mPtuamZJVSnff+7kvP3Otq5hTRTOMWbGt7fewp+94/pshyIiIiKS8/IiEWisLuPN1y/lePcwv3FbC5+8dxPvvHE5Tx3qoGtwZF5+Z1vXMKvqNVE412xYVk1jdVm2wxARERHJeXmRCAB89G0b+OsHNvPHb9tAUZFx703NjMecf37+9GX/zG0vnOZVf/YEn3/iRQbOjyW2D46Mc3ZwhFUN6hEQERERkYUpbxKBxYtKuWvjUswMgLWNVWxqruGRXRcPD3J39p3q4/H97UzEpi872jkwwkd+uI+JmPP5Jw7z2k8+xRefPMyOo13sOdELoMXERERERGTBWvCThS/l3s3N/PG2/Rw43c+GZdW0953n7589zqMvnObo2aD856ta6vjMr7ySlVPG+3/sX/ZzbnSC7e97LedGJ/j0//4Fn/nJi5Oes1IVg0RERERkgcrrROCeG5bxpz86wDeefoma8hL+bkcbYxMxbr16Mb/1umuIFBl/8ugB3vyX/8aH7l7PfZubKY9GeOLAyzy65wwf+KW1rL5qEQDf/M2bebn/PIfaB3ixfYDh0Qk2LK3O8h6KiIiIiFyevE4E6iqj3Lm+kYd3naTI4J2bm/m9N62Z9O3/a1Y38PsPv8BHfriPT2w/yC9taGTH0W7WNVXx26+/dtLPa6wuo7G6jNevXZLpXRERERERSau8TgQA3n/nWhqry3jg1lWJb/eTLast59tbb2HH0S7+Zc9ptu9tZ3BknL/+tZuIFufNFAoRERERkUnyPhG4rqmKj97ziks+p6jIuG11A7etbuBj92yke2iUphqVoBQRERGR/KWvvKeIFhcpCRARERGRvKdEQERERESkACkREBEREREpQEoEREREREQKkBIBEREREZECpERARERERKQAKREQERERESlASgRERCStzOwuM/uFmbWa2YemebzUzP4xfPwZM2vJfJQiIqJEQERE0sbMIsCXgDcDG4D7zWzDlKdtBXrcfTXwOeAvMhuliIiAEgEREUmvm4FWdz/q7qPAPwBvn/KctwPfDG8/AtxhZpbBGEVEBCUCIiKSXsuBE0n3T4bbpn2Ou48DfcDijEQnIiIJxdkOAGDXrl1nzaztMl/eAJxNZzwLQKHts/Y3vxXa/sLc93nVfAWSy8zsPcB7wrsjZrYvm/HkiEL8e5mO2kFtEKd2gOsu94U5kQi4+5LLfa2Z7XT3LemMJ9cV2j5rf/Nboe0v5P0+nwJWJN1vDrdN95yTZlYM1ABdU3+Qu38V+CrkfZulTO0QUDuoDeLUDkEbXO5rNTRIRETS6efAGjO72syiwLuAbVOesw14MLx9H/BTd/cMxigiIuRIj4CIiOQHdx83s/cCjwMR4Ovuvt/MPg7sdPdtwNeAb5lZK9BNkCyIiEiG5UMi8NVsB5AFhbbP2t/8Vmj7C3m+z+6+Hdg+ZdsfJd0+D/zKHH9sXrfZHKgdAmoHtUGc2uEK2sDUGysiIiIiUng0R0BEREREpAAt6ERgtmXsFzozW2FmT5nZATPbb2bvC7fXm9lPzOxw+H9dtmNNJzOLmNluM3s0vH+1mT0THud/DCcg5g0zqzWzR8zskJkdNLNX5/MxNrP/Hr6f95nZd82sLJ+OsZl93cw6kktdznQ8LfCFcL/3mNnm7EWeG2b7XDez0vA90hq+Z1oyH+X8S6Ed/kd4bthjZk+aWd6VmE31HG9m95qZm1leVo5JpR3M7D8lXSv8faZjzIQU/iZWhtdMu8O/i7uzEed8mu78MuXxOZ9TFmwiYKktY7/QjQMfcPcNwK3A74b7+CHgSXdfAzwZ3s8n7wMOJt3/C+Bz7r4a6AG2ZiWq+fOXwGPuvg64gWDf8/IYm9ly4PeALe6+kWAy6bvIr2P8EHDXlG0zHc83A2vCf+8BvpyhGHNSip/rW4Ge8L3yOYL3Tl5JsR12E/wdbSJYnfmTmY1yfqV6jjezKoJzxjOZjTAzUmkHM1sD/AFwu7u/Anh/xgOdZym+H/4n8D13v5HgvPJXmY0yIx7i4vNLsjmfUxZsIkBqy9gvaO5+xt2fC28PEFwgLifYz2+GT/sm8B+zE2H6mVkz8Bbgb8P7BryJ4EQH+be/NcDrCKqo4O6j7t5LHh9jgiIF5RbUj68AzpBHx9jd/42gEk6ymY7n24G/88AOoNbMlmYm0pyUyud6cls+AtwRfk7kk1nbwd2fcvfh8O4OgvUa8kmq5/g/IUgGz2cyuAxKpR1+C/iSu/cAuHtHhmPMhFTawYHq8HYNcDqD8WXEDOeXZHM+pyzkRCCVZezzRtj9fSPBtx6N7n4mfKgdaMxSWPPh88AHgVh4fzHQ6+7j4f18O85XA53AN8LuzL81s0ry9Bi7+yng08BxggSgD9hFfh9jmPl4FtTnWApSaY/Ec8L3TB/B50Q+mev7Yivw43mNKPNmbYNw2MMKd/9RJgPLsFTeC2uBtWb2tJntMLNLfWO8UKXSDh8FHjCzkwRVy/5bZkLLKXM+pyzkRKBgmNki4PvA+929P/mxcBGevCj9ZGZvBTrcfVe2Y8mgYmAz8OWwO3OIKcOA8uwY1xF8Y3E1sAyo5NLdnHknn46nZJ+ZPQBsAT6V7VgyycyKgM8CH8h2LDmgmGAoyBuA+4G/MbParEaUHfcDD7l7M3A3wVolus6dxUJuoFSWsV/wzKyEIAn4jrv/INz8cryrJ/w/X7oBbwfuMbNjBN1+byIYP18bDiOB/DvOJ4GT7h4f3/oIQWKQr8f4TuAld+909zHgBwTHPZ+PMcx8PAvic2wOUmmPxHPC90wN0JWR6DInpfeFmd0JfBi4x91HMhRbpszWBlXARuBfw3PGrcC2PJwwnMp74SSwzd3H3P0l4EWCxCCfpNIOW4HvAbj7/wPKgIaMRJc75nxOWciJQCrL2C9o4bjXrwEH3f2zSQ9tAx4Mbz8I/HOmY5sP7v4H7t7s7i0Ex/On7v6rwFPAfeHT8mZ/Ady9HThhZteFm+4ADpCnx5hgSNCtZlYRvr/j+5u3xzg00/HcBvx6WOnhVqAvaQhRIUrlcz25Le8j+JzItx6WWdvBzG4EvkKQBOTLFwXJLtkG7t7n7g3u3hKeM3YQtMXO7IQ7b1L5m/ghQW8AZtZAMFToaCaDzIBU2uE4wTkFM1tPkAh0ZjTK7Jv7OcXdF+w/gq6fF4EjwIezHc887N9rCIYQ7AGeD//dTTAe9kngMPAEUJ/tWOdh398APBrevgZ4FmgFHgZKsx1fmvf1lcDO8Dj/EKjL52MMfAw4BOwDvgWU5tMxBr5LMP9hjOCbuq0zHU/ACCphHAH2ElSByfo+ZLn9LvpcBz5OcJEHwcn94fC98ixwTbZjzlI7PAG8nHRu2JbtmDPdBlOe+6/5+veTwnvBCIZJHQg/R96V7Ziz1A4bgKeBF8K/iV/Odszz0AbTnV9+B/idpPfCnM4pWllYRERERKQALeShQSIiIiIicpmUCIiIiIiIFCAlAiIiIiIiBUiJgIiIiIhIAVIiICIiIiJSgJQIiIiIiIgUICUCIiIiIiIFSImAiIiIiEgB+v/blJhoWQLvkgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "frame_idx    = 0\n",
    "max_frames   = 10000\n",
    "num_steps    = 5\n",
    "log_interval = 100\n",
    "test_rewards = []\n",
    "state = env.reset()\n",
    "\n",
    "while frame_idx < max_frames:\n",
    "    \n",
    "    q_values = []\n",
    "    values   = []\n",
    "    policies = []\n",
    "    actions  = []\n",
    "    rewards  = []\n",
    "    masks    = []\n",
    "    \n",
    "    for step in range(num_steps):\n",
    "    \n",
    "        state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "        policy, q_value, value = model(state)\n",
    "        \n",
    "        action = policy.multinomial(1)\n",
    "        next_state, reward, done, _ = env.step(action.item())\n",
    "        \n",
    "        reward = torch.FloatTensor([reward]).unsqueeze(1).to(device)\n",
    "        mask   = torch.FloatTensor(1 - np.float32([done])).unsqueeze(1).to(device)\n",
    "        replay_buffer.push(state.detach(), action, reward, policy.detach(), mask, done)\n",
    "\n",
    "        q_values.append(q_value)\n",
    "        policies.append(policy)\n",
    "        actions.append(action)\n",
    "        rewards.append(reward)\n",
    "        values.append(value)\n",
    "        masks.append(mask)\n",
    "        \n",
    "        state = next_state\n",
    "        if done:\n",
    "            state = env.reset()\n",
    "    \n",
    "\n",
    "\n",
    "    \n",
    "    next_state = torch.FloatTensor(state).unsqueeze(0).to(device)\n",
    "    _, _, retrace = model(next_state)\n",
    "    retrace = retrace.detach()\n",
    "    compute_acer_loss(policies, q_values, values, actions, rewards, retrace, masks, policies)\n",
    "    \n",
    "    off_policy_update(128)\n",
    "    \n",
    "    if frame_idx % log_interval == 0:\n",
    "        test_rewards.append(np.mean([test_env() for _ in range(5)]))\n",
    "        plot(frame_idx, test_rewards)\n",
    "        \n",
    "    frame_idx += num_steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_env(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:pytorch4]",
   "language": "python",
   "name": "conda-env-pytorch4-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}