Poloicy_Gradient_Continuous_mountain_car.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import sys\n",
    "sys.path.append('environment/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 建力Policy Gradient模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Policy_Gradient:\n",
    "    \n",
    "    def __init__(\n",
    "        self, \n",
    "        n_actions, \n",
    "        n_states, \n",
    "        gamma = 0.9, #遠見程度\n",
    "        epsilon = None,  #保守程度，越大就越容易用Q值大小來採取行動；越小則越容易產生隨機行動\n",
    "        epsilon_increase = None,\n",
    "        learning_rate = 0.001, #神經網路的更新率\n",
    "        #memory_size = 50, #####\n",
    "        batch_size = 32, #####\n",
    "        nueron_num = 10\n",
    "    ):\n",
    "    \n",
    "        self.n_actions = n_actions\n",
    "        self.n_states = n_states\n",
    "        self.gamma = gamma\n",
    "        #self.epsilon_max = epsilon #####\n",
    "        #self.epsilon_increase = epsilon_increase #####\n",
    "        #self.epsilon = 0 if epsilon_increase is not None else epsilon #####\n",
    "        self.lr = learning_rate\n",
    "        #self.memory_size = memory_size #####\n",
    "        #self.memory_counter = 0 #####\n",
    "        self.batch_size = batch_size ####\n",
    "        self.nueron_num = nueron_num\n",
    "        \n",
    "        ##### initialize memory\n",
    "        \"\"\"\n",
    "        \n",
    "        \"\"\"\n",
    "        self.past_state, self.past_action, self.past_reward = [], [], []\n",
    "        #self.action_one_hot = np.zeros(self.n_actions, dtype=np.int32)\n",
    "\n",
    "        \"\"\"\n",
    "\n",
    "        \"\"\"\n",
    "        \n",
    "        tf.reset_default_graph() ## 重新 build graph 需要跑這行\n",
    "        self.sess = tf.Session() #宣告session\n",
    "        #輸入current state\n",
    "        self.state_input = tf.placeholder(shape = [None, self.n_states], \n",
    "                                          name = 'state_input',\n",
    "                                          dtype = tf.float32)\n",
    "        \"\"\"\n",
    "        輸入real action和神經網路的output act_proba算cross entropy當作更新方向\n",
    "        以超級瑪莉的遊戲為例 action = [上, 下, 左, 右] 如果實際action為向左則\n",
    "        action = [0, 0, 1, 0]。\n",
    "        也可以將這四個動作用0, 1, 2 ,3代表，如此的話只需要用一維來存取動作，也就是輸\n",
    "        入shape = [None, 1]，那後面再算cross entropy的話就要用tf.nn.sparse_\n",
    "        softmax_cross_entropy_with_logits，大家也可以試著改寫看看。\n",
    "        \"\"\"    \n",
    "        self.real_action = tf.placeholder(shape = [None, self.n_actions], \n",
    "                                          name = 'real_action',\n",
    "                                          dtype = tf.float32)\n",
    "        \"\"\"\n",
    "        但是有時候產生的動作會帶來好的效果或壞的效果並且程度不一，因此loss不能光用神經網路的\n",
    "        輸出action_proba和real action的cross entropy代表，因此這邊乘上action_reward\n",
    "        來校正loss。例如某個動作很有幫助那必然會產生很大的action_reward，因此乘上很大的\n",
    "        action_reward即可加大loss讓此動作之後產生的機率被放大；相反的，某個動作如果產生很\n",
    "        好的效果反而會帶來負的action_reward使得loss變負的，讓更新方向相反使得之後輸出此動\n",
    "        作的機會減少。\n",
    "        \"\"\"\n",
    "        self.Vt = tf.placeholder(shape= [None, ], \n",
    "                                            name=\"action_reward\",\n",
    "                                            dtype = tf.float32)\n",
    "        #搭建神經網路\n",
    "        with tf.variable_scope('Actor'):\n",
    "            self.act_mean_std = self.build_network(self.nueron_num, Trainable = True, \\\n",
    "                             scope = 'net_eval') \n",
    "            \n",
    "        \n",
    "        #管理神經網路的parameters\n",
    "        self.Actor_eval_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/net_eval')\n",
    "        \n",
    "        \n",
    "        #loss\n",
    "        \"\"\"\n",
    "        算出 “神經網路輸出的動作機率”與 “實際動作”的cross entropy當作loss，但是更新的方向和力道就利用action\n",
    "        reward來決定。例如，這一回合產生的所有動作組合如果得到很好的reward，那就應該讓神經網路的輸出機率更靠近實\n",
    "        際輸出的結果，因此cross_entropy和action_reward相乘的到的loss就更大，更新力度就更大。相反的，這一回\n",
    "        合產生的所有動作組合如果得到負的reward，那就應該讓神經網路輸出動作的機率更遠離實際輸出結果，在這樣的狀況\n",
    "        下，cross_entropy和action_reward相乘的到的loss就會得到負的，神經網路的參數更新方向就會往反方向。\n",
    "        \"\"\"\n",
    "        #self.cross_entropy = tf.reduce_sum(-tf.log(self.act_proba)*self.real_action, axis=1)\n",
    "        #self.loss = tf.reduce_sum(self.cross_entropy*self.action_reward)\n",
    "        dist = tf.contrib.distributions.Normal(loc=self.act_mean_std[:,0:self.n_actions], \n",
    "                             scale=self.act_mean_std[:,self.n_actions:2*self.n_actions])\n",
    "        #log_prob = dist.log_prob()\n",
    "        self.cross_entropy = -dist.log_prob(self.real_action)\n",
    "        \n",
    "        \n",
    "        self.loss = self.cross_entropy*self.Vt[:, np.newaxis]\n",
    "        \n",
    "        \n",
    "        self.train = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss, var_list=self.Actor_eval_params)\n",
    "    \n",
    "        self.sess.run(tf.global_variables_initializer()) #將神經網路初始化\n",
    "    \n",
    "    def write_memory(self, current_state, reward, action): #####\n",
    "        \n",
    "        #action_one_hot = self.action_one_hot.copy()\n",
    "        #action_one_hot[action] = 1\n",
    "        self.past_state.append(current_state)\n",
    "        self.past_action.append(action)\n",
    "        self.past_reward.append(reward)\n",
    "    \n",
    "    \n",
    "    def build_network(self, neuron_num, Trainable, scope): \n",
    "        with tf.variable_scope(scope):\n",
    "            init_w = tf.random_normal_initializer(0., 0.3)\n",
    "            init_b = tf.constant_initializer(0.1)\n",
    "            \n",
    "            x1_h1 = tf.layers.dense(inputs = self.state_input, units = neuron_num, \\\n",
    "                   activation = tf.nn.tanh, kernel_initializer=init_w, \\\n",
    "                   bias_initializer=init_b, trainable=Trainable) \n",
    "            x1_h2 = tf.layers.dense(inputs = x1_h1, units = neuron_num, \\\n",
    "                   activation = tf.nn.tanh, kernel_initializer=init_w, \\\n",
    "                   bias_initializer=init_b, trainable=Trainable)\n",
    "            \n",
    "            x2_h1 = tf.layers.dense(inputs = self.state_input, units = neuron_num, \\\n",
    "                   activation = tf.nn.relu, kernel_initializer=init_w, \\\n",
    "                   bias_initializer=init_b, trainable=Trainable) \n",
    "  \n",
    "            \n",
    "             \n",
    "            #在continuous的版本output為動作的機率分佈，這邊以機率分布的mean值和std\n",
    "            #當成神經網路的輸出值。這邊輸出值為mean1, mean2,... std1, std2,...。\n",
    "             \n",
    "            mean = tf.layers.dense(inputs = x1_h2,\n",
    "                                   units = self.n_actions, \n",
    "                                     activation = tf.nn.tanh, \n",
    "                                     kernel_initializer=init_w, \n",
    "                                     bias_initializer=init_b, \n",
    "                                     trainable=Trainable)*1\n",
    "            std = tf.layers.dense(inputs = x2_h1,\n",
    "                                     units = self.n_actions, \n",
    "                                     activation = tf.nn.sigmoid, \n",
    "                                     kernel_initializer=init_w, \n",
    "                                     bias_initializer=init_b, \n",
    "                                     trainable=Trainable)*1\n",
    "            output = tf.concat(axis=1,values=[mean, std])\n",
    "            \n",
    "            #tf.stack([mean, std], axis=1)\n",
    "        return output #輸出‘不同動作’對應的Q值 \n",
    "               \n",
    " \n",
    "            \n",
    "    def choose_action(self, current_state):\n",
    "        \n",
    "        act_mean_std = self.sess.run(self.act_mean_std, feed_dict={self.state_input: current_state[np.newaxis, :]})\n",
    "        \"\"\"\n",
    "        \n",
    "        \"\"\"\n",
    "        self.action = np.zeros(self.n_actions)\n",
    "        \n",
    "        for i in range(self.n_actions):\n",
    "            #輸出動作從normal distribution去隨機抽樣ㄙ\n",
    "            self.action[i] = np.random.normal(act_mean_std[0,i], \n",
    "                                              act_mean_std[0,self.n_actions+i])\n",
    "   \n",
    "        return self.action\n",
    "    \n",
    "    def learn(self): #####\n",
    "        \n",
    "        #用reward算出更新向量\n",
    "        Vt = self.calculate_Vt()\n",
    "        \n",
    "        self.past_state = np.vstack(self.past_state)\n",
    "        self.past_action = np.array(self.past_action)\n",
    "        Vt = np.array(Vt)\n",
    "        size = self.past_state.shape[0]\n",
    "        #將整個episode的資料放進來做更新\n",
    "        for index in self.index_generator(size):\n",
    "            cost, _ =self.sess.run([self.loss , self.train], feed_dict={\n",
    "                 self.state_input: self.past_state[index,:],  # shape=[None, n_state]\n",
    "                 self.real_action: self.past_action[index,:],  # shape=[None, n_actions]\n",
    "                 self.Vt: Vt[index]  # shape=[None, ]\n",
    "            })\n",
    "\n",
    "        #更新完後將記憶庫清空\n",
    "        self.past_state, self.past_action, self.past_reward = [], [], []    \n",
    "    \n",
    "    def calculate_Vt(self):\n",
    "        # discount episode rewards\n",
    "        Vt = np.zeros_like(self.past_reward, dtype=np.float64)\n",
    "        Vt_temp = 0\n",
    "        for t in reversed(range(0, len(self.past_reward))):\n",
    "            Vt_temp = self.past_reward[t] + Vt_temp * self.gamma\n",
    "            Vt[t] = Vt_temp\n",
    "\n",
    "        # normalize episode rewards\n",
    "        \n",
    "        Vt -= np.mean(Vt)\n",
    "        Vt /= np.std(Vt)\n",
    "        \n",
    "        return Vt \n",
    "    #前幾回合所要花的步數比較多，因此要一次將所有資訊餵進神經是有困難的，因此利用\n",
    "    # index generator將資料分batch送進神經網路去更新參數\n",
    "    def index_generator(self, size):\n",
    "        i = 0\n",
    "        while True:\n",
    "            if (i+1)*self.batch_size< size:\n",
    "                index = np.arange(i*self.batch_size,(i+1)*self.batch_size)\n",
    "                i+=1\n",
    "                yield index\n",
    "            else:\n",
    "                index= np.arange(i*self.batch_size,size)\n",
    "                yield index\n",
    "                break  \n",
    "            \n",
    "        \n",
    "    def model_save(self, model_name):\n",
    "        \n",
    "        saver = tf.train.Saver()\n",
    "        saver.save(self.sess, \"saved_models/{}.ckpt\".format(model_name))\n",
    "    \n",
    "    def model_restore(self, model_name):\n",
    "        \n",
    "        saver = tf.train.Saver()\n",
    "        saver.restore(self.sess, \"saved_models/{}.ckpt\".format(model_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/Users/Jacklin/anaconda3/lib/python3.6/site-packages/')\n",
    "import gym\n",
    "\n",
    "def training(save_model, model_name):\n",
    "    step_record = []\n",
    "    reward_record = []\n",
    "    for episode in range(1000):\n",
    "        # initial environment並給出起始的state\n",
    "        current_state = env.reset()\n",
    "        step = 0\n",
    "        total_reward = 0\n",
    "        while True:\n",
    "            # 產生環境視窗\n",
    "            env.render()\n",
    "\n",
    "            # 根據現在的狀態選擇動作\n",
    "            action = RL.choose_action(current_state)\n",
    "\n",
    "            # 產生動作和環境互動後產生下一個狀態、獎勵值及遊戲是否結束\n",
    "            next_state, reward, done, _ = env.step(action)\n",
    "            \n",
    "            position, velocity = next_state\n",
    "            reward = abs(position + 0.5) + 15*abs(velocity) + reward \n",
    "\n",
    "            \n",
    "            total_reward+= reward\n",
    "            \n",
    "            \n",
    "            # 將資訊存至記憶體\n",
    "            RL.write_memory(current_state, reward, action)\n",
    "            \n",
    "            \n",
    "            current_state = next_state\n",
    "\n",
    "            # break while loop when end of this episode\n",
    "            if done:\n",
    "                mean_reward = total_reward/step\n",
    "                RL.learn()\n",
    "                print('episode:{} steps:{} mean_reward:{}'.format(episode, step, mean_reward))\n",
    "                step_record.append(total_reward)\n",
    "                reward_record.append(mean_reward)\n",
    "                break\n",
    "            step += 1\n",
    "\n",
    "    # end of game\n",
    "    if save_model:\n",
    "        RL.model_save(model_name)\n",
    "    print('game over')\n",
    "    env.close()\n",
    "    return step_record, reward_record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:0 steps:9950 mean_reward:0.6020045720155871\n",
      "episode:1 steps:5715 mean_reward:0.4123724949533009\n",
      "episode:2 steps:2721 mean_reward:0.4398937272143751\n",
      "episode:3 steps:5061 mean_reward:0.3428407445954397\n",
      "episode:4 steps:15025 mean_reward:0.45841664036274965\n",
      "episode:5 steps:13959 mean_reward:0.4406210406557515\n",
      "episode:6 steps:6596 mean_reward:0.5425318492545446\n",
      "episode:7 steps:4981 mean_reward:0.3952645014301262\n",
      "episode:8 steps:6370 mean_reward:0.4798319561107541\n",
      "episode:9 steps:6835 mean_reward:0.4405422745389752\n",
      "episode:10 steps:3082 mean_reward:0.5038348597255634\n",
      "episode:11 steps:3222 mean_reward:0.4825209932703235\n",
      "episode:12 steps:4969 mean_reward:0.569692367295589\n",
      "episode:13 steps:2737 mean_reward:0.641988523100794\n",
      "episode:14 steps:6773 mean_reward:0.4885507718038348\n",
      "episode:15 steps:4770 mean_reward:0.5013102184289637\n",
      "episode:16 steps:5289 mean_reward:0.4368332129614751\n",
      "episode:17 steps:5390 mean_reward:0.5436012716286243\n",
      "episode:18 steps:16223 mean_reward:0.46032928981884563\n",
      "episode:19 steps:6803 mean_reward:0.4183260137934502\n",
      "episode:20 steps:4271 mean_reward:0.34617103863253224\n",
      "episode:21 steps:3748 mean_reward:0.3561823573086684\n",
      "episode:22 steps:5592 mean_reward:0.5022492591965401\n",
      "episode:23 steps:7169 mean_reward:0.5329838904606775\n",
      "episode:24 steps:5523 mean_reward:0.48422315377193653\n",
      "episode:25 steps:9769 mean_reward:0.436281220449428\n",
      "episode:26 steps:3513 mean_reward:0.5850847628358965\n",
      "episode:27 steps:10757 mean_reward:0.6340414176041266\n",
      "episode:28 steps:10685 mean_reward:0.5728974095944286\n",
      "episode:29 steps:3633 mean_reward:0.6281926416353603\n",
      "episode:30 steps:7813 mean_reward:0.3552713910684861\n",
      "episode:31 steps:6762 mean_reward:0.5052849359387735\n",
      "episode:32 steps:19108 mean_reward:0.5216477195027038\n",
      "episode:33 steps:3511 mean_reward:0.40814406239989776\n",
      "episode:34 steps:3942 mean_reward:0.49292809881363103\n",
      "episode:35 steps:4963 mean_reward:0.49761323100825844\n",
      "episode:36 steps:9441 mean_reward:0.39761308753363794\n",
      "episode:37 steps:3894 mean_reward:0.46562054181158324\n",
      "episode:38 steps:2003 mean_reward:0.4240897899977376\n",
      "episode:39 steps:6210 mean_reward:0.5354336071169724\n",
      "episode:40 steps:5002 mean_reward:0.3606336080335332\n",
      "episode:41 steps:5134 mean_reward:0.5038880255832694\n",
      "episode:42 steps:4057 mean_reward:0.44143269433126914\n",
      "episode:43 steps:13629 mean_reward:0.3738996691742083\n",
      "episode:44 steps:2773 mean_reward:0.5416469444810817\n",
      "episode:45 steps:7805 mean_reward:0.37328977171054406\n",
      "episode:46 steps:4018 mean_reward:0.45508929276539395\n",
      "episode:47 steps:6422 mean_reward:0.40288676880309776\n",
      "episode:48 steps:3117 mean_reward:0.4247665089412768\n",
      "episode:49 steps:2822 mean_reward:0.4061921573968456\n",
      "episode:50 steps:5230 mean_reward:0.4863606029017022\n",
      "episode:51 steps:10578 mean_reward:0.5346954641364497\n",
      "episode:52 steps:11353 mean_reward:0.5063578306059415\n",
      "episode:53 steps:2690 mean_reward:0.5414271374068936\n",
      "episode:54 steps:3540 mean_reward:0.4964673655019501\n",
      "episode:55 steps:6245 mean_reward:0.5166515965663832\n",
      "episode:56 steps:5680 mean_reward:0.4884411309795044\n",
      "episode:57 steps:2970 mean_reward:0.5547778318058332\n",
      "episode:58 steps:7483 mean_reward:0.5067971335196011\n",
      "episode:59 steps:8680 mean_reward:0.5110666126453722\n",
      "episode:60 steps:1471 mean_reward:0.5577660412551144\n",
      "episode:61 steps:2981 mean_reward:0.6302997723753536\n",
      "episode:62 steps:2435 mean_reward:0.4775663720190496\n",
      "episode:63 steps:6302 mean_reward:0.5137911448078367\n",
      "episode:64 steps:3843 mean_reward:0.5637819526511937\n",
      "episode:65 steps:7707 mean_reward:0.5732806452734179\n",
      "episode:66 steps:5077 mean_reward:0.2999449483903348\n",
      "episode:67 steps:3700 mean_reward:0.5532385965784543\n",
      "episode:68 steps:6110 mean_reward:0.5448734640715037\n",
      "episode:69 steps:6605 mean_reward:0.4106563176341427\n",
      "episode:70 steps:3465 mean_reward:0.6513681821516022\n",
      "episode:71 steps:8978 mean_reward:0.3163363195393759\n",
      "episode:72 steps:15035 mean_reward:0.49854460933443084\n",
      "episode:73 steps:7952 mean_reward:0.3378169825641502\n",
      "episode:74 steps:12246 mean_reward:0.39863932776591393\n",
      "episode:75 steps:5106 mean_reward:0.4749218918158356\n",
      "episode:76 steps:3129 mean_reward:0.5248465805943587\n",
      "episode:77 steps:11330 mean_reward:0.3487558250437434\n",
      "episode:78 steps:3173 mean_reward:0.5833474729417303\n",
      "episode:79 steps:5595 mean_reward:0.4899880864760339\n",
      "episode:80 steps:5414 mean_reward:0.34601358670034155\n",
      "episode:81 steps:5675 mean_reward:0.5797846119825736\n",
      "episode:82 steps:6497 mean_reward:0.3846407298615151\n",
      "episode:83 steps:1784 mean_reward:0.5940001201387858\n",
      "episode:84 steps:4026 mean_reward:0.4033885316481974\n",
      "episode:85 steps:2822 mean_reward:0.6495962743143044\n",
      "episode:86 steps:3165 mean_reward:0.3895267315958619\n",
      "episode:87 steps:5579 mean_reward:0.4556614267022055\n",
      "episode:88 steps:6993 mean_reward:0.3455865490766256\n",
      "episode:89 steps:3411 mean_reward:0.43087479228129694\n",
      "episode:90 steps:9255 mean_reward:0.39493461883028097\n",
      "episode:91 steps:5165 mean_reward:0.512071101790893\n",
      "episode:92 steps:3032 mean_reward:0.5966961638094576\n",
      "episode:93 steps:3806 mean_reward:0.3374551110931433\n",
      "episode:94 steps:7419 mean_reward:0.422645857990546\n",
      "episode:95 steps:5654 mean_reward:0.5694429673734454\n",
      "episode:96 steps:4500 mean_reward:0.2655504606926945\n",
      "episode:97 steps:2714 mean_reward:0.5051606189563568\n",
      "episode:98 steps:3985 mean_reward:0.47766246046242156\n",
      "episode:99 steps:8647 mean_reward:0.40107400732778786\n",
      "episode:100 steps:3766 mean_reward:0.48544693987561016\n",
      "episode:101 steps:3627 mean_reward:0.2896915112175304\n",
      "episode:102 steps:2993 mean_reward:0.4728475000040302\n",
      "episode:103 steps:8308 mean_reward:0.2950165120297052\n",
      "episode:104 steps:2526 mean_reward:0.5121453512269644\n",
      "episode:105 steps:3172 mean_reward:0.4738215047024267\n",
      "episode:106 steps:1794 mean_reward:0.5276939458153255\n",
      "episode:107 steps:5434 mean_reward:0.5412353719383762\n",
      "episode:108 steps:3049 mean_reward:0.48721559302915796\n",
      "episode:109 steps:2735 mean_reward:0.5177511505727994\n",
      "episode:110 steps:2977 mean_reward:0.5172822657829108\n",
      "episode:111 steps:2250 mean_reward:0.7344336115600205\n",
      "episode:112 steps:7644 mean_reward:0.42619562645416753\n",
      "episode:113 steps:4469 mean_reward:0.617729828549127\n",
      "episode:114 steps:7153 mean_reward:0.36060523028734853\n",
      "episode:115 steps:4611 mean_reward:0.3544110974730295\n",
      "episode:116 steps:6068 mean_reward:0.3411123866935422\n",
      "episode:117 steps:2036 mean_reward:0.5204713830341665\n",
      "episode:118 steps:3267 mean_reward:0.41738309729047374\n",
      "episode:119 steps:1847 mean_reward:0.5155616438955737\n",
      "episode:120 steps:2963 mean_reward:0.4816632529515403\n",
      "episode:121 steps:4764 mean_reward:0.4637543598280148\n",
      "episode:122 steps:5426 mean_reward:0.4043121139237717\n",
      "episode:123 steps:3922 mean_reward:0.4823224198429592\n",
      "episode:124 steps:1125 mean_reward:0.6302908085718099\n",
      "episode:125 steps:3785 mean_reward:0.5419250771237221\n",
      "episode:126 steps:2682 mean_reward:0.4725609186828323\n",
      "episode:127 steps:6182 mean_reward:0.34340400456610015\n",
      "episode:128 steps:3381 mean_reward:0.4382896261995487\n",
      "episode:129 steps:3217 mean_reward:0.4483819788566505\n",
      "episode:130 steps:4849 mean_reward:0.33330059766446524\n",
      "episode:131 steps:2101 mean_reward:0.47631236517858727\n",
      "episode:132 steps:4405 mean_reward:0.4234840068442341\n",
      "episode:133 steps:5950 mean_reward:0.46541224730403663\n",
      "episode:134 steps:3528 mean_reward:0.5860062288054418\n",
      "episode:135 steps:6513 mean_reward:0.2600164216109872\n",
      "episode:136 steps:2939 mean_reward:0.35846723087173876\n",
      "episode:137 steps:2444 mean_reward:0.5277612545138182\n",
      "episode:138 steps:2081 mean_reward:0.49864530885031494\n",
      "episode:139 steps:3781 mean_reward:0.5614281661337104\n",
      "episode:140 steps:3217 mean_reward:0.3212817619725507\n",
      "episode:141 steps:5803 mean_reward:0.3421690836264745\n",
      "episode:142 steps:4892 mean_reward:0.40831587075956804\n",
      "episode:143 steps:2061 mean_reward:0.45466578578204425\n",
      "episode:144 steps:4836 mean_reward:0.4240215878961697\n",
      "episode:145 steps:2564 mean_reward:0.37375812826703453\n",
      "episode:146 steps:2837 mean_reward:0.5736190914367997\n",
      "episode:147 steps:3287 mean_reward:0.42043191426677845\n",
      "episode:148 steps:3098 mean_reward:0.4064168147549178\n",
      "episode:149 steps:4772 mean_reward:0.49104242185926283\n",
      "episode:150 steps:2316 mean_reward:0.4796090472043875\n",
      "episode:151 steps:2319 mean_reward:0.35275369851185706\n",
      "episode:152 steps:3502 mean_reward:0.5741651212244746\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:153 steps:3689 mean_reward:0.6112353051997741\n",
      "episode:154 steps:2316 mean_reward:0.4655865079160895\n",
      "episode:155 steps:8323 mean_reward:0.22209315737720345\n",
      "episode:156 steps:1339 mean_reward:0.5647073327657076\n",
      "episode:157 steps:3031 mean_reward:0.44267633738228324\n",
      "episode:158 steps:2066 mean_reward:0.43692904362507534\n",
      "episode:159 steps:2875 mean_reward:0.35756726583154896\n",
      "episode:160 steps:4529 mean_reward:0.4932544662698502\n",
      "episode:161 steps:3024 mean_reward:0.4661633879777137\n",
      "episode:162 steps:4984 mean_reward:0.3395984787310331\n",
      "episode:163 steps:2561 mean_reward:0.41777935411579986\n",
      "episode:164 steps:7278 mean_reward:0.2952783709190341\n",
      "episode:165 steps:3877 mean_reward:0.3750557746968958\n",
      "episode:166 steps:4903 mean_reward:0.4769355230972744\n",
      "episode:167 steps:10191 mean_reward:0.42337329300864024\n",
      "episode:168 steps:3478 mean_reward:0.5447547713395307\n",
      "episode:169 steps:4723 mean_reward:0.34701628476144303\n",
      "episode:170 steps:3703 mean_reward:0.4112802177631583\n",
      "episode:171 steps:2384 mean_reward:0.4255422442620168\n",
      "episode:172 steps:2244 mean_reward:0.40107961054791397\n",
      "episode:173 steps:3953 mean_reward:0.5601119792863574\n",
      "episode:174 steps:4525 mean_reward:0.3722983567031276\n",
      "episode:175 steps:3675 mean_reward:0.4577078759978766\n",
      "episode:176 steps:5565 mean_reward:0.4285033289045928\n",
      "episode:177 steps:1275 mean_reward:0.6228289119672309\n",
      "episode:178 steps:4071 mean_reward:0.37040079199364656\n",
      "episode:179 steps:3295 mean_reward:0.44501890112060694\n",
      "episode:180 steps:4419 mean_reward:0.42288196622674445\n",
      "episode:181 steps:3682 mean_reward:0.35793332403026396\n",
      "episode:182 steps:4977 mean_reward:0.45925710526184516\n",
      "episode:183 steps:4713 mean_reward:0.3826342381399305\n",
      "episode:184 steps:3958 mean_reward:0.39934620795835524\n",
      "episode:185 steps:4811 mean_reward:0.3485262135222439\n",
      "episode:186 steps:2539 mean_reward:0.47864984910126973\n",
      "episode:187 steps:4591 mean_reward:0.346776130300395\n",
      "episode:188 steps:4086 mean_reward:0.502630437693785\n",
      "episode:189 steps:2807 mean_reward:0.5141268441282425\n",
      "episode:190 steps:4226 mean_reward:0.29643388044378627\n",
      "episode:191 steps:3128 mean_reward:0.5201660919044906\n",
      "episode:192 steps:2828 mean_reward:0.4493605706303452\n",
      "episode:193 steps:2551 mean_reward:0.5503144242463324\n",
      "episode:194 steps:2069 mean_reward:0.45398736709347787\n",
      "episode:195 steps:2551 mean_reward:0.4670341390001233\n",
      "episode:196 steps:2735 mean_reward:0.5028979339903125\n",
      "episode:197 steps:3187 mean_reward:0.47041882097665794\n",
      "episode:198 steps:3247 mean_reward:0.5147952564164148\n",
      "episode:199 steps:3881 mean_reward:0.4213773533016443\n",
      "episode:200 steps:1404 mean_reward:0.5530073592762428\n",
      "episode:201 steps:3052 mean_reward:0.5277779027553122\n",
      "episode:202 steps:2499 mean_reward:0.517564783180346\n",
      "episode:203 steps:2392 mean_reward:0.6187908720756846\n",
      "episode:204 steps:3431 mean_reward:0.5245828531002735\n",
      "episode:205 steps:5077 mean_reward:0.36071931280443065\n",
      "episode:206 steps:2936 mean_reward:0.38008461123832993\n",
      "episode:207 steps:2013 mean_reward:0.609340400854244\n",
      "episode:208 steps:1731 mean_reward:0.5059354434987678\n",
      "episode:209 steps:3668 mean_reward:0.37971564371110067\n",
      "episode:210 steps:3446 mean_reward:0.4539241295270309\n",
      "episode:211 steps:2599 mean_reward:0.46029200289859357\n",
      "episode:212 steps:1648 mean_reward:0.4153750404179948\n",
      "episode:213 steps:2709 mean_reward:0.49174466533945377\n",
      "episode:214 steps:5040 mean_reward:0.37973360407127416\n",
      "episode:215 steps:991 mean_reward:0.5576202857078577\n",
      "episode:216 steps:1658 mean_reward:0.4368271792906876\n",
      "episode:217 steps:4621 mean_reward:0.3875148315564949\n",
      "episode:218 steps:4198 mean_reward:0.491210857995967\n",
      "episode:219 steps:5110 mean_reward:0.2960987616929167\n",
      "episode:220 steps:2631 mean_reward:0.47775961796113614\n",
      "episode:221 steps:2866 mean_reward:0.52087136613326\n",
      "episode:222 steps:3866 mean_reward:0.45114173704654387\n",
      "episode:223 steps:2198 mean_reward:0.3881970107503774\n",
      "episode:224 steps:3449 mean_reward:0.5209366773244769\n",
      "episode:225 steps:2506 mean_reward:0.3600598001581259\n",
      "episode:226 steps:2493 mean_reward:0.3916964478596416\n",
      "episode:227 steps:5426 mean_reward:0.3683168962731443\n",
      "episode:228 steps:1963 mean_reward:0.4356747739592191\n",
      "episode:229 steps:3358 mean_reward:0.40352105226286256\n",
      "episode:230 steps:1988 mean_reward:0.5952702585935381\n",
      "episode:231 steps:2107 mean_reward:0.47379716425980684\n",
      "episode:232 steps:3992 mean_reward:0.41483552812647934\n",
      "episode:233 steps:2176 mean_reward:0.52545101894948\n",
      "episode:234 steps:1371 mean_reward:0.5682853741211665\n",
      "episode:235 steps:2563 mean_reward:0.47404788552825966\n",
      "episode:236 steps:2626 mean_reward:0.4051351714707094\n",
      "episode:237 steps:2693 mean_reward:0.4518009463679516\n",
      "episode:238 steps:3608 mean_reward:0.4282979736924492\n",
      "episode:239 steps:2395 mean_reward:0.4962488383894448\n",
      "episode:240 steps:5070 mean_reward:0.3850264350534831\n",
      "episode:241 steps:2932 mean_reward:0.4683078053273961\n",
      "episode:242 steps:3625 mean_reward:0.42530150879631745\n",
      "episode:243 steps:3285 mean_reward:0.34842879371329777\n",
      "episode:244 steps:3457 mean_reward:0.4256262262616294\n",
      "episode:245 steps:2530 mean_reward:0.3521698371914689\n",
      "episode:246 steps:2519 mean_reward:0.5054287191360084\n",
      "episode:247 steps:3367 mean_reward:0.4237436496870443\n",
      "episode:248 steps:2091 mean_reward:0.39123042514425976\n",
      "episode:249 steps:2042 mean_reward:0.47375107911718456\n",
      "episode:250 steps:2982 mean_reward:0.49071311271988005\n",
      "episode:251 steps:4840 mean_reward:0.37784963058843457\n",
      "episode:252 steps:3474 mean_reward:0.4558146673815815\n",
      "episode:253 steps:2144 mean_reward:0.5092676418325744\n",
      "episode:254 steps:3045 mean_reward:0.33644411802816365\n",
      "episode:255 steps:2891 mean_reward:0.3995895904199372\n",
      "episode:256 steps:3336 mean_reward:0.312507161585271\n",
      "episode:257 steps:2240 mean_reward:0.4531138170432413\n",
      "episode:258 steps:2130 mean_reward:0.49077585140180674\n",
      "episode:259 steps:1813 mean_reward:0.40896334637454806\n",
      "episode:260 steps:2610 mean_reward:0.40344491320207737\n",
      "episode:261 steps:3703 mean_reward:0.4077840754854518\n",
      "episode:262 steps:4135 mean_reward:0.34635673582518145\n",
      "episode:263 steps:1278 mean_reward:0.5600438088094324\n",
      "episode:264 steps:2780 mean_reward:0.49922883416681607\n",
      "episode:265 steps:2907 mean_reward:0.40497591149607465\n",
      "episode:266 steps:2731 mean_reward:0.4258606733875465\n",
      "episode:267 steps:2312 mean_reward:0.4948926212093825\n",
      "episode:268 steps:2363 mean_reward:0.3890276562557797\n",
      "episode:269 steps:1900 mean_reward:0.4459547750553538\n",
      "episode:270 steps:2867 mean_reward:0.43701804038136577\n",
      "episode:271 steps:1897 mean_reward:0.5538084954750071\n",
      "episode:272 steps:3549 mean_reward:0.45445148695107984\n",
      "episode:273 steps:1774 mean_reward:0.4606816722238919\n",
      "episode:274 steps:2447 mean_reward:0.47265348353481995\n",
      "episode:275 steps:2686 mean_reward:0.3522645448483568\n",
      "episode:276 steps:4102 mean_reward:0.43735072012458975\n",
      "episode:277 steps:3173 mean_reward:0.27429347833818324\n",
      "episode:278 steps:5425 mean_reward:0.3150846689751085\n",
      "episode:279 steps:4771 mean_reward:0.32054660712963484\n",
      "episode:280 steps:2551 mean_reward:0.4947384409041062\n",
      "episode:281 steps:3968 mean_reward:0.35822110216127967\n",
      "episode:282 steps:1856 mean_reward:0.46347307933160703\n",
      "episode:283 steps:2186 mean_reward:0.46357960491703337\n",
      "episode:284 steps:2742 mean_reward:0.4684247460874973\n",
      "episode:285 steps:1557 mean_reward:0.6279759834626678\n",
      "episode:286 steps:2882 mean_reward:0.3872534903935278\n",
      "episode:287 steps:2029 mean_reward:0.5153521381652387\n",
      "episode:288 steps:4278 mean_reward:0.4302475862889875\n",
      "episode:289 steps:2266 mean_reward:0.44926375649789035\n",
      "episode:290 steps:2832 mean_reward:0.3906269101089277\n",
      "episode:291 steps:2362 mean_reward:0.4253661132677148\n",
      "episode:292 steps:1386 mean_reward:0.641710431104362\n",
      "episode:293 steps:3521 mean_reward:0.3315178020576819\n",
      "episode:294 steps:2635 mean_reward:0.4422347647134811\n",
      "episode:295 steps:3350 mean_reward:0.46084299281515256\n",
      "episode:296 steps:2863 mean_reward:0.4014828873526296\n",
      "episode:297 steps:3205 mean_reward:0.38250844740412693\n",
      "episode:298 steps:2413 mean_reward:0.4440614608785545\n",
      "episode:299 steps:2307 mean_reward:0.3378488170737043\n",
      "episode:300 steps:2101 mean_reward:0.4222341528582525\n",
      "episode:301 steps:1250 mean_reward:0.549168070558615\n",
      "episode:302 steps:1261 mean_reward:0.6632355042853877\n",
      "episode:303 steps:3427 mean_reward:0.4980816526577895\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:304 steps:1582 mean_reward:0.4730312528684122\n",
      "episode:305 steps:2570 mean_reward:0.42595707592899\n",
      "episode:306 steps:4084 mean_reward:0.2593485023406741\n",
      "episode:307 steps:2125 mean_reward:0.5446997838752212\n",
      "episode:308 steps:3182 mean_reward:0.4141930618310597\n",
      "episode:309 steps:3323 mean_reward:0.44129208996333896\n",
      "episode:310 steps:2107 mean_reward:0.3549906120308114\n",
      "episode:311 steps:4237 mean_reward:0.3155672675526536\n",
      "episode:312 steps:2034 mean_reward:0.4721888099314462\n",
      "episode:313 steps:2697 mean_reward:0.358511368562818\n",
      "episode:314 steps:1569 mean_reward:0.4665973337170142\n",
      "episode:315 steps:2157 mean_reward:0.3766151453503014\n",
      "episode:316 steps:3170 mean_reward:0.33504505171483906\n",
      "episode:317 steps:2559 mean_reward:0.44916952373123104\n",
      "episode:318 steps:3211 mean_reward:0.35643201739906055\n",
      "episode:319 steps:2151 mean_reward:0.45827803360414116\n",
      "episode:320 steps:1961 mean_reward:0.4411434912668961\n",
      "episode:321 steps:3429 mean_reward:0.3898130991059481\n",
      "episode:322 steps:1665 mean_reward:0.47762234581369856\n",
      "episode:323 steps:3331 mean_reward:0.39223755644205893\n",
      "episode:324 steps:2717 mean_reward:0.36211274383447006\n",
      "episode:325 steps:2273 mean_reward:0.42858068927038506\n",
      "episode:326 steps:2371 mean_reward:0.45605276971947245\n",
      "episode:327 steps:1280 mean_reward:0.5915473198260064\n",
      "episode:328 steps:1694 mean_reward:0.5590524344357869\n",
      "episode:329 steps:1739 mean_reward:0.3849274671741937\n",
      "episode:330 steps:2891 mean_reward:0.34497201282465717\n",
      "episode:331 steps:3063 mean_reward:0.43841022230565935\n",
      "episode:332 steps:1691 mean_reward:0.5698725384785919\n",
      "episode:333 steps:3899 mean_reward:0.360478625758831\n",
      "episode:334 steps:1910 mean_reward:0.45013036619396546\n",
      "episode:335 steps:4005 mean_reward:0.32171158081059287\n",
      "episode:336 steps:3171 mean_reward:0.3209831245788071\n",
      "episode:337 steps:3082 mean_reward:0.43107363021431816\n",
      "episode:338 steps:3107 mean_reward:0.3801843049513612\n",
      "episode:339 steps:6445 mean_reward:0.2598853468326321\n",
      "episode:340 steps:2432 mean_reward:0.37402249956359096\n",
      "episode:341 steps:1777 mean_reward:0.52624101093969\n",
      "episode:342 steps:1080 mean_reward:0.6449307936703012\n",
      "episode:343 steps:1951 mean_reward:0.41697388841872846\n",
      "episode:344 steps:2400 mean_reward:0.36275006571248375\n",
      "episode:345 steps:4068 mean_reward:0.2664480612505483\n",
      "episode:346 steps:1424 mean_reward:0.43314402482649067\n",
      "episode:347 steps:2294 mean_reward:0.3891014244111375\n",
      "episode:348 steps:5216 mean_reward:0.23021034592222409\n",
      "episode:349 steps:3661 mean_reward:0.26356145872914005\n",
      "episode:350 steps:3212 mean_reward:0.36041276190423466\n",
      "episode:351 steps:3339 mean_reward:0.3716564458000393\n",
      "episode:352 steps:2293 mean_reward:0.3376501301022964\n",
      "episode:353 steps:1158 mean_reward:0.5857015836464519\n",
      "episode:354 steps:1779 mean_reward:0.5314237907815603\n",
      "episode:355 steps:2461 mean_reward:0.3371261052245945\n",
      "episode:356 steps:2082 mean_reward:0.45413352132526474\n",
      "episode:357 steps:3508 mean_reward:0.3436320802856979\n",
      "episode:358 steps:2038 mean_reward:0.43071061913874026\n",
      "episode:359 steps:2315 mean_reward:0.4508130083742451\n",
      "episode:360 steps:3685 mean_reward:0.32365396533151763\n",
      "episode:361 steps:3672 mean_reward:0.30924396433755724\n",
      "episode:362 steps:2120 mean_reward:0.49646240984571877\n",
      "episode:363 steps:2461 mean_reward:0.3652941040065956\n",
      "episode:364 steps:1730 mean_reward:0.4238622130436479\n",
      "episode:365 steps:2473 mean_reward:0.3182038616851473\n",
      "episode:366 steps:1109 mean_reward:0.5048662804707733\n",
      "episode:367 steps:3828 mean_reward:0.35698842116762325\n",
      "episode:368 steps:2231 mean_reward:0.42968068181486996\n",
      "episode:369 steps:1834 mean_reward:0.40036575501761645\n",
      "episode:370 steps:2586 mean_reward:0.37970647078489306\n",
      "episode:371 steps:1718 mean_reward:0.5113988424323233\n",
      "episode:372 steps:1901 mean_reward:0.43783743426875743\n",
      "episode:373 steps:2348 mean_reward:0.38729815513834326\n",
      "episode:374 steps:2198 mean_reward:0.3928743989176265\n",
      "episode:375 steps:1994 mean_reward:0.46090664370413714\n",
      "episode:376 steps:2106 mean_reward:0.3624938115193291\n",
      "episode:377 steps:2154 mean_reward:0.3398288693296342\n",
      "episode:378 steps:3563 mean_reward:0.23104073767136166\n",
      "episode:379 steps:1513 mean_reward:0.4052870131856072\n",
      "episode:380 steps:2413 mean_reward:0.38640467055721595\n",
      "episode:381 steps:1906 mean_reward:0.4530994585702553\n",
      "episode:382 steps:1366 mean_reward:0.5420135953136801\n",
      "episode:383 steps:1110 mean_reward:0.5051478945248513\n",
      "episode:384 steps:2187 mean_reward:0.42066819837385744\n",
      "episode:385 steps:2316 mean_reward:0.34332559560015913\n",
      "episode:386 steps:3379 mean_reward:0.32085473612584753\n",
      "episode:387 steps:2636 mean_reward:0.2840347963548066\n",
      "episode:388 steps:1765 mean_reward:0.5211590104026258\n",
      "episode:389 steps:3939 mean_reward:0.26157137524202717\n",
      "episode:390 steps:1798 mean_reward:0.42298347697416117\n",
      "episode:391 steps:1590 mean_reward:0.46919037864365415\n",
      "episode:392 steps:1806 mean_reward:0.4576250552723033\n",
      "episode:393 steps:2076 mean_reward:0.47256990824308487\n",
      "episode:394 steps:2038 mean_reward:0.440699062892003\n",
      "episode:395 steps:2238 mean_reward:0.4146517814760581\n",
      "episode:396 steps:1804 mean_reward:0.43025444161513926\n",
      "episode:397 steps:2125 mean_reward:0.4979941019348418\n",
      "episode:398 steps:994 mean_reward:0.5771938993280461\n",
      "episode:399 steps:1122 mean_reward:0.5615693574851887\n",
      "episode:400 steps:2680 mean_reward:0.31580739007206043\n",
      "episode:401 steps:2299 mean_reward:0.4450971500171492\n",
      "episode:402 steps:2606 mean_reward:0.3228099888759382\n",
      "episode:403 steps:1642 mean_reward:0.4216992768926035\n",
      "episode:404 steps:2514 mean_reward:0.4108218651838525\n",
      "episode:405 steps:1822 mean_reward:0.43736364151540713\n",
      "episode:406 steps:2590 mean_reward:0.4594355959508387\n",
      "episode:407 steps:2345 mean_reward:0.38921092273128194\n",
      "episode:408 steps:1554 mean_reward:0.5235093162443547\n",
      "episode:409 steps:2803 mean_reward:0.32928673438583045\n",
      "episode:410 steps:2696 mean_reward:0.3558361192414522\n",
      "episode:411 steps:1679 mean_reward:0.46059912831876954\n",
      "episode:412 steps:1338 mean_reward:0.4568315045334382\n",
      "episode:413 steps:2825 mean_reward:0.49758133284717404\n",
      "episode:414 steps:2087 mean_reward:0.4586896553962485\n",
      "episode:415 steps:2002 mean_reward:0.41105235964857606\n",
      "episode:416 steps:1648 mean_reward:0.4578969884091643\n",
      "episode:417 steps:1660 mean_reward:0.46966764464115424\n",
      "episode:418 steps:2213 mean_reward:0.4299225675566388\n",
      "episode:419 steps:1620 mean_reward:0.455754197939226\n",
      "episode:420 steps:2220 mean_reward:0.39186402521998653\n",
      "episode:421 steps:3060 mean_reward:0.29458370183414123\n",
      "episode:422 steps:1892 mean_reward:0.4223353091537978\n",
      "episode:423 steps:1918 mean_reward:0.4297559977361836\n",
      "episode:424 steps:3113 mean_reward:0.2776842705973939\n",
      "episode:425 steps:1510 mean_reward:0.45767205646757664\n",
      "episode:426 steps:1886 mean_reward:0.41290535758229824\n",
      "episode:427 steps:2488 mean_reward:0.2864222915637143\n",
      "episode:428 steps:919 mean_reward:0.6473078496180049\n",
      "episode:429 steps:1298 mean_reward:0.51949882265346\n",
      "episode:430 steps:1482 mean_reward:0.5545409818596196\n",
      "episode:431 steps:2232 mean_reward:0.4375640408259565\n",
      "episode:432 steps:2206 mean_reward:0.4007555145079804\n",
      "episode:433 steps:2115 mean_reward:0.3834092823377075\n",
      "episode:434 steps:2472 mean_reward:0.3109219541536622\n",
      "episode:435 steps:2639 mean_reward:0.32680211463716363\n",
      "episode:436 steps:2764 mean_reward:0.33587381096398705\n",
      "episode:437 steps:2006 mean_reward:0.4629804133751639\n",
      "episode:438 steps:802 mean_reward:0.620185261384686\n",
      "episode:439 steps:2096 mean_reward:0.4359032984774864\n",
      "episode:440 steps:2693 mean_reward:0.4711928814024714\n",
      "episode:441 steps:1189 mean_reward:0.5105448053281372\n",
      "episode:442 steps:2394 mean_reward:0.299916808309799\n",
      "episode:443 steps:2205 mean_reward:0.3493418988439838\n",
      "episode:444 steps:1728 mean_reward:0.535931810973943\n",
      "episode:445 steps:2253 mean_reward:0.4642091667236286\n",
      "episode:446 steps:2284 mean_reward:0.3618584455975239\n",
      "episode:447 steps:2025 mean_reward:0.45540096165203603\n",
      "episode:448 steps:2349 mean_reward:0.4458248461317749\n",
      "episode:449 steps:2164 mean_reward:0.295976287895121\n",
      "episode:450 steps:2621 mean_reward:0.2706905648283459\n",
      "episode:451 steps:3452 mean_reward:0.30281639066390537\n",
      "episode:452 steps:2301 mean_reward:0.39127326607128393\n",
      "episode:453 steps:2741 mean_reward:0.26566058481123495\n",
      "episode:454 steps:2140 mean_reward:0.411236077360317\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:455 steps:3014 mean_reward:0.24868179909285595\n",
      "episode:456 steps:1903 mean_reward:0.40499128033509146\n",
      "episode:457 steps:1746 mean_reward:0.44068991778806926\n",
      "episode:458 steps:2259 mean_reward:0.4238430043557465\n",
      "episode:459 steps:2051 mean_reward:0.45332956385620315\n",
      "episode:460 steps:1521 mean_reward:0.46104521373398216\n",
      "episode:461 steps:2556 mean_reward:0.41183940170795924\n",
      "episode:462 steps:1742 mean_reward:0.53166136962473\n",
      "episode:463 steps:1386 mean_reward:0.523272637765209\n",
      "episode:464 steps:2704 mean_reward:0.3033526468879606\n",
      "episode:465 steps:1222 mean_reward:0.4588564805464439\n",
      "episode:466 steps:2499 mean_reward:0.3308373087702082\n",
      "episode:467 steps:921 mean_reward:0.598955603989339\n",
      "episode:468 steps:3155 mean_reward:0.29514022776801085\n",
      "episode:469 steps:1582 mean_reward:0.4380314074118108\n",
      "episode:470 steps:1090 mean_reward:0.5442538129065454\n",
      "episode:471 steps:1720 mean_reward:0.3815652378872341\n",
      "episode:472 steps:1436 mean_reward:0.49843449827868735\n",
      "episode:473 steps:1356 mean_reward:0.46391679685755477\n",
      "episode:474 steps:2000 mean_reward:0.42031786903005175\n",
      "episode:475 steps:1395 mean_reward:0.4275753031238903\n",
      "episode:476 steps:1039 mean_reward:0.5176268025428076\n",
      "episode:477 steps:1836 mean_reward:0.4207520294362322\n",
      "episode:478 steps:2925 mean_reward:0.4097220406662915\n",
      "episode:479 steps:1361 mean_reward:0.4631554485468682\n",
      "episode:480 steps:1517 mean_reward:0.4417481412948061\n",
      "episode:481 steps:1820 mean_reward:0.3839006361585385\n",
      "episode:482 steps:2269 mean_reward:0.3752465864909685\n",
      "episode:483 steps:4752 mean_reward:0.2311703838234032\n",
      "episode:484 steps:2203 mean_reward:0.33753572784719743\n",
      "episode:485 steps:1468 mean_reward:0.49820680496620284\n",
      "episode:486 steps:1897 mean_reward:0.39132077079531286\n",
      "episode:487 steps:2237 mean_reward:0.29629987238533045\n",
      "episode:488 steps:1227 mean_reward:0.532010127464495\n",
      "episode:489 steps:2066 mean_reward:0.34935239052215966\n",
      "episode:490 steps:1481 mean_reward:0.4899557323261982\n",
      "episode:491 steps:1517 mean_reward:0.4703295290857514\n",
      "episode:492 steps:1740 mean_reward:0.4091311098529911\n",
      "episode:493 steps:1998 mean_reward:0.3975051809295979\n",
      "episode:494 steps:1033 mean_reward:0.501235914486085\n",
      "episode:495 steps:1135 mean_reward:0.5373898423428358\n",
      "episode:496 steps:2145 mean_reward:0.4559946910789455\n",
      "episode:497 steps:2010 mean_reward:0.5068090416000799\n",
      "episode:498 steps:1952 mean_reward:0.35536262914693845\n",
      "episode:499 steps:1460 mean_reward:0.4726413064475054\n",
      "episode:500 steps:967 mean_reward:0.5609979889366542\n",
      "episode:501 steps:1243 mean_reward:0.5743324948374957\n",
      "episode:502 steps:1407 mean_reward:0.5140078310580639\n",
      "episode:503 steps:1880 mean_reward:0.3518845245020517\n",
      "episode:504 steps:964 mean_reward:0.5384783792653506\n",
      "episode:505 steps:2184 mean_reward:0.345139009223135\n",
      "episode:506 steps:899 mean_reward:0.6239695195388275\n",
      "episode:507 steps:2899 mean_reward:0.2809522767196659\n",
      "episode:508 steps:1146 mean_reward:0.5260286379176199\n",
      "episode:509 steps:2011 mean_reward:0.2848001729691528\n",
      "episode:510 steps:1657 mean_reward:0.4223263618639741\n",
      "episode:511 steps:2040 mean_reward:0.37681575276017976\n",
      "episode:512 steps:1539 mean_reward:0.42229430436565546\n",
      "episode:513 steps:1068 mean_reward:0.5533610314884485\n",
      "episode:514 steps:2175 mean_reward:0.34421118314433174\n",
      "episode:515 steps:2595 mean_reward:0.3099718084659362\n",
      "episode:516 steps:1578 mean_reward:0.408480342609281\n",
      "episode:517 steps:1220 mean_reward:0.4949784294067004\n",
      "episode:518 steps:976 mean_reward:0.5753695387888116\n",
      "episode:519 steps:980 mean_reward:0.585542969407934\n",
      "episode:520 steps:1509 mean_reward:0.44733956046406215\n",
      "episode:521 steps:1849 mean_reward:0.38356206067512977\n",
      "episode:522 steps:1608 mean_reward:0.5098931380252799\n",
      "episode:523 steps:2365 mean_reward:0.28216853642652445\n",
      "episode:524 steps:1870 mean_reward:0.41665765796351856\n",
      "episode:525 steps:3184 mean_reward:0.2979710702987411\n",
      "episode:526 steps:1241 mean_reward:0.5581144282248887\n",
      "episode:527 steps:2273 mean_reward:0.3280881302648925\n",
      "episode:528 steps:2289 mean_reward:0.3439647833139403\n",
      "episode:529 steps:1668 mean_reward:0.3647309907106565\n",
      "episode:530 steps:1779 mean_reward:0.4106676178258702\n",
      "episode:531 steps:2174 mean_reward:0.3444487902670356\n",
      "episode:532 steps:2595 mean_reward:0.33566038111337687\n",
      "episode:533 steps:1705 mean_reward:0.4538625181282429\n",
      "episode:534 steps:1354 mean_reward:0.4448980231211872\n",
      "episode:535 steps:2761 mean_reward:0.316192490960708\n",
      "episode:536 steps:2150 mean_reward:0.3290579051673812\n",
      "episode:537 steps:1255 mean_reward:0.5442459428309911\n",
      "episode:538 steps:1491 mean_reward:0.4770773843943623\n",
      "episode:539 steps:1315 mean_reward:0.5056211046425\n",
      "episode:540 steps:1372 mean_reward:0.44159800187434095\n",
      "episode:541 steps:1275 mean_reward:0.5687790631817795\n",
      "episode:542 steps:2540 mean_reward:0.2640325261388008\n",
      "episode:543 steps:1900 mean_reward:0.3489801165818802\n",
      "episode:544 steps:1574 mean_reward:0.4506011947737483\n",
      "episode:545 steps:991 mean_reward:0.5326868984281157\n",
      "episode:546 steps:1599 mean_reward:0.376860125022135\n",
      "episode:547 steps:2142 mean_reward:0.3281505320141537\n",
      "episode:548 steps:1555 mean_reward:0.4383081194684955\n",
      "episode:549 steps:2378 mean_reward:0.3222435049756829\n",
      "episode:550 steps:2061 mean_reward:0.305593076700845\n",
      "episode:551 steps:2360 mean_reward:0.35224281853009504\n",
      "episode:552 steps:983 mean_reward:0.5376864632350311\n",
      "episode:553 steps:3075 mean_reward:0.27932462796955415\n",
      "episode:554 steps:1251 mean_reward:0.5075500631756116\n",
      "episode:555 steps:853 mean_reward:0.6024763230507352\n",
      "episode:556 steps:1237 mean_reward:0.44248788480033474\n",
      "episode:557 steps:1903 mean_reward:0.3418289259397035\n",
      "episode:558 steps:907 mean_reward:0.5056313775720925\n",
      "episode:559 steps:978 mean_reward:0.5010775064351722\n",
      "episode:560 steps:2439 mean_reward:0.31327013892842676\n",
      "episode:561 steps:1335 mean_reward:0.4490026505593071\n",
      "episode:562 steps:906 mean_reward:0.5369768644958731\n",
      "episode:563 steps:1096 mean_reward:0.5325999030769563\n",
      "episode:564 steps:1173 mean_reward:0.521200214423235\n",
      "episode:565 steps:1411 mean_reward:0.47870216359142653\n",
      "episode:566 steps:1078 mean_reward:0.5329156292345428\n",
      "episode:567 steps:2764 mean_reward:0.29379785029422334\n",
      "episode:568 steps:924 mean_reward:0.5623482459544037\n",
      "episode:569 steps:1695 mean_reward:0.36648411491595145\n",
      "episode:570 steps:2625 mean_reward:0.32372206062007014\n",
      "episode:571 steps:569 mean_reward:0.6624044286392322\n",
      "episode:572 steps:997 mean_reward:0.526516356054728\n",
      "episode:573 steps:2541 mean_reward:0.3672165675997468\n",
      "episode:574 steps:2801 mean_reward:0.2661736688182916\n",
      "episode:575 steps:1260 mean_reward:0.5181105358571816\n",
      "episode:576 steps:1001 mean_reward:0.5062690299483817\n",
      "episode:577 steps:1234 mean_reward:0.4930461585745474\n",
      "episode:578 steps:1473 mean_reward:0.4362954371468891\n",
      "episode:579 steps:1193 mean_reward:0.5416976302552206\n",
      "episode:580 steps:3592 mean_reward:0.2735104692065536\n",
      "episode:581 steps:2706 mean_reward:0.3225913409674739\n",
      "episode:582 steps:1241 mean_reward:0.4789826851528595\n",
      "episode:583 steps:1400 mean_reward:0.45552376659806976\n",
      "episode:584 steps:2242 mean_reward:0.33585757489898205\n",
      "episode:585 steps:1118 mean_reward:0.5610736871252416\n",
      "episode:586 steps:1061 mean_reward:0.49029379763289427\n",
      "episode:587 steps:4566 mean_reward:0.1917045072249054\n",
      "episode:588 steps:1072 mean_reward:0.4647489274533106\n",
      "episode:589 steps:1441 mean_reward:0.4047473627842078\n",
      "episode:590 steps:2386 mean_reward:0.40807429201400414\n",
      "episode:591 steps:3082 mean_reward:0.2858876232060499\n",
      "episode:592 steps:1172 mean_reward:0.4749226724129202\n",
      "episode:593 steps:1068 mean_reward:0.5061734579618197\n",
      "episode:594 steps:2058 mean_reward:0.47499395523360277\n",
      "episode:595 steps:853 mean_reward:0.6092031758088169\n",
      "episode:596 steps:1496 mean_reward:0.47080757161990444\n",
      "episode:597 steps:1293 mean_reward:0.4382537669024316\n",
      "episode:598 steps:1217 mean_reward:0.4124006714656306\n",
      "episode:599 steps:2379 mean_reward:0.3547442358575582\n",
      "episode:600 steps:1512 mean_reward:0.32109204190821133\n",
      "episode:601 steps:1653 mean_reward:0.39634177304739476\n",
      "episode:602 steps:1877 mean_reward:0.3766475990145592\n",
      "episode:603 steps:1435 mean_reward:0.3668918687740416\n",
      "episode:604 steps:794 mean_reward:0.635754579501216\n",
      "episode:605 steps:1105 mean_reward:0.5488970924241996\n",
      "episode:606 steps:1000 mean_reward:0.5293181742957489\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:607 steps:2176 mean_reward:0.31972681802200714\n",
      "episode:608 steps:2497 mean_reward:0.265419143247525\n",
      "episode:609 steps:752 mean_reward:0.5925981538269903\n",
      "episode:610 steps:1290 mean_reward:0.5126020014192649\n",
      "episode:611 steps:1707 mean_reward:0.40863086886037153\n",
      "episode:612 steps:970 mean_reward:0.4938319040563005\n",
      "episode:613 steps:1535 mean_reward:0.4539821010758692\n",
      "episode:614 steps:1565 mean_reward:0.46441128323735437\n",
      "episode:615 steps:868 mean_reward:0.6046069872392191\n",
      "episode:616 steps:2002 mean_reward:0.30654366396900873\n",
      "episode:617 steps:1122 mean_reward:0.5720038814082948\n",
      "episode:618 steps:2404 mean_reward:0.3428813017006818\n",
      "episode:619 steps:1300 mean_reward:0.40954349939589896\n",
      "episode:620 steps:2369 mean_reward:0.3205739857889855\n",
      "episode:621 steps:1248 mean_reward:0.5025730258689864\n",
      "episode:622 steps:1563 mean_reward:0.46761805115409866\n",
      "episode:623 steps:2023 mean_reward:0.3704868525205085\n",
      "episode:624 steps:1667 mean_reward:0.42608346608903036\n",
      "episode:625 steps:1128 mean_reward:0.45299121834790057\n",
      "episode:626 steps:1109 mean_reward:0.5318287335296777\n",
      "episode:627 steps:1292 mean_reward:0.4136986767535364\n",
      "episode:628 steps:2572 mean_reward:0.29064621353636894\n",
      "episode:629 steps:1799 mean_reward:0.3724945378760414\n",
      "episode:630 steps:1264 mean_reward:0.45007633084089266\n",
      "episode:631 steps:3081 mean_reward:0.25697167189309705\n",
      "episode:632 steps:1291 mean_reward:0.47304287248315047\n",
      "episode:633 steps:1213 mean_reward:0.48326380836627286\n",
      "episode:634 steps:1853 mean_reward:0.3676767709729966\n",
      "episode:635 steps:2645 mean_reward:0.2853909139535848\n",
      "episode:636 steps:1804 mean_reward:0.3090982630291267\n",
      "episode:637 steps:1745 mean_reward:0.3160972481340787\n",
      "episode:638 steps:753 mean_reward:0.6215114307161909\n",
      "episode:639 steps:1904 mean_reward:0.35514875563306825\n",
      "episode:640 steps:2299 mean_reward:0.3100630472277494\n",
      "episode:641 steps:1838 mean_reward:0.39616993037378023\n",
      "episode:642 steps:2716 mean_reward:0.31517796785686236\n",
      "episode:643 steps:1676 mean_reward:0.44654980521054755\n",
      "episode:644 steps:816 mean_reward:0.575891703272985\n",
      "episode:645 steps:823 mean_reward:0.6118380739121414\n",
      "episode:646 steps:1290 mean_reward:0.4783396237438403\n",
      "episode:647 steps:1078 mean_reward:0.5555439138384368\n",
      "episode:648 steps:1657 mean_reward:0.37112947137897706\n",
      "episode:649 steps:1420 mean_reward:0.3604588004901938\n",
      "episode:650 steps:1846 mean_reward:0.380431974859197\n",
      "episode:651 steps:2350 mean_reward:0.3220453617895519\n",
      "episode:652 steps:1230 mean_reward:0.47147475561051466\n",
      "episode:653 steps:1420 mean_reward:0.4435079544117223\n",
      "episode:654 steps:1129 mean_reward:0.488115941454448\n",
      "episode:655 steps:2075 mean_reward:0.3277429748517775\n",
      "episode:656 steps:1803 mean_reward:0.4594585627637561\n",
      "episode:657 steps:1465 mean_reward:0.3458588384457464\n",
      "episode:658 steps:1560 mean_reward:0.40183277120513494\n",
      "episode:659 steps:1034 mean_reward:0.6143376084766371\n",
      "episode:660 steps:1524 mean_reward:0.4177644279794251\n",
      "episode:661 steps:2330 mean_reward:0.34764678461531284\n",
      "episode:662 steps:1973 mean_reward:0.34312607601101996\n",
      "episode:663 steps:1148 mean_reward:0.4891302641705647\n",
      "episode:664 steps:1379 mean_reward:0.4781164166045938\n",
      "episode:665 steps:1957 mean_reward:0.33073758039830564\n",
      "episode:666 steps:1991 mean_reward:0.38576934648819694\n",
      "episode:667 steps:1786 mean_reward:0.395663467319012\n",
      "episode:668 steps:1670 mean_reward:0.397866407059099\n",
      "episode:669 steps:815 mean_reward:0.6019707527331047\n",
      "episode:670 steps:1847 mean_reward:0.37994628701466615\n",
      "episode:671 steps:1456 mean_reward:0.45465934595171864\n",
      "episode:672 steps:1168 mean_reward:0.5205266425890647\n",
      "episode:673 steps:1137 mean_reward:0.5379036584659357\n",
      "episode:674 steps:2297 mean_reward:0.348125552159627\n",
      "episode:675 steps:2210 mean_reward:0.35788925681212297\n",
      "episode:676 steps:1347 mean_reward:0.4089547600624103\n",
      "episode:677 steps:2153 mean_reward:0.29524874920163263\n",
      "episode:678 steps:630 mean_reward:0.5920488906363284\n",
      "episode:679 steps:941 mean_reward:0.6217088300117612\n",
      "episode:680 steps:1490 mean_reward:0.4122607503946712\n",
      "episode:681 steps:1028 mean_reward:0.467419891961432\n",
      "episode:682 steps:1917 mean_reward:0.30008784873339794\n",
      "episode:683 steps:2110 mean_reward:0.33910620774169076\n",
      "episode:684 steps:1849 mean_reward:0.39953075249624725\n",
      "episode:685 steps:1711 mean_reward:0.4455316162027282\n",
      "episode:686 steps:1352 mean_reward:0.4033662199525751\n",
      "episode:687 steps:2075 mean_reward:0.31093110949157354\n",
      "episode:688 steps:818 mean_reward:0.570886673981503\n",
      "episode:689 steps:717 mean_reward:0.6021090119440776\n",
      "episode:690 steps:1067 mean_reward:0.5903638275524715\n",
      "episode:691 steps:1046 mean_reward:0.5349887659675061\n",
      "episode:692 steps:1289 mean_reward:0.4132983267510723\n",
      "episode:693 steps:1280 mean_reward:0.4333694422594351\n",
      "episode:694 steps:907 mean_reward:0.5883359369147311\n",
      "episode:695 steps:1125 mean_reward:0.45803332597896484\n",
      "episode:696 steps:1262 mean_reward:0.4313640333494279\n",
      "episode:697 steps:1902 mean_reward:0.3130622948625881\n",
      "episode:698 steps:737 mean_reward:0.6079476875536589\n",
      "episode:699 steps:1293 mean_reward:0.5071597770656436\n",
      "episode:700 steps:1043 mean_reward:0.5310832332932575\n",
      "episode:701 steps:638 mean_reward:0.6143564868399446\n",
      "episode:702 steps:2644 mean_reward:0.2426829552244446\n",
      "episode:703 steps:1418 mean_reward:0.4032734476985308\n",
      "episode:704 steps:1919 mean_reward:0.33860045334275307\n",
      "episode:705 steps:1682 mean_reward:0.4018091281955936\n",
      "episode:706 steps:968 mean_reward:0.5605867872424026\n",
      "episode:707 steps:1459 mean_reward:0.46502172027773064\n",
      "episode:708 steps:2180 mean_reward:0.38940162446751336\n",
      "episode:709 steps:877 mean_reward:0.5249695916947342\n",
      "episode:710 steps:1454 mean_reward:0.40743669388035725\n",
      "episode:711 steps:969 mean_reward:0.543328220325623\n",
      "episode:712 steps:1599 mean_reward:0.43914080752087203\n",
      "episode:713 steps:939 mean_reward:0.48889440669961254\n",
      "episode:714 steps:1396 mean_reward:0.4363883351051909\n",
      "episode:715 steps:2118 mean_reward:0.306615266594077\n",
      "episode:716 steps:970 mean_reward:0.4870322822441074\n",
      "episode:717 steps:1224 mean_reward:0.4772659745410688\n",
      "episode:718 steps:1230 mean_reward:0.4355077557256499\n",
      "episode:719 steps:936 mean_reward:0.46930725161919157\n",
      "episode:720 steps:967 mean_reward:0.5034774432062458\n",
      "episode:721 steps:906 mean_reward:0.5778973251812309\n",
      "episode:722 steps:1258 mean_reward:0.4255399249836312\n",
      "episode:723 steps:746 mean_reward:0.6217520938432871\n",
      "episode:724 steps:1486 mean_reward:0.47281021880178564\n",
      "episode:725 steps:1438 mean_reward:0.40578700971755827\n",
      "episode:726 steps:912 mean_reward:0.5293256315511903\n",
      "episode:727 steps:1574 mean_reward:0.36336612417589936\n",
      "episode:728 steps:1371 mean_reward:0.44949082351893316\n",
      "episode:729 steps:1577 mean_reward:0.43816626191132013\n",
      "episode:730 steps:1746 mean_reward:0.3674701717703343\n",
      "episode:731 steps:1075 mean_reward:0.42513405228698903\n",
      "episode:732 steps:1597 mean_reward:0.41023280508440013\n",
      "episode:733 steps:1154 mean_reward:0.4858368935044271\n",
      "episode:734 steps:1118 mean_reward:0.44450516406009627\n",
      "episode:735 steps:2272 mean_reward:0.28800907448708785\n",
      "episode:736 steps:1682 mean_reward:0.3201407834016134\n",
      "episode:737 steps:1002 mean_reward:0.5816779718685895\n",
      "episode:738 steps:1113 mean_reward:0.4117976169853453\n",
      "episode:739 steps:1373 mean_reward:0.4287756607943988\n",
      "episode:740 steps:1298 mean_reward:0.41255587511456937\n",
      "episode:741 steps:1673 mean_reward:0.3577314944487598\n",
      "episode:742 steps:867 mean_reward:0.48793436264635626\n",
      "episode:743 steps:820 mean_reward:0.5625691844060497\n",
      "episode:744 steps:1307 mean_reward:0.4845555778014204\n",
      "episode:745 steps:2572 mean_reward:0.27105329829745234\n",
      "episode:746 steps:918 mean_reward:0.5610973527073666\n",
      "episode:747 steps:806 mean_reward:0.5102408417071143\n",
      "episode:748 steps:1933 mean_reward:0.320896429669657\n",
      "episode:749 steps:983 mean_reward:0.48802589075527725\n",
      "episode:750 steps:1603 mean_reward:0.37526447210913816\n",
      "episode:751 steps:897 mean_reward:0.5041458501907163\n",
      "episode:752 steps:1916 mean_reward:0.2873713758713013\n",
      "episode:753 steps:952 mean_reward:0.4431101994598401\n",
      "episode:754 steps:826 mean_reward:0.5053651191225336\n",
      "episode:755 steps:1975 mean_reward:0.31589343694885197\n",
      "episode:756 steps:736 mean_reward:0.6093680895232837\n",
      "episode:757 steps:836 mean_reward:0.5890533813018782\n",
      "episode:758 steps:1706 mean_reward:0.3652431548098253\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:759 steps:1134 mean_reward:0.40916038325987625\n",
      "episode:760 steps:1266 mean_reward:0.34199062510569644\n",
      "episode:761 steps:762 mean_reward:0.6206369910886609\n",
      "episode:762 steps:767 mean_reward:0.6373989852359961\n",
      "episode:763 steps:742 mean_reward:0.568529876382942\n",
      "episode:764 steps:1480 mean_reward:0.461624818932431\n",
      "episode:765 steps:998 mean_reward:0.4660547442242221\n",
      "episode:766 steps:1355 mean_reward:0.5088439096658374\n",
      "episode:767 steps:753 mean_reward:0.6157147490724469\n",
      "episode:768 steps:904 mean_reward:0.5173779220373557\n",
      "episode:769 steps:1382 mean_reward:0.38991093162488777\n",
      "episode:770 steps:1787 mean_reward:0.32136125588087555\n",
      "episode:771 steps:1897 mean_reward:0.32192179057982234\n",
      "episode:772 steps:970 mean_reward:0.4923928773443559\n",
      "episode:773 steps:569 mean_reward:0.6396398487838971\n",
      "episode:774 steps:1504 mean_reward:0.360421589939546\n",
      "episode:775 steps:737 mean_reward:0.5634843848431166\n",
      "episode:776 steps:704 mean_reward:0.6544050821565444\n",
      "episode:777 steps:730 mean_reward:0.5167212806086927\n",
      "episode:778 steps:738 mean_reward:0.5856847580494485\n",
      "episode:779 steps:1396 mean_reward:0.38444412628053376\n",
      "episode:780 steps:1284 mean_reward:0.4271307887332814\n",
      "episode:781 steps:872 mean_reward:0.4777178468009157\n",
      "episode:782 steps:1144 mean_reward:0.460620245979653\n",
      "episode:783 steps:1219 mean_reward:0.4382974424464716\n",
      "episode:784 steps:1317 mean_reward:0.3677078420085748\n",
      "episode:785 steps:983 mean_reward:0.5008447770886347\n",
      "episode:786 steps:833 mean_reward:0.5834711109036107\n",
      "episode:787 steps:646 mean_reward:0.6233459351945598\n",
      "episode:788 steps:668 mean_reward:0.6205875437809836\n",
      "episode:789 steps:1589 mean_reward:0.3288848421401713\n",
      "episode:790 steps:774 mean_reward:0.6329291612781162\n",
      "episode:791 steps:1472 mean_reward:0.3421314756467131\n",
      "episode:792 steps:1879 mean_reward:0.33483486375797067\n",
      "episode:793 steps:1226 mean_reward:0.4339888494950885\n",
      "episode:794 steps:966 mean_reward:0.47538768972682605\n",
      "episode:795 steps:1140 mean_reward:0.4515939454182277\n",
      "episode:796 steps:1077 mean_reward:0.49565972917821666\n",
      "episode:797 steps:1244 mean_reward:0.4582383773402888\n",
      "episode:798 steps:1914 mean_reward:0.341669263010487\n",
      "episode:799 steps:777 mean_reward:0.6630992943111873\n",
      "episode:800 steps:1285 mean_reward:0.4382630471615534\n",
      "episode:801 steps:967 mean_reward:0.481068374196237\n",
      "episode:802 steps:1662 mean_reward:0.37096642786974326\n",
      "episode:803 steps:1298 mean_reward:0.3762679170478746\n",
      "episode:804 steps:1324 mean_reward:0.43555106667080684\n",
      "episode:805 steps:1292 mean_reward:0.4689768206248722\n",
      "episode:806 steps:1400 mean_reward:0.37359684216072553\n",
      "episode:807 steps:599 mean_reward:0.6731360339772714\n",
      "episode:808 steps:1299 mean_reward:0.41966159400462033\n",
      "episode:809 steps:2241 mean_reward:0.2533033235911876\n",
      "episode:810 steps:506 mean_reward:0.6991252690308997\n",
      "episode:811 steps:1406 mean_reward:0.402253332656057\n",
      "episode:812 steps:1414 mean_reward:0.316505657665236\n",
      "episode:813 steps:750 mean_reward:0.5902394686434035\n",
      "episode:814 steps:763 mean_reward:0.585800240612207\n",
      "episode:815 steps:992 mean_reward:0.5077546379199445\n",
      "episode:816 steps:1597 mean_reward:0.34004525960864845\n",
      "episode:817 steps:901 mean_reward:0.4685891709661076\n",
      "episode:818 steps:2500 mean_reward:0.2590393427374453\n",
      "episode:819 steps:1452 mean_reward:0.35761653624293577\n",
      "episode:820 steps:909 mean_reward:0.5599174878189186\n",
      "episode:821 steps:1011 mean_reward:0.5327291240707726\n",
      "episode:822 steps:568 mean_reward:0.5865687257750402\n",
      "episode:823 steps:1256 mean_reward:0.4109280258123522\n",
      "episode:824 steps:995 mean_reward:0.48532168761603717\n",
      "episode:825 steps:1293 mean_reward:0.38026030995909565\n",
      "episode:826 steps:758 mean_reward:0.5265822041689519\n",
      "episode:827 steps:1004 mean_reward:0.4708218384338164\n",
      "episode:828 steps:1150 mean_reward:0.4437548313555189\n",
      "episode:829 steps:991 mean_reward:0.4607439439800948\n",
      "episode:830 steps:1654 mean_reward:0.31111817216445575\n",
      "episode:831 steps:1232 mean_reward:0.39945436582520033\n",
      "episode:832 steps:1381 mean_reward:0.3871594779672111\n",
      "episode:833 steps:767 mean_reward:0.604466622901814\n",
      "episode:834 steps:565 mean_reward:0.6457477047355172\n",
      "episode:835 steps:1088 mean_reward:0.4517737390043282\n",
      "episode:836 steps:1964 mean_reward:0.2948804120547818\n",
      "episode:837 steps:578 mean_reward:0.6577420138203142\n",
      "episode:838 steps:677 mean_reward:0.566747194222685\n",
      "episode:839 steps:1596 mean_reward:0.3144545999364835\n",
      "episode:840 steps:833 mean_reward:0.49053015111750675\n",
      "episode:841 steps:665 mean_reward:0.6087783113666749\n",
      "episode:842 steps:1138 mean_reward:0.3889465607373134\n",
      "episode:843 steps:1243 mean_reward:0.4263261027999587\n",
      "episode:844 steps:772 mean_reward:0.5704320184509648\n",
      "episode:845 steps:565 mean_reward:0.6331391518106959\n",
      "episode:846 steps:481 mean_reward:0.688811483545175\n",
      "episode:847 steps:989 mean_reward:0.50020486092954\n",
      "episode:848 steps:1738 mean_reward:0.3436758433805812\n",
      "episode:849 steps:868 mean_reward:0.42064502994252406\n",
      "episode:850 steps:1474 mean_reward:0.3363821499461428\n",
      "episode:851 steps:817 mean_reward:0.5401334550703809\n",
      "episode:852 steps:882 mean_reward:0.48552572487859474\n",
      "episode:853 steps:499 mean_reward:0.6754053166226429\n",
      "episode:854 steps:776 mean_reward:0.6123672681842895\n",
      "episode:855 steps:1128 mean_reward:0.3980737741161646\n",
      "episode:856 steps:1135 mean_reward:0.4446349264033457\n",
      "episode:857 steps:905 mean_reward:0.515849337926191\n",
      "episode:858 steps:1158 mean_reward:0.44718523648588754\n",
      "episode:859 steps:988 mean_reward:0.48108209860272144\n",
      "episode:860 steps:929 mean_reward:0.5352883232502824\n",
      "episode:861 steps:1740 mean_reward:0.38030935163204665\n",
      "episode:862 steps:1284 mean_reward:0.35332484066783604\n",
      "episode:863 steps:822 mean_reward:0.5092401156379326\n",
      "episode:864 steps:1483 mean_reward:0.39882644867367756\n",
      "episode:865 steps:1069 mean_reward:0.3952471611962757\n",
      "episode:866 steps:1065 mean_reward:0.4566001299003257\n",
      "episode:867 steps:499 mean_reward:0.6984043380869619\n",
      "episode:868 steps:1109 mean_reward:0.5017618674368383\n",
      "episode:869 steps:821 mean_reward:0.5782343085669174\n",
      "episode:870 steps:978 mean_reward:0.46989108912709143\n",
      "episode:871 steps:670 mean_reward:0.5986170682376556\n",
      "episode:872 steps:1792 mean_reward:0.3412315917862362\n",
      "episode:873 steps:747 mean_reward:0.5655941016983909\n",
      "episode:874 steps:1068 mean_reward:0.49090686330372524\n",
      "episode:875 steps:952 mean_reward:0.45312356646686125\n",
      "episode:876 steps:671 mean_reward:0.6025578731238005\n",
      "episode:877 steps:655 mean_reward:0.5735677484234799\n",
      "episode:878 steps:1371 mean_reward:0.39214378175597414\n",
      "episode:879 steps:1469 mean_reward:0.4205376926063367\n",
      "episode:880 steps:1292 mean_reward:0.34990928717710873\n",
      "episode:881 steps:1125 mean_reward:0.40847690553202826\n",
      "episode:882 steps:653 mean_reward:0.5939191517547897\n",
      "episode:883 steps:1173 mean_reward:0.5043808942827986\n",
      "episode:884 steps:2300 mean_reward:0.3120601280765349\n",
      "episode:885 steps:670 mean_reward:0.6272370065166437\n",
      "episode:886 steps:887 mean_reward:0.5085949649602772\n",
      "episode:887 steps:584 mean_reward:0.6639604735345669\n",
      "episode:888 steps:873 mean_reward:0.5848605513387435\n",
      "episode:889 steps:1611 mean_reward:0.37413105852261125\n",
      "episode:890 steps:738 mean_reward:0.5722413509191375\n",
      "episode:891 steps:825 mean_reward:0.5027825250533241\n",
      "episode:892 steps:1158 mean_reward:0.5299652546614024\n",
      "episode:893 steps:843 mean_reward:0.5094005269937315\n",
      "episode:894 steps:1597 mean_reward:0.30768627774391927\n",
      "episode:895 steps:841 mean_reward:0.5839252629673443\n",
      "episode:896 steps:811 mean_reward:0.5083606514376792\n",
      "episode:897 steps:730 mean_reward:0.5281597650586324\n",
      "episode:898 steps:1004 mean_reward:0.4222320750444226\n",
      "episode:899 steps:922 mean_reward:0.48263226803938214\n",
      "episode:900 steps:825 mean_reward:0.4965666479321011\n",
      "episode:901 steps:832 mean_reward:0.5320077064251442\n",
      "episode:902 steps:1582 mean_reward:0.32982284902801695\n",
      "episode:903 steps:645 mean_reward:0.5533983319529622\n",
      "episode:904 steps:959 mean_reward:0.437846402167495\n",
      "episode:905 steps:925 mean_reward:0.48539632509230546\n",
      "episode:906 steps:644 mean_reward:0.5575714112623925\n",
      "episode:907 steps:834 mean_reward:0.545155158684967\n",
      "episode:908 steps:848 mean_reward:0.547833287639936\n",
      "episode:909 steps:1006 mean_reward:0.4750417642482219\n",
      "episode:910 steps:848 mean_reward:0.5441724611412151\n",
      "episode:911 steps:737 mean_reward:0.5401295469561852\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode:912 steps:896 mean_reward:0.4731460906597544\n",
      "episode:913 steps:656 mean_reward:0.5832592116639368\n",
      "episode:914 steps:880 mean_reward:0.4962227861614367\n",
      "episode:915 steps:947 mean_reward:0.5546880194211501\n",
      "episode:916 steps:748 mean_reward:0.5335046836595909\n",
      "episode:917 steps:754 mean_reward:0.6026983541408906\n",
      "episode:918 steps:568 mean_reward:0.6314068090968902\n",
      "episode:919 steps:1227 mean_reward:0.41130963188321856\n",
      "episode:920 steps:1318 mean_reward:0.4296729330371179\n",
      "episode:921 steps:1376 mean_reward:0.36960830150487956\n",
      "episode:922 steps:484 mean_reward:0.6578385821783642\n",
      "episode:923 steps:1516 mean_reward:0.3556442031243171\n",
      "episode:924 steps:1627 mean_reward:0.3447658068398719\n",
      "episode:925 steps:788 mean_reward:0.4636914866927075\n",
      "episode:926 steps:811 mean_reward:0.5018240801969321\n",
      "episode:927 steps:830 mean_reward:0.5391656764613357\n",
      "episode:928 steps:826 mean_reward:0.5561860429270069\n",
      "episode:929 steps:1355 mean_reward:0.38324426158490954\n",
      "episode:930 steps:1298 mean_reward:0.40606211215391247\n",
      "episode:931 steps:980 mean_reward:0.5174268785489223\n",
      "episode:932 steps:1408 mean_reward:0.37702896975709943\n",
      "episode:933 steps:1015 mean_reward:0.4098092462045796\n",
      "episode:934 steps:816 mean_reward:0.5078052526291851\n",
      "episode:935 steps:1460 mean_reward:0.37189292088548226\n",
      "episode:936 steps:1140 mean_reward:0.3793352906639588\n",
      "episode:937 steps:970 mean_reward:0.5155071742658053\n",
      "episode:938 steps:824 mean_reward:0.5122649520272679\n",
      "episode:939 steps:911 mean_reward:0.5077037380793364\n",
      "episode:940 steps:1069 mean_reward:0.43624205420820344\n",
      "episode:941 steps:1239 mean_reward:0.38394989352679293\n",
      "episode:942 steps:1509 mean_reward:0.2757422874422778\n",
      "episode:943 steps:1245 mean_reward:0.41032131325998844\n",
      "episode:944 steps:1455 mean_reward:0.3704955937467503\n",
      "episode:945 steps:1056 mean_reward:0.396843012574592\n",
      "episode:946 steps:1549 mean_reward:0.36713937873408525\n",
      "episode:947 steps:1114 mean_reward:0.39670868766708695\n",
      "episode:948 steps:1051 mean_reward:0.40891169744709943\n",
      "episode:949 steps:1017 mean_reward:0.5124153330178564\n",
      "episode:950 steps:856 mean_reward:0.6339814795642912\n",
      "episode:951 steps:1046 mean_reward:0.413158688524304\n",
      "episode:952 steps:872 mean_reward:0.5017845719139176\n",
      "episode:953 steps:1139 mean_reward:0.4356324682344498\n",
      "episode:954 steps:1631 mean_reward:0.32233280631969624\n",
      "episode:955 steps:1357 mean_reward:0.3289200799106369\n",
      "episode:956 steps:1359 mean_reward:0.35494162599324525\n",
      "episode:957 steps:1246 mean_reward:0.4742969005290598\n",
      "episode:958 steps:1241 mean_reward:0.37586495663093517\n",
      "episode:959 steps:1149 mean_reward:0.40135302885339\n",
      "episode:960 steps:788 mean_reward:0.4930071318169055\n",
      "episode:961 steps:984 mean_reward:0.4462066957701091\n",
      "episode:962 steps:1641 mean_reward:0.2970943071024904\n",
      "episode:963 steps:967 mean_reward:0.5208665468381051\n",
      "episode:964 steps:897 mean_reward:0.5199947990979243\n",
      "episode:965 steps:1189 mean_reward:0.3947333946514997\n",
      "episode:966 steps:1281 mean_reward:0.47014586096809663\n",
      "episode:967 steps:1076 mean_reward:0.4262243916734143\n",
      "episode:968 steps:1380 mean_reward:0.3429854030334726\n",
      "episode:969 steps:1315 mean_reward:0.3344936618030975\n",
      "episode:970 steps:1586 mean_reward:0.34607989694064617\n",
      "episode:971 steps:1180 mean_reward:0.3617348975317186\n",
      "episode:972 steps:893 mean_reward:0.3964977171353151\n",
      "episode:973 steps:837 mean_reward:0.5856725275073228\n",
      "episode:974 steps:1593 mean_reward:0.33660937716241174\n",
      "episode:975 steps:940 mean_reward:0.5113992021765242\n",
      "episode:976 steps:979 mean_reward:0.49535227268673104\n",
      "episode:977 steps:995 mean_reward:0.43306476876525435\n",
      "episode:978 steps:1381 mean_reward:0.3794780477042043\n",
      "episode:979 steps:1264 mean_reward:0.3699096026191661\n",
      "episode:980 steps:1560 mean_reward:0.3556326860925939\n",
      "episode:981 steps:1085 mean_reward:0.4088145791433207\n",
      "episode:982 steps:655 mean_reward:0.5866611152174062\n",
      "episode:983 steps:837 mean_reward:0.4945393667708288\n",
      "episode:984 steps:902 mean_reward:0.5235056451960638\n",
      "episode:985 steps:1173 mean_reward:0.41683834919755514\n",
      "episode:986 steps:931 mean_reward:0.565089624370628\n",
      "episode:987 steps:655 mean_reward:0.5905780921766024\n",
      "episode:988 steps:1232 mean_reward:0.3739867536316923\n",
      "episode:989 steps:828 mean_reward:0.5751635918784862\n",
      "episode:990 steps:1870 mean_reward:0.2775914679213606\n",
      "episode:991 steps:906 mean_reward:0.5234409227441951\n",
      "episode:992 steps:1146 mean_reward:0.4160720610825708\n",
      "episode:993 steps:1390 mean_reward:0.3292171324203536\n",
      "episode:994 steps:1204 mean_reward:0.39898039332418056\n",
      "episode:995 steps:644 mean_reward:0.5681734607214244\n",
      "episode:996 steps:1105 mean_reward:0.44027512517140194\n",
      "episode:997 steps:956 mean_reward:0.4759886896727949\n",
      "episode:998 steps:567 mean_reward:0.6139827727956068\n",
      "episode:999 steps:861 mean_reward:0.5010460302805567\n",
      "game over\n"
     ]
    }
   ],
   "source": [
    "step_result = []\n",
    "reward_result = []\n",
    "env = gym.make('MountainCarContinuous-v0')\n",
    "env.seed(1)     # reproducible, general Policy gradient has high variance\n",
    "env = env.unwrapped\n",
    "RL = Policy_Gradient(n_actions = 1, \n",
    "                   n_states = 2,\n",
    "                   gamma = 0.9,\n",
    "                   learning_rate = 0.001,\n",
    "                   batch_size = 1000\n",
    "                 )\n",
    "step_record, reward_record = training(save_model = True, model_name='PG_MountainCarContinuous')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 訓練完的結果\n",
    "因為是連續動作相比之下較非連續動作難訓練，耗費的時間比較長。當然這個問題本身\n",
    "並不需要控制到這麼細微的動作才能破關，因此將動作切割成discrete的動作就可以\n",
    "，不過一般在真實世界，動作其實是連續的，而且差一點點結果就會差很多，因此用這\n",
    "個例子讓大家感受一下連續動作有多難訓練，還有policy gradeint如何處理連續動\n",
    "作。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAEKCAYAAAAiizNaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xd4lFXa+PHvnUJCD12aBKSKVCNF\n/AnSRPQVe8F9QfTVddfuqivqWhDruvayqAh2VCygYKOoIEiVXiRAgFADKSRA+vn9Mc9MpmcmmZkk\nzP25rlyZ5zzneebMMMyd08UYg1JKKRUpMVVdAKWUUtFFA49SSqmI0sCjlFIqojTwKKWUiigNPEop\npSJKA49SSqmI0sCjlFIqojTwKKWUiigNPEoppSIqrqoLEGlNmzY1ycnJVV0MpZSqMVatWnXYGNMs\nVPeLusCTnJzMypUrq7oYSilVY4jIrlDeT5valFJKRZQGHqWUUhGlgUcppVRERV0fj1IqvIqKikhP\nTyc/P7+qi6KClJiYSJs2bYiPjw/r84Qt8IjIu8BFwCFjzBlWWmPgUyAZSAOuMsZkiYgALwOjgePA\n9caY1dY144GHrdtONsa8Z6WfCUwHagNzgTuNbi6kVJVLT0+nfv36JCcnY/uvrWoCYwxHjhwhPT2d\n9u3bh/W5wtnUNh0Y5Zb2ADDfGNMJmG8dA1wAdLJ+bgbeBEegehToD/QDHhWRRtY1bwI3OV3n/lxK\nqSqQn59PkyZNNOjUMCJCkyZNIlJTDVvgMcb8CmS6JY8B3rMevwdc4pT+vrH5HUgSkZbA+cBPxphM\nY0wW8BMwyjrXwBjzu1XLed/pXkqpKqZBp2aK1L9bpAcXtDDG7LceHwBaWI9bA3uc8qVbaf7S072k\nh8SRvAK+W7+//IxKKaWCVmWj2qyaSkT6ZETkZhFZKSIrMzIyys1/0/sr+dtHq8k8VhiB0imlVHSJ\ndOA5aDWTYf0+ZKXvBdo65WtjpflLb+Ml3StjzFvGmBRjTEqzZuWv+rA78wQAxSWl5eZVStV8Q4YM\ncaxoMnr0aLKzs6u4RN499thjPP/88wA88sgjzJs3r0L3WbNmDXPnzg1l0YIS6cAzGxhvPR4PzHJK\nHyc2A4Acq0nuB2CkiDSyBhWMBH6wzh0VkQHWiLhxTvdSSqkKmzt3LklJSRF7vuLi4gpdN2nSJIYP\nH16ha6s68IRzOPUnwBCgqYikYxud9gzwmYjcCOwCrrKyz8U2lDoV23DqCQDGmEwReQJYYeWbZIyx\nD1j4O2XDqb+zfpRS1cjj32xk076jIb3n6a0a8Oj/dPebJy0tjVGjRnHmmWeyevVqunfvzvvvv8/S\npUu59957KS4u5qyzzuLNN98kISHB5Vr7eo5Nmzbl/fff5/nnn0dE6NmzJ2+88QY9e/bkzz//JD4+\nnqNHj9KrVy/HsbsVK1Zw4403EhMTw4gRI/juu+/YsGED06dP58svvyQvL4+SkhLmzJnDmDFjyMrK\noqioiMmTJzNmzBgAnnzySd577z2aN29O27ZtOfPMMwG4/vrrueiii7jiiitYtWoV99xzD3l5eTRt\n2pTp06fTsmVLhgwZQv/+/Vm4cCHZ2dlMnTqV/v3788gjj3DixAkWL17MxIkTufrqq0P0rxOYsAUe\nY8y1Pk4N85LXALf6uM+7wLte0lcCZ1SmjOXSgTlK1Vhbt25l6tSpDBo0iBtuuIEXXniBKVOmMH/+\nfDp37sy4ceN48803ueuuu7xev3HjRiZPnsySJUto2rQpmZmZ1K9fnyFDhjBnzhwuueQSZsyYwWWX\nXeZzwuWECRN4++23GThwIA888IDLudWrV7Nu3ToaN25McXExX331FQ0aNODw4cMMGDCAiy++mNWr\nVzNjxgzWrFlDcXExffv2dQQeu6KiIm6//XZmzZpFs2bN+PTTT3nooYd4913b12ZxcTHLly9n7ty5\nPP7448ybN49JkyaxcuVKXnvttRC808HTlQuUUmFTXs0knNq2bcugQYMA+Mtf/sITTzxB+/bt6dy5\nMwDjx4/n9ddf9xl4FixYwJVXXknTpk0BaNy4MQD/93//x3PPPccll1zCtGnTePvtt71en52dTW5u\nLgMHDgRg7NixfPvtt47zI0aMcNzTGMODDz7Ir7/+SkxMDHv37uXgwYMsWrSISy+9lDp16gBw8cUX\nezzP1q1b2bBhAyNGjACgpKSEli1bOs5fdtllAJx55pmkpaUF8M6FnwYepdRJyX1OSlJSEkeOHKn0\nfQcNGkRaWho///wzJSUlnHFGxRpe6tat63j80UcfkZGRwapVq4iPjyc5OTngiZzGGLp3787SpUu9\nnrc3JcbGxla4PynUdJFQpdRJaffu3Y4v448//piUlBTS0tJITU0F4IMPPmDw4ME+rx86dCiff/65\nI1hlZpbNhx83bhxjx45lwoQJPq9PSkqifv36LFu2DIAZM2b4zJuTk0Pz5s2Jj49n4cKF7Npl2/7m\n3HPP5euvv+bEiRPk5ubyzTffeFzbpUsXMjIyHK+1qKiIjRs3+nwugPr165Obm+s3Tzhp4FFKnZS6\ndOnC66+/Trdu3cjKyuLuu+9m2rRpXHnllfTo0YOYmBhuueUWn9d3796dhx56iMGDB9OrVy/uuece\nx7nrrruOrKwsrr3WV1e2zdSpU7npppvo3bs3x44do2HDhl7zXXfddaxcuZIePXrw/vvv07VrVwD6\n9u3L1VdfTa9evbjgggs466yzPK6tVasWM2fO5J///Ce9evWid+/eLFmyxG+5zjvvPDZt2kTv3r35\n9NNP/eYNB4m2dTVTUlJMeTuQpkz+icN5hSx/aBjN6ydGqGRKnRw2b95Mt27dqrQMaWlpXHTRRWzY\nsCEs9585cyazZs3igw8+8JsvLy+PevXqAfDMM8+wf/9+Xn755bCUKVS8/fuJyCpjTEqonkP7eJRS\nKgi333473333XUDzYObMmcPTTz9NcXEx7dq1Y/r06eEvYA2ggUcpddJJTk4OW23n1Vdf9Ui79dZb\n+e2331zS7rzzTiZMmBDxOTI1gQYeP0Qn8ihVIcaYqFqh+vXXX6/qIoREpLpedHCBUiqkEhMTOXLk\nSMS+xFRo2DeCS0wMf7+21niUUiHVpk0b0tPTCWQleFW92Le+DjcNPF7oH2pKVVx8fHzYt05WNZs2\ntSmllIooDTxKKaUiSgOPH1E0KEcppSJGA49SSqmI0sDjhw4yUEqp0NPAo5RSKqI08Phh0CqPUkqF\nmgYeLzTcKKVU+Gjg8UcjkFJKhZwGHqWUUhGlgccPrfAopVToaeBRSikVURp4vNDl3JVSKnw08Pih\n8UcppUJPA49SSqmI0sDjh04gVUqp0NPAo5RSKqI08PihfTxKKRV6Gni80HijlFLho4HHDw1ASikV\nehp4lFJKRVSVBB4RuVtENorIBhH5REQSRaS9iCwTkVQR+VREall5E6zjVOt8stN9JlrpW0Xk/FCX\nUyeSKqVU6EU88IhIa+AOIMUYcwYQC1wDPAu8aIzpCGQBN1qX3AhkWekvWvkQkdOt67oDo4A3RCQ2\nkq9FKaVU8KqqqS0OqC0icUAdYD8wFJhpnX8PuMR6PMY6xjo/TETESp9hjCkwxuwEUoF+oSykVniU\nUir0Ih54jDF7geeB3dgCTg6wCsg2xhRb2dKB1tbj1sAe69piK38T53Qv11SyjKG4i1JKKW+qoqmt\nEbbaSnugFVAXW1NZOJ/zZhFZKSIrMzIywvlUSimlylEVTW3DgZ3GmAxjTBHwJTAISLKa3gDaAHut\nx3uBtgDW+YbAEed0L9e4MMa8ZYxJMcakNGvWLNSvRymlVBCqIvDsBgaISB2rr2YYsAlYCFxh5RkP\nzLIez7aOsc4vMLbhZrOBa6xRb+2BTsDyUBZUm9yUUir04srPElrGmGUiMhNYDRQDfwBvAXOAGSIy\n2Uqbal0yFfhARFKBTGwj2TDGbBSRz7AFrWLgVmNMSURfjFJKqaBFPPAAGGMeBR51S96Bl1Fpxph8\n4Eof93kSeDLkBbTfX9cuUEqpkNOVC5RSSkWUBh4/tI9HKaVCTwOPF7pUjlJKhY8GHj80/CilVOhp\n4AG2HcylpLRyYaa4pJQxr//GL3/qBFWllPIn6gPPlgNHGfHir7y6YJvHuWCa3DKPFbJ2Tzb3fr42\nlMVTSqmTTtQHnv05+QD8sTu7ikuilFLRIeoDjz/ax6OUUqGngccLDThKKRU+Gnj80FHVSikVehp4\nNLgopVREaeCxiHhL1aiklFKhpoHHos1qSikVGRp4vNZ0bIIJRhq3lFIqMBp4QsxPHFNKKYUGHr8q\nUovRmo9SSvkXdYEnv6iU9Kzj/jNVIHpoTUcppQITdYFn26Fcznl2YVUXQymlolaVbH1dXR3OKyBl\n8jzHsY50U0qp0Iu6Go8/2w/lVfhajVFKKRUYDTxOESMuNsbtVODhRGtHSikVGA08FhGoFVvxtyOY\nIKWUUtFMA4+TuFjXsWlBTSC18uroNqWU8k8Dj5P42IqHDa3vKKVUYDTwOImRitd4HNeEqCxKKXWy\n0sDjpDJBw+joAqWUCogGHosxnjUcHdWmlFKhp4HHpXVNo4dSSoWbBh4nHjWeCoxqU0op5Z8GnhDR\neTxKKRUYDTzG68Pgb6PzeJRSKiAaeCwilWsu0/qOUkoFpkoCj4gkichMEdkiIptFZKCINBaRn0Rk\nm/W7kZVXROQVEUkVkXUi0tfpPuOt/NtEZHxly+XeXBZcH4+GHqWUCkRV1XheBr43xnQFegGbgQeA\n+caYTsB86xjgAqCT9XMz8CaAiDQGHgX6A/2AR+3BKhDeAkUoYoeGH6WU8i/igUdEGgLnAlMBjDGF\nxphsYAzwnpXtPeAS6/EY4H1j8zuQJCItgfOBn4wxmcaYLOAnYFSg5Sj1EiEqNY8n4JxKKRXdqqLG\n0x7IAKaJyB8i8o6I1AVaGGP2W3kOAC2sx62BPU7Xp1tpvtIDUuqtxlOJ8KEtbUopFZiqCDxxQF/g\nTWNMH+AYZc1qABhbO1jIvspF5GYRWSkiK8uewzNfZebxaJ1HKaUCUxWBJx1IN8Yss45nYgtEB60m\nNKzfh6zze4G2Tte3sdJ8pXswxrxljEkxxqTY07zVeCpDazxKKRWYiAceY8wBYI+IdLGShgGbgNmA\nfWTaeGCW9Xg2MM4a3TYAyLGa5H4ARopII2tQwUgrLcByBJAn0Js55dV5PEop5V9cFT3v7cBHIlIL\n2AFMwBYEPxORG4FdwFVW3rnAaCAVOG7lxRiTKSJPACusfJOMMZmBFsDb0OmAgpExiHiGF63xKKVU\nYKok8Bhj1gApXk4N85LXALf6uM+7wLsVKYPXUW0ewcgz0+B//0yjOvHMuu0cv9cqpZTyrqpqPFXO\nPagEunLB7szj7PZSr9Iaj1JKBSboPh6rT6VnOAoTSd5rPP6Pc04UlXtfjT9KKeVfQIFHRH4WkQbW\nagGrgbdF5IXwFi28vK9c4D9sFBaX+rlfpYuklFJRIdAaT0NjzFHgMmyrCPQHhoevWOHndR5POXn8\n9eNoH49SSgUm0MATZ82tuQr4NozliRivKxeUEzv8ndcaj1JKBSbQwDMJ2xyZ7caYFSLSAdgWvmKF\nn72Px39NJfjVqnUej1JK+RfQqDZjzOfA507HO4DLw1WoSLAHHNdg4j+y+G1q0xqPUkoFJNDBBR1E\n5BsRyRCRQyIyy6r11Fj2QOEcMMpbq83bSDhHXu3jUUqpgATa1PYx8BnQEmiFrfbzSbgKFQn2Ph7n\ncFFe6PA36k1rPEopFZhAA08dY8wHxphi6+dDIDGcBQu3shqP7YHgbRSb92u83i9kJVNKqZNboCsX\nfCciDwAzsH3HXg3Mteb1EMwaadWF1xpPOdWWUC8sqpRS0SjQwGNfsPOvbunXYPuurXH9Pd5qN5Wa\nx6NtbUopFZBAR7W1D3dBIi2QwQV2s9fuIz5G6Nqyge/7hbBsSil1Mgt0VFsdEXlYRN6yjjuJyEXh\nLVp4lU0gLT9k3PHJH/zto9UBDS7QeTxKKeVfoIMLpgGFwNnW8V5gclhKFCGOsONc4ylnW4TyZvko\npZQqX6CB5zRjzHNAEYAx5jg1/I97b4MLyosdOpxaKaUqL9DAUygitbG+mkXkNKAgbKWKAHsQca3x\nuOXxuMbP/azfh3Jr9NuilFJhF2jgeQz4HmgrIh8B84F/hqtQkeAYXOAUXspdJDSA+wGkHsqreMGU\nUuokF+ioth9FZBUwAFsT253GmMNhLVmYlXob1VbOoqDeVrQuy1t27nBeAR2b16t0GZVS6mQU6Ki2\n+caYI8aYOcaYb40xh0VkfrgLF06ORUKtY28rF3hcoysXKKVUpfmt8YhIIlAHaCoijSgbUNAAaB3m\nsoVVqbWZqHNNxbOPJ/htEZRSSvlXXlPbX4G7sC0MugqrYgDkAq+Gt2jh5X0jOP+RxX9TW6WLpJRS\nUcFvU5sx5mVr1YIngd7W42nADmBpBMoXdn4DRhDBxLl2VKPHmSulVJgFOqrtCmPMURE5BxgKvAO8\nGb5ihV/ZPB7fTW3uQhWklFIqmgUaeEqs3xcCbxtj5gC1wlOkyPA2qs09eLjHEr9NbSEplVJKnfwC\nDTx7RWQKZdshJARxbbXkbTmc8nYRdT5bWmrIPl7odL+ycyLa2KaUUr4EGjyuAn4AzjfGZAONgfvC\nVqoI8DqPp5ytr52D1asLUuk96Sd+3nqI6975nYLiEpRSSpUv0Amkx4EvnY73A/vDVahIMF7Waitv\nZFqp0/k56/cBcMuHq8gvKqV/+yYhLqFSSp2canRzWWWUrU4d+Dwe5xxFJbbH8bEx1n1CXMAg5ReV\n8NjsjeScKKragiilVDmiNvCUlnquXDBv00H/1zgFl8Ji2wxUe+BxVhVdPDNXpTN9SRov/vRn5J9c\nKaWCELWBx3g8gE9X7nHN46fPp9ha+iDGijLOwaYqaj8lVlQsKdXxdUqp6i1qA4+3eTzlcW6WKy4x\nLmlV3dRmpwPqlFLVXZUFHhGJFZE/RORb67i9iCwTkVQR+VREalnpCdZxqnU+2ekeE630rSJyfjDP\nb7yMavPI43bsXJkoKin1msdWrmBKEhrlLfejlFLVRVXWeO4ENjsdPwu8aIzpCGQBN1rpNwJZVvqL\nVj5E5HTgGqA7MAp4Q0RiA33ysv14Ame8DC6oSM0pnLTCo5Sq7qok8IhIG2yrILxjHQu2pXhmWlne\nAy6xHo+xjrHOD7PyjwFmGGMKjDE7gVSgX6BlKA2gicyjFuGlj8fb9Vr5UEop36qqxvMScD9gbU5A\nEyDbGFNsHadTtu1Ca2APgHU+x8rvSPdyjQsRuVlEVorISntaRWoqrk1tvvt4dh05FvA9lVIq2kQ8\n8IjIRcAhY8yqSD2nMeYtY0yKMSbFkeY45+c6j2NvWyl45r1v5jp2ZER2+2utZCmlaoqqqPEMAi4W\nkTRgBrYmtpeBJBGxr6TQBthrPd4LtAWwzjcEjjine7mmXN5WLij/Gs+0Uh+jFA7k5Adx58qzP72u\nE6eUqu4iHniMMRONMW2MMcnYBgcsMMZcBywErrCyjQdmWY9nW8dY5xcYW9SYDVxjjXprD3QClgda\nDvsOpHb7sr0ECrdA43XzuECfUCmlFBDgWm0R8k9ghohMBv4AplrpU4EPRCQVyMQWrDDGbBSRz4BN\nQDFwqzEm4JU6yyaQ2h5tPZjrkScjr8CxwoHLNU5Ky6oagT51WGgAVErVFFUaeIwxPwM/W4934GVU\nmjEmH7jSx/VPYtsdNWilATS13T9znWuTmdemNkdhKlIMpZSKOlG7ckEgE0gBFm495HjsdSM4H/OB\nnI9zThRRXOLWthcm2sWjlKruojjwuA6H9iXG6Zvc3+ACX7uTlpQaej3+Iw9+tb6CJVVKqZNL1Aae\nUh81FXfOFQhvee1pvtbmtC/a+cXqgAfcVYgumaOUqimiOPAEtrinc9OVt1qNo8bjI/LY5/5EatVo\n0UVzlFLVnAaecvI5f5H7Wx7HV1Obe/KmfUfJzdfN2pRS0UsDTzlVHtfOet953ccOeAtIxhhGv7KI\nG6avCKaoSil1UonawBPoIDPXpjbf+XzVeJyvsT9ekZYV2JMHoZpMJ1JKqXJFbeAJtI/n9x2Zjsf+\n8n71h/fBA84BKRL9PBp3lFLVXXVauSCiSksNz36/hTd/3h7wNf5Wss454dpv8/yPW2lWP4FTGiSW\nPaeOPFNKqWiu8cC7i3cGfU2g1uzJ5qopS12CTeohzxWr//bhKro/8n1Q5VBKqZosams8JcYE3R8S\n7FyZwuJSl8Bz0auLPfJ8t+FAcIXwobrsgKqUUuWJ3hpPqXFZlSAQwbaUlRoTVC0pFHRwgVKquove\nwGMqEHiCrFUYE7kVBbT7SClVU0Rt4Hn8m01BjzIL9su9xESuAUzjjlKqpojawANwoijg7XuA4AYX\n2PKbgEeyFYVo9WrdgVQpVd1F7eCCigi22cwY38HqRGEJn67Y7TguKC4lPrbyfwdo2FFKVXcaeIIw\ne+2+oK/xtXjocz9sYdpvaY7jgqIS6iXoP4dS6uQX1U1twVq07XDQ1/iqJOUcd51wml9cuaY2HVyg\nlKopNPCEmc8+Hrc2sYIg+5t80rY2pVQ1p4EnzAIdXJBf5FnjuX/mWjo9NDeg63UCqVKqptBOhTAL\ndCRc9olCj7TPVqYH/jyRnqmqlFIVpDWeMPM1Es59p9DDeZ6BJxj2uKM7kCqlqjsNPGHmqyLiPt3m\ncG5BJZ/HeL2vUkpVNxp4wmzVLu+bvrnHhyPHKht4bL91dJtSqrrTwBNmD3613mu6e82k0M9w6kCW\n9jGOje008iilqjcNPFVk+c5Ml+NiP8ElkOV07E1tutmcUqq608BTRdKOHHc5Li6pbOCx/bbHnbyC\n4gqXTSmlwkkDTzXhv8bjem7jvhyPJrWyGg9s2JvDGY/+wJx1+0NfUKWUqiQNPNVEsZ9ajXON5/sN\n+7nwlcUe68bZ5/GUGsO69BwAFqdmkB+qFRGUUipENPBUE/4GEDgPPNh2MA+Aab+lkXb4mCPdXisy\nxlBSasu/PeMYXf/1Pd9UYHFTpZQKFw08wIU9Wroc/3NU1wrdZ3i3FhUuQ1GpYe2ebD5bucfznJfa\n0Jo92Qx5/mce/Go9yQ/McQSuwpJSJs/ZDMCfB3MB+GHjgQqXSymlQk0DDxAT4zq2eeBpTSp0n3fG\np1S4DCWlpTz+zUbun7mOrQdyXc459/G414s+Xmbb08c+X2j2mn0UWDWkWtb+PsHutKqUUuEU8cAj\nIm1FZKGIbBKRjSJyp5XeWER+EpFt1u9GVrqIyCsikioi60Skr9O9xlv5t4nI+IqWKdZtTk1VzIUp\nLjHszT4BwPQlaTz93WbHuW/X7WPyt5v8Xr9p/1HAtt22XWJ8rO3eGniUUtVIVdR4ioF/GGNOBwYA\nt4rI6cADwHxjTCdgvnUMcAHQyfq5GXgTbIEKeBToD/QDHrUHq2C513iq4mv6x00HHeusfbJ8N1N+\n2eE49+qCVN5ZvJPdR45z9ESRr1sAEOM0M7VWXFmNZ+n2I34HMIAt4H60bJfHXkFKKRVKEQ88xpj9\nxpjV1uNcYDPQGhgDvGdlew+4xHo8Bnjf2PwOJIlIS+B84CdjTKYxJgv4CRhVkTLFui0jULdW1Sza\nfeBovt/z5/57Ie8s3uk4jnevquH6WhKswPNb6mGufft3Xl+43e/916bn8NBXG3jwa9tqC3uzT2gz\nnVIq5Kq0j0dEkoE+wDKghTHGPvHkAGDvqW8NOPe4p1tpvtK9Pc/NIrJSRFZ6Ox9r1Xj+em4H0p65\n0PGFXd3Zm9KcOdfe7K/L3uez43Ce3/vlWLWpoyeK2J9zgkHPLOD5H7c6zmfkFpBRycVMlVKqyr5h\nRaQe8AVwlzHmqPM5Y+tkCdmf2saYt4wxKcaYFICGteNdztu/rN2b3Ko7b6WNc3oN7l1VMX6Wrj6c\nV8DXf+wFbDUle4BZ7LTd91lPzuOsJ+dVvMBKKUUVBR4RiccWdD4yxnxpJR+0mtCwfh+y0vcCbZ0u\nb2Ol+Uov1xOXnOFybG+esv+uKVsLHM33vyzO+r05LsexfgLrhGkr+MoKPLVqSI1PKVUzVcWoNgGm\nApuNMS84nZoN2EemjQdmOaWPs0a3DQByrCa5H4CRItLIGlQw0korvwxux/YvZH9fzDXF0XzfAwPc\n+7J++TODWz9ejTGGtCNlk1ET4mI9lulxV1pqmPjlejbtO+o3n7PbP/mD//fcgoDzK6VOTlXxp+0g\n4H+BoSKyxvoZDTwDjBCRbcBw6xhgLrADSAXeBv4OYIzJBJ4AVlg/k6y0cvmq0dgDTyh28bxhUPtK\n36Mi/AUM96bE8e8uZ866/eQXlboMIsjNL+LyN5cAvt+rjLwCPlm+m/HTlvt8vrFv/07fJ35yHH+z\ndh97Mk8E8jKUUiexiA/fMsYsxnv3BMAwL/kNcKuPe70LvBtsGZwDS51aZR30oazxJMZXv+aqWB9F\nKixxDTzzNh/yntGJfW5QgZ+14JZsPwLA8cJi6vgYKXg0v4gThSW0aJBY7nMqpU4O1e/bMQLinIYh\nL7r/PMdjR42nAvEnzi1o+evIryr2praFWw+xaldZ5bDILfA4Kyk1HMjxHOZ9otAWcAoD2LLhpve9\nDiYEYMQLv9D/qfnl3sNu076jXpcVUkrVHFEZeJyHSzv/Je7eBxKMeLfqRHUcIffe0l1kHy9kwrQV\nXP7mUkd6UUmpz9UNNu47yoCn53Mkz3UYtX3Va187pzr3/fyWesRnmQ4eDW549uhXFnH/zHVBXaOU\nql6iNPB4b16zB4tAVsw52209N/tkzlm3DmLGzQMC2rzNrkuL+gHnrawnvt3skbb90DEvOV1lHS90\nObYHHm/xqqC4hNGvLKpYAZVSJ72oDDy14mKoa/XtxMaIY202ewwqLCl/D5sLzjjF454AvdomMaBD\nk6Bm/I8/OzngvJWVkedZw/jL1GXlXjdnXdkK10tSD3PCrW/naH4RWcdswclXLchuw94cujz8HQfL\nWalBKXVyisrAkxAXw5d/H8QXfxvoUuOxP8ovcv3i7NS8nsc96ie6TkK9/3zXrRSCqfF0aFY34LyV\ndaiCX/YvzvvT8XjsO8scfTyhTWEMAAAc+UlEQVR2I1/4lT7WCLbyAs+039IoKC7l1z8zHGlH8grY\nnuF/ZQVnupSPUjVX1SxKVsUS42Po2LyseSvBWnrmmI8O8xev7s2mfUe5/4uyvgXnUVgL/jGYDs1c\ng1MwX4xtG9cJvPCVtDcrNMOZJ7mtlm1fZ+5EYYljiR5n9toQgLEWpRCnPrVznl3IiaIS0p650Ovz\nGWNcmkCLSkqJjfFcMigU7M9VHfvplDoZRGmNx/ULq3n9BADHMjGnt2zAgA6NHecbJMYzqFNTx/E3\nt51Dm0a1HcfeRrA5z6d5fWxfj/Ou5QnNP8O0688qN09ugf/VDgKV7iOAvTTvT681nps/KBvZVuxl\nrpF70527O2asocODc8vuUU5g37z/aIW3tzjryfmc+++FFbpWKVW+KA08ri+7uVV7OZRr+6s9MT6W\nGTcPdJwXcR3x1qNNQ5ch196+3uzbT4P/+UEbHj/fYyh2RbVvGrkmO1+m/LrDa41ny/6yze2KS303\nxf2x27ah3fKdmby3JA2wrSPnvn23vy0eVu3K5IKXFzHVaSXvYBzOK/AZWJVSlReVTW3ua5EN7dqc\nAR0ac8+Izi7psTFCSalBxDNYOTcTeWtWc/6r3l9gqZcQR16IaiF1EsLT9BQsbzUe55qWvTZY6uV9\nu/SNJaQ9cyFXTbEN996wN4eWSbU98vmbP7Q32/YHxB97soMreBAO5xVQWmocf7QopQIXpTUe1y/o\neglxzLh5oEu/D5TVcmJEPFa0dg4lpV6adJybguJihR6tG/osT6hqPN72EVr32MiQ3Nsf9yatO2f8\n4Te/feDFsp3lr3D0+ap0r+Pb7YF9+c5MXvjpT5dz9vezvI3vKiNl8jz6BTHxVSlVJioDT6CrL9sr\nNTEiHh3Nzv063vosnGtBvdsmMevWQT6fx1tT3OV92wRURme1vezP08Bp9N3gzs1czrVokBD0c3jj\nXvvYcdj/vKA9mccB+GJ1ekD399adU1RSyifLd3PVlKW8Mn8br87fRvIDc3h67mbHZF5v/y4+n6PU\n8PR3m9mXHb4mtnHvLue79fvLz6jUSS7qmtpaNkwMeE22GD/bJDin1UvwfBvtf9X/9y99SapTC4C7\nhndi4ZZDrE13267AyxM4ryEXKF+jsJY9OIyEuBiWbD/CL05DmH2t1NCiQUJQKwqMm+p7oVBvtmf4\nD0x3f7rG5dh46UUrKrGtjm33H6vWM+XXHZzS0Nb8VRTEyML1e3OY8ssO1uwOT/NcSanh1z8z+PXP\nDJ8j95SKFlFX42laL/C/8t3Xbnvl2j68NraPLc0p36lNPIdD25va4mLK3uK7hnf2aM4D7wEjlAuW\ntmiQSFKdWh4LlyZ4qSGB79W52zet63XV7UCazIJh3xfIztsQ8J1+alXLrfJ4a2pbtuMIG/fleKTb\n/41D1d/mLt9p1N74d5f73b5CqZNd1AWeYMQ4NbUBXNyrFRf1bGVLtM41qhPv5Uocf3U3qut6Pj42\nsIASjjVGE936toIdBdenbRJXnRV8E6A357o1+/nz9Zp9Hmn+Fh5dYw0q8NbUdvVbv3PhK4vZdeQY\nmccKSX5gDp+v3OOYzOrcRBrIcOzHv9nIlf9dwl/eWeZYn+6dRTtcamPgOlz8lz8z+H592UoQhcWl\n5GogUlFEA48fMY79eXzzVTP514Wn89rYPpzZrrFL+gMXdOWWwad55HeeFwRl/VBXnFn+F33XU+qz\ndOLQcvN5E8zAhicv7eExMKOiWnsZqRYq+63VtJenZfpcQWLwv39m6uIdANw3cx3P/2hrqnPO7763\n0aGj+ezPca19TfstjRVpWSxOPczV1ki8yXM288ny3S758t3mKcXHlb3vF7+2mJ6P/xjw66usTfuO\nssNaJaKwuJT0rONs2JtT7ooTSoWKBh4/7H0g3roK7FNRfG1/ULtWbFntyElSnVo8cEFXj/QRp7dw\nOb7tvI5cf3Yyky85g7RnLuSOoR19lrNNozq0bFj+F7n7JE0huNn5tWvFetTYVj48PODrnbVqWLFh\nyHWD7Pt67vstnCgsYfP+o+w64to89/rC7R75nfufbvlwFVsOlK2y3e+p+Qx8egEr07w3LeYWFPPG\nz6llx/lFLNl+GPAMPM5NsFsO5Ja7MG1ufhH/+Gwt2W6LtQZj4ZZDjgVch/7nFwD+/cMWznl2IRe9\nupin5nouIKtUOGjg8aNstWrPbwX7EOqK9sW8dHVv7h5eNm/oruGdufGcsv6T+onxPHZxdxKtfpjR\nPVsC8A+3uUYArZMC+xLv3TaJhLgY/jq4gyMt2KHc7m9F03oJpLRrFNC1myeNYkgXWxNboo/+pfJ0\nOSW4lbzfXrSTbo98zwUvL2Lwv38O6toFWw4x6qVFHn1CV/x3qY8r4LnvtzoeP/TVBsa+vYyl249w\notC1NmGfRFsaYNPe+0t38cXqdKYu3smJwhKyjhV6BDN/1qfnMGH6Cp6cUxZcikpKmevU5BfOeU9K\nOdPA44f9O7nEyxeCvS+gohu+XdKnNXcO7+Q4blg7nn9ddLrP/F1PaUDaMxdy+7BOHufuGdnF8fiz\nvw7k1vM8m/IAmtRLYOvkCzjLqflvdI+WHvn8DTdvVr9scIa99hPoW1C7Viy3D+1IXIxwjtMSRMEY\n0qW56z3dAliyl4EelXXhK4uZv/lg0NfNtlZb2J6R52jWsztWYAsa+cVlwcPbig92ufm2QQ+xMUK3\nR76nzxM/OSbZluUpYuSLv/DyvG0u6YXFpdw3cy1g21/J7oEv1rPXafj4oaP5TP/N+2oPmccKfa4/\nWFpquPXj1SxJPexIM8aQeijXa/5gvLt4J7/v8L2fk6qZNPD48c64s7isb2ta1PesUdS2mnx6tvE9\nMTRc6rsN33ae3NqvfWPuO9+zKc+Xpy7twXNX9HRJmzbhLP4+5DT++5e+vD0uxeVcYnwsf/xrBAAD\nOrjuSWQv20U9y4LZF387m9m3DWLaBNs6cme2a0zqU6PLXZG7z6lJHmkPje7GLYNPo0ndWo60eoll\n78WkMd299p+FwiOzNlb42qfmbvYYIHG80BZIPvx9lyMtr6CYTfuOkvzAHLYdzOXHjQd4eu5mVu/O\n4r+/2JoFf9xYFgDXuQ3Lv37aCv48mOeykviRvAI6P/wdWw7YgsAxp1F77vOo9ufk89g3mxzzrOxy\njhfR94mf+M+PZbW5denZLLMCwqHcAuas2891TttrvLNoJ8Nf+JXbPl7N7Z+UTSjOPl7IQ1+t93gO\nb4wxTPp2E9e89Xu5eVXNEnXzeILRo01DXriqt9dzTesl8OXfz6bbKQ0iXCqY/4/BpGef4LI3lvjM\nM7hzM5+rJbS2BjL0bdeIWnExtHLrHzqtWT3uH+U7eDWqW4sPbuxHn1NtTWzOw69jYoTXxvblHyOP\ncaygmDN8lKG8QQovXd2bK/+7lEO5ZfOJbjrX1kR4xGml64a14x2Lu44bmMxPm1xrJtOuP4sJ01f4\nfS5nnVvU48+Dntsz7PUxsbRpvQQOe9njyNnxQs8msY+X7WbE6afw1NwtjrSsY4WODfRGvPirI925\nOXfT/rIai+3excTFxLA/5wSrdmU50j9dsZt//7CVxy8+wyW/vebkT7Fb898Gq6nxjZ+3U1hcyqCO\nTR3v6SW9W3FlSlsrr632c+/Mtcy2Au2362wTZhduOcSEQcl0blGfj5btJiO3gLfGpfCfH7ey4/Ax\nrwvpHg2grKpm0hpPJfQ9tZGj5hNJzRsk0vfURkwa050ZNw/wmue9G/px7/ldvJ7r1rIB8/8xmL9Z\ntYPymsrWPTaS9W5L7/y/Ts0cE2ftAQHKRue1b1rXZ9CxG92jbDO9r/5+Nh2tfY+GdGlGuyZ1Wf7Q\ncGbeMtDX5QCc5lZzch8iPqijZ5PeqY3r0KttEl+7rSZxTsem/N85HVzSurdy/cPi6ct6uEwYtvdZ\n2TWvH9g8sbQjxznv+Z9d0pyDjbNdfmoHpz/yA50f/s6j/+pfszZyOK+QPVmu1/oKoM7+9uEqFm3L\n4LMVe2g/cS7XveNUk1m80yWQf71mH5OtfqNasTFs2JfDl6v3eqwenldQzKsLUh3LGxWWlLLlwFFe\nXZDKnHXeV3Nw3p/prV/LBoL8sTuL1xZsY/nOTI89nHYePsZHy3bhzfKdmS57QFU3vR7/kWe/31J+\nxpOA1niqmZm3DCx3yRm7cQOTK/w8pzntH9TOqV/E2zyjBone5yrZjTi9BWnPXMg3a/fRv0Njv3md\nXXlmW+auP8CcO86he6uGzLtnMDknilz6bVKSbfdznvz6zGU9eMCaJ9PEmhBsD0AdnTbtW/PICGrF\nxfD4xd15dHZZU9mv95/ntTzjz052DBoZ1rU5U61tJsa9u9zxhTW4czO+XJ3OijRb7aJ7qwbMXGW7\nfs4d53B6ywbcOWONo3/HmzuGduSVBak+z7vz9cXsj31o9DPfBf9FtuVALv8bxGoUm61aWMM68S6r\nkHtjn/i7Lj2HDXvLam/Ld2bSq21DEuJieWX+No7kFfDe0rIA8tTcLdRNiOO6/u24esrvLss0/b9O\nTZkwKJmhXVsw6qVfKSgu5fK+bUiMj8UYg4iQeijX0Se2/rGRHhs52tmbIudtPkhJqeEyL0tXfbxs\nN0O6NKNVkFMCPvx9F+vTc3jWrWkbbKMec04U8ebP2/mnn9aGk4UGnmomJbmx48s2Uto0qsM3t53D\n/7y2mMZO/SfB+p9ensPH/Tmva3O2Th7l0uzmvhgrwCc3DaBt47L/5Nf0O5XYGOG+metoWi+BJQ8M\npa6XZYvsNZNAviA2Pn4+dRPiWLXLWvXA6S/292/oR/IDcwDbkkvOY01aJ9VmxOktWJeeTfdWthre\nK9f2Ib+ohB83eR+Q8LchHYmPjeFYYYmj76Y8Azs0YfKlZzDMGgZd3dSpFUtGboHLZon+ZB4r5N7P\n1zqOr5qylP7tG/P0ZT08Fn21e+irDbYpCm5/Gy3adphF2w5zx9COjgEaXf/1Pc9d0ZP7Z9rKM25g\nO0f+R2dvpFebJK9bzp/z7AIKiksdzaP3fLaWi3u14j9X9SI+NoacE0U8+NV6OjSty4J7h7hcezS/\niDrxscRZawXOXruPPm2THBs9Pvz1BgCuG3AqPdsk8c+Z6xjSpRlbD+Yy3doCBGyL7K5My2LGzQNC\nuklkbn4R69NzOLtjU/ZkHufrP/Zy29COLivtR4oGHuWiSd3QLBwaqEAmpA48zXMQw2V925BXUMzY\n/qf6vIe9b2Ro1+bcPbyzS6e7O3vg6tO2EXcM7cg1/U51Of8/vVpxrKAYEXGsHNexeT2GdGnOyO6n\neAyFdl6OqFn9BEc/1O8Th9lG91mjEwMNPDed257TmtXjwdFdaV4/kQ9/38WkMWfw+s9lTVWDOzdz\nWYvPl9ZJtcnILfC7tUSwTm1cxzGAoaKW7cx0zC/yZcBT831OdHWvRdqDDsCMFXscj79cvZcvV+/l\n3M7NSD2UR0lpKf3bN6FR3VpkHfdcQWL22n2OGuwr19qWzNpx2NaHWTchDmMMWw/mMuqlRYwf2I4H\nL+zG0RPF3PHJHyQ3qcPP97nWsC9+7Teu7Xcqn67cw6cr93g83yyrf+zDZbuYeEE3n+/F/TPXsvPw\nMdo1qctzl/fk8LEC9mQep0ndBJKb1mXVriwuf3MJEy/oyrBuLZj07SZ+/TOD1f8awS0frmLjvqPk\nFRZzaZ/WNK5TixVpWRgMF5zRkhghrAFJKrpLY02VkpJiVq70vdxKtMovKuGqKUt5/OLujkEDNdW6\n9GwWbDnEXcNd5zx9tmIPTerVYli3ssm6m/YdJSE+xqXpsTyXvvEbf+zOZuYtA33WTjNyCzjryXlc\n0rsVL13Th5Ev/sKOjGOkPjXaJZ+9JjWmdyvHF079hDimjDuTAe2bsDwtk0XbMrh7eGfHX9Luxr79\nO0u2Hwl4IEW/9o357K8DHc9dEa2TavPu9Wdx/ku2fqnzu7fgB6cRd60aJrLPWkHivvO70KdtEmOt\nvqLrz052+Qu/uhjUsQm/pQY3dPvSPq091hZsUreWywCYlQ8Pp9+T87xORPenTaPa/HLfeYx48RdG\ndT+F+87vQmFJKS/89CdTfnEdnn9el2Ys3Fr2R8fvE4cx4Gnv23Y4N1V707lFPbYdyqND07pMHX8W\nxaWldGrRYJUxJsXnRUHSwKNUkL7fcIBbPlzFmkdGOFYeL09BcQklpYY6bnsmbdibw+7M44zu0ZKC\n4hI++n034wa28xlkvMk+Xkh61gnOaN2QJamHefaHray1JoO2aVSbZy/v6RggMKr7Kdw/qgsdmtWz\njTSbvoK3/vdMateK5Z7P1jpqZqc1q+tYxSGpTjzf3HYOtWvFsnT7EbZn5HHrebbmwhunr2D+lkNc\n2LMlc9btJ6VdIy4/sw3X9juViV+u5396tuTsjk0xxtB+4lyu7Xcqdw3vRH9rL6Oup9THGNh6MJcb\nz2nvsmvswxd2cwxciFb/O6AdH1hD7hvXrYUxxmutLNx2PXuRBp7K0MCjTnbHCoq557M13HxuB7qe\n0oC6CXGO2k15WzLszT7BKQ0SOV5YTI/HfuTyvm2YfMkZPkdvZh0r5JMVuxnb71S+WL2XcQPbOfZD\ncpdfVEKt2BgKikvp9sj3AKx9dCRTF+3glQWpzLh5AA0S4x1Dyu8d2Zm+pzbix00HOXjUVnv6bsMB\nerVN4rr+pzKwQxMe/noDT13Wg5kr0/02pdo1rZfA4M7NPOYw2WspNwxqT/bxQm44pz37sk9wOK+Q\nB7/yXTsIxqjup/D9xgPlZ6yg3m2THAvklqdWXEy5a/O1TqrtGAWpgaeSNPCoaBRo4IkEe+2nTq1Y\nNk0ahTGGlbuySGnXCBFhSephxr6zjI9v6s/Zp7kOh8/ILXBZPcNZetZxVu/O5mBOPi2TEomLiWHn\n4WNcf3YyS3cc5kheIVemtGXJ9sOMfbtsiPg9IzqTktyIJ77dzKvX9nEZGQm2Fcin/ZYGwPIHh/HT\n5oM89NUGx/llDw5z1OB8eenq3lzSpzXvLNoRUC3u0j6tWbr9CAesgFue01s24LkrevLy/G20aVSb\nrqfUZ0VaFm0a1aZf+8bExcRwVnIj/v3DVt74eTvLHhxG8/oJtJ84F4Cp41OoUyuOa98um6y78N4h\njiH/GngqSQOPikZr92RTNyHW635QVeG79fs5vVUD2jXxvoJFbn6RzyHPlVVUUspjszcyqGNT2jet\nS7eW5U8Cf+ir9eScKOI1a6KreyD/YeMBVu/OYkCHJgzp3MzxhW6fkOzcH/iPz9Yyb/NB1j46kvyi\nEmat2cvoHi0di/jWS4ijTq048gqKycgtYOuBo9wxYw2X9WnN05f1YNnOTI4XFnPLB6sdA0SWThwa\n0ELBxSWlZOQVOPIu2HKQZ77bwuzbziExPpYNe3M4rVk9DuXm065JXb7fsJ9bPlytgaeyNPAopSpr\nf84JCotLfQbOPZnHyTlRRPdWDdh2KI/OLUIf8AuLS/nPj1v525DTAu5rrCgR0cBTGRp4lFIqOKEO\nPLpkjlJKqYiq8YFHREaJyFYRSRWRB6q6PEoppfyr0YFHRGKB14ELgNOBa0XE96Y2SimlqlyNDjxA\nPyDVGLPDGFMIzADGVHGZlFJK+VHTA09rwHmxo3QrzYWI3CwiK0VkZUZG9V0WXSmlokFNDzwBMca8\nZYxJMcakNGvWrPwLlFJKhU1NDzx7gbZOx22sNKWUUtVUTQ88K4BOItJeRGoB1wCzq7hMSiml/Kjx\nE0hFZDTwEhALvGuMebKc/LnA1kiUrQZoChyu6kJUE/pelNH3ooy+FzbtgIeMMW+F4mY1PvAES0RW\nhnIGbk2m70UZfS/K6HtRRt+LMqF8L2p6U5tSSqkaRgOPUkqpiIrGwBOSNsqThL4XZfS9KKPvRRl9\nL8qE7L2Iuj4epZRSVSsaazxKKaWqUNQEnmhbxVpE2orIQhHZJCIbReROK72xiPwkItus342sdBGR\nV6z3Z52I9K3aVxB6IhIrIn+IyLfWcXsRWWa95k+tuWCISIJ1nGqdT67KcoeaiCSJyEwR2SIim0Vk\nYLR+LkTkbuv/xwYR+UREEqPlcyEi74rIIRHZ4JQW9OdARMZb+beJyPhAnjsqAk+UrmJdDPzDGHM6\nMAC41XrNDwDzjTGdgPnWMdjem07Wz83Am5EvctjdCThveP8s8KIxpiOQBdxopd8IZFnpL1r5TiYv\nA98bY7oCvbC9J1H3uRCR1sAdQIox5gxscwGvIXo+F9OBUW5pQX0ORKQx8CjQH9uizY/ag5VfxpiT\n/gcYCPzgdDwRmFjV5YrwezALGIFt8mxLK60lsNV6PAW41im/I9/J8INtOaX5wFDgW0CwTQyMc/+M\nAD8AA63HcVY+qerXEKL3oSGw0/31ROPngrJFhhtb/87fAudH0+cCSAY2VPRzAFwLTHFKd8nn6ycq\najwEuIr1ycpqEugDLANaGGP2W6cOAC2sxyf7e/QScD9Qah03AbKNMcXWsfPrdbwX1vkcK//JoD2Q\nAUyzmh3fEZG6ROHnwhizF3ge2A3sx/bvvIro/FzYBfs5qNDnI1oCT9QSkXrAF8BdxpijzueM7U+U\nk35Yo4hcBBwyxqyq6rJUA3FAX+BNY0wf4BhlzSlAVH0uGmHbv6s90Aqoi2fTU9QK5+cgWgJPVK5i\nLSLx2ILOR8aYL63kgyLS0jrfEjhkpZ/M79Eg4GIRScO2WeBQbP0cSSISZ+Vxfr2O98I63xA4EskC\nh1E6kG6MWWYdz8QWiKLxczEc2GmMyTDGFAFfYvusROPnwi7Yz0GFPh/REniibhVrERFgKrDZGPOC\n06nZgH3kyXhsfT/29HHW6JUBQI5TlbtGM8ZMNMa0McYkY/u3X2CMuQ5YCFxhZXN/L+zv0RVW/pOi\nBmCMOQDsEZEuVtIwYBNR+LnA1sQ2QETqWP9f7O9F1H0unAT7OfgBGCkijawa5Egrzb+q7tyKYCfa\naOBPYDu2VVarvExhfr3nYKsmrwPWWD+jsbVJzwe2AfOAxlZ+wTbybzuwHttInyp/HWF4X4YA31qP\nOwDLgVTgcyDBSk+0jlOt8x2qutwhfg96Ayutz8bXQKNo/VwAjwNbgA3AB0BCtHwugE+w9W0VYasJ\n31iRzwFwg/WepAITAnluXblAKaVUREVLU5tSSqlqQgOPUkqpiNLAo5RSKqI08CillIooDTxKKaUi\nSgOPUhEkIpNEZHgI7pMXivIoVRV0OLVSNZCI5Blj6lV1OZSqCK3xKFVJIvIXEVkuImtEZIq170+e\niLxo7fUyX0SaWXmni8gV1uNnxLZf0joRed5KSxaRBVbafBE51UpvLyJLRWS9iEx2e/77RGSFdc3j\nkX79SgVLA49SlSAi3YCrgUHGmN5ACXAdtgUnVxpjugO/YNuzxPm6JsClQHdjTE/AHkxeBd6z0j4C\nXrHSX8a2sGcPbLPN7fcZiW2PlH7YViQ4U0TODcdrVSpUNPAoVTnDgDOBFSKyxjrugG37hU+tPB9i\nW8LIWQ6QD0wVkcuA41b6QOBj6/EHTtcNwrbEiT3dbqT18wewGuiKLRApVW3FlZ9FKeWHYKuhTHRJ\nFPmXWz6XzlRjTLGI9MMWqK4AbsO2arY/3jpkBXjaGDMlqFIrVYW0xqNU5cwHrhCR5uDYs74dtv9b\n9hWOxwKLnS+y9klqaIyZC9yNbQtqgCXYVtAGW5PdIuvxb27pdj8AN1j3Q0Ra28uiVHWlNR6lKsEY\ns0lEHgZ+FJEYbCv93optg7V+1rlD2PqBnNUHZolIIrZayz1W+u3Ydge9D9tOoROs9DuBj0Xkn5Qt\nVY8x5kern2mpbWV/8oC/ULaPilLVjg6nVioMdLizUr5pU5tSSqmI0hqPUkqpiNIaj1JKqYjSwKOU\nUiqiNPAopZSKKA08SimlIkoDj1JKqYjSwKOUUiqi/j9UTTbemX89JgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10950bcc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "step_result = pd.DataFrame(step_record, columns=['policy_gradient'])\n",
    "step_result.plot()\n",
    "plt.xlabel('episode')\n",
    "plt.ylabel('steps')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}