car_torch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/alexpc/anaconda3/envs/rlenv/lib/python3.6/site-packages/gym/envs/registration.py:14: PkgResourcesDeprecationWarning: Parameters to load are deprecated.  Call .resolve and .require separately.\n",
      "  result = entry_point.load(False)\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import math\n",
    "import random\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import namedtuple\n",
    "from itertools import count\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torchvision.transforms as T\n",
    "\n",
    "# XVFB will be launched if you run on a server\n",
    "import os\n",
    "if type(os.environ.get(\"DISPLAY\")) is not str or len(os.environ.get(\"DISPLAY\")) == 0:\n",
    "    !bash ../xvfb start\n",
    "    %env DISPLAY = : 1\n",
    "\n",
    "from gym import wrappers\n",
    "\n",
    "env = gym.make('CarRacing-v0').unwrapped\n",
    "\n",
    "# set up matplotlib\n",
    "is_ipython = 'inline' in matplotlib.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()\n",
    "\n",
    "# if gpu is to be used\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Transition = namedtuple('Transition',\n",
    "                        ('state', 'action', 'next_state', 'reward'))\n",
    "\n",
    "\n",
    "class ReplayMemory(object):\n",
    "\n",
    "    def __init__(self, capacity):\n",
    "        self.capacity = capacity\n",
    "        self.memory = []\n",
    "        self.position = 0\n",
    "\n",
    "    def push(self, *args):\n",
    "        \"\"\"Saves a transition.\"\"\"\n",
    "        if len(self.memory) < self.capacity:\n",
    "            self.memory.append(None)\n",
    "        self.memory[self.position] = Transition(*args)\n",
    "        self.position = (self.position + 1) % self.capacity\n",
    "\n",
    "    def sample(self, batch_size):\n",
    "        return random.sample(self.memory, batch_size)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.memory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQN(nn.Module):\n",
    "\n",
    "    def __init__(self, h, w):\n",
    "        super(DQN, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 16, kernel_size=5, stride=2)\n",
    "        self.bn1 = nn.BatchNorm2d(16)\n",
    "        self.conv2 = nn.Conv2d(16, 32, kernel_size=5, stride=2)\n",
    "        self.bn2 = nn.BatchNorm2d(32)\n",
    "        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2)\n",
    "        self.bn3 = nn.BatchNorm2d(32)\n",
    "\n",
    "        # Number of Linear input connections depends on output of conv2d layers\n",
    "        # and therefore the input image size, so compute it.\n",
    "        def conv2d_size_out(size, kernel_size = 5, stride = 2):\n",
    "            return (size - (kernel_size - 1) - 1) // stride  + 1\n",
    "        convw = conv2d_size_out(conv2d_size_out(conv2d_size_out(w)))\n",
    "        convh = conv2d_size_out(conv2d_size_out(conv2d_size_out(h)))\n",
    "        linear_input_size = convw * convh * 32\n",
    "        self.head = nn.Linear(linear_input_size, 6) # 448 or 512\n",
    "\n",
    "    # Called with either one element to determine next action, or a batch\n",
    "    # during optimization. Returns tensor([[left0exp,right0exp]...]).\n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.bn1(self.conv1(x)))\n",
    "        x = F.relu(self.bn2(self.conv2(x)))\n",
    "        x = F.relu(self.bn3(self.conv3(x)))\n",
    "        return self.head(x.view(x.size(0), -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DQN(\n",
      "  (conv1): Conv2d(3, 16, kernel_size=(5, 5), stride=(2, 2))\n",
      "  (bn1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (conv2): Conv2d(16, 32, kernel_size=(5, 5), stride=(2, 2))\n",
      "  (bn2): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (conv3): Conv2d(32, 32, kernel_size=(5, 5), stride=(2, 2))\n",
      "  (bn3): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "  (head): Linear(in_features=96, out_features=6, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "net = DQN(35, 50)\n",
    "print(net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Track generation: 1145..1435 -> 290-tiles track\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEICAYAAACZJtWMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGytJREFUeJzt3X2QXNV55/HvT/Oi0cuAXgBZSCJyAAdDyha1WoHLdhaT2NE6yUJqU8RsQiDljexdOzG1JA5m/4B47VpTsSGuypY3whDL8auCjcHsJrGCsQmVBCNAvBk7vBgWySPJvCiSQG8z8+wf98q05p473T09PaMz8/tUdan79Ln3Pud26+k795xzryICMzM7vs2Z7gDMzKw5J2szsww4WZuZZcDJ2swsA07WZmYZcLI2M8uAk7V1naQrJN073XEcTyStlhSSeqc7FsuDk3XmJD0r6YCk/Q2PP5/uuKabpAskbe/i+q+T9IVurd9sLP+qzwy/FhF/P91B5EZSb0QMT3cc3TCT2zZb+ch6BpP0GUlfa3h9vaS7VFgs6U5JP5H0cvl8ZUPd70j6mKR/LI/WvylpqaQvStor6X5Jqxvqh6Q/kPSMpBck/amk5PdL0lmStkh6SdIPJV0yThtOlHSzpCFJO8qYepq0bwHwN8CpDX9tnFoeDd8q6QuS9gJXSFon6Z8k7Sm38eeS+hvWeU5DrLskXSNpPXAN8Jvluh9uIdYeSZ8s980zwK80+ez+uFzHvnIf/WLDeq6R9HT53gOSVjV8Bh+Q9CTwZLN9LWluGdP/K9v2vyXNK9+7QNJ2SVdJ2l226XfHi9m6LCL8yPgBPAv8Us1784F/Aa4A3g68AKws31sK/MeyziDw18A3Gpb9DvAUcDpwIvD9cl2/RPEX2eeBv2yoH8DdwBLgtLLufy7fuwK4t3y+AHge+N1yPeeWcZ1d04bbgL8olzsF+B7wvhbadwGwfcy6rgOOABdTHKjMA/4NcH4Zy2rgCeDKsv4gMARcBQyUr89rWNcX2oj1/cAPgFXlPrq73Ge9iTb/XLmPTi1frwZOL5//EfBoWUfAm4GlDZ/BlnL985rta+BG4I6y/iDwTeB/Nuy/YeCjQB/wbuBVYPF0f+dn62PaA/Cjww+wSNb7gT0Nj99reP884CXgOeDScdazBni54fV3gP/e8PpTwN80vP41YFvD6wDWN7z+r8Bd5fMreC1Z/ybwD2O2/RfAtYmYlgGHgHkNZZcCdzdrH/XJ+p4m+/NK4LaGbT1UU+86GpJ1s1iBbwPvb3jvXdQn6zOA3RQ/jH1j3vshcFFNTAFc2PC6dl9TJPpXKH8EyvfeAvyoYf8daIyvjOn86f7Oz9aHz1nPDBdHzTnriLiv/LP7FGDz0XJJ8ymOrNYDi8viQUk9ETFSvt7VsKoDidcLx2zu+YbnzwGnJkL6GeA8SXsaynqBv6qp2wcMSTpaNqdxO3XtG0djjEh6A3ADsJbiSL0XeKB8exXwdAvrbCXWU6nun6SIeErSlRQ/COdI+jvgv0XEj1uIqXEb4+3rkyna+0BDvAJ6Guq+GMee936V6mduU8TnrGc4SR8A5gI/Bj7c8NZVFH9KnxcRJwC/cHSRDja3quH5aeU2x3oe+G5ELGp4LIyI/1JT9xBwUkPdEyLinKMVxmlf3eUkx5Z/huL0xJnlfriG1/bB88DPtrieZrEOUd0/tSLiSxHxNoqEG8D1Dds5fbxFx8RUt69foPjBPafhvRMjwsn4OOVkPYOVR40fA34buAz4sKQ15duDFP9Z90haQvGncaf+qOy4XAV8CPhqos6dwBskXSapr3z8W0lvHFsxIoaAbwGfknSCpDmSTpf071po3y5gqaQTm8Q8COwF9ks6C2j80bgTWC7pyrIzblDSeQ3rX320E7VZrBRH/X8gaaWkxcDVdQFJ+jlJF0qaCxyk+JxGy7c/C/wPSWeq8CZJS2tWVbuvI2IUuAm4UdIp5XZXSPrlJvvLpomT9czwTR07zvo2FZMtvgBcHxEPR8STFEeNf1UmgT+j6IR6Afhn4G8nIY7bKU4hbAP+D3Dz2AoRsY/ifO17KI6Gd1IcNc6tWefvAP0UHZwvA7dSJNBx2xcRPwC+DDxTjvRInZIB+EPgPwH7KJLXT39gyljfSXF+fifFCIt3lG//dfnvi5IeHC/W8r2bgL8DHgYeBL5eEw/lvvgExWezk+IUz0fK926gSPzfoviRuZnic6xoYV//MUUn8j+Xo2P+nuKvLTsOKcI3H7DOSQqKUwlPTXcsZjORj6zNzDLgZG1mlgGfBjEzy4CPrM3MMtDRpJjyGgmfphhI/9mI+EST+j6MNzMbIyKazm+Y8GmQ8gI1/0IxtGk7cD/FdN/vj7OMk7WZ2RitJOtOToOsA56KiGci4jDwFeCiDtZnZmY1OknWKzj2OgTby7JjSNogaaukrR1sy8xsVuv6hZwiYiOwEXwaxMxsojo5st7BsRemWVmWmZnZJOskWd8PnCnp9SrurPEeiguZm5nZJJvwaZCIGJb0QYqL0/QAt0TE45MWmZmZ/dSUzmD0OWszs6puD90zM7Mp4mRtZpYBJ2szsww4WZuZZcDJ2swsA07WZmYZcLI2M8uAk7WZWQacrM3MMuBkbWaWASdrM7MMOFmbmWXAydrMLANO1mZmGXCyNjPLgJO1mVkGnKzNzDLgZG1mloEJ34MRQNKzwD5gBBiOiLWTEZSZmR2ro2RdekdEvDAJ6zEzsxo+DWJmloFOk3UA35L0gKQNqQqSNkjaKmlrh9syM5u1FBETX1haERE7JJ0CbAF+PyLuGaf+xDdmZjZDRYSa1enoyDoidpT/7gZuA9Z1sj4zM0ubcLKWtEDS4NHnwLuAxyYrMDMze00no0GWAbdJOrqeL0XE305KVGZmdoyOzlm3vTGfszYzq+j6OWszM5saTtZmZhlwsjYzy4CTtZlZBpyszcwy4GRtZpYBJ2szsww4WZuZZcDJ2swsA07WZmYZcLI2M8uAk7WZWQacrM3MMuBkbWaWASdrM7MMOFmbmWXAydrMLANO1mZmGWiarCXdImm3pMcaypZI2iLpyfLfxd0N08xsdmvlyPpzwPoxZVcDd0XEmcBd5WszM+uSpsk6Iu4BXhpTfBGwqXy+Cbh4kuMyM7MGvRNcbllEDJXPdwLL6ipK2gBsmOB2zMyMiSfrn4qIkBTjvL8R2AgwXj0zM6s30dEguyQtByj/3T15IZmZ2VgTTdZ3AJeXzy8Hbp+ccMzMLEUR45+ZkPRl4ALgJGAXcC3wDWAzcBrwHHBJRIzthEyty6dBzMzGiAg1q9M0WU8mJ2szs6pWkrVnMJqZZcDJ2swsA07WZmYZcLI2M8uAk7WZWQacrM3MMuBkbWaWASdrM7MMOFmbmWXAydrMLANO1mZmGXCyNjPLgJO1mVkGnKzNzDLgZG1mlgEnazOzDDhZm5llwMnazCwDTZO1pFsk7Zb0WEPZdZJ2SNpWPt7d3TDNzGa3Vm6Y+wvAfuDzEfHzZdl1wP6I+GRbG/M9GG0C5vRUjyl6+nuSdXsHeitlfQv6KmX9C/uTy6fKBxYNJOvu3b63UvbCD15I1jUbz6TcgzEi7gGa3rnczMy6p5Nz1h+U9Eh5mmTxpEVkZmYVE03WnwFOB9YAQ8Cn6ipK2iBpq6StE9yWmdmsN6FkHRG7ImIkIkaBm4B149TdGBFrI2LtRIM0M5vtqr0xLZC0PCKGype/Djw2Xn1rbk5v9Xezd27640l2os2vdqIB9A9WO8zmDs5N1010rs1dVK07b/G85PJzT6zWTS0PMHBitdNu7gnpun2DiQ7CBekOwt551X3Tu6Ba1tOb7qBMlfedmN63j9z0SKXsux/9brKuWaeaJmtJXwYuAE6StB24FrhA0hoggGeB93UxRjOzWa9pso6ISxPFN3chFjMzq+EZjGZmGXCyNjPLgJO1mVkGJjQaZLoMvm4wWT7/5PmVstQoiLry1MgESE8zHljS+iiG5OiImrqp0Q2pURBQM+KhP/1R9syrjm5ITd8GkKozXtWXmAVbd9GAVHnN4UCMJCqP1tRNXaVgpKZu6vIJqSaktg+MjlaDGD44nKw7/6Tq986sW3xkbWaWASdrM7MMOFmbmWXAydrMLANZdTCe/5Hzk+Vn/PIZlbK663SrJ9GJVncp2fSM5Kq6zq5Ux1hN51yyw6smrE4750aH6yonylJtq1k8qe5wILXeuiv6ptZRs89bPvyo6yRNxVCzrVTHtuakGxGjvpS7dcZH1mZmGXCyNjPLgJO1mVkGnKzNzDLgZG1mloGsRoMceOlAsjzVAz9yqKYL/0iirK6jPjUaJFW3bvnUT2HdSIrUOupGo7QzkqLVbdWto532trp8nbo2dDqQosMYUlPQIX3pgdSlAACOvJL64pm1zkfWZmYZcLI2M8uAk7WZWQacrM3MMtDKDXNXAZ8HllF01WyMiE9LWgJ8FVhNcdPcSyLi5e6FCvt+vK/1yt3qrEr1NXWrcy83x2sbOu3krOkUTl2vfOCE9LXR3cFonWrlyHoYuCoizgbOBz4g6WzgauCuiDgTuKt8bWZmXdA0WUfEUEQ8WD7fBzwBrAAuAjaV1TYBF3crSDOz2a6tcdaSVgPnAvcByyJiqHxrJ8VpktQyG4ANEw/RzMxa7mCUtBD4GnBlROxtfC+K65EmzwxGxMaIWBsRazuK1MxsFmvpyFpSH0Wi/mJEfL0s3iVpeUQMSVoO7O5WkEe9svOV9BvtzLJLlXerM7Idx8Psv9RP91TOHpyMdbS6H9voDK27FnXv/Op/n3lL5iXr7htqo3PcLKHpkbWKW17fDDwRETc0vHUHcHn5/HLg9skPz8zMoLUj67cClwGPStpWll0DfALYLOm9wHPAJd0J0czMmibriLiX+j8af3FywzEzsxTPYDQzy4CTtZlZBrK6nvX+of3J8pHhxAWe25n63Ok06brlU9OU2xnZkNu1oNtZvtMp+t1avo2RRT0D1QuOp+54bjYZfGRtZpYBJ2szsww4WZuZZcDJ2swsA1l1MB54MX3D3OFDw5Wy3rnppkWqt6idDsJO1W2rnQ6zdn5i27n+dquda5PRhm5d+7rVDtFJ2L56qitZuGxh5ys2S/CRtZlZBpyszcwy4GRtZpYBJ2szsww4WZuZZSCr0SAHXz6YLD+8/3ClrG40SMdSowjaGV3RziiEarOKVeyrriSW1M2JTpR1OoX8eL1Zw2Qs3+EIoIXLPRrEusNH1mZmGXCyNjPLgJO1mVkGnKzNzDLQtBdO0irg88Ayii6kjRHxaUnXAb8H/KSsek1E/N9uBQpwZP+RZPnBPdWOxwVLFyTrhtroBZvK6ygnOgJ7fpj+Le3512r5kQWJa3oDLE2EtaJmH/TXxDZW3U98p1PT27n2dacdhJNwrfCI6koWvC79vTPrVCtDJoaBqyLiQUmDwAOStpTv3RgRn+xeeGZmBq3dMHcIGCqf75P0BLCi24GZmdlr2jpnLWk1cC5wX1n0QUmPSLpF0uKaZTZI2ippa0eRmpnNYi0na0kLga8BV0bEXuAzwOnAGooj70+llouIjRGxNiLWTkK8ZmazUkvJWlIfRaL+YkR8HSAidkXESESMAjcB67oXppnZ7NbKaBABNwNPRMQNDeXLy/PZAL8OPNadEF8zOpzu6k/dlEBvqOnCTw2amMo7gLcxmmS0ZsWjp1cbEaek6+qp6gZ1f00QqRNZg9Wi6K9pcF8qgHRV5taUp6SmzNdJhZZavi6u6n0s2hr9svAUTze37mhlNMhbgcuARyVtK8uuAS6VtIbiK/ss8L6uRGhmZi2NBrmX9HFIV8dUm5nZazyD0cwsA07WZmYZyOp61nVe2f1KpUx1PUjtTHNudfl2tDNNum7m8t5E2cp01XhjonE1M9N/euGABqlrZ2tPlzpv26lbd5iRuAZ4T1TjHT6rZmMLWg8iRqp1B5YOJOv29Fd7OUcO130QZlU+sjYzy4CTtZlZBpyszcwy4GRtZpYBJ2szswzMiNEg+4f2Vwvrfoa6MRqknTti120rUR4npivPfbw6suDQG2tGFqRWUbdvTkksvjyxgroRLakQ2jkcaGdwRDv78alqENqVXkGc0UYIo4nRIIvTo0H6F1bv7HDgpeplEszq+MjazCwDTtZmZhlwsjYzy4CTtZlZBmZGB+PORAfjZGh1avlk3JU71bmWuDM5wJGl1R5N3Zdecfx8Irj5NTG001Ha6vLt3N28Tjudwolv9HBP9SLVSkxBb3tbifb2LUxd1BvmLZpXKXMHo7XDR9ZmZhlwsjYzy4CTtZlZBpyszcwy0MoNcweAeyhucdoL3BoR10p6PfAVim6wB4DLIiJxNeHu27+j2sFYd3PdZGdRXQdSpz9lnc6WrDH6pupK5uxId5j13l+d7XjkLTVTBVvtbm6nQ7WmD23Oj6qV48SazaVuBpzux2v55roRNTcYTjWipr2pGYy9A+mdOO/kagcjz9SGZ1bRSjo6BFwYEW8G1gDrJZ0PXA/cGBFnAC8D7+1emGZms1vTZB2Fo4eufeUjgAuBW8vyTcDFXYnQzMxa+0NfUo+kbcBuYAvwNLAnIo4OYN0OrKhZdoOkrZK2TkbAZmazUUvJOiJGImINxZ3+1gFntbqBiNgYEWsjYu0EYzQzm/Xa6kKLiD3A3cBbgEWSjvamrAR2THJsZmZWamU0yMnAkYjYI2ke8E6KzsW7gd+gGBFyOXB7NwMdz6svvFopGzmSHvGgng5vTz6V176uC7U6e5rRlTXXZ96TmJq+vWZq+pmJdaTirbvudOLbNGdbelsaSIwGeaVmhMZDiXXUjAaJ1yWqHqwOERntT38Qo+psuM6cvvTxz4KT625Vb9aaVgZrLQc2SeqhOBLfHBF3Svo+8BVJHwMeAm7uYpxmZrNa02QdEY8A5ybKn6E4f21mZl3mGYxmZhlwsjYzy8CMuJ71wZcOVsqOHDiSrDt3cG6lLNrpNZyE6eJJHfZ7pjodAWJVYkr0tvTHPqJqz2Hypr0118NO9s39a7phI+ckOvjqrrOdatuemhheTHRcHq4GNnpa3RzymhhaVXP4s3D5wg5XbLOdj6zNzDLgZG1mlgEnazOzDDhZm5llwMnazCwDM2I0yKG9hyplh/en74OQusv0aNTMAU/9lLVzB+92fgrbuRN6asRC3bYWVYtG1tZMxd9T3aB2JoKo7m4A+g5WgxipmwY/r/WL/CfbtqSm7inVotHEjQY00sbdzes+h8SNDjQnXXnw1MGalZi1xkfWZmYZcLI2M8uAk7WZWQacrM3MMjAjOhiHD1TnI+95Lj0fuX9+f6Ws9k7oqb6iVN9cO1PF665n3a0OxlaXh/R071RZzdT2ZNvqvmE/SZTVtSG13nbusN7OHe1Td0dv4zPreTV9e/W+eXW3YzdrjY+szcwy4GRtZpYBJ2szsww4WZuZZaBpspY0IOl7kh6W9LikPynLPyfpR5K2lY813Q/XzGx2UiSm4h5TQRKwICL2S+oD7gU+BLwfuDMibm15Y+rw1tFt6FuQ7n1Xb7ULf46Ogz8wOr35QDvLdzqSoh1T9omX2hlV06p29lfNtuJIdSWpyyTY7BQRTb+lrdwwN4D95cu+8jHV/wXNzGa1lg4pJfVI2gbsBrZExH3lWx+X9IikGyVV75dVLLtB0lZJWycpZjOzWafpaZBjKkuLgNuA3wdeBHYC/cBG4OmI+GiT5X0apI5Pg0wOnwaxDLVyGqStLBURe4C7gfURMRSFQ8BfAusmFqaZmTXT9Jy1pJOBIxGxR9I84J3A9ZKWR8RQ2QF5MfDYRAJYtChxwWVgeLg6p7m3Nx1u6q+DI4fTdzfvGalOB963f994IVruqpcwT08rh9d6Z8yOM61cG2Q5sElSD8WR+OaIuFPSt8tELmAbxegQMzPrglZGgzwCnJsov7ArEZmZWcVx0LNmZmbNOFmbmWXAydrMLANTevMBScyde+zcmbe//e3JugsWLKiUHT6cvmN5ysDAQLK8v79684HNmzcn67766qstb8+OY6mRH3WjQcyOUz6yNjPLgJO1mVkGnKzNzDLgZG1mloEp7WCMCA4dOvbiNY8//niybmq6eV9f+uJMo6PV20+PjKRuQw7z51dv133kSHpqus0Qr0x3AGad85G1mVkGnKzNzDLgZG1mlgEnazOzDDhZm5lloK3benW8sSm8rZeZWS4m/bZeZmY2PZyszcwy4GRtZpYBJ2szswxM6XRz4AXgufL5SeXrmcbtys9MbZvblYefaaXSlI4GOWbD0taIWDstG+8itys/M7VtbtfM4tMgZmYZcLI2M8vAdCbrjdO47W5yu/IzU9vmds0g03bO2szMWufTIGZmGXCyNjPLwJQna0nrJf1Q0lOSrp7q7U8mSbdI2i3psYayJZK2SHqy/HfxdMY4EZJWSbpb0vclPS7pQ2V51m2TNCDpe5IeLtv1J2X56yXdV34nvyqpf7pjnQhJPZIeknRn+XqmtOtZSY9K2iZpa1mW9XdxIqY0WUvqAf4X8O+Bs4FLJZ09lTFMss8B68eUXQ3cFRFnAneVr3MzDFwVEWcD5wMfKD+n3Nt2CLgwIt4MrAHWSzofuB64MSLOAF4G3juNMXbiQ8ATDa9nSrsA3hERaxrGV+f+XWzbVB9ZrwOeiohnIuIw8BXgoimOYdJExD3AS2OKLwI2lc83ARdPaVCTICKGIuLB8vk+igSwgszbFoX95cu+8hHAhcCtZXl27QKQtBL4FeCz5WsxA9o1jqy/ixMx1cl6BfB8w+vtZdlMsiwihsrnO4Fl0xlMpyStBs4F7mMGtK08VbAN2A1sAZ4G9kTEcFkl1+/knwEfBkbL10uZGe2C4gf1W5IekLShLMv+u9iuqb42yKwSEZHzDRckLQS+BlwZEXuLg7VCrm2LiBFgjaRFwG3AWdMcUsck/SqwOyIekHTBdMfTBW+LiB2STgG2SPpB45u5fhfbNdVH1juAVQ2vV5ZlM8kuScsByn93T3M8EyKpjyJRfzEivl4Wz4i2AUTEHuBu4C3AIklHD1xy/E6+FfgPkp6lOLV4IfBp8m8XABGxo/x3N8UP7Dpm0HexVVOdrO8Hzix7qfuB9wB3THEM3XYHcHn5/HLg9mmMZULK8503A09ExA0Nb2XdNkknl0fUSJoHvJPifPzdwG+U1bJrV0R8JCJWRsRqiv9T346I3yLzdgFIWiBp8Ohz4F3AY2T+XZyIKZ/BKOndFOfXeoBbIuLjUxrAJJL0ZeACiks27gKuBb4BbAZOo7gc7CURMbYT8rgm6W3APwCP8to50Gsozltn2zZJb6LojOqhOFDZHBEflfSzFEekS4CHgN+OiEPTF+nEladB/jAifnUmtKtsw23ly17gSxHxcUlLyfi7OBGebm5mlgHPYDQzy4CTtZlZBpyszcwy4GRtZpYBJ2szsww4WZuZZcDJ2swsA/8f0TPb1+uvx7YAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "resize = T.Compose([T.ToPILImage(),\n",
    "                    T.Resize(40, interpolation=Image.CUBIC),\n",
    "                    T.ToTensor()])\n",
    "\n",
    "\n",
    "\n",
    "def get_screen():\n",
    "    # Returned screen requested by gym is 400x600x3, but is sometimes larger\n",
    "    # such as 800x1200x3. Transpose it into torch order (CHW).\n",
    "    screen = env.render(mode='rgb_array').transpose((2, 0, 1))\n",
    "\n",
    "    # Strip off the edges, so that we have a square image centered on a cart\n",
    "    #screen = screen[:, :, slice_range]\n",
    "    # Convert to float, rescale, convert to torch tensor\n",
    "    # (this doesn't require a copy)\n",
    "    screen = np.ascontiguousarray(screen, dtype=np.float32) / 255\n",
    "    screen = torch.from_numpy(screen)\n",
    "    # Resize, and add a batch dimension (BCHW)\n",
    "    return resize(screen).unsqueeze(0).to(device)\n",
    "\n",
    "\n",
    "env.reset()\n",
    "plt.figure()\n",
    "plt.imshow(get_screen().cpu().squeeze(0).permute(1, 2, 0).numpy(),\n",
    "           interpolation='none')\n",
    "plt.title('Example extracted screen')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 128\n",
    "GAMMA = 0.999\n",
    "EPS_START = 0.9\n",
    "EPS_END = 0.05\n",
    "EPS_DECAY = 200\n",
    "TARGET_UPDATE = 10\n",
    "\n",
    "# Get screen size so that we can initialize layers correctly based on shape\n",
    "# returned from AI gym. Typical dimensions at this point are close to 3x40x90\n",
    "# which is the result of a clamped and down-scaled render buffer in get_screen()\n",
    "init_screen = get_screen()\n",
    "_, _, screen_height, screen_width = init_screen.shape\n",
    "\n",
    "policy_net = DQN(screen_height, screen_width).to(device)\n",
    "target_net = DQN(screen_height, screen_width).to(device)\n",
    "target_net.load_state_dict(policy_net.state_dict())\n",
    "target_net.eval()\n",
    "\n",
    "optimizer = optim.RMSprop(policy_net.parameters())\n",
    "memory = ReplayMemory(10000)\n",
    "\n",
    "\n",
    "steps_done = 0\n",
    "\n",
    "\n",
    "def select_action(state):\n",
    "    global steps_done\n",
    "    sample = random.random()\n",
    "    eps_threshold = EPS_END + (EPS_START - EPS_END) * \\\n",
    "        math.exp(-1. * steps_done / EPS_DECAY)\n",
    "    steps_done += 1\n",
    "    if sample > eps_threshold:\n",
    "        with torch.no_grad():\n",
    "            # t.max(1) will return largest value for column of each row.\n",
    "            # second column on max result is index of where max element was\n",
    "            # found, so we pick action with the larger expected reward.\n",
    "            return policy_net(state).max(1)[1].view(1, 1)\n",
    "    else:\n",
    "        return torch.tensor([[random.randrange(2)]], device=device, dtype=torch.long)\n",
    "\n",
    "\n",
    "episode_durations = []\n",
    "\n",
    "\n",
    "def plot_durations():\n",
    "    plt.figure(2)\n",
    "    plt.clf()\n",
    "    durations_t = torch.tensor(episode_durations, dtype=torch.float)\n",
    "    plt.title('Training...')\n",
    "    plt.xlabel('Episode')\n",
    "    plt.ylabel('Duration')\n",
    "    plt.plot(durations_t.numpy())\n",
    "    # Take 100 episode averages and plot them too\n",
    "    if len(durations_t) >= 100:\n",
    "        means = durations_t.unfold(0, 100, 1).mean(1).view(-1)\n",
    "        means = torch.cat((torch.zeros(99), means))\n",
    "        plt.plot(means.numpy())\n",
    "\n",
    "    plt.pause(0.001)  # pause a bit so that plots are updated\n",
    "    if is_ipython:\n",
    "        display.clear_output(wait=True)\n",
    "        display.display(plt.gcf())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimize_model():\n",
    "    if len(memory) < BATCH_SIZE:\n",
    "        return\n",
    "    transitions = memory.sample(BATCH_SIZE)\n",
    "    # Transpose the batch (see http://stackoverflow.com/a/19343/3343043 for\n",
    "    # detailed explanation). This converts batch-array of Transitions\n",
    "    # to Transition of batch-arrays.\n",
    "    batch = Transition(*zip(*transitions))\n",
    "   # print(batch)\n",
    "    # Compute a mask of non-final states and concatenate the batch elements\n",
    "    # (a final state would've been the one after which simulation ended)\n",
    "    non_final_mask = torch.tensor(tuple(map(lambda s: s is not None,\n",
    "                                          batch.next_state)), device=device, dtype=torch.uint8)\n",
    "    non_final_next_states = torch.cat([s for s in batch.next_state\n",
    "                                                if s is not None])\n",
    "    state_batch = torch.cat(batch.state)\n",
    "    #print(batch.action)\n",
    "    action_batch = torch.cat(batch.action)\n",
    "    reward_batch = torch.cat(batch.reward)\n",
    "\n",
    "    # Compute Q(s_t, a) - the model computes Q(s_t), then we select the\n",
    "    # columns of actions taken. These are the actions which would've been taken\n",
    "    # for each batch state according to policy_net\n",
    "    state_action_values = policy_net(state_batch).gather(1, action_batch)\n",
    "\n",
    "    # Compute V(s_{t+1}) for all next states.\n",
    "    # Expected values of actions for non_final_next_states are computed based\n",
    "    # on the \"older\" target_net; selecting their best reward with max(1)[0].\n",
    "    # This is merged based on the mask, such that we'll have either the expected\n",
    "    # state value or 0 in case the state was final.\n",
    "    next_state_values = torch.zeros(BATCH_SIZE, device=device)\n",
    "    next_state_values[non_final_mask] = target_net(non_final_next_states).max(1)[0].detach()\n",
    "    # Compute the expected Q values\n",
    "    expected_state_action_values = (next_state_values * GAMMA) + reward_batch\n",
    "\n",
    "    # Compute Huber loss\n",
    "    loss = F.smooth_l1_loss(state_action_values, expected_state_action_values.unsqueeze(1))\n",
    "\n",
    "    # Optimize the model\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    for param in policy_net.parameters():\n",
    "        param.grad.data.clamp_(-1, 1)\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Complete\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_episodes = 50\n",
    "for i_episode in range(num_episodes):\n",
    "    # Initialize the environment and state\n",
    "    env.reset()\n",
    "    score_buf = 0 \n",
    "    print('current episode: ', i_episode)\n",
    "    last_screen = get_screen()\n",
    "    current_screen = get_screen()\n",
    "    state = current_screen - last_screen\n",
    "    for t in count():\n",
    "        # Select and perform an action\n",
    "        action_space = {\n",
    "                        0 : [1, 0, 0],\n",
    "                        1 : [-1, 0 , 0],\n",
    "                        2 : [0, 1, 0],\n",
    "                        3 : [0, 0, 1],\n",
    "                        4 : [1, 1, 0],\n",
    "                        5 : [-1, 1 , 0],\n",
    "        }\n",
    "        action = select_action(state)\n",
    "        _, reward, done, _ = env.step(action_space[int(action)])\n",
    "        score_buf += reward\n",
    "        reward = torch.tensor([reward], device=device)\n",
    "        print(\"score_buf:\", score_buf)\n",
    "        # Observe new state\n",
    "        last_screen = current_screen\n",
    "        current_screen = get_screen()\n",
    "        if not done:\n",
    "            next_state = current_screen - last_screen\n",
    "        else:\n",
    "            next_state = None\n",
    "\n",
    "        # Store the transition in memory\n",
    "        memory.push(state, action, next_state, reward)\n",
    "        #print(\"type(action): \", type(action))\n",
    "        #print(action)\n",
    "        # Move to the next state\n",
    "        state = next_state\n",
    "\n",
    "        # Perform one step of the optimization (on the target network)\n",
    "        optimize_model()\n",
    "        \n",
    "        if done or score_buf < -1:\n",
    "            episode_durations.append(t + 1)\n",
    "            plot_durations()\n",
    "            break\n",
    "    # Update the target network, copying all weights and biases in DQN\n",
    "    if i_episode % TARGET_UPDATE == 0:\n",
    "        target_net.load_state_dict(policy_net.state_dict())\n",
    "\n",
    "print('Complete')\n",
    "env.render()\n",
    "env.close()\n",
    "plt.ioff()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Track generation: 1207..1513 -> 306-tiles track\n",
      "current episode:  49\n",
      "score_buf: 6.457377049180328\n",
      "score_buf: 6.357377049180329\n",
      "score_buf: 6.257377049180329\n",
      "score_buf: 6.1573770491803295\n",
      "score_buf: 6.05737704918033\n",
      "score_buf: 5.95737704918033\n",
      "score_buf: 5.8573770491803305\n",
      "score_buf: 5.757377049180331\n",
      "score_buf: 5.657377049180331\n",
      "score_buf: 5.557377049180332\n",
      "score_buf: 5.457377049180332\n",
      "score_buf: 5.357377049180332\n",
      "score_buf: 5.257377049180333\n",
      "score_buf: 5.157377049180333\n",
      "score_buf: 5.057377049180333\n",
      "score_buf: 4.957377049180334\n",
      "score_buf: 4.857377049180334\n",
      "score_buf: 4.757377049180334\n",
      "score_buf: 4.657377049180335\n",
      "score_buf: 4.557377049180335\n",
      "score_buf: 4.4573770491803355\n",
      "score_buf: 4.357377049180336\n",
      "score_buf: 4.257377049180336\n",
      "score_buf: 4.157377049180337\n",
      "score_buf: 4.057377049180337\n",
      "score_buf: 3.957377049180337\n",
      "score_buf: 3.8573770491803367\n",
      "score_buf: 3.7573770491803367\n",
      "score_buf: 3.6573770491803366\n",
      "score_buf: 3.5573770491803365\n",
      "score_buf: 3.4573770491803364\n",
      "score_buf: 3.3573770491803363\n",
      "score_buf: 3.257377049180336\n",
      "score_buf: 3.157377049180336\n",
      "score_buf: 3.057377049180336\n",
      "score_buf: 2.957377049180336\n",
      "score_buf: 2.857377049180336\n",
      "score_buf: 2.7573770491803358\n",
      "score_buf: 2.6573770491803357\n",
      "score_buf: 2.5573770491803356\n",
      "score_buf: 2.4573770491803355\n",
      "score_buf: 2.3573770491803354\n",
      "score_buf: 2.2573770491803353\n",
      "score_buf: 2.1573770491803352\n",
      "score_buf: 2.057377049180335\n",
      "score_buf: 1.957377049180335\n",
      "score_buf: 1.857377049180335\n",
      "score_buf: 1.7573770491803349\n",
      "score_buf: 1.6573770491803348\n",
      "score_buf: 1.5573770491803347\n",
      "score_buf: 1.4573770491803346\n",
      "score_buf: 1.3573770491803345\n",
      "score_buf: 1.2573770491803344\n",
      "score_buf: 1.1573770491803343\n",
      "score_buf: 1.0573770491803343\n",
      "score_buf: 0.9573770491803343\n",
      "score_buf: 0.8573770491803343\n",
      "score_buf: 0.7573770491803343\n",
      "score_buf: 0.6573770491803343\n",
      "score_buf: 0.5573770491803344\n",
      "score_buf: 0.4573770491803344\n",
      "score_buf: 0.3573770491803344\n",
      "score_buf: 0.25737704918033444\n",
      "score_buf: 0.15737704918033443\n",
      "score_buf: 0.05737704918033443\n",
      "score_buf: -0.04262295081966558\n",
      "score_buf: -0.14262295081966558\n",
      "score_buf: -0.2426229508196656\n",
      "score_buf: -0.3426229508196656\n",
      "score_buf: -0.44262295081966563\n",
      "score_buf: -0.5426229508196656\n",
      "score_buf: -0.6426229508196656\n",
      "score_buf: -0.7426229508196656\n",
      "score_buf: -0.8426229508196655\n",
      "score_buf: -0.9426229508196655\n",
      "score_buf: -1.0426229508196656\n",
      "score_buf: -1.1426229508196657\n",
      "score_buf: -1.2426229508196658\n",
      "score_buf: -1.3426229508196659\n",
      "score_buf: -1.442622950819666\n",
      "score_buf: -1.542622950819666\n",
      "score_buf: -1.6426229508196661\n",
      "score_buf: -1.7426229508196662\n",
      "score_buf: -1.8426229508196663\n",
      "score_buf: -1.9426229508196664\n",
      "score_buf: -2.0426229508196663\n",
      "score_buf: -2.1426229508196664\n",
      "score_buf: -2.2426229508196664\n",
      "score_buf: -2.3426229508196665\n",
      "score_buf: -2.4426229508196666\n",
      "score_buf: -2.5426229508196667\n",
      "score_buf: -2.642622950819667\n",
      "score_buf: -2.742622950819667\n",
      "score_buf: -2.842622950819667\n",
      "score_buf: -2.942622950819667\n",
      "score_buf: -3.042622950819667\n",
      "score_buf: -3.1426229508196672\n",
      "score_buf: -3.2426229508196673\n",
      "score_buf: -3.3426229508196674\n",
      "score_buf: -3.4426229508196675\n",
      "score_buf: -3.5426229508196676\n",
      "score_buf: -3.6426229508196677\n",
      "score_buf: -3.742622950819668\n",
      "score_buf: -3.842622950819668\n",
      "score_buf: -3.942622950819668\n",
      "score_buf: -4.042622950819668\n",
      "score_buf: -4.142622950819668\n",
      "score_buf: -4.242622950819667\n",
      "score_buf: -4.342622950819667\n",
      "score_buf: -4.442622950819667\n",
      "score_buf: -4.542622950819666\n",
      "score_buf: -4.642622950819666\n",
      "score_buf: -4.742622950819666\n",
      "score_buf: -4.842622950819665\n",
      "score_buf: -4.942622950819665\n",
      "score_buf: -5.0426229508196645\n",
      "score_buf: -5.142622950819664\n",
      "score_buf: -5.242622950819664\n",
      "score_buf: -5.342622950819663\n",
      "score_buf: -5.442622950819663\n",
      "score_buf: -5.542622950819663\n",
      "score_buf: -5.642622950819662\n",
      "score_buf: -5.742622950819662\n",
      "score_buf: -5.842622950819662\n",
      "score_buf: -5.942622950819661\n",
      "score_buf: -6.042622950819661\n",
      "score_buf: -6.142622950819661\n",
      "score_buf: -6.24262295081966\n",
      "score_buf: -6.34262295081966\n",
      "score_buf: -6.4426229508196595\n",
      "score_buf: -6.542622950819659\n",
      "score_buf: -6.642622950819659\n",
      "score_buf: -6.7426229508196585\n",
      "score_buf: -6.842622950819658\n",
      "score_buf: -6.942622950819658\n",
      "score_buf: -7.042622950819657\n",
      "score_buf: -7.142622950819657\n",
      "score_buf: -7.242622950819657\n",
      "score_buf: -7.342622950819656\n",
      "score_buf: -7.442622950819656\n",
      "score_buf: -7.542622950819656\n",
      "score_buf: -7.642622950819655\n",
      "score_buf: -7.742622950819655\n",
      "score_buf: -7.8426229508196545\n",
      "score_buf: -7.942622950819654\n",
      "score_buf: -8.042622950819654\n",
      "score_buf: -8.142622950819653\n",
      "score_buf: -8.242622950819653\n",
      "score_buf: -8.342622950819653\n",
      "score_buf: -8.442622950819652\n",
      "score_buf: -8.542622950819652\n",
      "score_buf: -8.642622950819652\n",
      "score_buf: -8.742622950819651\n",
      "score_buf: -8.842622950819651\n",
      "score_buf: -8.94262295081965\n",
      "score_buf: -9.04262295081965\n",
      "score_buf: -9.14262295081965\n",
      "score_buf: -9.24262295081965\n",
      "score_buf: -9.34262295081965\n",
      "score_buf: -9.442622950819649\n",
      "score_buf: -9.542622950819649\n",
      "score_buf: -9.642622950819648\n",
      "score_buf: -9.742622950819648\n",
      "score_buf: -9.842622950819647\n",
      "score_buf: -9.942622950819647\n",
      "score_buf: -10.042622950819647\n",
      "score_buf: -10.142622950819646\n",
      "score_buf: -10.242622950819646\n",
      "score_buf: -10.342622950819646\n",
      "score_buf: -10.442622950819645\n",
      "score_buf: -10.542622950819645\n",
      "score_buf: -10.642622950819645\n",
      "score_buf: -10.742622950819644\n",
      "score_buf: -10.842622950819644\n",
      "score_buf: -10.942622950819644\n",
      "score_buf: -11.042622950819643\n",
      "score_buf: -11.142622950819643\n",
      "score_buf: -11.242622950819642\n",
      "score_buf: -11.342622950819642\n",
      "score_buf: -11.442622950819642\n",
      "score_buf: -11.542622950819641\n",
      "score_buf: -11.642622950819641\n",
      "score_buf: -11.74262295081964\n",
      "score_buf: -11.84262295081964\n",
      "score_buf: -11.94262295081964\n",
      "score_buf: -12.04262295081964\n",
      "score_buf: -12.14262295081964\n",
      "score_buf: -12.242622950819639\n",
      "score_buf: -12.342622950819639\n",
      "score_buf: -12.442622950819638\n",
      "score_buf: -12.542622950819638\n",
      "score_buf: -12.642622950819637\n",
      "score_buf: -12.742622950819637\n",
      "score_buf: -12.842622950819637\n",
      "score_buf: -12.942622950819636\n",
      "score_buf: -13.042622950819636\n",
      "score_buf: -13.142622950819636\n",
      "score_buf: -13.242622950819635\n",
      "score_buf: -13.342622950819635\n",
      "score_buf: -13.442622950819635\n",
      "score_buf: -13.542622950819634\n",
      "score_buf: -13.642622950819634\n",
      "score_buf: -13.742622950819634\n",
      "score_buf: -13.842622950819633\n",
      "score_buf: -13.942622950819633\n",
      "score_buf: -14.042622950819633\n",
      "score_buf: -14.142622950819632\n",
      "score_buf: -14.242622950819632\n",
      "score_buf: -14.342622950819631\n",
      "score_buf: -14.442622950819631\n",
      "score_buf: -14.54262295081963\n",
      "score_buf: -14.64262295081963\n",
      "score_buf: -14.74262295081963\n",
      "score_buf: -14.84262295081963\n",
      "score_buf: -14.94262295081963\n",
      "score_buf: -15.042622950819629\n",
      "score_buf: -15.142622950819629\n",
      "score_buf: -15.242622950819628\n",
      "score_buf: -15.342622950819628\n",
      "score_buf: -15.442622950819628\n",
      "score_buf: -15.542622950819627\n",
      "score_buf: -15.642622950819627\n",
      "score_buf: -15.742622950819626\n",
      "score_buf: -15.842622950819626\n",
      "score_buf: -15.942622950819626\n",
      "score_buf: -16.042622950819627\n",
      "score_buf: -16.14262295081963\n",
      "score_buf: -16.24262295081963\n",
      "score_buf: -16.34262295081963\n",
      "score_buf: -16.442622950819633\n",
      "score_buf: -16.542622950819634\n",
      "score_buf: -16.642622950819636\n",
      "score_buf: -16.742622950819637\n",
      "score_buf: -16.84262295081964\n",
      "score_buf: -16.94262295081964\n",
      "score_buf: -17.04262295081964\n",
      "score_buf: -17.142622950819643\n",
      "score_buf: -17.242622950819644\n",
      "score_buf: -17.342622950819646\n",
      "score_buf: -17.442622950819647\n",
      "score_buf: -17.54262295081965\n",
      "score_buf: -17.64262295081965\n",
      "score_buf: -17.74262295081965\n",
      "score_buf: -17.842622950819653\n",
      "score_buf: -17.942622950819654\n",
      "score_buf: -18.042622950819656\n",
      "score_buf: -18.142622950819657\n",
      "score_buf: -18.24262295081966\n",
      "score_buf: -18.34262295081966\n",
      "score_buf: -18.44262295081966\n",
      "score_buf: -18.542622950819663\n",
      "score_buf: -18.642622950819664\n",
      "score_buf: -18.742622950819666\n",
      "score_buf: -18.842622950819667\n",
      "score_buf: -18.94262295081967\n",
      "score_buf: -19.04262295081967\n",
      "score_buf: -19.14262295081967\n",
      "score_buf: -19.242622950819673\n",
      "score_buf: -19.342622950819674\n",
      "score_buf: -19.442622950819676\n",
      "score_buf: -19.542622950819677\n",
      "score_buf: -19.64262295081968\n",
      "score_buf: -19.74262295081968\n",
      "score_buf: -19.84262295081968\n",
      "score_buf: -19.942622950819683\n",
      "score_buf: -20.042622950819684\n",
      "score_buf: -20.142622950819685\n",
      "score_buf: -20.242622950819687\n",
      "score_buf: -20.34262295081969\n",
      "score_buf: -20.44262295081969\n",
      "score_buf: -20.54262295081969\n",
      "score_buf: -20.642622950819693\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_buf: -20.742622950819694\n",
      "score_buf: -20.842622950819695\n",
      "score_buf: -20.942622950819697\n",
      "score_buf: -21.0426229508197\n",
      "score_buf: -21.1426229508197\n",
      "score_buf: -21.2426229508197\n",
      "score_buf: -21.342622950819703\n",
      "score_buf: -21.442622950819704\n",
      "score_buf: -21.542622950819705\n",
      "score_buf: -21.642622950819707\n",
      "score_buf: -21.742622950819708\n",
      "score_buf: -21.84262295081971\n",
      "score_buf: -21.94262295081971\n",
      "score_buf: -22.042622950819712\n",
      "score_buf: -22.142622950819714\n",
      "score_buf: -22.242622950819715\n",
      "score_buf: -22.342622950819717\n",
      "score_buf: -22.442622950819718\n",
      "score_buf: -22.54262295081972\n",
      "score_buf: -22.64262295081972\n",
      "score_buf: -22.742622950819722\n",
      "score_buf: -22.842622950819724\n",
      "score_buf: -22.942622950819725\n",
      "score_buf: -23.042622950819727\n",
      "score_buf: -23.142622950819728\n",
      "score_buf: -23.24262295081973\n",
      "score_buf: -23.34262295081973\n",
      "score_buf: -23.442622950819732\n",
      "score_buf: -23.542622950819734\n",
      "score_buf: -23.642622950819735\n",
      "score_buf: -23.742622950819737\n",
      "score_buf: -23.842622950819738\n",
      "score_buf: -23.94262295081974\n",
      "score_buf: -24.04262295081974\n",
      "score_buf: -24.142622950819742\n",
      "score_buf: -24.242622950819744\n",
      "score_buf: -24.342622950819745\n",
      "score_buf: -24.442622950819747\n",
      "score_buf: -24.542622950819748\n",
      "score_buf: -24.64262295081975\n",
      "score_buf: -24.74262295081975\n",
      "score_buf: -24.842622950819752\n",
      "score_buf: -24.942622950819754\n",
      "score_buf: -25.042622950819755\n",
      "score_buf: -25.142622950819757\n",
      "score_buf: -25.242622950819758\n",
      "score_buf: -25.34262295081976\n",
      "score_buf: -25.44262295081976\n",
      "score_buf: -25.542622950819762\n",
      "score_buf: -25.642622950819764\n",
      "score_buf: -25.742622950819765\n",
      "score_buf: -25.842622950819766\n",
      "score_buf: -25.942622950819768\n",
      "score_buf: -26.04262295081977\n",
      "score_buf: -26.14262295081977\n",
      "score_buf: -26.242622950819772\n",
      "score_buf: -26.342622950819774\n",
      "score_buf: -26.442622950819775\n",
      "score_buf: -26.542622950819776\n",
      "score_buf: -26.642622950819778\n",
      "score_buf: -26.74262295081978\n",
      "score_buf: -26.84262295081978\n",
      "score_buf: -26.942622950819782\n",
      "score_buf: -27.042622950819784\n",
      "score_buf: -27.142622950819785\n",
      "score_buf: -27.242622950819786\n",
      "score_buf: -27.342622950819788\n",
      "score_buf: -27.44262295081979\n",
      "score_buf: -27.54262295081979\n",
      "score_buf: -27.642622950819792\n",
      "score_buf: -27.742622950819793\n",
      "score_buf: -27.842622950819795\n",
      "score_buf: -27.942622950819796\n",
      "score_buf: -28.042622950819798\n",
      "score_buf: -28.1426229508198\n",
      "score_buf: -28.2426229508198\n",
      "score_buf: -28.342622950819802\n",
      "score_buf: -28.442622950819803\n",
      "score_buf: -28.542622950819805\n",
      "score_buf: -28.642622950819806\n",
      "score_buf: -28.742622950819808\n",
      "score_buf: -28.84262295081981\n",
      "score_buf: -28.94262295081981\n",
      "score_buf: -29.042622950819812\n",
      "score_buf: -29.142622950819813\n",
      "score_buf: -29.242622950819815\n",
      "score_buf: -29.342622950819816\n",
      "score_buf: -29.442622950819818\n",
      "score_buf: -29.54262295081982\n",
      "score_buf: -29.64262295081982\n",
      "score_buf: -29.742622950819822\n",
      "score_buf: -29.842622950819823\n",
      "score_buf: -29.942622950819825\n",
      "score_buf: -30.042622950819826\n",
      "score_buf: -30.142622950819828\n",
      "score_buf: -30.24262295081983\n",
      "score_buf: -30.34262295081983\n",
      "score_buf: -30.442622950819832\n",
      "score_buf: -30.542622950819833\n",
      "score_buf: -30.642622950819835\n",
      "score_buf: -30.742622950819836\n",
      "score_buf: -30.842622950819838\n",
      "score_buf: -30.94262295081984\n",
      "score_buf: -31.04262295081984\n",
      "score_buf: -31.14262295081984\n",
      "score_buf: -31.242622950819843\n",
      "score_buf: -31.342622950819845\n",
      "score_buf: -31.442622950819846\n",
      "score_buf: -31.542622950819847\n",
      "score_buf: -31.64262295081985\n",
      "score_buf: -31.74262295081985\n",
      "score_buf: -31.84262295081985\n",
      "score_buf: -31.942622950819853\n",
      "score_buf: -32.042622950819855\n",
      "score_buf: -32.142622950819856\n",
      "score_buf: -32.24262295081986\n",
      "score_buf: -32.34262295081986\n",
      "score_buf: -32.44262295081986\n",
      "score_buf: -32.54262295081986\n",
      "score_buf: -32.64262295081986\n",
      "score_buf: -32.742622950819865\n",
      "score_buf: -32.842622950819866\n",
      "score_buf: -32.94262295081987\n",
      "score_buf: -33.04262295081987\n",
      "score_buf: -33.14262295081987\n",
      "score_buf: -33.24262295081987\n",
      "score_buf: -33.34262295081987\n",
      "score_buf: -33.442622950819874\n",
      "score_buf: -33.542622950819876\n",
      "score_buf: -33.64262295081988\n",
      "score_buf: -33.74262295081988\n",
      "score_buf: -33.84262295081988\n",
      "score_buf: -33.94262295081988\n",
      "score_buf: -34.04262295081988\n",
      "score_buf: -34.142622950819884\n",
      "score_buf: -34.242622950819886\n",
      "score_buf: -34.34262295081989\n",
      "score_buf: -34.44262295081989\n",
      "score_buf: -34.54262295081989\n",
      "score_buf: -34.64262295081989\n",
      "score_buf: -34.74262295081989\n",
      "score_buf: -34.842622950819894\n",
      "score_buf: -34.942622950819896\n",
      "score_buf: -35.0426229508199\n",
      "score_buf: -35.1426229508199\n",
      "score_buf: -35.2426229508199\n",
      "score_buf: -35.3426229508199\n",
      "score_buf: -35.4426229508199\n",
      "score_buf: -35.542622950819904\n",
      "score_buf: -35.642622950819906\n",
      "score_buf: -35.74262295081991\n",
      "score_buf: -35.84262295081991\n",
      "score_buf: -35.94262295081991\n",
      "score_buf: -36.04262295081991\n",
      "score_buf: -36.14262295081991\n",
      "score_buf: -36.242622950819914\n",
      "score_buf: -36.342622950819916\n",
      "score_buf: -36.44262295081992\n",
      "score_buf: -36.54262295081992\n",
      "score_buf: -36.64262295081992\n",
      "score_buf: -36.74262295081992\n",
      "score_buf: -36.84262295081992\n",
      "score_buf: -36.942622950819924\n",
      "score_buf: -37.042622950819926\n",
      "score_buf: -37.14262295081993\n",
      "score_buf: -37.24262295081993\n",
      "score_buf: -37.34262295081993\n",
      "score_buf: -37.44262295081993\n",
      "score_buf: -37.54262295081993\n",
      "score_buf: -37.642622950819934\n",
      "score_buf: -37.742622950819936\n",
      "score_buf: -37.84262295081994\n",
      "score_buf: -37.94262295081994\n",
      "score_buf: -38.04262295081994\n",
      "score_buf: -38.14262295081994\n",
      "score_buf: -38.24262295081994\n",
      "score_buf: -38.342622950819944\n",
      "score_buf: -38.442622950819946\n",
      "score_buf: -38.54262295081995\n",
      "score_buf: -38.64262295081995\n",
      "score_buf: -38.74262295081995\n",
      "score_buf: -38.84262295081995\n",
      "score_buf: -38.94262295081995\n",
      "score_buf: -39.042622950819954\n",
      "score_buf: -39.142622950819955\n",
      "score_buf: -39.24262295081996\n",
      "score_buf: -39.34262295081996\n",
      "score_buf: -39.44262295081996\n",
      "score_buf: -39.54262295081996\n",
      "score_buf: -39.64262295081996\n",
      "score_buf: -39.742622950819964\n",
      "score_buf: -39.842622950819965\n",
      "score_buf: -39.94262295081997\n",
      "score_buf: -40.04262295081997\n",
      "score_buf: -40.14262295081997\n",
      "score_buf: -40.24262295081997\n",
      "score_buf: -40.34262295081997\n",
      "score_buf: -40.442622950819974\n",
      "score_buf: -40.542622950819975\n",
      "score_buf: -40.64262295081998\n",
      "score_buf: -40.74262295081998\n",
      "score_buf: -40.84262295081998\n",
      "score_buf: -40.94262295081998\n",
      "score_buf: -41.04262295081998\n",
      "score_buf: -41.142622950819984\n",
      "score_buf: -41.242622950819985\n",
      "score_buf: -41.34262295081999\n",
      "score_buf: -41.44262295081999\n",
      "score_buf: -41.54262295081999\n",
      "score_buf: -41.64262295081999\n",
      "score_buf: -41.74262295081999\n",
      "score_buf: -41.842622950819994\n",
      "score_buf: -41.942622950819995\n",
      "score_buf: -42.04262295082\n",
      "score_buf: -42.14262295082\n",
      "score_buf: -42.24262295082\n",
      "score_buf: -42.34262295082\n",
      "score_buf: -42.44262295082\n",
      "score_buf: -42.542622950820004\n",
      "score_buf: -42.642622950820005\n",
      "score_buf: -42.74262295082001\n",
      "score_buf: -42.84262295082001\n",
      "score_buf: -42.94262295082001\n",
      "score_buf: -43.04262295082001\n",
      "score_buf: -43.14262295082001\n",
      "score_buf: -43.242622950820014\n",
      "score_buf: -43.342622950820015\n",
      "score_buf: -43.44262295082002\n",
      "score_buf: -43.54262295082002\n",
      "score_buf: -43.64262295082002\n",
      "score_buf: -43.74262295082002\n",
      "score_buf: -43.84262295082002\n",
      "score_buf: -43.942622950820024\n",
      "score_buf: -44.042622950820025\n",
      "score_buf: -44.14262295082003\n",
      "score_buf: -44.24262295082003\n",
      "score_buf: -44.34262295082003\n",
      "score_buf: -44.44262295082003\n",
      "score_buf: -44.54262295082003\n",
      "score_buf: -44.642622950820034\n",
      "score_buf: -44.742622950820035\n",
      "score_buf: -44.84262295082004\n",
      "score_buf: -44.94262295082004\n",
      "score_buf: -45.04262295082004\n",
      "score_buf: -45.14262295082004\n",
      "score_buf: -45.24262295082004\n",
      "score_buf: -45.342622950820044\n",
      "score_buf: -45.442622950820045\n",
      "score_buf: -45.542622950820046\n",
      "score_buf: -45.64262295082005\n",
      "score_buf: -45.74262295082005\n",
      "score_buf: -45.84262295082005\n",
      "score_buf: -45.94262295082005\n",
      "score_buf: -46.04262295082005\n",
      "score_buf: -46.142622950820055\n",
      "score_buf: -46.242622950820056\n",
      "score_buf: -46.34262295082006\n",
      "score_buf: -46.44262295082006\n",
      "score_buf: -46.54262295082006\n",
      "score_buf: -46.64262295082006\n",
      "score_buf: -46.74262295082006\n",
      "score_buf: -46.842622950820065\n",
      "score_buf: -46.942622950820066\n",
      "score_buf: -47.04262295082007\n",
      "score_buf: -47.14262295082007\n",
      "score_buf: -47.24262295082007\n",
      "score_buf: -47.34262295082007\n",
      "score_buf: -47.44262295082007\n",
      "score_buf: -47.542622950820075\n",
      "score_buf: -47.642622950820076\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "score_buf: -47.74262295082008\n",
      "score_buf: -47.84262295082008\n",
      "score_buf: -47.94262295082008\n",
      "score_buf: -48.04262295082008\n",
      "score_buf: -48.14262295082008\n",
      "score_buf: -48.242622950820085\n",
      "score_buf: -48.342622950820086\n",
      "score_buf: -48.44262295082009\n",
      "score_buf: -48.54262295082009\n",
      "score_buf: -48.64262295082009\n",
      "score_buf: -48.74262295082009\n",
      "score_buf: -48.84262295082009\n",
      "score_buf: -48.942622950820095\n",
      "score_buf: -49.042622950820096\n",
      "score_buf: -49.1426229508201\n",
      "score_buf: -49.2426229508201\n",
      "score_buf: -49.3426229508201\n",
      "score_buf: -49.4426229508201\n",
      "score_buf: -49.5426229508201\n",
      "score_buf: -49.642622950820105\n",
      "score_buf: -49.742622950820106\n",
      "score_buf: -49.84262295082011\n",
      "score_buf: -49.94262295082011\n",
      "score_buf: -50.04262295082011\n",
      "score_buf: -50.14262295082011\n",
      "score_buf: -50.24262295082011\n",
      "score_buf: -50.342622950820115\n",
      "score_buf: -50.442622950820116\n",
      "score_buf: -50.54262295082012\n",
      "score_buf: -50.64262295082012\n",
      "score_buf: -50.74262295082012\n",
      "score_buf: -50.84262295082012\n",
      "score_buf: -50.94262295082012\n",
      "score_buf: -51.042622950820125\n",
      "score_buf: -51.142622950820126\n",
      "score_buf: -51.24262295082013\n",
      "score_buf: -51.34262295082013\n",
      "score_buf: -51.44262295082013\n",
      "score_buf: -51.54262295082013\n",
      "score_buf: -51.64262295082013\n",
      "score_buf: -51.742622950820135\n",
      "score_buf: -51.842622950820136\n",
      "score_buf: -51.94262295082014\n",
      "score_buf: -52.04262295082014\n",
      "score_buf: -52.14262295082014\n",
      "score_buf: -52.24262295082014\n",
      "score_buf: -52.34262295082014\n",
      "score_buf: -52.442622950820144\n",
      "score_buf: -52.542622950820146\n",
      "score_buf: -52.64262295082015\n",
      "score_buf: -52.74262295082015\n",
      "score_buf: -52.84262295082015\n",
      "score_buf: -52.94262295082015\n",
      "score_buf: -53.04262295082015\n",
      "score_buf: -53.142622950820154\n",
      "score_buf: -53.242622950820156\n",
      "score_buf: -53.34262295082016\n",
      "score_buf: -53.44262295082016\n",
      "score_buf: -53.54262295082016\n",
      "score_buf: -53.64262295082016\n",
      "score_buf: -53.74262295082016\n",
      "score_buf: -53.842622950820164\n",
      "score_buf: -53.942622950820166\n",
      "score_buf: -54.04262295082017\n",
      "score_buf: -54.14262295082017\n",
      "score_buf: -54.24262295082017\n",
      "score_buf: -54.34262295082017\n",
      "score_buf: -54.44262295082017\n",
      "score_buf: -54.542622950820174\n",
      "score_buf: -54.642622950820176\n",
      "score_buf: -54.74262295082018\n",
      "score_buf: -54.84262295082018\n",
      "score_buf: -54.94262295082018\n",
      "score_buf: -55.04262295082018\n",
      "score_buf: -55.14262295082018\n",
      "score_buf: -55.242622950820184\n",
      "score_buf: -55.342622950820186\n",
      "score_buf: -55.44262295082019\n",
      "score_buf: -55.54262295082019\n",
      "score_buf: -55.64262295082019\n",
      "score_buf: -55.74262295082019\n",
      "score_buf: -55.84262295082019\n",
      "score_buf: -55.942622950820194\n",
      "score_buf: -56.042622950820196\n",
      "score_buf: -56.1426229508202\n",
      "score_buf: -56.2426229508202\n",
      "score_buf: -56.3426229508202\n",
      "score_buf: -56.4426229508202\n",
      "score_buf: -56.5426229508202\n",
      "score_buf: -56.642622950820204\n",
      "score_buf: -56.742622950820206\n",
      "score_buf: -56.84262295082021\n",
      "score_buf: -56.94262295082021\n",
      "score_buf: -57.04262295082021\n",
      "score_buf: -57.14262295082021\n",
      "score_buf: -57.24262295082021\n",
      "score_buf: -57.342622950820214\n",
      "score_buf: -57.442622950820216\n",
      "score_buf: -57.54262295082022\n",
      "score_buf: -57.64262295082022\n",
      "score_buf: -57.74262295082022\n",
      "score_buf: -57.84262295082022\n",
      "score_buf: -57.94262295082022\n",
      "score_buf: -58.042622950820224\n",
      "score_buf: -58.142622950820225\n",
      "score_buf: -58.24262295082023\n",
      "score_buf: -58.34262295082023\n",
      "score_buf: -58.44262295082023\n",
      "score_buf: -58.54262295082023\n",
      "score_buf: -58.64262295082023\n",
      "score_buf: -58.742622950820234\n",
      "score_buf: -58.842622950820235\n",
      "score_buf: -58.94262295082024\n",
      "score_buf: -59.04262295082024\n",
      "score_buf: -59.14262295082024\n",
      "score_buf: -59.24262295082024\n",
      "score_buf: -59.34262295082024\n",
      "score_buf: -59.442622950820244\n",
      "score_buf: -59.542622950820245\n",
      "score_buf: -59.64262295082025\n",
      "score_buf: -59.74262295082025\n",
      "score_buf: -59.84262295082025\n",
      "score_buf: -59.94262295082025\n",
      "score_buf: -60.04262295082025\n",
      "score_buf: -60.142622950820254\n",
      "score_buf: -60.242622950820255\n",
      "score_buf: -60.34262295082026\n",
      "score_buf: -60.44262295082026\n",
      "score_buf: -60.54262295082026\n",
      "score_buf: -60.64262295082026\n",
      "score_buf: -60.74262295082026\n",
      "score_buf: -60.842622950820264\n",
      "score_buf: -60.942622950820265\n",
      "score_buf: -61.04262295082027\n",
      "score_buf: -61.14262295082027\n",
      "score_buf: -61.24262295082027\n",
      "score_buf: -61.34262295082027\n",
      "score_buf: -61.44262295082027\n",
      "score_buf: -61.542622950820274\n",
      "score_buf: -61.642622950820275\n",
      "score_buf: -61.74262295082028\n",
      "score_buf: -61.84262295082028\n",
      "score_buf: -61.94262295082028\n",
      "score_buf: -62.04262295082028\n",
      "score_buf: -62.14262295082028\n",
      "score_buf: -62.242622950820284\n",
      "score_buf: -62.342622950820285\n",
      "score_buf: -62.44262295082029\n",
      "score_buf: -62.54262295082029\n",
      "score_buf: -62.64262295082029\n",
      "score_buf: -62.74262295082029\n",
      "score_buf: -62.84262295082029\n",
      "score_buf: -62.942622950820294\n",
      "score_buf: -63.042622950820295\n",
      "score_buf: -63.1426229508203\n",
      "score_buf: -63.2426229508203\n",
      "score_buf: -63.3426229508203\n",
      "score_buf: -63.4426229508203\n",
      "score_buf: -63.5426229508203\n",
      "score_buf: -63.642622950820304\n",
      "score_buf: -63.742622950820305\n",
      "score_buf: -63.84262295082031\n",
      "score_buf: -63.94262295082031\n",
      "score_buf: -64.0426229508203\n",
      "score_buf: -64.1426229508203\n",
      "score_buf: -64.24262295082029\n",
      "score_buf: -64.34262295082029\n",
      "score_buf: -64.44262295082028\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-05389fa0c991>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m     }\n\u001b[1;32m     17\u001b[0m     \u001b[0maction\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mselect_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction_space\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maction\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mscore_buf\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m     \u001b[0mreward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mreward\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/gym/envs/box2d/car_racing.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, action)\u001b[0m\n\u001b[1;32m    303\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mt\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1.0\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mFPS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 305\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"state_pixels\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    306\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m         \u001b[0mstep_reward\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/gym/envs/box2d/car_racing.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, mode)\u001b[0m\n\u001b[1;32m    367\u001b[0m             \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglViewport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVP_W\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVP_H\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m             \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender_road\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    370\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mgeom\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mviewer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0monetime_geoms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m                 \u001b[0mgeom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/gym/envs/box2d/car_racing.py\u001b[0m in \u001b[0;36mrender_road\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    414\u001b[0m                 \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglVertex3f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    415\u001b[0m                 \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglVertex3f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 416\u001b[0;31m                 \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglVertex3f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    417\u001b[0m                 \u001b[0mgl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mglVertex3f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mk\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    418\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mpoly\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mroad_poly\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/pyglet/gl/lib.py\u001b[0m in \u001b[0;36merrcheck\u001b[0;34m(result, func, arguments)\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 85\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0merrcheck\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0marguments\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     86\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0m_debug_gl_trace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "env.reset()\n",
    "score_buf = 0 \n",
    "print('current episode: ', i_episode)\n",
    "last_screen = get_screen()\n",
    "current_screen = get_screen()\n",
    "state = current_screen - last_screen\n",
    "\n",
    "action_space = {\n",
    "                        0 : [1, 0, 0],\n",
    "                        1 : [-1, 0 , 0],\n",
    "                        2 : [0, 1, 0],\n",
    "                        3 : [0, 0, 1],\n",
    "                        4 : [1, 1, 0],\n",
    "                        5 : [-1, 1 , 0],\n",
    "        }\n",
    "action = policy_net(state).max(1)[1].view(1, 1)\n",
    "_, reward, done, _ = env.step(action_space[int(action)])\n",
    "score_buf += reward\n",
    "reward = torch.tensor([reward], device=device)\n",
    "print(\"score_buf:\", score_buf)\n",
    "    # Observe new state\n",
    "last_screen = current_screen\n",
    "current_screen = get_screen()\n",
    "    if not done:\n",
    "        next_state = current_screen - last_screen\n",
    "        else:\n",
    "            next_state = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Track generation: 1216..1524 -> 308-tiles track\n",
      "current episode:  0\n"
     ]
    },
    {
     "ename": "ArgumentError",
     "evalue": "argument 2: <class 'TypeError'>: wrong type",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mArgumentError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-21939e11ba40>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mscore_buf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'current episode: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi_episode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mlast_screen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_screen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     \u001b[0mcurrent_screen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_screen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcurrent_screen\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlast_screen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-73e1ef772c68>\u001b[0m in \u001b[0;36mget_screen\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m# Returned screen requested by gym is 400x600x3, but is sometimes larger\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# such as 800x1200x3. Transpose it into torch order (CHW).\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m     \u001b[0mscreen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrender\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'rgb_array'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0;31m# Strip off the edges, so that we have a square image centered on a cart\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/gym/envs/box2d/car_racing.py\u001b[0m in \u001b[0;36mrender\u001b[0;34m(self, mode)\u001b[0m\n\u001b[1;32m    354\u001b[0m         \u001b[0mwin\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mviewer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    355\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'state_pixels'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m             \u001b[0mwin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mswitch_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m             \u001b[0mwin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_events\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m\"rgb_array\"\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m\"state_pixels\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/pyglet/window/xlib/__init__.py\u001b[0m in \u001b[0;36mswitch_to\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    497\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mswitch_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    498\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 499\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_current\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    500\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    501\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mflip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/rlenv/lib/python3.6/site-packages/pyglet/gl/xlib.py\u001b[0m in \u001b[0;36mset_current\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    325\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mset_current\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m         glx.glXMakeContextCurrent(\n\u001b[0;32m--> 327\u001b[0;31m             self.x_display, self.glx_window, self.glx_window, self.glx_context)\n\u001b[0m\u001b[1;32m    328\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXlibContext13\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_current\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    329\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mArgumentError\u001b[0m: argument 2: <class 'TypeError'>: wrong type"
     ]
    }
   ],
   "source": [
    "num_episodes = 2\n",
    "for i_episode in range(num_episodes):\n",
    "    # Initialize the environment and state\n",
    "    env.reset()\n",
    "    score_buf = 0 \n",
    "    print('current episode: ', i_episode)\n",
    "    last_screen = get_screen()\n",
    "    current_screen = get_screen()\n",
    "    state = current_screen - last_screen\n",
    "    for t in count():\n",
    "        # Select and perform an action\n",
    "        action_space = {\n",
    "                        0 : [1, 0, 0],\n",
    "                        1 : [-1, 0 , 0],\n",
    "                        2 : [0, 1, 0],\n",
    "                        3 : [0, 0, 1],\n",
    "                        4 : [1, 1, 0],\n",
    "                        5 : [-1, 1 , 0],\n",
    "        }\n",
    "        action = policy_net(state).max(1)[1].view(1, 1)\n",
    "        _, reward, done, _ = env.step(action_space[int(action)])\n",
    "        score_buf += reward\n",
    "        print(\"score_buf:\", score_buf)\n",
    "        # Observe new state\n",
    "        last_screen = current_screen\n",
    "        current_screen = get_screen()\n",
    "        if not done:\n",
    "            next_state = current_screen - last_screen\n",
    "        else:\n",
    "            next_state = None\n",
    "            \n",
    "            \n",
    "        state = next_state\n",
    "        \n",
    "        if done or score_buf < -1:\n",
    "            episode_durations.append(t + 1)\n",
    "            plot_durations()\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}