Google_Dopamine_RL.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Google Dopamine RL.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "[View in Colaboratory](https://colab.research.google.com/github/ZER-0-NE/ML_problems/blob/master/Google_Dopamine_RL.ipynb)"
      ]
    },
    {
      "metadata": {
        "id": "WKRT7t2dJojN",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "!pip install PyDrive\n",
        "\n",
        "from pydrive.auth import GoogleAuth\n",
        "from pydrive.drive import GoogleDrive\n",
        "from google.colab import auth\n",
        "from oauth2client.client import GoogleCredentials\n",
        "\n",
        "# Authenticate and create the PyDrive client.\n",
        "# This only needs to be done once in a notebook.\n",
        "auth.authenticate_user()\n",
        "gauth = GoogleAuth()\n",
        "gauth.credentials = GoogleCredentials.get_application_default()\n",
        "drive = GoogleDrive(gauth)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Bdw15NYsV8-H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734
        },
        "outputId": "1e762c92-7d74-4dc8-8734-315da0be9ce7"
      },
      "cell_type": "code",
      "source": [
        "!pip install --upgrade --no-cache-dir dopamine-rl\n",
        "!pip install cmake\n",
        "!pip install atari_py"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: dopamine-rl in /usr/local/lib/python3.6/dist-packages (1.0.2)\n",
            "Requirement already satisfied, skipping upgrade: gin-config>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.1.1.2)\n",
            "Requirement already satisfied, skipping upgrade: absl-py>=0.2.2 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.4.1)\n",
            "Requirement already satisfied, skipping upgrade: gym>=0.10.5 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (0.10.5)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (1.10.1)\n",
            "Requirement already satisfied, skipping upgrade: opencv-python>=3.4.1.15 in /usr/local/lib/python3.6/dist-packages (from dopamine-rl) (3.4.3.18)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from gin-config>=0.1.1->dopamine-rl) (1.11.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.10.4 in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (1.14.5)\n",
            "Requirement already satisfied, skipping upgrade: requests>=2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (2.18.4)\n",
            "Requirement already satisfied, skipping upgrade: pyglet>=1.2.0 in /usr/local/lib/python3.6/dist-packages (from gym>=0.10.5->dopamine-rl) (1.3.2)\n",
            "Requirement already satisfied, skipping upgrade: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (0.2.0)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (3.6.1)\n",
            "Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (0.31.1)\n",
            "Requirement already satisfied, skipping upgrade: setuptools<=39.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (39.1.0)\n",
            "Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (0.7.1)\n",
            "Requirement already satisfied, skipping upgrade: tensorboard<1.11.0,>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (1.10.0)\n",
            "Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow->dopamine-rl) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (2018.8.24)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.7,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (2.6)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.23,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests>=2.0->gym>=0.10.5->dopamine-rl) (1.22)\n",
            "Requirement already satisfied, skipping upgrade: future in /usr/local/lib/python3.6/dist-packages (from pyglet>=1.2.0->gym>=0.10.5->dopamine-rl) (0.16.0)\n",
            "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.11.0,>=1.10.0->tensorflow->dopamine-rl) (2.6.11)\n",
            "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.10 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.11.0,>=1.10.0->tensorflow->dopamine-rl) (0.14.1)\n",
            "Collecting cmake\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/af/ee/36a244e43fc9ea64a2aab9ec854616a1771c11e91ae97d7b34246f922668/cmake-3.12.0-cp36-cp36m-manylinux1_x86_64.whl (17.7MB)\n",
            "\u001b[K    100% |████████████████████████████████| 17.7MB 1.9MB/s \n",
            "\u001b[?25hInstalling collected packages: cmake\n",
            "Successfully installed cmake-3.12.0\n",
            "Collecting atari_py\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8b/38/3c6716ac9031a686cc3228f3855e48c08a40e4d7c490dd4c21c65b465205/atari-py-0.1.1.tar.gz (760kB)\n",
            "\u001b[K    100% |████████████████████████████████| 768kB 4.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from atari_py) (1.14.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from atari_py) (1.11.0)\n",
            "Building wheels for collected packages: atari-py\n",
            "  Running setup.py bdist_wheel for atari-py ... \u001b[?25l-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \b/\b \b-\b \b\\\b \b|\b \bdone\n",
            "\u001b[?25h  Stored in directory: /root/.cache/pip/wheels/ac/79/85/b21b404d3469c3028aea3b7a1dfe9d9bf9827a20cee6a118bd\n",
            "Successfully built atari-py\n",
            "Installing collected packages: atari-py\n",
            "Successfully installed atari-py-0.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "ChDLP3PgWeGk",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "#importing packages\n",
        "import numpy as np\n",
        "import os\n",
        "from dopamine.agents.dqn import dqn_agent\n",
        "from dopamine.atari import run_experiment\n",
        "from dopamine.colab import utils as colab_utils\n",
        "from absl import flags\n",
        "import random\n",
        "\n",
        "BASE_PATH = '/tmp/colab_dope_run'\n",
        "GAME = 'Asterix'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "wIZpOmayWqHM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Creating new agent from scratch\n",
        "\n",
        "LOG_PATH = os.path.join(BASE_PATH, 'basic_agent', GAME)\n",
        "\n",
        "class BasicAgent(object):\n",
        "  \n",
        "  def __init__(self, sess, num_actions, switch_prob=0.1):\n",
        "    self._sess = sess\n",
        "    self._num_actions = num_actions\n",
        "    self._switch_prob = switch_prob\n",
        "    self._last_action = np.random.randint(num_actions)\n",
        "    self.eval_mode = False\n",
        "    \n",
        "  def _choose_actions(self):\n",
        "    #defining our policy\n",
        "    if random.random() <= self._switch_prob:\n",
        "      self._last_action = np.random.randint(self._num_actions)\n",
        "    return self._last_action\n",
        "    \n",
        "  def bundle_and_checkpoint(self, unused_checkpoint_dir, unused_iteration):\n",
        "    pass\n",
        "  \n",
        "  def unbundle(self, unused_checkpoint_dir, unused_checkpoint_version, unused_data):\n",
        "    pass\n",
        "  \n",
        "  def begin_episode(self, unused_operation):\n",
        "    return self._choose_actions()\n",
        "  \n",
        "  def end_episode(self, unused_reward):\n",
        "    pass\n",
        "  \n",
        "  def step(self, reward, observation):\n",
        "    return self._choose_actions()\n",
        "      \n",
        "def create_basic_agent(sess, environment):\n",
        "  return BasicAgent(sess, num_actions=environment.action_space.n, switch_prob=0.2)\n",
        "\n",
        "basic_runner = run_experiment.Runner(LOG_PATH, create_basic_agent, game_name=GAME, num_iterations = 200, evaluation_steps=10, max_steps_per_episode = 100)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "Rlu4gjTfXWGC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2108
        },
        "outputId": "1fa5b0e2-c9d2-4410-edec-a59d52092cd9"
      },
      "cell_type": "code",
      "source": [
        "#Training basic agent\n",
        "basic_runner.run_experiment()\n",
        "print(\"Done Training\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Beginning training...\n",
            "INFO:tensorflow:Starting iteration 0\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 107.54\n",
            "INFO:tensorflow:Average training steps per second: 727.19\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 1\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.00\n",
            "INFO:tensorflow:Average training steps per second: 725.39\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 2\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.34\n",
            "INFO:tensorflow:Average training steps per second: 722.13\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 3\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.46\n",
            "INFO:tensorflow:Average training steps per second: 731.84\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 4\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.68\n",
            "INFO:tensorflow:Average training steps per second: 738.82\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 200.00\n",
            "INFO:tensorflow:Starting iteration 5\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.08\n",
            "INFO:tensorflow:Average training steps per second: 739.90\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 6\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 111.88\n",
            "INFO:tensorflow:Average training steps per second: 738.62\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 7\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 108.92\n",
            "INFO:tensorflow:Average training steps per second: 739.26\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 8\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 112.74\n",
            "INFO:tensorflow:Average training steps per second: 739.11\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 200.00\n",
            "INFO:tensorflow:Starting iteration 9\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.96\n",
            "INFO:tensorflow:Average training steps per second: 745.55\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 10\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.20\n",
            "INFO:tensorflow:Average training steps per second: 746.44\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 250.00\n",
            "INFO:tensorflow:Starting iteration 11\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 107.50\n",
            "INFO:tensorflow:Average training steps per second: 742.32\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 12\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.68\n",
            "INFO:tensorflow:Average training steps per second: 719.90\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 13\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.42\n",
            "INFO:tensorflow:Average training steps per second: 737.28\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 14\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.16\n",
            "INFO:tensorflow:Average training steps per second: 737.85\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 200.00\n",
            "INFO:tensorflow:Starting iteration 15\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.70\n",
            "INFO:tensorflow:Average training steps per second: 738.02\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 250.00\n",
            "INFO:tensorflow:Starting iteration 16\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.90\n",
            "INFO:tensorflow:Average training steps per second: 732.51\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 250.00\n",
            "INFO:tensorflow:Starting iteration 17\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 111.96\n",
            "INFO:tensorflow:Average training steps per second: 732.81\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 18\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.76\n",
            "INFO:tensorflow:Average training steps per second: 735.63\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 19\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 108.80\n",
            "INFO:tensorflow:Average training steps per second: 736.16\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 20\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 109.78\n",
            "INFO:tensorflow:Average training steps per second: 741.31\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 21\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.92\n",
            "INFO:tensorflow:Average training steps per second: 745.61\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 22\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.34\n",
            "INFO:tensorflow:Average training steps per second: 742.53\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 23\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.16\n",
            "INFO:tensorflow:Average training steps per second: 743.65\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 24\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 107.46\n",
            "INFO:tensorflow:Average training steps per second: 742.59\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 25\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 111.06\n",
            "INFO:tensorflow:Average training steps per second: 741.77\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 26\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.06\n",
            "INFO:tensorflow:Average training steps per second: 741.48\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 200.00\n",
            "INFO:tensorflow:Starting iteration 27\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.48\n",
            "INFO:tensorflow:Average training steps per second: 742.02\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 100.00\n",
            "INFO:tensorflow:Starting iteration 28\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.80\n",
            "INFO:tensorflow:Average training steps per second: 716.74\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 50.00\n",
            "INFO:tensorflow:Starting iteration 29\n",
            "INFO:tensorflow:Average undiscounted return per training episode: 110.24\n",
            "INFO:tensorflow:Average training steps per second: 733.20\n",
            "INFO:tensorflow:Average undiscounted return per evaluation episode: 150.00\n",
            "INFO:tensorflow:Starting iteration 30\n",
            "Buffered data was truncated after reaching the output size limit."
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "bj8QdaxvhG93",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}