MAML_TRPO.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0u8jyVEHU90_",
        "outputId": "13be74b4-8d60-4528-e88e-a7e0a6b9eaac"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "libgl1-mesa-dev is already the newest version (20.0.8-0ubuntu1~18.04.1).\n",
            "libgl1-mesa-dev set to manually installed.\n",
            "software-properties-common is already the newest version (0.96.24.32.18).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "Suggested packages:\n",
            "  glew-utils\n",
            "The following NEW packages will be installed:\n",
            "  libgl1-mesa-glx libglew-dev libglew2.0 libosmesa6 libosmesa6-dev\n",
            "0 upgraded, 5 newly installed, 0 to remove and 19 not upgraded.\n",
            "Need to get 2,916 kB of archives.\n",
            "After this operation, 12.6 MB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libgl1-mesa-glx amd64 20.0.8-0ubuntu1~18.04.1 [5,532 B]\n",
            "Get:2 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libglew2.0 amd64 2.0.0-5 [140 kB]\n",
            "Get:3 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libglew-dev amd64 2.0.0-5 [120 kB]\n",
            "Get:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libosmesa6 amd64 20.0.8-0ubuntu1~18.04.1 [2,641 kB]\n",
            "Get:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 libosmesa6-dev amd64 20.0.8-0ubuntu1~18.04.1 [8,828 B]\n",
            "Fetched 2,916 kB in 0s (23.9 MB/s)\n",
            "Selecting previously unselected package libgl1-mesa-glx:amd64.\n",
            "(Reading database ... 155680 files and directories currently installed.)\n",
            "Preparing to unpack .../libgl1-mesa-glx_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
            "Unpacking libgl1-mesa-glx:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Selecting previously unselected package libglew2.0:amd64.\n",
            "Preparing to unpack .../libglew2.0_2.0.0-5_amd64.deb ...\n",
            "Unpacking libglew2.0:amd64 (2.0.0-5) ...\n",
            "Selecting previously unselected package libglew-dev:amd64.\n",
            "Preparing to unpack .../libglew-dev_2.0.0-5_amd64.deb ...\n",
            "Unpacking libglew-dev:amd64 (2.0.0-5) ...\n",
            "Selecting previously unselected package libosmesa6:amd64.\n",
            "Preparing to unpack .../libosmesa6_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
            "Unpacking libosmesa6:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Selecting previously unselected package libosmesa6-dev:amd64.\n",
            "Preparing to unpack .../libosmesa6-dev_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
            "Unpacking libosmesa6-dev:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Setting up libosmesa6:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Setting up libgl1-mesa-glx:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Setting up libglew2.0:amd64 (2.0.0-5) ...\n",
            "Setting up libglew-dev:amd64 (2.0.0-5) ...\n",
            "Setting up libosmesa6-dev:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
            "Processing triggers for libc-bin (2.27-3ubuntu1.5) ...\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-460\n",
            "Use 'apt autoremove' to remove it.\n",
            "The following NEW packages will be installed:\n",
            "  patchelf\n",
            "0 upgraded, 1 newly installed, 0 to remove and 19 not upgraded.\n",
            "Need to get 46.5 kB of archives.\n",
            "After this operation, 130 kB of additional disk space will be used.\n",
            "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 patchelf amd64 0.9-1 [46.5 kB]\n",
            "Fetched 46.5 kB in 0s (801 kB/s)\n",
            "Selecting previously unselected package patchelf.\n",
            "(Reading database ... 155718 files and directories currently installed.)\n",
            "Preparing to unpack .../patchelf_0.9-1_amd64.deb ...\n",
            "Unpacking patchelf (0.9-1) ...\n",
            "Setting up patchelf (0.9-1) ...\n",
            "Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting free-mujoco-py\n",
            "  Downloading free_mujoco_py-2.1.6-py3-none-any.whl (14.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 14.1 MB 10.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cffi<2.0.0,>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from free-mujoco-py) (1.15.1)\n",
            "Collecting glfw<2.0.0,>=1.4.0\n",
            "  Downloading glfw-1.12.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl (203 kB)\n",
            "\u001b[K     |████████████████████████████████| 203 kB 70.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: imageio<3.0.0,>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from free-mujoco-py) (2.9.0)\n",
            "Requirement already satisfied: numpy<2.0.0,>=1.21.3 in /usr/local/lib/python3.7/dist-packages (from free-mujoco-py) (1.21.6)\n",
            "Requirement already satisfied: Cython<0.30.0,>=0.29.24 in /usr/local/lib/python3.7/dist-packages (from free-mujoco-py) (0.29.32)\n",
            "Collecting fasteners==0.15\n",
            "  Downloading fasteners-0.15-py2.py3-none-any.whl (23 kB)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from fasteners==0.15->free-mujoco-py) (1.15.0)\n",
            "Collecting monotonic>=0.1\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi<2.0.0,>=1.15.0->free-mujoco-py) (2.21)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from imageio<3.0.0,>=2.9.0->free-mujoco-py) (7.1.2)\n",
            "Installing collected packages: monotonic, glfw, fasteners, free-mujoco-py\n",
            "Successfully installed fasteners-0.15 free-mujoco-py-2.1.6 glfw-1.12.0 monotonic-1.6\n"
          ]
        }
      ],
      "source": [
        "!apt-get install -y \\\n",
        "    libgl1-mesa-dev \\\n",
        "    libgl1-mesa-glx \\\n",
        "    libglew-dev \\\n",
        "    libosmesa6-dev \\\n",
        "    software-properties-common\n",
        "\n",
        "!apt-get install -y patchelf\n",
        "!pip install free-mujoco-py"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EvBkQ0-2TIGI"
      },
      "source": [
        "## Cherry TRPO MAML"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wZleUELMTDCf"
      },
      "outputs": [],
      "source": [
        "!pip install cherry-rl learn2learn &> /dev/null"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "b1cjFZVuS3U2"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "import math\n",
        "import time\n",
        "\n",
        "from copy import deepcopy\n",
        "\n",
        "import cherry as ch\n",
        "import gym\n",
        "import numpy as np\n",
        "import torch\n",
        "from cherry.algorithms import a2c, trpo\n",
        "from cherry.models.robotics import LinearValue\n",
        "from tqdm import tqdm\n",
        "\n",
        "import learn2learn as l2l\n",
        "\n",
        "import torch as th\n",
        "import torch.nn as nn\n",
        "from torch import autograd\n",
        "from torch.distributions.kl import kl_divergence\n",
        "from torch.nn.utils import parameters_to_vector, vector_to_parameters\n",
        "from torch.distributions import Normal, Categorical\n",
        "from torch.utils.tensorboard import SummaryWriter"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "env = gym.make('HalfCheetahForwardBackward-v1')\n",
        "env.reset()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rG2tDzKRf7_d",
        "outputId": "302ea660-2a1a-4262-a42d-337201cddde3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Compiling /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.pyx because it changed.\n",
            "[1/1] Cythonizing /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.pyx\n",
            "running build_ext\n",
            "building 'mujoco_py.cymj' extension\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/mujoco_py -I/usr/local/lib/python3.7/dist-packages/mujoco_py/binaries/linux/mujoco210/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.c -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.o -fopenmp -w\n",
            "x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/mujoco_py -I/usr/local/lib/python3.7/dist-packages/mujoco_py/binaries/linux/mujoco210/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/include/python3.7m -c /usr/local/lib/python3.7/dist-packages/mujoco_py/gl/osmesashim.c -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl/osmesashim.o -fopenmp -w\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/lib.linux-x86_64-3.7\n",
            "creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/lib.linux-x86_64-3.7/mujoco_py\n",
            "x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -g -fwrapv -O2 -Wl,-Bsymbolic-functions -g -fwrapv -O2 -g -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl/osmesashim.o -L/usr/local/lib/python3.7/dist-packages/mujoco_py/binaries/linux/mujoco210/bin -Wl,--enable-new-dtags,-R/usr/local/lib/python3.7/dist-packages/mujoco_py/binaries/linux/mujoco210/bin -lmujoco210 -lglewosmesa -lOSMesa -lGL -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.0.2.13_37_linuxcpuextensionbuilder/lib.linux-x86_64-3.7/mujoco_py/cymj.cpython-37m-x86_64-linux-gnu.so -fopenmp\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([-0.08513404,  0.08605558, -0.08362168, -0.07544462,  0.08005954,\n",
              "        0.04693294,  0.03585278, -0.04871276,  0.16829012,  0.03828093,\n",
              "       -0.04322466, -0.0109521 , -0.03779795,  0.06077007, -0.17164099,\n",
              "       -0.0534065 ,  0.21574007,  0.0142859 ,  0.        ,  0.61486596],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lARajYUKTQO_"
      },
      "outputs": [],
      "source": [
        "EPSILON = 1e-6\n",
        "\n",
        "def linear_init(module):\n",
        "    if isinstance(module, nn.Linear):\n",
        "        nn.init.xavier_uniform_(module.weight)\n",
        "        module.bias.data.zero_()\n",
        "    return module\n",
        "\n",
        "\n",
        "class DiagNormalPolicy(nn.Module):\n",
        "\n",
        "    def __init__(self, input_size, output_size, hiddens=None, activation='relu', device='cpu'):\n",
        "        super(DiagNormalPolicy, self).__init__()\n",
        "        self.device = device\n",
        "        if hiddens is None:\n",
        "            hiddens = [100, 100]\n",
        "        if activation == 'relu':\n",
        "            activation = nn.ReLU\n",
        "        elif activation == 'tanh':\n",
        "            activation = nn.Tanh\n",
        "        layers = [linear_init(nn.Linear(input_size, hiddens[0])), activation()]\n",
        "        for i, o in zip(hiddens[:-1], hiddens[1:]):\n",
        "            layers.append(linear_init(nn.Linear(i, o)))\n",
        "            layers.append(activation())\n",
        "        layers.append(linear_init(nn.Linear(hiddens[-1], output_size)))\n",
        "        self.mean = nn.Sequential(*layers)\n",
        "        self.sigma = nn.Parameter(torch.Tensor(output_size))\n",
        "        self.sigma.data.fill_(math.log(1))\n",
        "\n",
        "    # def forward(self, state):\n",
        "    #     state = state.to(self.device, non_blocking=True)\n",
        "    #     loc = self.mean(state)\n",
        "    #     scale = torch.exp(torch.clamp(self.sigma, min=math.log(EPSILON)))\n",
        "    #     return Normal(loc=loc, scale=scale)\n",
        "\n",
        "    def density(self, state):\n",
        "        state = state.to(self.device, non_blocking=True)\n",
        "        loc = self.mean(state)\n",
        "        scale = torch.exp(torch.clamp(self.sigma, min=math.log(EPSILON)))\n",
        "        return Normal(loc=loc, scale=scale)\n",
        "\n",
        "    def log_prob(self, state, action):\n",
        "        density = self.density(state)\n",
        "        return density.log_prob(action).mean(dim=1, keepdim=True)\n",
        "\n",
        "    def forward(self, state):\n",
        "        density = self.density(state)\n",
        "        action = density.sample()\n",
        "        return action\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9x6TJOktOIcF"
      },
      "outputs": [],
      "source": [
        "def layer_init(layer, std=np.sqrt(2), bias_const=0.0):\n",
        "    torch.nn.init.orthogonal_(layer.weight, std)\n",
        "    torch.nn.init.constant_(layer.bias, bias_const)\n",
        "    return layer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MDEKGt5KONHM"
      },
      "outputs": [],
      "source": [
        "class Actor(nn.Module):\n",
        "    def __init__(self, env, hidden_size=100):\n",
        "        super().__init__()\n",
        "        self.input_size = env.observation_space.shape[0]\n",
        "        self.actor_output_size = env.action_space.shape[0]\n",
        "\n",
        "        self.l1 = layer_init(nn.Linear(self.input_size, hidden_size))\n",
        "        self.l2 = layer_init(nn.Linear(hidden_size, hidden_size))\n",
        "        self.output = layer_init(nn.Linear(hidden_size, self.actor_output_size), std=0.01)\n",
        "        self.activation = nn.ReLU()\n",
        "        self.distribution = ch.distributions.ActionDistribution(env)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.activation(self.l1(x))\n",
        "        x = self.activation(self.l2(x))\n",
        "        x = self.output(x)\n",
        "        mass = self.distribution(x)\n",
        "\n",
        "        return mass"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dbv0dy_WOOf9"
      },
      "outputs": [],
      "source": [
        "class Critic(nn.Module):\n",
        "    def __init__(self, env, lr, hidden_size=32):\n",
        "        super().__init__()\n",
        "        self.input_size = env.observation_space.shape[0]\n",
        "        self.critic_output_size = 1\n",
        "\n",
        "        self.l1 = layer_init(nn.Linear(self.input_size, hidden_size))\n",
        "        self.l2 = layer_init(nn.Linear(hidden_size, hidden_size))\n",
        "        self.critic_head = layer_init(nn.Linear(hidden_size, self.critic_output_size), std=1.)\n",
        "        self.activation = nn.ReLU()\n",
        "\n",
        "        self.optimizer = torch.optim.Adam(self.parameters(), lr=lr, eps=1e-5)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.activation(self.l1(x))\n",
        "        x = self.activation(self.l2(x))\n",
        "        value = self.critic_head(x)\n",
        "\n",
        "        return value"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YIPEWKrndmIl"
      },
      "outputs": [],
      "source": [
        "class MAMLTRPO():\n",
        "    def __init__(self, env_name,\n",
        "                 actor_class=Actor, critic_class=Critic, \n",
        "                 actor_args=dict(), critic_args=dict(),\n",
        "                 adapt_lr=0.1, meta_lr=1.0, \n",
        "                 adapt_steps=3,\n",
        "                 adapt_batch_size=20, meta_batch_size=20,\n",
        "                 gamma=0.95, tau=1.0,\n",
        "                 backtrack_factor=0.5, ls_max_steps=15, max_kl=0.01,\n",
        "                 num_workers=10,\n",
        "                 seed=42,\n",
        "                 device=None, name=\"MAMLTRPO\", tensorboard_log=\"./logs\"):\n",
        "        \n",
        "        random.seed(seed)\n",
        "        np.random.seed(seed)\n",
        "        torch.manual_seed(seed)\n",
        "\n",
        "        if torch.cuda.is_available():\n",
        "            self.device = torch.device(\"cuda\")\n",
        "            torch.cuda.manual_seed(seed)\n",
        "        else:\n",
        "            self.device = torch.device(\"cpu\")\n",
        "        if device:\n",
        "            self.device = torch.device(device)\n",
        "        print(\"Running on: \" + str(self.device))\n",
        "\n",
        "        def make_env():\n",
        "            env = gym.make(env_name)\n",
        "            env = ch.envs.ActionSpaceScaler(env)\n",
        "            return env\n",
        "\n",
        "        env = l2l.gym.AsyncVectorEnv([make_env for _ in range(num_workers)])\n",
        "        env.seed(seed)\n",
        "        env.set_task(env.sample_tasks(1)[0])\n",
        "        self.env = ch.envs.Torch(env)\n",
        "\n",
        "        self.gamma = gamma\n",
        "        self.tau = tau\n",
        "        self.adapt_lr = adapt_lr\n",
        "        self.meta_lr = meta_lr\n",
        "        self.adapt_steps = adapt_steps\n",
        "        self.adapt_batch_size = adapt_batch_size\n",
        "        self.meta_batch_size = meta_batch_size\n",
        "        self.backtrack_factor = backtrack_factor\n",
        "        self.ls_max_steps = ls_max_steps\n",
        "        self.max_kl = max_kl\n",
        "        self.global_iteration = 0\n",
        "\n",
        "        # self.policy = Actor(env, **actor_args).to(device)\n",
        "        # self.baseline = Critic(env, lr=0.001, **critic_args).to(device)\n",
        "        self.policy = DiagNormalPolicy(self.env.state_size, self.env.action_size, device=self.device)\n",
        "        self.baseline = LinearValue(self.env.state_size, self.env.action_size)\n",
        "\n",
        "        self.policy.to(self.device)\n",
        "        self.baseline.to(self.device)\n",
        "\n",
        "        if tensorboard_log is not None:\n",
        "            self.run_name = name + \"_\" + str(int(time.time()))\n",
        "            self.writer = SummaryWriter(f\"{tensorboard_log}/{self.run_name}\")\n",
        "        else:\n",
        "            self.writer = None\n",
        "\n",
        "    \n",
        "    def save(self, path=\"./\"):\n",
        "        torch.save(self.baseline.state_dict(), path + \"/baseline.pt\")\n",
        "        torch.save(self.policy.state_dict(), path + \"/policy.pt\")\n",
        "\n",
        "\n",
        "    def load(self, path=\"./\"):\n",
        "        self.baseline.load_state_dict(torch.load(path + \"/baseline.pt\"))\n",
        "        self.policy.load_state_dict(torch.load(path + \"/policy.pt\"))\n",
        "\n",
        "\n",
        "    def collect_steps(self, policy, n_episodes):\n",
        "        self.env.reset()\n",
        "        task = ch.envs.Runner(self.env)\n",
        "        replay = task.run(policy, episodes=n_episodes).to(self.device)\n",
        "\n",
        "        returns = ch.td.discount(self.gamma, replay.reward(), replay.done())\n",
        "        self.baseline.fit(replay.state(), returns)\n",
        "        values = self.baseline(replay.state())\n",
        "        next_values = self.baseline(replay.next_state())\n",
        "        bootstraps = values * (1.0 - replay.done()) + next_values * replay.done()\n",
        "        next_value = torch.zeros(1, device=values.device)\n",
        "        advantages = ch.pg.generalized_advantage(tau=self.tau,\n",
        "                                        gamma=self.gamma,\n",
        "                                        rewards=replay.reward(),\n",
        "                                        dones=replay.done(),\n",
        "                                        values=bootstraps,\n",
        "                                        next_value=next_value)\n",
        "        advantages = ch.normalize(advantages, epsilon=1e-8).detach()\n",
        "\n",
        "        for i, sars in enumerate(replay):\n",
        "            sars.returns = returns[i]\n",
        "            sars.advantage = advantages[i]\n",
        "\n",
        "        return replay\n",
        "\n",
        "\n",
        "    # def collect_steps(self, policy, n_episodes):\n",
        "    #     # replay = ch.ExperienceReplay(device=self.device)\n",
        "    #     # for i in range(n_episodes):\n",
        "    #     #     state = self.env.reset()\n",
        "\n",
        "    #     #     while True:\n",
        "    #     #         with torch.no_grad():\n",
        "    #     #             mass = policy(state)\n",
        "    #     #         action = mass.sample()\n",
        "    #     #         log_prob = mass.log_prob(action).mean(dim=1, keepdim=True)\n",
        "    #     #         next_state, reward, done, _ = self.env.step(action)\n",
        "\n",
        "    #     #         replay.append(state,\n",
        "    #     #                     action,\n",
        "    #     #                     reward,\n",
        "    #     #                     next_state,\n",
        "    #     #                     done,\n",
        "    #     #                     log_prob=log_prob)\n",
        "                \n",
        "    #     #         if done.any():\n",
        "    #     #             break\n",
        "\n",
        "    #     #         state = next_state\n",
        "        \n",
        "    #     self.env.reset()\n",
        "    #     task = ch.envs.Runner(self.env)\n",
        "    #     replay = task.run(policy, episodes=n_episodes).to(self.device)\n",
        "\n",
        "    #     with torch.no_grad():\n",
        "    #         next_state_value = self.baseline(replay[-1].next_state)\n",
        "    #     values = self.baseline(replay.state())\n",
        "\n",
        "    #     advantages = ch.generalized_advantage(self.gamma,\n",
        "    #                                             self.tau,\n",
        "    #                                             replay.reward(),\n",
        "    #                                             replay.done(),\n",
        "    #                                             values.detach(),\n",
        "    #                                             next_state_value)\n",
        "    #     returns = advantages + values.detach()\n",
        "    #     advantages = ch.normalize(advantages, epsilon=1e-8)\n",
        "\n",
        "    #     for i, sars in enumerate(replay):\n",
        "    #         sars.returns = returns[i]\n",
        "    #         sars.advantage = advantages[i]\n",
        "\n",
        "    #     # value_loss = a2c.state_value_loss(returns, values)\n",
        "    #     # self.baseline.optimizer.zero_grad()\n",
        "    #     # value_loss.backward()\n",
        "    #     # self.baseline.optimizer.step()\n",
        "\n",
        "    #     self.baseline.fit(replay.state(), returns)\n",
        "    #     return replay\n",
        "\n",
        "\n",
        "    def maml_a2c_loss(self, train_episodes, learner):\n",
        "        # Update policy and baseline\n",
        "        states = train_episodes.state()\n",
        "        actions = train_episodes.action()\n",
        "        density = learner.density(states)\n",
        "        log_probs = density.log_prob(actions).mean(dim=1, keepdim=True)\n",
        "\n",
        "        advantages = train_episodes.advantage()\n",
        "        return a2c.policy_loss(log_probs, train_episodes.advantage())\n",
        "\n",
        "\n",
        "    def fast_adapt(self, clone, train_episodes, first_order=False):\n",
        "        second_order = not first_order\n",
        "        loss = self.maml_a2c_loss(train_episodes, clone)\n",
        "        gradients = autograd.grad(loss,\n",
        "                                clone.parameters(),\n",
        "                                retain_graph=second_order,\n",
        "                                create_graph=second_order)\n",
        "        return l2l.algorithms.maml.maml_update(clone, self.adapt_lr, gradients)\n",
        "\n",
        "\n",
        "    def meta_surrogate_loss(self, iteration_replays, iteration_policies, policy):\n",
        "        mean_loss = 0.0\n",
        "        mean_kl = 0.0\n",
        "        for task_replays, old_policy in tqdm(zip(iteration_replays, iteration_policies),\n",
        "                                            total=len(iteration_replays),\n",
        "                                            desc='Surrogate Loss',\n",
        "                                            leave=False):\n",
        "            train_replays = task_replays[:-1]\n",
        "            valid_episodes = task_replays[-1]\n",
        "            new_policy = l2l.clone_module(policy)\n",
        "\n",
        "            # Fast Adapt\n",
        "            for train_episodes in train_replays:\n",
        "                new_policy = self.fast_adapt(new_policy, train_episodes, first_order=False)\n",
        "\n",
        "            # Useful values\n",
        "            states = valid_episodes.state()\n",
        "            actions = valid_episodes.action()\n",
        "\n",
        "            # Compute KL\n",
        "            old_densities = old_policy.density(states)\n",
        "            new_densities = new_policy.density(states)\n",
        "            kl = kl_divergence(new_densities, old_densities).mean()\n",
        "            mean_kl += kl\n",
        "\n",
        "            # Compute Surrogate Loss\n",
        "            advantages = valid_episodes.advantage()\n",
        "            old_log_probs = old_densities.log_prob(actions).mean(dim=1, keepdim=True).detach()\n",
        "            new_log_probs = new_densities.log_prob(actions).mean(dim=1, keepdim=True)\n",
        "            mean_loss += trpo.policy_loss(new_log_probs, old_log_probs, advantages)\n",
        "        mean_kl /= len(iteration_replays)\n",
        "        mean_loss /= len(iteration_replays)\n",
        "        return mean_loss, mean_kl\n",
        "\n",
        "\n",
        "    def meta_optimize(self, iteration_replays, iteration_policies):\n",
        "        # Compute CG step direction\n",
        "        old_loss, old_kl = self.meta_surrogate_loss(iteration_replays, iteration_policies, self.policy)\n",
        "\n",
        "        grad = autograd.grad(old_loss,\n",
        "                                self.policy.parameters(),\n",
        "                                retain_graph=True)\n",
        "        grad = parameters_to_vector([g.detach() for g in grad])\n",
        "        Fvp = trpo.hessian_vector_product(old_kl, self.policy.parameters())\n",
        "        step = trpo.conjugate_gradient(Fvp, grad)\n",
        "        shs = 0.5 * torch.dot(step, Fvp(step))\n",
        "        lagrange_multiplier = torch.sqrt(shs / self.max_kl)\n",
        "        step = step / lagrange_multiplier\n",
        "        step_ = [torch.zeros_like(p.data) for p in self.policy.parameters()]\n",
        "        vector_to_parameters(step, step_)\n",
        "        step = step_\n",
        "        del old_kl, Fvp, grad\n",
        "        old_loss.detach_()\n",
        "\n",
        "        # Line-search\n",
        "        for ls_step in range(self.ls_max_steps):\n",
        "            stepsize = self.backtrack_factor ** ls_step * self.meta_lr\n",
        "            clone = deepcopy(self.policy)\n",
        "            for p, u in zip(clone.parameters(), step):\n",
        "                p.data.add_(-stepsize, u.data)\n",
        "            new_loss, kl = self.meta_surrogate_loss(iteration_replays, iteration_policies, clone)\n",
        "            if new_loss < old_loss and kl < self.max_kl:\n",
        "                for p, u in zip(self.policy.parameters(), step):\n",
        "                    p.data.add_(-stepsize, u.data)\n",
        "                break\n",
        "\n",
        "        if self.writer is not None:\n",
        "            self.writer.add_scalar(\"loss\", new_loss, self.global_iteration)\n",
        "            self.writer.add_scalar(\"kl\", kl, self.global_iteration)\n",
        "\n",
        "\n",
        "    def train(self, num_iterations=100):\n",
        "        for iteration in range(num_iterations):\n",
        "            self.global_iteration += 1\n",
        "            iteration_reward = 0.0\n",
        "            iteration_replays = []\n",
        "            iteration_policies = []\n",
        "\n",
        "            for task_config in tqdm(self.env.sample_tasks(self.meta_batch_size), leave=False, desc='Data'):\n",
        "                clone = deepcopy(self.policy)\n",
        "                self.env.set_task(task_config)\n",
        "                task_replay = []\n",
        "\n",
        "                # Fast Adapt\n",
        "                for step in range(self.adapt_steps):\n",
        "                    train_episodes = self.collect_steps(clone, n_episodes=self.adapt_batch_size)\n",
        "                    self.fast_adapt(clone, train_episodes, first_order=True)\n",
        "                    # self.fast_adapt(clone, train_episodes, first_order=True)\n",
        "                    task_replay.append(train_episodes)\n",
        "\n",
        "                # Compute Validation Loss\n",
        "                valid_episodes = self.collect_steps(clone, n_episodes=self.adapt_batch_size)\n",
        "                task_replay.append(valid_episodes)\n",
        "                iteration_reward += valid_episodes.reward().sum().item() / self.adapt_batch_size\n",
        "                iteration_replays.append(task_replay)\n",
        "                iteration_policies.append(clone)\n",
        "\n",
        "            # Print statistics\n",
        "            print('\\nIteration', self.global_iteration)\n",
        "            adaptation_reward = iteration_reward / self.meta_batch_size\n",
        "            print('adaptation_reward', adaptation_reward)\n",
        "\n",
        "            if self.writer is not None:\n",
        "                self.writer.add_scalar(\"adaptation_reward\", adaptation_reward, self.global_iteration)\n",
        "\n",
        "            self.meta_optimize(iteration_replays, iteration_policies)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qhHiZBGTPkJZ",
        "outputId": "abf56e2d-499f-4fdd-b26a-7eaa6c9e5a77"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Running on: cuda\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
            "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
          ]
        }
      ],
      "source": [
        "metalearner = MAMLTRPO(\"HalfCheetahForwardBackward-v1\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1UAU7YG-Pn34",
        "outputId": "d18faf1e-b4c6-4d80-8c67-50d0ffde4238"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 1\n",
            "adaptation_reward -8.28092391014099\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:234: UserWarning: This overload of add_ is deprecated:\n",
            "\tadd_(Number alpha, Tensor other)\n",
            "Consider using one of the following signatures instead:\n",
            "\tadd_(Tensor other, *, Number alpha) (Triggered internally at  ../torch/csrc/utils/python_arg_parser.cpp:1174.)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 2\n",
            "adaptation_reward -0.34856413841247547\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 3\n",
            "adaptation_reward 2.080847492218017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 4\n",
            "adaptation_reward -0.5971503734588618\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 5\n",
            "adaptation_reward -1.7480440235137933\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 6\n",
            "adaptation_reward 1.8982534980773924\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 7\n",
            "adaptation_reward 11.04148238182068\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 8\n",
            "adaptation_reward 10.341353340148926\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 9\n",
            "adaptation_reward 17.28411301612854\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 10\n",
            "adaptation_reward 14.976770763397216\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 11\n",
            "adaptation_reward 19.711226229667666\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 12\n",
            "adaptation_reward 13.943080797195435\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 13\n",
            "adaptation_reward 17.968124423027035\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 14\n",
            "adaptation_reward 20.62150827407837\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 15\n",
            "adaptation_reward 17.79564896583557\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 16\n",
            "adaptation_reward 22.115227460861206\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 17\n",
            "adaptation_reward 23.308125534057613\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 18\n",
            "adaptation_reward 25.411133508682248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 19\n",
            "adaptation_reward 30.91326232910156\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 20\n",
            "adaptation_reward 17.19877754688263\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 21\n",
            "adaptation_reward 26.66049586296081\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 22\n",
            "adaptation_reward 26.200458589792255\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 23\n",
            "adaptation_reward 28.52642942428589\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 24\n",
            "adaptation_reward 29.991001434326172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 25\n",
            "adaptation_reward 36.24410778045654\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 26\n",
            "adaptation_reward 25.23564931869507\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 27\n",
            "adaptation_reward 37.48611532211304\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 28\n",
            "adaptation_reward 36.27908054351807\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 29\n",
            "adaptation_reward 34.98307246685028\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 30\n",
            "adaptation_reward 36.3601388168335\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 31\n",
            "adaptation_reward 48.249526987075804\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 32\n",
            "adaptation_reward 48.20226264953613\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 33\n",
            "adaptation_reward 55.135260162353504\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 34\n",
            "adaptation_reward 37.87402187347412\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 35\n",
            "adaptation_reward 42.60499000549316\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 36\n",
            "adaptation_reward 55.382625274658196\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 37\n",
            "adaptation_reward 50.53114169120788\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 38\n",
            "adaptation_reward 53.6613903427124\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 39\n",
            "adaptation_reward 53.0059388923645\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 40\n",
            "adaptation_reward 71.97300415039062\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 41\n",
            "adaptation_reward 64.63682746887207\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 42\n",
            "adaptation_reward 71.24325111389162\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 43\n",
            "adaptation_reward 72.58702003479003\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 44\n",
            "adaptation_reward 70.17856033325195\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 45\n",
            "adaptation_reward 72.90884925842285\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 46\n",
            "adaptation_reward 71.92757469177248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 47\n",
            "adaptation_reward 77.81718391418457\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 48\n",
            "adaptation_reward 89.65004516601563\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 49\n",
            "adaptation_reward 38.41054718017578\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 50\n",
            "adaptation_reward 79.5917359161377\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 51\n",
            "adaptation_reward 79.2995180130005\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 52\n",
            "adaptation_reward 100.92394172668459\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 53\n",
            "adaptation_reward 109.95098472595217\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 54\n",
            "adaptation_reward 92.09172195434573\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 55\n",
            "adaptation_reward 75.43868385314943\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 56\n",
            "adaptation_reward 84.92533065795898\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 57\n",
            "adaptation_reward 103.89073081970214\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 58\n",
            "adaptation_reward 80.22187103271483\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 59\n",
            "adaptation_reward 99.12112686157226\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 60\n",
            "adaptation_reward 94.24565719604493\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 61\n",
            "adaptation_reward 97.87074371337891\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 62\n",
            "adaptation_reward 97.86229125976561\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 63\n",
            "adaptation_reward 99.85719619750978\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 64\n",
            "adaptation_reward 119.92230667114256\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 65\n",
            "adaptation_reward 99.23022994995118\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 66\n",
            "adaptation_reward 109.11082901000978\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 67\n",
            "adaptation_reward 103.466604309082\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 68\n",
            "adaptation_reward 95.31423065185545\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 69\n",
            "adaptation_reward 101.64668262481689\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 70\n",
            "adaptation_reward 105.79614547729493\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 71\n",
            "adaptation_reward 131.46594146728518\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 72\n",
            "adaptation_reward 93.83898712158204\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 73\n",
            "adaptation_reward 119.8161575317383\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 74\n",
            "adaptation_reward 129.66812576293944\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 75\n",
            "adaptation_reward 114.6803157043457\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 76\n",
            "adaptation_reward 126.83172302246093\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 77\n",
            "adaptation_reward 126.06679794311526\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 78\n",
            "adaptation_reward 118.24338623046876\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 79\n",
            "adaptation_reward 106.25312591552736\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 80\n",
            "adaptation_reward 108.03101013183593\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 81\n",
            "adaptation_reward 129.00718734741213\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 82\n",
            "adaptation_reward 136.9614614868164\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 83\n",
            "adaptation_reward 125.6036264038086\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 84\n",
            "adaptation_reward 126.58405395507812\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 85\n",
            "adaptation_reward 132.63417694091794\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 86\n",
            "adaptation_reward 112.66816223144531\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 87\n",
            "adaptation_reward 123.53140808105468\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 88\n",
            "adaptation_reward 152.90267028808594\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 89\n",
            "adaptation_reward 101.66463790893553\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 90\n",
            "adaptation_reward 120.25252044677737\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 91\n",
            "adaptation_reward 122.0551712036133\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 92\n",
            "adaptation_reward 107.14428894042969\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 93\n",
            "adaptation_reward 121.33952697753907\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 94\n",
            "adaptation_reward 115.83540771484374\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 95\n",
            "adaptation_reward 135.24816589355473\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 96\n",
            "adaptation_reward 118.84354217529297\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 97\n",
            "adaptation_reward 122.44311462402341\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 98\n",
            "adaptation_reward 150.19780670166017\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 99\n",
            "adaptation_reward 124.96628875732422\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 100\n",
            "adaptation_reward 143.41025878906248\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 101\n",
            "adaptation_reward 128.2265719604492\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 102\n",
            "adaptation_reward 136.34365600585934\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 103\n",
            "adaptation_reward 142.16400939941408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 104\n",
            "adaptation_reward 133.19400268554688\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 105\n",
            "adaptation_reward 129.6954800415039\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 106\n",
            "adaptation_reward 136.14401367187503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 107\n",
            "adaptation_reward 122.77431167602538\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 108\n",
            "adaptation_reward 133.239375\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 109\n",
            "adaptation_reward 134.83941772460938\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 110\n",
            "adaptation_reward 147.72953155517578\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 111\n",
            "adaptation_reward 128.5400161743164\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 112\n",
            "adaptation_reward 149.35482818603512\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 113\n",
            "adaptation_reward 125.15053131103517\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 114\n",
            "adaptation_reward 126.9016830444336\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 115\n",
            "adaptation_reward 148.26574218750002\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 116\n",
            "adaptation_reward 143.71024047851563\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 117\n",
            "adaptation_reward 136.3840689086914\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 118\n",
            "adaptation_reward 126.05021697998048\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 119\n",
            "adaptation_reward 109.76238739013672\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 120\n",
            "adaptation_reward 140.29455780029295\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 121\n",
            "adaptation_reward 145.6919470214844\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 122\n",
            "adaptation_reward 136.8255487060547\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 123\n",
            "adaptation_reward 145.49706024169922\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 124\n",
            "adaptation_reward 149.16690704345703\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 125\n",
            "adaptation_reward 122.23747314453121\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 126\n",
            "adaptation_reward 134.81828521728514\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 127\n",
            "adaptation_reward 139.4824737548828\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 128\n",
            "adaptation_reward 118.47483276367186\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 129\n",
            "adaptation_reward 125.37716156005858\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 130\n",
            "adaptation_reward 146.32498046875003\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 131\n",
            "adaptation_reward 130.77628509521486\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 132\n",
            "adaptation_reward 132.6172528076172\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 133\n",
            "adaptation_reward 136.2187088012695\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 134\n",
            "adaptation_reward 152.79599586486816\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 135\n",
            "adaptation_reward 111.99664566040039\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 136\n",
            "adaptation_reward 138.6254864501953\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 137\n",
            "adaptation_reward 123.96364501953124\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 138\n",
            "adaptation_reward 147.68623992919922\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 139\n",
            "adaptation_reward 114.55794769287111\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 140\n",
            "adaptation_reward 146.8922604370117\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 141\n",
            "adaptation_reward 125.73159271240232\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 142\n",
            "adaptation_reward 146.35274490356443\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 143\n",
            "adaptation_reward 145.07391937255858\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 144\n",
            "adaptation_reward 138.98472991943356\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 145\n",
            "adaptation_reward 139.694778137207\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 146\n",
            "adaptation_reward 130.5045462036133\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 147\n",
            "adaptation_reward 155.26862365722656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 148\n",
            "adaptation_reward 151.41507919311522\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 149\n",
            "adaptation_reward 153.05163543701173\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 150\n",
            "adaptation_reward 124.78086975097658\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 151\n",
            "adaptation_reward 148.61216430664064\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 152\n",
            "adaptation_reward 141.75603240966797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 153\n",
            "adaptation_reward 134.48184783935545\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 154\n",
            "adaptation_reward 139.4728946495056\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 155\n",
            "adaptation_reward 132.10548950195312\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 156\n",
            "adaptation_reward 130.5357879638672\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 157\n",
            "adaptation_reward 135.4056819152832\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 158\n",
            "adaptation_reward 148.00340179443361\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 159\n",
            "adaptation_reward 137.65231994628905\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 160\n",
            "adaptation_reward 159.55751281738281\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 161\n",
            "adaptation_reward 123.18832916259764\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 162\n",
            "adaptation_reward 122.95544776916503\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 163\n",
            "adaptation_reward 140.3839978027344\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 164\n",
            "adaptation_reward 130.6942874145508\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 165\n",
            "adaptation_reward 136.51457015991213\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 166\n",
            "adaptation_reward 148.25066467285154\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 167\n",
            "adaptation_reward 121.42385833740234\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 168\n",
            "adaptation_reward 146.7616128540039\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 169\n",
            "adaptation_reward 128.13209068298343\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 170\n",
            "adaptation_reward 132.6983905029297\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 171\n",
            "adaptation_reward 141.76994323730472\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 172\n",
            "adaptation_reward 162.26768615722654\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 173\n",
            "adaptation_reward 139.9373455810547\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 174\n",
            "adaptation_reward 148.18710937500003\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 175\n",
            "adaptation_reward 146.22262979507448\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 176\n",
            "adaptation_reward 145.0174154663086\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 177\n",
            "adaptation_reward 153.28139739990235\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 178\n",
            "adaptation_reward 123.85475067138671\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 179\n",
            "adaptation_reward 157.55834472656252\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 180\n",
            "adaptation_reward 156.54908020019533\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 181\n",
            "adaptation_reward 135.69312530517576\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 182\n",
            "adaptation_reward 146.76217895507813\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 183\n",
            "adaptation_reward 125.34659515380858\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 184\n",
            "adaptation_reward 141.78922424316406\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 185\n",
            "adaptation_reward 154.67402893066406\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 186\n",
            "adaptation_reward 131.17592987060544\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 187\n",
            "adaptation_reward 155.29208404541015\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 188\n",
            "adaptation_reward 149.42689178466796\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 189\n",
            "adaptation_reward 138.42941284179685\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 190\n",
            "adaptation_reward 145.46338439941408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 191\n",
            "adaptation_reward 147.56166473388672\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 192\n",
            "adaptation_reward 137.19742553710935\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 193\n",
            "adaptation_reward 129.47724914550784\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 194\n",
            "adaptation_reward 144.3224789428711\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 195\n",
            "adaptation_reward 154.6925854492187\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 196\n",
            "adaptation_reward 147.83824768066407\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 197\n",
            "adaptation_reward 162.41989044189452\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 198\n",
            "adaptation_reward 142.7805499267578\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 199\n",
            "adaptation_reward 142.9035186767578\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 200\n",
            "adaptation_reward 124.37760345458985\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 201\n",
            "adaptation_reward 153.69385101318358\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 202\n",
            "adaptation_reward 141.27465972900393\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 203\n",
            "adaptation_reward 169.20961242675784\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 204\n",
            "adaptation_reward 164.15261322021485\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 205\n",
            "adaptation_reward 141.84031677246094\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 206\n",
            "adaptation_reward 147.3411013793945\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 207\n",
            "adaptation_reward 123.78508422851561\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 208\n",
            "adaptation_reward 151.3322821044922\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 209\n",
            "adaptation_reward 150.3736212158203\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 210\n",
            "adaptation_reward 137.52212036132815\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 211\n",
            "adaptation_reward 156.87165374755858\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 212\n",
            "adaptation_reward 146.26910766601563\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 213\n",
            "adaptation_reward 150.00325378417972\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 214\n",
            "adaptation_reward 142.3878579711914\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 215\n",
            "adaptation_reward 145.81812622070314\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 216\n",
            "adaptation_reward 159.06810913085934\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 217\n",
            "adaptation_reward 151.41071960449221\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 218\n",
            "adaptation_reward 156.3450506591797\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 219\n",
            "adaptation_reward 163.73144134521485\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 220\n",
            "adaptation_reward 134.52774917602534\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 221\n",
            "adaptation_reward 159.0822589111328\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 222\n",
            "adaptation_reward 151.38710998535154\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 223\n",
            "adaptation_reward 146.38701202392576\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 224\n",
            "adaptation_reward 154.287197265625\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 225\n",
            "adaptation_reward 154.50495422363278\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 226\n",
            "adaptation_reward 147.7494387817383\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 227\n",
            "adaptation_reward 142.8959823608398\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 228\n",
            "adaptation_reward 151.56300567626954\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 229\n",
            "adaptation_reward 161.15469848632813\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 230\n",
            "adaptation_reward 143.10298706054687\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 231\n",
            "adaptation_reward 183.70361480712893\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 232\n",
            "adaptation_reward 148.4655429077148\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 233\n",
            "adaptation_reward 156.69342071533202\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 234\n",
            "adaptation_reward 163.63286682128907\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 235\n",
            "adaptation_reward 151.98829040527343\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 236\n",
            "adaptation_reward 162.68623535156252\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 237\n",
            "adaptation_reward 163.16104125976562\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 238\n",
            "adaptation_reward 140.95082336425781\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 239\n",
            "adaptation_reward 146.93634796142575\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 240\n",
            "adaptation_reward 155.5125048828125\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 241\n",
            "adaptation_reward 139.80081008911134\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 242\n",
            "adaptation_reward 141.05599273681642\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 243\n",
            "adaptation_reward 160.35296409606934\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 244\n",
            "adaptation_reward 154.09287811279302\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 245\n",
            "adaptation_reward 160.75592712402343\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 246\n",
            "adaptation_reward 156.80777893066409\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 247\n",
            "adaptation_reward 160.21907653808597\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 248\n",
            "adaptation_reward 145.6268603515625\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 249\n",
            "adaptation_reward 153.14061279296874\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 250\n",
            "adaptation_reward 161.16396972656247\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 251\n",
            "adaptation_reward 174.97597076416014\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 252\n",
            "adaptation_reward 129.29023071289063\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 253\n",
            "adaptation_reward 159.71306335449214\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 254\n",
            "adaptation_reward 156.6143997192383\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 255\n",
            "adaptation_reward 138.94787590026854\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 256\n",
            "adaptation_reward 140.1086343383789\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 257\n",
            "adaptation_reward 160.68875671386718\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 258\n",
            "adaptation_reward 157.9180520629883\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 259\n",
            "adaptation_reward 131.0152032470703\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 260\n",
            "adaptation_reward 161.23407012939452\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 261\n",
            "adaptation_reward 139.00652648925782\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 262\n",
            "adaptation_reward 158.52531951904297\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 263\n",
            "adaptation_reward 174.00380737304687\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 264\n",
            "adaptation_reward 161.88855804443364\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 265\n",
            "adaptation_reward 150.66029998779297\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 266\n",
            "adaptation_reward 154.4602914428711\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 267\n",
            "adaptation_reward 140.4063949584961\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 268\n",
            "adaptation_reward 148.39608032226562\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 269\n",
            "adaptation_reward 143.58118453979492\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 270\n",
            "adaptation_reward 146.07932678222656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 271\n",
            "adaptation_reward 155.29214904785152\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 272\n",
            "adaptation_reward 154.03672103881837\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 273\n",
            "adaptation_reward 153.18362350463866\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 274\n",
            "adaptation_reward 151.95979553222656\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 275\n",
            "adaptation_reward 155.2203369140625\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 276\n",
            "adaptation_reward 149.58294219970702\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 277\n",
            "adaptation_reward 134.40002822875977\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 278\n",
            "adaptation_reward 154.60589904785155\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 279\n",
            "adaptation_reward 146.1709017944336\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 280\n",
            "adaptation_reward 153.31843170166013\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 281\n",
            "adaptation_reward 153.01593872070316\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 282\n",
            "adaptation_reward 148.1270901489258\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 283\n",
            "adaptation_reward 157.2264254760742\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 284\n",
            "adaptation_reward 175.57748901367188\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 285\n",
            "adaptation_reward 144.26090423583986\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 286\n",
            "adaptation_reward 166.17122589111327\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 287\n",
            "adaptation_reward 151.31627212524413\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 288\n",
            "adaptation_reward 156.9271664428711\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 289\n",
            "adaptation_reward 154.16096450805665\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 290\n",
            "adaptation_reward 153.01113563537595\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 291\n",
            "adaptation_reward 155.10679199218754\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 292\n",
            "adaptation_reward 160.29535888671876\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 293\n",
            "adaptation_reward 154.7941296386719\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 294\n",
            "adaptation_reward 163.816259765625\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 295\n",
            "adaptation_reward 167.83766113281249\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 296\n",
            "adaptation_reward 170.13136779785157\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 297\n",
            "adaptation_reward 158.50057220458982\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 298\n",
            "adaptation_reward 153.27674530029296\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 299\n",
            "adaptation_reward 162.37435668945315\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Iteration 300\n",
            "adaptation_reward 138.08816619873045\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            ""
          ]
        }
      ],
      "source": [
        "metalearner.train(300)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "metalearner.save()"
      ],
      "metadata": {
        "id": "3tzQ1hJ-uxEF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('/content/logs/MAMLTRPO_1659815445/events.out.tfevents.1659815447.214887446a74.73.0')\n",
        "files.download('baseline.pt') \n",
        "files.download('policy.pt')"
      ],
      "metadata": {
        "id": "Gh4peXhF2qYv"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "New MAML TRPO.ipynb",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}