notebooks/linear-typing-using-atom-features-only-student-t-loss.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:54:37.543932Z",
     "start_time": "2019-03-07T15:54:29.384039Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of molecules being considered: 642\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:54:37.743117Z",
     "start_time": "2019-03-07T15:54:37.546567Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:54:37.749058Z",
     "start_time": "2019-03-07T15:54:37.745034Z"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:20.673094Z",
     "start_time": "2019-03-07T15:55:19.081967Z"
    }
   },
   "outputs": [],
   "source": [
    "from bayes_implicit_solvent.constants import beta\n",
    "def unreduce(value):\n",
    "    \"\"\"Input value is in units of kB T, turn it into units of kilocalorie_per_mole\"\"\"\n",
    "    return value / (beta * unit.kilocalorie_per_mole)\n",
    "\n",
    "from bayes_implicit_solvent.continuous_parameter_experiments.gradient_free import mols as all_mols\n",
    "# at this stage, let's eliminate any molecule that has an experimental value outside of [-15, +5]\n",
    "from autograd import numpy as np\n",
    "mols = []\n",
    "    \n",
    "from bayes_implicit_solvent.utils import get_charges\n",
    "from scipy.spatial.distance import pdist, squareform\n",
    "from simtk import unit\n",
    "elements = []\n",
    "charges = []\n",
    "distance_matrices = []\n",
    "expt_means = []\n",
    "expt_uncs = []\n",
    "vacuum_trajs = []\n",
    "for mol in all_mols:\n",
    "    \n",
    "    expt_mean = unreduce(mol.experimental_value)\n",
    "    expt_unc = unreduce(mol.experimental_uncertainty)\n",
    "    \n",
    "    if (expt_mean > -15) and (expt_mean < 5):\n",
    "        \n",
    "        mols.append(mol)\n",
    "        expt_means.append(expt_mean)\n",
    "        expt_uncs.append(expt_unc)\n",
    "\n",
    "        elements.append(np.array([a.element.atomic_number for a in mol.top.atoms()]))\n",
    "        charges.append(get_charges(mol.sys))\n",
    "        distance_matrices.append([squareform(pdist(snapshot / unit.nanometer)) for snapshot in mol.vacuum_traj])\n",
    "expt_means = np.array(expt_means)\n",
    "expt_uncs = np.array(expt_uncs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:20.685737Z",
     "start_time": "2019-03-07T15:55:20.675441Z"
    }
   },
   "outputs": [],
   "source": [
    "# THE CODE IN THIS CELL IS ADAPTED FROM https://github.com/HIPS/neural-fingerprint\n",
    "# ACCOMPANYING \"Convolutional Networks on Graphs for Learning Molecular Fingerprints\"\n",
    "# by David Duvenaud, Dougal Maclaurin, Jorge Aguilera-Iparraguirre, Rafael Gómez-Bombarelli, Timothy Hirzel, Alán Aspuru-Guzik, and Ryan P. Adams.\n",
    "\n",
    "import autograd.numpy as np\n",
    "\n",
    "def one_of_k_encoding(x, allowable_set):\n",
    "    if x not in allowable_set:\n",
    "        raise Exception(\"input {0} not in allowable set{1}:\".format(x, allowable_set))\n",
    "    return list(map(lambda s: x == s, allowable_set))\n",
    "\n",
    "def one_of_k_encoding_unk(x, allowable_set):\n",
    "    \"\"\"Maps inputs not in the allowable set to the last element.\"\"\"\n",
    "    if x not in allowable_set:\n",
    "        x = allowable_set[-1]\n",
    "    return list(map(lambda s: x == s, allowable_set))\n",
    "\n",
    "def atom_features(atom):\n",
    "    \"\"\"Use some OpenEye features instead\"\"\"\n",
    "    return np.array(one_of_k_encoding_unk(atom.GetAtomicNum(),\n",
    "                                      [1, 35, 6, 7, 8, 9, 15, 16, 17, 53, -1]) +\n",
    "                    one_of_k_encoding(atom.GetDegree(), [1, 2, 3, 4]) +\n",
    "                    one_of_k_encoding(atom.GetValence(), [1, 2, 3, 4, 5, 6]) +\n",
    "                    one_of_k_encoding(atom.GetHvyValence(), [0, 1, 2, 3, 4, 5, 6]) +\n",
    "                    one_of_k_encoding(atom.GetHvyDegree(), [0, 1, 2, 3, 4]) +\n",
    "                    one_of_k_encoding(atom.GetTotalHCount(), [0, 1, 2, 3, 4]) +\n",
    "                    [atom.IsAromatic(), atom.IsHalogen(), atom.IsPolarHydrogen(), atom.IsMetal(), atom.IsInRing()] # binary features\n",
    "                   )\n",
    "def num_atom_features():\n",
    "    atoms = list(mols[0].mol.GetAtoms())\n",
    "    return len(atom_features(atoms[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.616838Z",
     "start_time": "2019-03-07T15:55:20.687969Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "631"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compute_features(mol):\n",
    "    return np.vstack([atom_features(a) for a in mol.GetAtoms()])\n",
    "\n",
    "computed_features = list(map(compute_features, [mol.mol for mol in mols]))\n",
    "len(computed_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.624123Z",
     "start_time": "2019-03-07T15:55:22.620259Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "N = num_atom_features()\n",
    "N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.639229Z",
     "start_time": "2019-03-07T15:55:22.627493Z"
    }
   },
   "outputs": [],
   "source": [
    "from autograd.scipy.stats import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.644834Z",
     "start_time": "2019-03-07T15:55:22.641137Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(distance_matrices[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.652065Z",
     "start_time": "2019-03-07T15:55:22.646528Z"
    }
   },
   "outputs": [],
   "source": [
    "N_atoms = np.array(list(map(len, charges)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.658670Z",
     "start_time": "2019-03-07T15:55:22.653577Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "631"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(mols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.665063Z",
     "start_time": "2019-03-07T15:55:22.660766Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_features = N\n",
    "n_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.672493Z",
     "start_time": "2019-03-07T15:55:22.666922Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.15, 0.8 ])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "default_radius = 0.15\n",
    "default_scale = 0.8\n",
    "baseline_bias = np.array([default_radius, default_scale])\n",
    "baseline_bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.677059Z",
     "start_time": "2019-03-07T15:55:22.674393Z"
    }
   },
   "outputs": [],
   "source": [
    "def relu(X):\n",
    "    \"Rectified linear activation function.\"\n",
    "    return X * (X > 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.687055Z",
     "start_time": "2019-03-07T15:55:22.678889Z"
    }
   },
   "outputs": [],
   "source": [
    "import autograd.numpy.random as npr\n",
    "\n",
    "class NeuralNet():\n",
    "\n",
    "    def __init__(self, layer_sizes, scale=0.01):\n",
    "        assert(layer_sizes[-1] == 2)\n",
    "        self.layer_sizes = layer_sizes\n",
    "        self.scale = scale\n",
    "        self.params = self.init_random_params(scale, layer_sizes)\n",
    "\n",
    "    def init_random_params(self, scale, layer_sizes, rs=npr.RandomState(0)):\n",
    "        return [(scale * rs.randn(m, n),  # weight matrix\n",
    "                 scale * rs.randn(n))  # bias vector\n",
    "                for m, n in zip(layer_sizes[:-1], layer_sizes[1:])]\n",
    "\n",
    "    def neural_net_predict(self, params, inputs):\n",
    "        for W, b in params:\n",
    "            outputs = np.dot(inputs, W) + b\n",
    "            inputs = relu(outputs)\n",
    "        return outputs + baseline_bias\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        return self.neural_net_predict(self.params, inputs)\n",
    "    \n",
    "npr.seed(0)\n",
    "layer_sizes = [n_features, 2]\n",
    "neural_net = NeuralNet(layer_sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.692248Z",
     "start_time": "2019-03-07T15:55:22.689288Z"
    }
   },
   "outputs": [],
   "source": [
    "params = neural_net.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.702031Z",
     "start_time": "2019-03-07T15:55:22.694505Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.20804834, 0.79478895],\n",
       "       [0.17475065, 0.79627458],\n",
       "       [0.16188043, 0.79518197],\n",
       "       [0.19613242, 0.82061456],\n",
       "       [0.13902075, 0.77676231],\n",
       "       [0.12769472, 0.78161642],\n",
       "       [0.14320772, 0.79528638],\n",
       "       [0.14320772, 0.79528638],\n",
       "       [0.13902075, 0.77676231],\n",
       "       [0.12769472, 0.78161642],\n",
       "       [0.14320772, 0.79528638],\n",
       "       [0.14320772, 0.79528638],\n",
       "       [0.16188043, 0.79518197],\n",
       "       [0.16188043, 0.79518197],\n",
       "       [0.16188043, 0.79518197],\n",
       "       [0.16188043, 0.79518197],\n",
       "       [0.16188043, 0.79518197]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neural_net(computed_features[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.707665Z",
     "start_time": "2019-03-07T15:55:22.703807Z"
    }
   },
   "outputs": [],
   "source": [
    "from bayes_implicit_solvent.gb_models.numpy_gb_models import compute_OBC_energy_vectorized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.713591Z",
     "start_time": "2019-03-07T15:55:22.708932Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0.20804834, 0.17475065, 0.16188043, 0.19613242, 0.13902075,\n",
       "        0.12769472, 0.14320772, 0.14320772, 0.13902075, 0.12769472,\n",
       "        0.14320772, 0.14320772, 0.16188043, 0.16188043, 0.16188043,\n",
       "        0.16188043, 0.16188043]),\n",
       " array([0.79478895, 0.79627458, 0.79518197, 0.82061456, 0.77676231,\n",
       "        0.78161642, 0.79528638, 0.79528638, 0.77676231, 0.78161642,\n",
       "        0.79528638, 0.79528638, 0.79518197, 0.79518197, 0.79518197,\n",
       "        0.79518197, 0.79518197]))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "radii, scales = neural_net.neural_net_predict(params, computed_features[0]).T\n",
    "radii, scales"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.719073Z",
     "start_time": "2019-03-07T15:55:22.715500Z"
    }
   },
   "outputs": [],
   "source": [
    "npr.seed(0)\n",
    "all_inds = np.arange(len(mols))\n",
    "np.random.shuffle(all_inds)\n",
    "train_inds = all_inds[::2]\n",
    "test_inds = all_inds[1::2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:22.727764Z",
     "start_time": "2019-03-07T15:55:22.723263Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.1382    ,  0.1522    ,  0.1023    ,  0.1233    , -0.3511    ,\n",
       "        0.37990001, -0.1655    , -0.1655    , -0.37040001,  0.3845    ,\n",
       "       -0.1723    , -0.1723    ,  0.0681    ,  0.0681    ,  0.0681    ,\n",
       "        0.0944    ,  0.0944    ])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "charges[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:23.206284Z",
     "start_time": "2019-03-07T15:55:23.202288Z"
    }
   },
   "outputs": [],
   "source": [
    "from autograd.scipy.misc import logsumexp\n",
    "from simtk import unit\n",
    "from bayes_implicit_solvent.constants import kB, temperature\n",
    "\n",
    "kj_mol_to_kT = 1.0 * unit.kilojoule_per_mole / (kB * temperature)\n",
    "\n",
    "def one_sided_exp(w_F):\n",
    "    DeltaF = - (logsumexp(- w_F) - np.log(len(w_F)))\n",
    "    return DeltaF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:23.645598Z",
     "start_time": "2019-03-07T15:55:23.535661Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_types = np.array(sorted(list(set([tuple(f) for f in np.vstack(computed_features)]))))\n",
    "len(all_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:23.844258Z",
     "start_time": "2019-03-07T15:55:23.839168Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, False, False, False, False, False, False,\n",
       "        True, False,  True, False, False, False,  True, False, False,\n",
       "       False, False, False, False,  True, False, False, False, False,\n",
       "       False, False,  True, False, False, False,  True, False, False,\n",
       "       False, False, False,  True, False, False, False])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_types[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:24.005102Z",
     "start_time": "2019-03-07T15:55:24.000304Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type_index = dict(zip([tuple(t) for t in all_types], range(len(all_types))))\n",
    "type_index[tuple(computed_features[0][0])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:24.284193Z",
     "start_time": "2019-03-07T15:55:24.215690Z"
    }
   },
   "outputs": [],
   "source": [
    "type_indices = []\n",
    "for i in range(len(computed_features)):\n",
    "    type_indices.append(np.array([type_index[tuple(f)] for f in computed_features[i]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:24.476912Z",
     "start_time": "2019-03-07T15:55:24.470153Z"
    }
   },
   "outputs": [],
   "source": [
    "def make_predictions(params, inds=train_inds, batch_size=5, randomized=True):\n",
    "    all_radii, all_scales = neural_net.neural_net_predict(params, all_types).T\n",
    "    \n",
    "    \n",
    "    predictions = []\n",
    "    for i in inds:\n",
    "        radii, scales= all_radii[type_indices[i]], all_scales[type_indices[i]]\n",
    "        #radii, scales, log_taus = neural_net.neural_net_predict(params, computed_features[i]).T\n",
    "        #taus = np.exp(log_taus)\n",
    "        \n",
    "        if randomized:\n",
    "            dmat_inds = np.random.randint(0, len(distance_matrices[i]), batch_size)\n",
    "        else:\n",
    "            dmat_inds = np.arange(min(batch_size, len(distance_matrices[i])))\n",
    "        W_F = np.array([compute_OBC_energy_vectorized(distance_matrices[i][j], radii, scales, charges=charges[i]) for j in dmat_inds])\n",
    "        \n",
    "        w_F = W_F * kj_mol_to_kT\n",
    "        pred_free_energy = unreduce(one_sided_exp(w_F))\n",
    "        predictions.append(pred_free_energy)\n",
    "    return np.array(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:25.159765Z",
     "start_time": "2019-03-07T15:55:24.702414Z"
    }
   },
   "outputs": [],
   "source": [
    "predictions = make_predictions(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:25.167710Z",
     "start_time": "2019-03-07T15:55:25.162202Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.10763877806275707"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def log_prior(params):\n",
    "    return - sum([np.linalg.norm(W) + np.linalg.norm(b) for (W,b) in params])\n",
    "log_prior(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:25.179477Z",
     "start_time": "2019-03-07T15:55:25.169945Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-4.950e+00, -4.590e+00, -7.400e-01, -9.840e+00, -2.400e+00,\n",
       "       -6.480e+00,  1.070e+00, -3.430e+00,  1.830e+00, -4.290e+00,\n",
       "       -3.200e+00, -2.220e+00, -2.370e+00, -7.000e-01, -1.960e+00,\n",
       "       -2.490e+00, -4.800e-01, -4.220e+00, -4.510e+00, -2.860e+00,\n",
       "       -6.190e+00, -9.630e+00, -1.620e+00, -2.490e+00, -3.800e+00,\n",
       "       -2.550e+00, -3.880e+00, -1.460e+00, -9.000e-01, -7.620e+00,\n",
       "       -1.343e+01, -1.114e+01, -6.400e-01, -1.920e+00,  2.500e-01,\n",
       "       -3.750e+00, -2.210e+00,  2.670e+00, -4.710e+00, -3.840e+00,\n",
       "        3.430e+00, -3.780e+00, -6.160e+00, -3.820e+00, -4.590e+00,\n",
       "       -3.560e+00, -2.810e+00, -5.000e-01, -4.330e+00,  1.160e+00,\n",
       "       -3.280e+00, -7.800e-01, -3.680e+00, -2.530e+00, -1.101e+01,\n",
       "       -2.640e+00, -7.980e+00, -7.030e+00, -1.100e+00, -6.120e+00,\n",
       "       -4.850e+00, -1.160e+00, -2.490e+00, -7.500e+00, -2.480e+00,\n",
       "       -4.230e+00, -3.810e+00, -2.340e+00, -2.690e+00, -5.220e+00,\n",
       "       -2.780e+00,  2.880e+00, -3.640e+00, -3.520e+00, -9.820e+00,\n",
       "        1.200e+00, -2.560e+00, -3.130e+00, -6.620e+00,  1.660e+00,\n",
       "       -1.960e+00, -6.800e-01, -2.870e+00,  3.120e+00, -2.380e+00,\n",
       "       -4.870e+00, -4.420e+00,  8.400e-01,  2.340e+00, -2.630e+00,\n",
       "       -4.690e+00, -9.200e+00, -6.230e+00, -4.210e+00, -3.370e+00,\n",
       "        2.900e-01, -3.670e+00, -1.000e+01, -3.640e+00, -5.600e-01,\n",
       "       -4.590e+00, -1.000e-01, -3.510e+00, -7.800e-01, -4.020e+00,\n",
       "        5.600e-01, -1.610e+00, -9.940e+00,  1.600e-01,  1.400e-01,\n",
       "       -1.500e+00, -5.910e+00, -1.200e+00, -5.310e+00, -1.022e+01,\n",
       "        1.920e+00, -4.020e+00, -5.660e+00, -4.290e+00, -6.010e+00,\n",
       "       -9.650e+00, -4.000e-01, -5.530e+00, -2.940e+00, -1.240e+00,\n",
       "       -9.730e+00, -5.730e+00, -5.210e+00, -6.340e+00,  2.110e+00,\n",
       "       -2.150e+00, -2.460e+00, -2.330e+00, -2.160e+00, -7.780e+00,\n",
       "       -9.010e+00, -1.078e+01, -8.210e+00, -6.130e+00, -3.500e+00,\n",
       "       -1.880e+00, -4.000e-01, -1.340e+00, -1.600e-01, -3.150e+00,\n",
       "       -4.860e+00, -4.040e+00, -1.290e+00, -3.520e+00, -4.000e-02,\n",
       "       -1.120e+00, -4.500e+00, -6.550e+00, -9.760e+00, -3.340e+00,\n",
       "       -4.840e+00,  2.880e+00, -1.910e+00, -2.880e+00, -4.270e+00,\n",
       "       -2.320e+00, -3.000e-01, -1.590e+00, -1.400e+00, -1.000e-01,\n",
       "       -4.130e+00, -4.500e-01, -3.880e+00, -8.720e+00, -7.000e+00,\n",
       "       -3.730e+00, -2.930e+00, -5.940e+00, -5.110e+00, -5.330e+00,\n",
       "       -1.100e-01, -1.274e+01, -9.130e+00, -2.920e+00, -5.260e+00,\n",
       "       -2.260e+00, -1.360e+00, -1.240e+00,  1.310e+00, -5.900e+00,\n",
       "       -3.040e+00, -5.490e+00, -6.160e+00, -5.660e+00, -9.500e-01,\n",
       "       -4.700e+00,  6.700e-01, -9.400e+00, -4.620e+00, -4.200e+00,\n",
       "       -4.840e+00, -6.720e+00, -9.000e-01, -7.400e-01, -5.290e+00,\n",
       "       -2.230e+00, -1.380e+00, -5.300e-01, -2.730e+00, -6.500e+00,\n",
       "       -3.110e+00, -4.090e+00, -3.300e-01, -4.090e+00, -2.670e+00,\n",
       "       -4.800e+00, -6.690e+00, -8.000e-01, -4.390e+00, -4.390e+00,\n",
       "       -1.600e-01, -2.010e+00,  1.000e-01, -5.990e+00, -5.490e+00,\n",
       "       -4.100e+00,  2.930e+00, -1.120e+00, -4.440e+00, -5.300e-01,\n",
       "       -4.550e+00,  2.830e+00, -7.770e+00, -8.680e+00, -3.300e+00,\n",
       "       -6.500e+00,  0.000e+00, -3.610e+00, -5.900e+00, -8.000e-01,\n",
       "        1.580e+00, -5.900e-01, -3.950e+00, -3.440e+00, -3.170e+00,\n",
       "       -9.800e-01, -6.690e+00, -6.250e+00, -1.017e+01, -1.140e+00,\n",
       "       -3.650e+00, -6.200e+00, -9.860e+00, -1.264e+01, -7.580e+00,\n",
       "       -3.200e+00, -4.420e+00,  2.000e+00,  6.000e-01,  2.480e+00,\n",
       "       -2.490e+00,  2.300e+00, -3.520e+00,  3.160e+00, -5.040e+00,\n",
       "       -5.820e+00, -4.820e+00,  2.560e+00,  2.060e+00, -4.580e+00,\n",
       "       -3.050e+00, -1.240e+00, -8.500e-01, -6.100e+00, -4.520e+00,\n",
       "        1.800e-01, -3.410e+00, -6.620e+00, -4.430e+00, -2.440e+00,\n",
       "       -5.100e+00, -4.470e+00, -5.560e+00, -1.790e+00, -9.620e+00,\n",
       "       -2.680e+00,  2.890e+00, -3.540e+00,  8.000e-02, -2.890e+00,\n",
       "       -1.210e+00, -2.920e+00, -4.720e+00,  1.590e+00, -3.100e+00,\n",
       "       -1.640e+00, -5.460e+00, -7.810e+00, -2.280e+00, -9.000e-01,\n",
       "       -3.350e+00, -9.280e+00, -2.300e+00,  1.790e+00, -5.490e+00,\n",
       "       -2.550e+00,  1.280e+00, -2.290e+00, -2.070e+00, -2.500e-01,\n",
       "       -1.027e+01, -8.610e+00, -4.680e+00,  2.550e+00,  2.510e+00,\n",
       "       -9.610e+00, -3.470e+00, -2.560e+00,  3.130e+00, -1.170e+00,\n",
       "       -5.330e+00,  1.000e-01, -1.340e+00, -6.000e+00, -6.790e+00,\n",
       "       -4.630e+00, -3.220e+00,  2.930e+00, -6.780e+00,  5.200e-01,\n",
       "       -5.800e+00, -7.000e+00,  1.680e+00, -5.880e+00,  0.000e+00,\n",
       "       -3.880e+00, -1.390e+00, -2.450e+00, -1.008e+01, -3.640e+00,\n",
       "       -8.260e+00, -1.450e+00, -4.370e+00, -5.300e-01, -8.700e+00,\n",
       "       -3.430e+00, -3.000e-02, -1.950e+00, -6.460e+00, -6.300e-01,\n",
       "       -4.400e+00, -7.280e+00, -4.570e+00, -6.320e+00, -1.210e+00,\n",
       "       -6.020e+00, -7.650e+00, -8.300e-01, -5.000e+00, -2.210e+00,\n",
       "       -4.740e+00, -3.220e+00, -6.210e+00, -1.064e+01, -3.710e+00,\n",
       "       -4.230e+00, -1.280e+00, -4.730e+00, -2.830e+00,  2.000e+00,\n",
       "        5.000e-01, -5.100e+00, -4.400e+00, -4.050e+00, -3.250e+00,\n",
       "       -7.660e+00, -2.820e+00, -3.130e+00, -8.900e-01, -1.010e+00,\n",
       "       -9.300e+00, -6.400e+00, -4.820e+00, -7.170e+00,  1.380e+00,\n",
       "       -5.910e+00, -7.630e+00, -2.220e+00, -2.470e+00, -1.195e+01,\n",
       "       -2.020e+00, -6.960e+00, -3.920e+00, -2.450e+00, -2.500e-01,\n",
       "       -4.290e+00, -4.400e+00, -5.720e+00, -7.070e+00, -3.880e+00,\n",
       "        7.100e-01, -4.720e+00, -4.400e-01, -2.100e+00, -2.280e+00,\n",
       "       -4.120e+00, -4.240e+00,  2.510e+00, -8.400e-01, -5.500e-01,\n",
       "       -4.690e+00, -7.290e+00, -5.450e+00, -8.180e+00, -1.140e+00,\n",
       "       -3.880e+00, -6.250e+00,  1.070e+00, -6.680e+00, -9.450e+00,\n",
       "       -4.910e+00,  1.830e+00, -2.640e+00, -6.270e+00, -9.400e+00,\n",
       "       -5.850e+00, -4.390e+00, -5.740e+00, -2.090e+00, -4.420e+00,\n",
       "       -4.610e+00, -8.150e+00, -1.100e-01, -4.910e+00, -7.370e+00,\n",
       "       -7.100e+00,  6.800e-01,  5.600e-01, -2.500e-01, -3.950e+00,\n",
       "        2.380e+00, -9.290e+00, -7.430e+00, -2.780e+00, -4.870e+00,\n",
       "       -2.200e-01, -2.740e+00,  2.710e+00, -1.660e+00, -8.830e+00,\n",
       "        1.680e+00, -4.450e+00, -7.470e+00, -4.590e+00, -7.190e+00,\n",
       "       -4.600e-01, -3.240e+00, -1.100e+00, -2.490e+00, -1.310e+00,\n",
       "       -3.120e+00, -4.800e-01, -2.930e+00, -1.100e+01, -6.090e+00,\n",
       "       -4.060e+00, -4.090e+00, -4.970e+00, -3.180e+00,  2.520e+00,\n",
       "       -2.640e+00,  1.470e+00, -9.710e+00, -1.230e+00, -7.290e+00,\n",
       "       -3.640e+00, -5.040e+00, -8.600e-01, -4.700e+00, -5.570e+00,\n",
       "        1.770e+00,  2.930e+00, -3.790e+00, -6.500e+00, -4.930e+00,\n",
       "       -9.510e+00, -4.380e+00,  2.560e+00,  1.580e+00, -6.790e+00,\n",
       "       -5.710e+00, -9.440e+00, -9.370e+00, -6.880e+00, -1.270e+00,\n",
       "        4.000e-01, -2.110e+00, -3.900e+00, -5.260e+00, -5.180e+00,\n",
       "        2.510e+00, -6.920e+00,  2.130e+00, -4.350e+00, -4.420e+00,\n",
       "       -1.740e+00, -6.750e+00, -5.510e+00, -4.780e+00, -4.500e+00,\n",
       "       -1.830e+00,  1.700e+00, -2.700e+00, -3.450e+00, -1.990e+00,\n",
       "       -6.350e+00, -3.300e+00, -9.800e+00,  2.300e+00, -8.000e-01,\n",
       "       -4.100e+00, -2.360e+00, -2.510e+00, -3.650e+00, -7.400e+00,\n",
       "       -4.610e+00, -1.185e+01,  7.500e-01, -4.530e+00, -6.440e+00,\n",
       "       -3.450e+00, -1.080e+00, -3.840e+00, -4.010e+00, -1.820e+00,\n",
       "        2.700e-01, -1.460e+00, -2.820e+00, -2.750e+00, -5.700e-01,\n",
       "       -6.400e+00, -4.770e+00, -1.430e+00, -2.130e+00, -9.300e+00,\n",
       "        1.010e+00,  3.400e-01, -5.220e+00, -4.150e+00, -4.000e+00,\n",
       "       -6.740e+00, -7.780e+00, -6.600e+00, -1.120e+00,  1.000e-02,\n",
       "       -1.003e+01,  2.100e+00, -6.130e+00, -8.410e+00, -7.480e+00,\n",
       "       -4.550e+00,  2.510e+00, -8.110e+00,  2.970e+00, -4.070e+00,\n",
       "       -1.153e+01, -2.300e-01, -1.690e+00, -7.900e-01, -2.400e+00,\n",
       "       -9.900e-01, -3.040e+00, -5.060e+00, -3.090e+00, -3.580e+00,\n",
       "       -7.670e+00, -2.330e+00, -9.410e+00, -4.160e+00, -3.150e+00,\n",
       "       -4.310e+00, -6.620e+00, -5.730e+00, -3.480e+00,  8.000e-02,\n",
       "       -9.530e+00, -8.300e-01, -2.490e+00, -2.040e+00, -4.400e-01,\n",
       "       -5.210e+00, -3.710e+00, -1.930e+00, -5.480e+00, -2.440e+00,\n",
       "       -9.520e+00, -8.840e+00,  9.300e-01, -1.400e-01, -9.340e+00,\n",
       "        6.000e-02, -5.480e+00, -9.610e+00, -5.030e+00,  1.310e+00,\n",
       "       -1.421e+01, -3.930e+00, -1.460e+00, -2.130e+00, -2.790e+00,\n",
       "       -5.730e+00, -3.030e+00, -3.280e+00, -2.980e+00, -7.700e-01,\n",
       "       -2.400e+00,  1.090e+00, -8.840e+00, -1.660e+00, -2.780e+00,\n",
       "       -3.920e+00, -4.580e+00, -1.021e+01, -9.310e+00, -5.230e+00,\n",
       "       -1.900e-01, -9.000e-01, -8.420e+00,  1.230e+00, -9.900e-01,\n",
       "       -8.200e-01, -4.780e+00, -1.890e+00, -3.240e+00,  1.320e+00,\n",
       "        2.900e-01])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "expt_means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:25.524254Z",
     "start_time": "2019-03-07T15:55:25.517988Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 2.51893830e+00,  6.52409645e-01,  1.94411378e+00, -1.24441321e+00,\n",
       "        1.23705650e+00, -1.38678078e+01, -1.52081783e+00, -1.56415353e+00,\n",
       "       -2.02868242e+00, -3.17965936e+00, -4.52874384e+00, -3.12796357e-01,\n",
       "        8.86823182e-01, -2.73879458e+00, -6.33325948e+00, -1.35129926e+00,\n",
       "       -3.33247576e-01, -1.51062071e+00, -6.37857467e+00, -7.33866413e+00,\n",
       "        7.54994923e-01, -7.28309026e+00, -4.10322680e+00, -1.69706483e+01,\n",
       "        3.27695927e-01,  6.73209847e-01, -1.52927707e+01,  1.50409483e+00,\n",
       "       -1.97465274e+01,  1.64600894e+00, -8.80045955e+00,  4.77334925e-01,\n",
       "        3.34902340e+00,  3.87637695e-01, -2.47521778e+00, -3.54404686e-01,\n",
       "       -8.65102994e+00, -2.34446969e+01, -5.87561857e+00, -2.93479429e-01,\n",
       "       -2.63249706e-01, -7.36548162e+00, -1.60139026e+00, -1.24279034e+00,\n",
       "        7.33277266e-01,  3.94079660e-01,  1.37585011e+00, -1.85344761e+00,\n",
       "       -3.55166002e+00, -2.47491267e+00, -7.31754671e-01,  4.04961411e-01,\n",
       "       -4.14331017e+00, -1.99392189e+00, -4.09856119e-01, -5.75876157e+00,\n",
       "       -1.07148359e+00, -3.22710969e-01, -3.84869632e+00,  1.18817535e+00,\n",
       "        2.34655774e-01, -2.20678184e+00, -9.07397390e+00, -8.31270720e+00,\n",
       "       -4.93865691e+00, -2.01156617e+00, -8.06683552e+00, -3.85038486e+00,\n",
       "       -4.93691631e+00, -9.39452619e-01, -1.15344863e-01, -1.93133613e+00,\n",
       "        1.66901281e+00, -2.07124316e+00,  6.36596095e-01,  2.57013112e-01,\n",
       "       -4.01880556e+00,  8.41788348e-01, -8.47356254e+00, -2.71680436e+00,\n",
       "        7.55809182e-01, -2.58313727e+00,  9.51305734e-03, -4.44114301e+00,\n",
       "        1.75375334e+00,  2.94189741e-01, -8.68623476e+00, -1.12407027e+01,\n",
       "       -4.21907697e+00,  2.68161247e+00, -2.19229964e-01, -1.07306542e+00,\n",
       "       -1.12401830e+00,  8.58777548e-01,  8.76543237e-01, -3.35484536e+00,\n",
       "       -1.22089655e+00, -1.45279945e+00, -7.56970445e-01,  2.59947095e-01,\n",
       "       -9.83734448e-01, -6.53282362e+00,  4.85816870e+00, -4.21021032e+00,\n",
       "       -6.09881544e-01,  1.29593425e+00,  1.35621067e+00, -1.00763196e+00,\n",
       "       -2.58429605e+00, -6.12201346e+00, -9.51625688e+00, -4.79210882e+00,\n",
       "       -2.17404019e+00,  8.13658321e-03, -7.47099256e+00, -3.11291236e-01,\n",
       "       -2.22034685e-01, -5.44515120e+00, -7.72498060e-01, -8.68892093e-01,\n",
       "       -3.53871316e+00,  2.73441488e+00, -7.97126646e+00, -9.65718058e+00,\n",
       "        7.04307122e-01, -7.05949055e+00, -2.27143359e+00, -3.62597108e+00,\n",
       "       -3.31101567e+00, -2.15397867e+00,  1.95820599e+00, -1.03698988e+01,\n",
       "       -1.63659814e+00, -4.06960792e+00, -1.96236098e+01, -1.97726295e+00,\n",
       "       -1.05275087e+01, -8.18064686e+00, -3.25005627e+00, -3.97365471e+00,\n",
       "       -1.24543196e+00, -1.06965618e+01, -5.80659724e-01, -1.39969818e+00,\n",
       "       -1.18269195e+00,  2.06929989e+00,  1.37272851e+00, -1.03766459e+00,\n",
       "       -3.12478066e+00,  1.38197107e+00, -3.88662080e+00, -4.50669736e+00,\n",
       "       -1.37593448e+01, -3.70095378e+00, -5.92330757e+00,  1.90158496e+00,\n",
       "       -8.31761207e-01, -3.82644568e+00, -8.93840756e+00, -6.53755820e+00,\n",
       "        2.75581036e+00,  5.53892285e-01, -5.80549769e+00, -6.39678139e+00,\n",
       "       -4.35821777e+00, -1.91105488e+00,  7.12219749e-01, -4.26161823e+00,\n",
       "       -2.50362434e-01,  8.51561729e-01,  1.06512109e+00, -1.99263722e+00,\n",
       "       -3.84822070e+00, -1.02762074e+00,  4.87040169e-02, -6.61154589e+00,\n",
       "        6.43878393e-02,  9.74999621e-01, -2.36441414e+00, -5.28428470e-04,\n",
       "        6.60218658e-01, -8.65165834e+00, -2.03926994e+00, -2.92002745e+00,\n",
       "        3.50188958e+00, -1.62860934e+00,  4.03115966e+00, -1.27639570e+01,\n",
       "       -1.07550740e+00, -9.24797275e+00, -1.63923185e+00, -9.52149898e-01,\n",
       "        1.40806422e-01, -1.43169776e+00,  8.09602860e-02, -7.10269188e+00,\n",
       "        3.95509789e-01,  5.06572084e-01,  1.26848342e+00,  1.40084932e+00,\n",
       "       -7.60701732e+00, -1.32935996e+00, -7.57447329e-01,  8.87699503e-01,\n",
       "       -1.24282824e+00, -1.08935879e+01, -6.73688802e-01, -3.37919922e+00,\n",
       "        8.97177118e-01,  3.03862517e-01, -1.49115698e+01, -5.62690837e+00,\n",
       "       -8.14605070e+00, -1.78035148e+00,  4.36884649e-01, -6.03207414e+00,\n",
       "        7.42489764e-01,  5.61780604e-01, -7.76402123e-01, -2.91014694e+00,\n",
       "       -7.67638593e+00, -1.47404783e+00,  1.10974687e+00,  1.31930400e+00,\n",
       "       -1.02508910e+01, -5.11202313e-01, -4.22943516e+00, -3.84520312e+00,\n",
       "       -8.10605796e-01,  4.62112312e-01,  1.31255088e+00,  4.10454366e-01,\n",
       "       -4.59006865e-03, -3.09411536e+00,  1.30575399e+00, -6.29161650e+00,\n",
       "        1.10771777e+00, -1.34384220e+00, -9.55654030e+00,  1.63829735e+00,\n",
       "        1.39033942e+00,  4.89148686e-01,  1.41120710e+00,  3.74446109e-01,\n",
       "       -1.40927848e+00, -1.33181962e+00,  2.12554673e-01, -3.94362531e-01,\n",
       "       -1.23662985e-01,  3.97893382e-01, -1.16329940e+00, -6.15404410e+00,\n",
       "       -2.64888622e-01,  1.19787899e+00,  1.62669059e+00, -6.46141514e+00,\n",
       "       -1.62874267e+01, -7.22009752e-02, -3.21299563e+00, -2.37820672e+00,\n",
       "        1.26955362e+00, -3.86344678e+00, -1.17013345e+01, -2.63018993e+00,\n",
       "       -1.61746528e-02, -2.10188134e+00,  1.41709678e+00, -1.01302209e+01,\n",
       "       -1.08521879e+01, -4.28556684e+00,  5.15532064e-01, -5.04951977e+00,\n",
       "       -4.01024689e+00,  1.47555438e-02, -4.35742897e+00, -3.66057769e+00,\n",
       "        3.84595644e-01,  2.41703621e-01, -5.31967068e+00,  9.23786999e-01,\n",
       "        6.79734670e-01, -6.23594806e+00, -1.38939936e+01, -8.51735805e+00,\n",
       "       -3.86134054e+00, -3.33871538e+00, -8.33364616e-01,  5.68384210e-01,\n",
       "       -8.33104049e+00, -1.01497355e+01, -1.25098674e+00,  3.91896242e-03,\n",
       "       -6.72043078e+00, -8.85403830e+00, -8.49088378e+00, -5.93145845e+00,\n",
       "       -5.34399379e+00, -8.39015343e+00,  2.13634210e+00, -5.09935259e+00,\n",
       "        2.43429032e-01, -9.16610026e+00,  1.56524565e-01, -4.19238621e+00,\n",
       "       -1.07067842e-01,  1.54050871e+00, -7.30031571e+00,  8.21140569e-01,\n",
       "       -5.78137346e-01, -1.17622074e+01, -7.21916938e-01,  1.71494057e+00,\n",
       "        1.73238234e+00, -1.82516942e+00, -6.44421346e+00, -2.42914121e-01])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:25.851938Z",
     "start_time": "2019-03-07T15:55:25.848140Z"
    }
   },
   "outputs": [],
   "source": [
    "from autograd.scipy.stats import t as student_t\n",
    "\n",
    "def log_likelihood(params):\n",
    "    predictions = make_predictions(params)\n",
    "    return np.sum(student_t.logpdf(predictions, loc=expt_means[train_inds], scale=expt_uncs[train_inds], df=7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:26.210264Z",
     "start_time": "2019-03-07T15:55:26.207421Z"
    }
   },
   "outputs": [],
   "source": [
    "def log_posterior(params):\n",
    "    return log_prior(params) + log_likelihood(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:27.010372Z",
     "start_time": "2019-03-07T15:55:26.519148Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-2171.1134610602717"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_posterior(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:33.988800Z",
     "start_time": "2019-03-07T15:55:27.012488Z"
    }
   },
   "outputs": [],
   "source": [
    "from autograd import grad\n",
    "g = grad(log_posterior)(params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T15:55:33.993764Z",
     "start_time": "2019-03-07T15:55:33.990465Z"
    }
   },
   "outputs": [],
   "source": [
    "def rmse(x, y):\n",
    "    squared_errors = (x - y)**2\n",
    "    mean_squared_errors = np.mean(squared_errors)\n",
    "    root_mean_squared_errors = np.sqrt(mean_squared_errors)\n",
    "    return root_mean_squared_errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:00.132052Z",
     "start_time": "2019-03-07T15:56:06.739441Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "initial parameters\n",
      "\ttrain RMSE = 3.632\n",
      "\ttest RMSE = 3.295\n",
      "Iteration 0\n",
      "\ttrain RMSE = 3.695\n",
      "\ttest RMSE = 3.414\n",
      "Iteration 1\n",
      "\ttrain RMSE = 3.328\n",
      "\ttest RMSE = 3.119\n",
      "Iteration 2\n",
      "\ttrain RMSE = 3.199\n",
      "\ttest RMSE = 3.019\n",
      "Iteration 3\n",
      "\ttrain RMSE = 3.099\n",
      "\ttest RMSE = 2.856\n",
      "Iteration 4\n",
      "\ttrain RMSE = 3.127\n",
      "\ttest RMSE = 2.738\n",
      "Iteration 5\n",
      "\ttrain RMSE = 3.343\n",
      "\ttest RMSE = 2.797\n",
      "Iteration 6\n",
      "\ttrain RMSE = 3.214\n",
      "\ttest RMSE = 2.694\n",
      "Iteration 7\n",
      "\ttrain RMSE = 2.967\n",
      "\ttest RMSE = 2.560\n",
      "Iteration 8\n",
      "\ttrain RMSE = 2.799\n",
      "\ttest RMSE = 2.510\n",
      "Iteration 9\n",
      "\ttrain RMSE = 2.696\n",
      "\ttest RMSE = 2.521\n",
      "Iteration 10\n",
      "\ttrain RMSE = 2.649\n",
      "\ttest RMSE = 2.547\n",
      "Iteration 11\n",
      "\ttrain RMSE = 2.620\n",
      "\ttest RMSE = 2.524\n",
      "Iteration 12\n",
      "\ttrain RMSE = 2.643\n",
      "\ttest RMSE = 2.500\n",
      "Iteration 13\n",
      "\ttrain RMSE = 2.633\n",
      "\ttest RMSE = 2.510\n",
      "Iteration 14\n",
      "\ttrain RMSE = 2.569\n",
      "\ttest RMSE = 2.516\n",
      "Iteration 15\n",
      "\ttrain RMSE = 2.540\n",
      "\ttest RMSE = 2.521\n",
      "Iteration 16\n",
      "\ttrain RMSE = 2.516\n",
      "\ttest RMSE = 2.521\n",
      "Iteration 17\n",
      "\ttrain RMSE = 2.505\n",
      "\ttest RMSE = 2.501\n",
      "Iteration 18\n",
      "\ttrain RMSE = 2.477\n",
      "\ttest RMSE = 2.480\n",
      "Iteration 19\n",
      "\ttrain RMSE = 2.443\n",
      "\ttest RMSE = 2.457\n",
      "Iteration 20\n",
      "\ttrain RMSE = 2.414\n",
      "\ttest RMSE = 2.422\n",
      "Iteration 21\n",
      "\ttrain RMSE = 2.390\n",
      "\ttest RMSE = 2.383\n",
      "Iteration 22\n",
      "\ttrain RMSE = 2.342\n",
      "\ttest RMSE = 2.341\n",
      "Iteration 23\n",
      "\ttrain RMSE = 2.297\n",
      "\ttest RMSE = 2.296\n",
      "Iteration 24\n",
      "\ttrain RMSE = 2.258\n",
      "\ttest RMSE = 2.247\n",
      "Iteration 25\n",
      "\ttrain RMSE = 2.215\n",
      "\ttest RMSE = 2.197\n",
      "Iteration 26\n",
      "\ttrain RMSE = 2.168\n",
      "\ttest RMSE = 2.151\n",
      "Iteration 27\n",
      "\ttrain RMSE = 2.128\n",
      "\ttest RMSE = 2.100\n",
      "Iteration 28\n",
      "\ttrain RMSE = 2.097\n",
      "\ttest RMSE = 2.046\n",
      "Iteration 29\n",
      "\ttrain RMSE = 2.051\n",
      "\ttest RMSE = 2.001\n",
      "Iteration 30\n",
      "\ttrain RMSE = 2.010\n",
      "\ttest RMSE = 1.968\n",
      "Iteration 31\n",
      "\ttrain RMSE = 1.977\n",
      "\ttest RMSE = 1.926\n",
      "Iteration 32\n",
      "\ttrain RMSE = 1.944\n",
      "\ttest RMSE = 1.863\n",
      "Iteration 33\n",
      "\ttrain RMSE = 1.916\n",
      "\ttest RMSE = 1.812\n",
      "Iteration 34\n",
      "\ttrain RMSE = 1.886\n",
      "\ttest RMSE = 1.791\n",
      "Iteration 35\n",
      "\ttrain RMSE = 1.875\n",
      "\ttest RMSE = 1.787\n",
      "Iteration 36\n",
      "\ttrain RMSE = 1.836\n",
      "\ttest RMSE = 1.718\n",
      "Iteration 37\n",
      "\ttrain RMSE = 1.838\n",
      "\ttest RMSE = 1.683\n",
      "Iteration 38\n",
      "\ttrain RMSE = 1.822\n",
      "\ttest RMSE = 1.664\n",
      "Iteration 39\n",
      "\ttrain RMSE = 1.778\n",
      "\ttest RMSE = 1.642\n",
      "Iteration 40\n",
      "\ttrain RMSE = 1.770\n",
      "\ttest RMSE = 1.659\n",
      "Iteration 41\n",
      "\ttrain RMSE = 1.757\n",
      "\ttest RMSE = 1.648\n",
      "Iteration 42\n",
      "\ttrain RMSE = 1.729\n",
      "\ttest RMSE = 1.607\n",
      "Iteration 43\n",
      "\ttrain RMSE = 1.735\n",
      "\ttest RMSE = 1.587\n",
      "Iteration 44\n",
      "\ttrain RMSE = 1.736\n",
      "\ttest RMSE = 1.581\n",
      "Iteration 45\n",
      "\ttrain RMSE = 1.699\n",
      "\ttest RMSE = 1.556\n",
      "Iteration 46\n",
      "\ttrain RMSE = 1.674\n",
      "\ttest RMSE = 1.545\n",
      "Iteration 47\n",
      "\ttrain RMSE = 1.662\n",
      "\ttest RMSE = 1.543\n",
      "Iteration 48\n",
      "\ttrain RMSE = 1.652\n",
      "\ttest RMSE = 1.523\n",
      "Iteration 49\n",
      "\ttrain RMSE = 1.655\n",
      "\ttest RMSE = 1.502\n",
      "Iteration 50\n",
      "\ttrain RMSE = 1.669\n",
      "\ttest RMSE = 1.498\n",
      "Iteration 51\n",
      "\ttrain RMSE = 1.652\n",
      "\ttest RMSE = 1.487\n",
      "Iteration 52\n",
      "\ttrain RMSE = 1.632\n",
      "\ttest RMSE = 1.496\n",
      "Iteration 53\n",
      "\ttrain RMSE = 1.635\n",
      "\ttest RMSE = 1.514\n",
      "Iteration 54\n",
      "\ttrain RMSE = 1.628\n",
      "\ttest RMSE = 1.497\n",
      "Iteration 55\n",
      "\ttrain RMSE = 1.642\n",
      "\ttest RMSE = 1.485\n",
      "Iteration 56\n",
      "\ttrain RMSE = 1.687\n",
      "\ttest RMSE = 1.510\n",
      "Iteration 57\n",
      "\ttrain RMSE = 1.690\n",
      "\ttest RMSE = 1.513\n",
      "Iteration 58\n",
      "\ttrain RMSE = 1.622\n",
      "\ttest RMSE = 1.487\n",
      "Iteration 59\n",
      "\ttrain RMSE = 1.613\n",
      "\ttest RMSE = 1.520\n",
      "Iteration 60\n",
      "\ttrain RMSE = 1.607\n",
      "\ttest RMSE = 1.514\n",
      "Iteration 61\n",
      "\ttrain RMSE = 1.603\n",
      "\ttest RMSE = 1.460\n",
      "Iteration 62\n",
      "\ttrain RMSE = 1.655\n",
      "\ttest RMSE = 1.470\n",
      "Iteration 63\n",
      "\ttrain RMSE = 1.629\n",
      "\ttest RMSE = 1.448\n",
      "Iteration 64\n",
      "\ttrain RMSE = 1.580\n",
      "\ttest RMSE = 1.435\n",
      "Iteration 65\n",
      "\ttrain RMSE = 1.586\n",
      "\ttest RMSE = 1.461\n",
      "Iteration 66\n",
      "\ttrain RMSE = 1.582\n",
      "\ttest RMSE = 1.442\n",
      "Iteration 67\n",
      "\ttrain RMSE = 1.582\n",
      "\ttest RMSE = 1.416\n",
      "Iteration 68\n",
      "\ttrain RMSE = 1.603\n",
      "\ttest RMSE = 1.415\n",
      "Iteration 69\n",
      "\ttrain RMSE = 1.598\n",
      "\ttest RMSE = 1.415\n",
      "Iteration 70\n",
      "\ttrain RMSE = 1.552\n",
      "\ttest RMSE = 1.414\n",
      "Iteration 71\n",
      "\ttrain RMSE = 1.555\n",
      "\ttest RMSE = 1.454\n",
      "Iteration 72\n",
      "\ttrain RMSE = 1.543\n",
      "\ttest RMSE = 1.436\n",
      "Iteration 73\n",
      "\ttrain RMSE = 1.551\n",
      "\ttest RMSE = 1.405\n",
      "Iteration 74\n",
      "\ttrain RMSE = 1.591\n",
      "\ttest RMSE = 1.419\n",
      "Iteration 75\n",
      "\ttrain RMSE = 1.552\n",
      "\ttest RMSE = 1.398\n",
      "Iteration 76\n",
      "\ttrain RMSE = 1.531\n",
      "\ttest RMSE = 1.405\n",
      "Iteration 77\n",
      "\ttrain RMSE = 1.535\n",
      "\ttest RMSE = 1.425\n",
      "Iteration 78\n",
      "\ttrain RMSE = 1.534\n",
      "\ttest RMSE = 1.407\n",
      "Iteration 79\n",
      "\ttrain RMSE = 1.542\n",
      "\ttest RMSE = 1.385\n",
      "Iteration 80\n",
      "\ttrain RMSE = 1.569\n",
      "\ttest RMSE = 1.390\n",
      "Iteration 81\n",
      "\ttrain RMSE = 1.575\n",
      "\ttest RMSE = 1.395\n",
      "Iteration 82\n",
      "\ttrain RMSE = 1.538\n",
      "\ttest RMSE = 1.384\n",
      "Iteration 83\n",
      "\ttrain RMSE = 1.531\n",
      "\ttest RMSE = 1.412\n",
      "Iteration 84\n",
      "\ttrain RMSE = 1.534\n",
      "\ttest RMSE = 1.424\n",
      "Iteration 85\n",
      "\ttrain RMSE = 1.526\n",
      "\ttest RMSE = 1.392\n",
      "Iteration 86\n",
      "\ttrain RMSE = 1.543\n",
      "\ttest RMSE = 1.376\n",
      "Iteration 87\n",
      "\ttrain RMSE = 1.574\n",
      "\ttest RMSE = 1.388\n",
      "Iteration 88\n",
      "\ttrain RMSE = 1.548\n",
      "\ttest RMSE = 1.375\n",
      "Iteration 89\n",
      "\ttrain RMSE = 1.527\n",
      "\ttest RMSE = 1.389\n",
      "Iteration 90\n",
      "\ttrain RMSE = 1.535\n",
      "\ttest RMSE = 1.410\n",
      "Iteration 91\n",
      "\ttrain RMSE = 1.532\n",
      "\ttest RMSE = 1.391\n",
      "Iteration 92\n",
      "\ttrain RMSE = 1.557\n",
      "\ttest RMSE = 1.377\n",
      "Iteration 93\n",
      "\ttrain RMSE = 1.603\n",
      "\ttest RMSE = 1.403\n",
      "Iteration 94\n",
      "\ttrain RMSE = 1.585\n",
      "\ttest RMSE = 1.391\n",
      "Iteration 95\n",
      "\ttrain RMSE = 1.546\n",
      "\ttest RMSE = 1.376\n",
      "Iteration 96\n",
      "\ttrain RMSE = 1.542\n",
      "\ttest RMSE = 1.403\n",
      "Iteration 97\n",
      "\ttrain RMSE = 1.548\n",
      "\ttest RMSE = 1.420\n",
      "Iteration 98\n",
      "\ttrain RMSE = 1.544\n",
      "\ttest RMSE = 1.395\n",
      "Iteration 99\n",
      "\ttrain RMSE = 1.570\n",
      "\ttest RMSE = 1.387\n",
      "Iteration 100\n",
      "\ttrain RMSE = 1.586\n",
      "\ttest RMSE = 1.393\n",
      "Iteration 101\n",
      "\ttrain RMSE = 1.556\n",
      "\ttest RMSE = 1.375\n",
      "Iteration 102\n",
      "\ttrain RMSE = 1.541\n",
      "\ttest RMSE = 1.374\n",
      "Iteration 103\n",
      "\ttrain RMSE = 1.538\n",
      "\ttest RMSE = 1.376\n",
      "Iteration 104\n",
      "\ttrain RMSE = 1.537\n",
      "\ttest RMSE = 1.370\n",
      "Iteration 105\n",
      "\ttrain RMSE = 1.541\n",
      "\ttest RMSE = 1.364\n",
      "Iteration 106\n",
      "\ttrain RMSE = 1.542\n",
      "\ttest RMSE = 1.365\n",
      "Iteration 107\n",
      "\ttrain RMSE = 1.547\n",
      "\ttest RMSE = 1.367\n",
      "Iteration 108\n",
      "\ttrain RMSE = 1.545\n",
      "\ttest RMSE = 1.371\n",
      "Iteration 109\n",
      "\ttrain RMSE = 1.543\n",
      "\ttest RMSE = 1.375\n",
      "Iteration 110\n",
      "\ttrain RMSE = 1.540\n",
      "\ttest RMSE = 1.380\n",
      "Iteration 111\n",
      "\ttrain RMSE = 1.544\n",
      "\ttest RMSE = 1.383\n",
      "Iteration 112\n",
      "\ttrain RMSE = 1.555\n",
      "\ttest RMSE = 1.383\n",
      "Iteration 113\n",
      "\ttrain RMSE = 1.561\n",
      "\ttest RMSE = 1.391\n",
      "Iteration 114\n",
      "\ttrain RMSE = 1.568\n",
      "\ttest RMSE = 1.397\n",
      "Iteration 115\n",
      "\ttrain RMSE = 1.571\n",
      "\ttest RMSE = 1.396\n",
      "Iteration 116\n",
      "\ttrain RMSE = 1.571\n",
      "\ttest RMSE = 1.392\n",
      "Iteration 117\n",
      "\ttrain RMSE = 1.567\n",
      "\ttest RMSE = 1.392\n",
      "Iteration 118\n",
      "\ttrain RMSE = 1.561\n",
      "\ttest RMSE = 1.402\n",
      "Iteration 119\n",
      "\ttrain RMSE = 1.558\n",
      "\ttest RMSE = 1.403\n",
      "Iteration 120\n",
      "\ttrain RMSE = 1.559\n",
      "\ttest RMSE = 1.387\n",
      "Iteration 121\n",
      "\ttrain RMSE = 1.560\n",
      "\ttest RMSE = 1.387\n",
      "Iteration 122\n",
      "\ttrain RMSE = 1.564\n",
      "\ttest RMSE = 1.387\n",
      "Iteration 123\n",
      "\ttrain RMSE = 1.564\n",
      "\ttest RMSE = 1.387\n",
      "Iteration 124\n",
      "\ttrain RMSE = 1.557\n",
      "\ttest RMSE = 1.405\n",
      "Iteration 125\n",
      "\ttrain RMSE = 1.563\n",
      "\ttest RMSE = 1.404\n",
      "Iteration 126\n",
      "\ttrain RMSE = 1.570\n",
      "\ttest RMSE = 1.396\n",
      "Iteration 127\n",
      "\ttrain RMSE = 1.580\n",
      "\ttest RMSE = 1.397\n",
      "Iteration 128\n",
      "\ttrain RMSE = 1.586\n",
      "\ttest RMSE = 1.397\n",
      "Iteration 129\n",
      "\ttrain RMSE = 1.592\n",
      "\ttest RMSE = 1.406\n",
      "Iteration 130\n",
      "\ttrain RMSE = 1.596\n",
      "\ttest RMSE = 1.414\n",
      "Iteration 131\n",
      "\ttrain RMSE = 1.590\n",
      "\ttest RMSE = 1.418\n",
      "Iteration 132\n",
      "\ttrain RMSE = 1.586\n",
      "\ttest RMSE = 1.405\n",
      "Iteration 133\n",
      "\ttrain RMSE = 1.586\n",
      "\ttest RMSE = 1.406\n",
      "Iteration 134\n",
      "\ttrain RMSE = 1.580\n",
      "\ttest RMSE = 1.417\n",
      "Iteration 135\n",
      "\ttrain RMSE = 1.580\n",
      "\ttest RMSE = 1.430\n",
      "Iteration 136\n",
      "\ttrain RMSE = 1.590\n",
      "\ttest RMSE = 1.419\n",
      "Iteration 137\n",
      "\ttrain RMSE = 1.611\n",
      "\ttest RMSE = 1.419\n",
      "Iteration 138\n",
      "\ttrain RMSE = 1.620\n",
      "\ttest RMSE = 1.424\n",
      "Iteration 139\n",
      "\ttrain RMSE = 1.631\n",
      "\ttest RMSE = 1.434\n",
      "Iteration 140\n",
      "\ttrain RMSE = 1.636\n",
      "\ttest RMSE = 1.454\n",
      "Iteration 141\n",
      "\ttrain RMSE = 1.636\n",
      "\ttest RMSE = 1.458\n",
      "Iteration 142\n",
      "\ttrain RMSE = 1.641\n",
      "\ttest RMSE = 1.451\n",
      "Iteration 143\n",
      "\ttrain RMSE = 1.653\n",
      "\ttest RMSE = 1.451\n",
      "Iteration 144\n",
      "\ttrain RMSE = 1.653\n",
      "\ttest RMSE = 1.457\n",
      "Iteration 145\n",
      "\ttrain RMSE = 1.640\n",
      "\ttest RMSE = 1.457\n",
      "Iteration 146\n",
      "\ttrain RMSE = 1.640\n",
      "\ttest RMSE = 1.454\n",
      "Iteration 147\n",
      "\ttrain RMSE = 1.638\n",
      "\ttest RMSE = 1.450\n",
      "Iteration 148\n",
      "\ttrain RMSE = 1.646\n",
      "\ttest RMSE = 1.448\n",
      "Iteration 149\n",
      "\ttrain RMSE = 1.640\n",
      "\ttest RMSE = 1.450\n",
      "Iteration 150\n",
      "\ttrain RMSE = 1.637\n",
      "\ttest RMSE = 1.454\n",
      "Iteration 151\n",
      "\ttrain RMSE = 1.631\n",
      "\ttest RMSE = 1.454\n",
      "Iteration 152\n",
      "\ttrain RMSE = 1.639\n",
      "\ttest RMSE = 1.442\n",
      "Iteration 153\n",
      "\ttrain RMSE = 1.667\n",
      "\ttest RMSE = 1.451\n",
      "Iteration 154\n",
      "\ttrain RMSE = 1.658\n",
      "\ttest RMSE = 1.444\n",
      "Iteration 155\n",
      "\ttrain RMSE = 1.667\n",
      "\ttest RMSE = 1.477\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 156\n",
      "\ttrain RMSE = 1.688\n",
      "\ttest RMSE = 1.506\n",
      "Iteration 157\n",
      "\ttrain RMSE = 1.693\n",
      "\ttest RMSE = 1.481\n",
      "Iteration 158\n",
      "\ttrain RMSE = 1.721\n",
      "\ttest RMSE = 1.478\n",
      "Iteration 159\n",
      "\ttrain RMSE = 1.749\n",
      "\ttest RMSE = 1.499\n",
      "Iteration 160\n",
      "\ttrain RMSE = 1.735\n",
      "\ttest RMSE = 1.492\n",
      "Iteration 161\n",
      "\ttrain RMSE = 1.707\n",
      "\ttest RMSE = 1.488\n",
      "Iteration 162\n",
      "\ttrain RMSE = 1.696\n",
      "\ttest RMSE = 1.508\n",
      "Iteration 163\n",
      "\ttrain RMSE = 1.674\n",
      "\ttest RMSE = 1.499\n",
      "Iteration 164\n",
      "\ttrain RMSE = 1.644\n",
      "\ttest RMSE = 1.478\n",
      "Iteration 165\n",
      "\ttrain RMSE = 1.631\n",
      "\ttest RMSE = 1.472\n",
      "Iteration 166\n",
      "\ttrain RMSE = 1.627\n",
      "\ttest RMSE = 1.484\n",
      "Iteration 167\n",
      "\ttrain RMSE = 1.632\n",
      "\ttest RMSE = 1.496\n",
      "Iteration 168\n",
      "\ttrain RMSE = 1.646\n",
      "\ttest RMSE = 1.521\n",
      "Iteration 169\n",
      "\ttrain RMSE = 1.670\n",
      "\ttest RMSE = 1.537\n",
      "Iteration 170\n",
      "\ttrain RMSE = 1.707\n",
      "\ttest RMSE = 1.538\n",
      "Iteration 171\n",
      "\ttrain RMSE = 1.730\n",
      "\ttest RMSE = 1.551\n",
      "Iteration 172\n",
      "\ttrain RMSE = 1.752\n",
      "\ttest RMSE = 1.578\n",
      "Iteration 173\n",
      "\ttrain RMSE = 1.754\n",
      "\ttest RMSE = 1.571\n",
      "Iteration 174\n",
      "\ttrain RMSE = 1.743\n",
      "\ttest RMSE = 1.542\n",
      "Iteration 175\n",
      "\ttrain RMSE = 1.749\n",
      "\ttest RMSE = 1.533\n",
      "Iteration 176\n",
      "\ttrain RMSE = 1.731\n",
      "\ttest RMSE = 1.522\n",
      "Iteration 177\n",
      "\ttrain RMSE = 1.721\n",
      "\ttest RMSE = 1.524\n",
      "Iteration 178\n",
      "\ttrain RMSE = 1.738\n",
      "\ttest RMSE = 1.533\n",
      "Iteration 179\n",
      "\ttrain RMSE = 1.765\n",
      "\ttest RMSE = 1.541\n",
      "Iteration 180\n",
      "\ttrain RMSE = 1.793\n",
      "\ttest RMSE = 1.553\n",
      "Iteration 181\n",
      "\ttrain RMSE = 1.813\n",
      "\ttest RMSE = 1.560\n",
      "Iteration 182\n",
      "\ttrain RMSE = 1.821\n",
      "\ttest RMSE = 1.563\n",
      "Iteration 183\n",
      "\ttrain RMSE = 1.803\n",
      "\ttest RMSE = 1.564\n",
      "Iteration 184\n",
      "\ttrain RMSE = 1.766\n",
      "\ttest RMSE = 1.552\n",
      "Iteration 185\n",
      "\ttrain RMSE = 1.729\n",
      "\ttest RMSE = 1.522\n",
      "Iteration 186\n",
      "\ttrain RMSE = 1.720\n",
      "\ttest RMSE = 1.509\n",
      "Iteration 187\n",
      "\ttrain RMSE = 1.715\n",
      "\ttest RMSE = 1.506\n",
      "Iteration 188\n",
      "\ttrain RMSE = 1.711\n",
      "\ttest RMSE = 1.518\n",
      "Iteration 189\n",
      "\ttrain RMSE = 1.725\n",
      "\ttest RMSE = 1.530\n",
      "Iteration 190\n",
      "\ttrain RMSE = 1.733\n",
      "\ttest RMSE = 1.516\n",
      "Iteration 191\n",
      "\ttrain RMSE = 1.755\n",
      "\ttest RMSE = 1.506\n",
      "Iteration 192\n",
      "\ttrain RMSE = 1.771\n",
      "\ttest RMSE = 1.511\n",
      "Iteration 193\n",
      "\ttrain RMSE = 1.760\n",
      "\ttest RMSE = 1.508\n",
      "Iteration 194\n",
      "\ttrain RMSE = 1.756\n",
      "\ttest RMSE = 1.528\n",
      "Iteration 195\n",
      "\ttrain RMSE = 1.768\n",
      "\ttest RMSE = 1.548\n",
      "Iteration 196\n",
      "\ttrain RMSE = 1.770\n",
      "\ttest RMSE = 1.530\n",
      "Iteration 197\n",
      "\ttrain RMSE = 1.766\n",
      "\ttest RMSE = 1.512\n",
      "Iteration 198\n",
      "\ttrain RMSE = 1.759\n",
      "\ttest RMSE = 1.507\n",
      "Iteration 199\n",
      "\ttrain RMSE = 1.726\n",
      "\ttest RMSE = 1.505\n",
      "Iteration 200\n",
      "\ttrain RMSE = 1.711\n",
      "\ttest RMSE = 1.537\n",
      "Iteration 201\n",
      "\ttrain RMSE = 1.705\n",
      "\ttest RMSE = 1.534\n",
      "Iteration 202\n",
      "\ttrain RMSE = 1.713\n",
      "\ttest RMSE = 1.515\n",
      "Iteration 203\n",
      "\ttrain RMSE = 1.741\n",
      "\ttest RMSE = 1.525\n",
      "Iteration 204\n",
      "\ttrain RMSE = 1.740\n",
      "\ttest RMSE = 1.524\n",
      "Iteration 205\n",
      "\ttrain RMSE = 1.753\n",
      "\ttest RMSE = 1.551\n",
      "Iteration 206\n",
      "\ttrain RMSE = 1.778\n",
      "\ttest RMSE = 1.576\n",
      "Iteration 207\n",
      "\ttrain RMSE = 1.770\n",
      "\ttest RMSE = 1.555\n",
      "Iteration 208\n",
      "\ttrain RMSE = 1.765\n",
      "\ttest RMSE = 1.529\n",
      "Iteration 209\n",
      "\ttrain RMSE = 1.760\n",
      "\ttest RMSE = 1.523\n",
      "Iteration 210\n",
      "\ttrain RMSE = 1.747\n",
      "\ttest RMSE = 1.522\n",
      "Iteration 211\n",
      "\ttrain RMSE = 1.740\n",
      "\ttest RMSE = 1.544\n",
      "Iteration 212\n",
      "\ttrain RMSE = 1.732\n",
      "\ttest RMSE = 1.546\n",
      "Iteration 213\n",
      "\ttrain RMSE = 1.723\n",
      "\ttest RMSE = 1.519\n",
      "Iteration 214\n",
      "\ttrain RMSE = 1.730\n",
      "\ttest RMSE = 1.510\n",
      "Iteration 215\n",
      "\ttrain RMSE = 1.726\n",
      "\ttest RMSE = 1.510\n",
      "Iteration 216\n",
      "\ttrain RMSE = 1.723\n",
      "\ttest RMSE = 1.525\n",
      "Iteration 217\n",
      "\ttrain RMSE = 1.716\n",
      "\ttest RMSE = 1.538\n",
      "Iteration 218\n",
      "\ttrain RMSE = 1.705\n",
      "\ttest RMSE = 1.515\n",
      "Iteration 219\n",
      "\ttrain RMSE = 1.727\n",
      "\ttest RMSE = 1.509\n",
      "Iteration 220\n",
      "\ttrain RMSE = 1.728\n",
      "\ttest RMSE = 1.505\n",
      "Iteration 221\n",
      "\ttrain RMSE = 1.722\n",
      "\ttest RMSE = 1.510\n",
      "Iteration 222\n",
      "\ttrain RMSE = 1.723\n",
      "\ttest RMSE = 1.505\n",
      "Iteration 223\n",
      "\ttrain RMSE = 1.734\n",
      "\ttest RMSE = 1.497\n",
      "Iteration 224\n",
      "\ttrain RMSE = 1.762\n",
      "\ttest RMSE = 1.500\n",
      "Iteration 225\n",
      "\ttrain RMSE = 1.776\n",
      "\ttest RMSE = 1.505\n",
      "Iteration 226\n",
      "\ttrain RMSE = 1.779\n",
      "\ttest RMSE = 1.512\n",
      "Iteration 227\n",
      "\ttrain RMSE = 1.784\n",
      "\ttest RMSE = 1.529\n",
      "Iteration 228\n",
      "\ttrain RMSE = 1.772\n",
      "\ttest RMSE = 1.523\n",
      "Iteration 229\n",
      "\ttrain RMSE = 1.762\n",
      "\ttest RMSE = 1.509\n",
      "Iteration 230\n",
      "\ttrain RMSE = 1.757\n",
      "\ttest RMSE = 1.506\n",
      "Iteration 231\n",
      "\ttrain RMSE = 1.752\n",
      "\ttest RMSE = 1.511\n",
      "Iteration 232\n",
      "\ttrain RMSE = 1.749\n",
      "\ttest RMSE = 1.529\n",
      "Iteration 233\n",
      "\ttrain RMSE = 1.751\n",
      "\ttest RMSE = 1.553\n",
      "Iteration 234\n",
      "\ttrain RMSE = 1.751\n",
      "\ttest RMSE = 1.546\n",
      "Iteration 235\n",
      "\ttrain RMSE = 1.750\n",
      "\ttest RMSE = 1.545\n",
      "Iteration 236\n",
      "\ttrain RMSE = 1.748\n",
      "\ttest RMSE = 1.551\n",
      "Iteration 237\n",
      "\ttrain RMSE = 1.750\n",
      "\ttest RMSE = 1.562\n",
      "Iteration 238\n",
      "\ttrain RMSE = 1.758\n",
      "\ttest RMSE = 1.556\n",
      "Iteration 239\n",
      "\ttrain RMSE = 1.782\n",
      "\ttest RMSE = 1.548\n",
      "Iteration 240\n",
      "\ttrain RMSE = 1.792\n",
      "\ttest RMSE = 1.548\n",
      "Iteration 241\n",
      "\ttrain RMSE = 1.787\n",
      "\ttest RMSE = 1.553\n",
      "Iteration 242\n",
      "\ttrain RMSE = 1.803\n",
      "\ttest RMSE = 1.564\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-39-0b4a7956fdb2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Iteration {}\\n\\ttrain RMSE = {:.3f}\\n\\ttest RMSE = {:.3f}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_rmse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_rmse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m     \u001b[0mtraj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madam\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_iters\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/misc/optimizers.py\u001b[0m in \u001b[0;36m_optimize\u001b[0;34m(grad, x0, callback, *args, **kwargs)\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m             \u001b[0m_callback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 28\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0munflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_grad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_x0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_callback\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/misc/optimizers.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(grad, x, callback, num_iters, step_size, b1, b2, eps)\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0mv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_iters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 64\u001b[0;31m         \u001b[0mg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     65\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0mm\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mb1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mg\u001b[0m      \u001b[0;34m+\u001b[0m \u001b[0mb1\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mm\u001b[0m  \u001b[0;31m# First  moment estimate.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/misc/optimizers.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x, i)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_optimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0m_x0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munflatten\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m         \u001b[0m_grad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0m_callback\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0munflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-39-0b4a7956fdb2>\u001b[0m in \u001b[0;36mgrad_loss\u001b[0;34m(x, i)\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mgrad_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/wrap_util.py\u001b[0m in \u001b[0;36mnary_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     18\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m                 \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0margnum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0munary_operator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0munary_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mnary_op_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mnary_op_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     21\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnary_f\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnary_operator\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/differential_operators.py\u001b[0m in \u001b[0;36mgrad\u001b[0;34m(fun, x)\u001b[0m\n\u001b[1;32m     26\u001b[0m         raise TypeError(\"Grad only applies to real scalar-output functions. \"\n\u001b[1;32m     27\u001b[0m                         \"Try jacobian or elementwise_grad.\")\n\u001b[0;32m---> 28\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mvjp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mans\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mones\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m@\u001b[0m\u001b[0munary_to_nary\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/core.py\u001b[0m in \u001b[0;36mvjp\u001b[0;34m(g)\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mvjp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mvspace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m         \u001b[0;32mdef\u001b[0m \u001b[0mvjp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mbackward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mvjp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/core.py\u001b[0m in \u001b[0;36mbackward_pass\u001b[0;34m(g, end_node)\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtoposort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mend_node\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0moutgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutgrads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mingrads\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvjp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutgrad\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mparent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mingrad\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparents\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mingrads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0moutgrads\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madd_outgrads\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutgrads\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mingrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/core.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(g)\u001b[0m\n\u001b[1;32m     59\u001b[0m                     \"VJP of {} wrt argnum 0 not defined\".format(fun.__name__))\n\u001b[1;32m     60\u001b[0m             \u001b[0mvjp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvjpfun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mans\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 61\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mvjp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     62\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0mL\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     63\u001b[0m             \u001b[0margnum_0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margnum_1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0margnums\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/numpy/numpy_vjps.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(g)\u001b[0m\n\u001b[1;32m    559\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0munbroadcast_f\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m     \u001b[0mtarget_meta\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmetadata\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0munbroadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_meta\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0munbroadcast_einsum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_meta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubscript\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/numpy/numpy_vjps.py\u001b[0m in \u001b[0;36munbroadcast\u001b[0;34m(x, target_meta, broadcast_idx)\u001b[0m\n\u001b[1;32m    548\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0munbroadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_meta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbroadcast_idx\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m     \u001b[0mtarget_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_ndim\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_iscomplex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtarget_meta\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m     \u001b[0;32mwhile\u001b[0m \u001b[0manp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mtarget_ndim\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0manp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbroadcast_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msize\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/autograd/tracer.py\u001b[0m in \u001b[0;36mf_wrapped\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mnotrace_primitive\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf_raw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m     \u001b[0;34m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf_raw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mf_wrapped\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m         \u001b[0margvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgetval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from autograd.misc.optimizers import adam, sgd\n",
    "\n",
    "def loss(x):\n",
    "    return - log_posterior(x)\n",
    "\n",
    "def grad_loss(x, i):\n",
    "    return grad(loss)(x)\n",
    "\n",
    "\n",
    "traj = [(params, -1, grad_loss(params, -1))]\n",
    "prediction_traj = [make_predictions(params, inds=np.arange(len(mols)))]\n",
    "train_rmse = rmse(prediction_traj[-1][train_inds], expt_means[train_inds])\n",
    "test_rmse = rmse(prediction_traj[-1][test_inds], expt_means[test_inds])\n",
    "print('initial parameters\\n\\ttrain RMSE = {:.3f}\\n\\ttest RMSE = {:.3f}'.format(train_rmse, test_rmse))\n",
    "\n",
    "def callback(x,i,g):\n",
    "    #if np.sum(np.isnan(x)) > 0:\n",
    "    #    raise(RuntimeError('NaNs encountered!'))\n",
    "    prediction_traj.append(make_predictions(x, inds=np.arange(len(mols)), batch_size=50, randomized=False))\n",
    "    train_rmse = rmse(prediction_traj[-1][train_inds], expt_means[train_inds])\n",
    "    test_rmse = rmse(prediction_traj[-1][test_inds], expt_means[test_inds])\n",
    "    print('Iteration {}\\n\\ttrain RMSE = {:.3f}\\n\\ttest RMSE = {:.3f}'.format(i, train_rmse, test_rmse))\n",
    "    traj.append((x,i,g))\n",
    "result = adam(grad_loss, params, callback=callback, num_iters=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:05.713498Z",
     "start_time": "2019-03-07T17:10:05.681951Z"
    }
   },
   "outputs": [],
   "source": [
    "np.savez('linear_typing_result_student_t_loss_df=7.npz',\n",
    "         prediction_traj=prediction_traj,\n",
    "         train_inds=train_inds,\n",
    "         test_inds=test_inds,\n",
    "         expt_means=expt_means,\n",
    "         expt_uncs=expt_uncs,\n",
    "         traj=traj,\n",
    "         layer_sizes=layer_sizes,\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:06.197190Z",
     "start_time": "2019-03-07T17:10:06.178370Z"
    }
   },
   "outputs": [],
   "source": [
    "train_rmses = [rmse(snapshot[train_inds], expt_means[train_inds]) for snapshot in prediction_traj]\n",
    "test_rmses = [rmse(snapshot[test_inds], expt_means[test_inds]) for snapshot in prediction_traj]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:06.684905Z",
     "start_time": "2019-03-07T17:10:06.681092Z"
    }
   },
   "outputs": [],
   "source": [
    "train_color = 'lightblue'\n",
    "test_color = 'green'\n",
    "train_style = '--'\n",
    "test_style = '-'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:07.327628Z",
     "start_time": "2019-03-07T17:10:07.128962Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'adam iterations')"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XeY1PW1+PH3mdnZ3lhYloUFFhDsUgUVNUSjUYOi0TSNLVhTTIzx3twk5ocmuSlqEr25iYkxttxoVKyoiBUsdOlF6i7ssr3Xqef3x8yOu7ANYXZ2d87reeZh5ltmzndnmDOfLqqKMcYYA+CIdgDGGGP6D0sKxhhjwiwpGGOMCbOkYIwxJsySgjHGmDBLCsYYY8IsKRhjjAmzpGCMMSbMkoIxxpiwuGgHcLiGDRum+fn50Q7DGGMGlLVr11aqanZPxw24pJCfn8+aNWuiHYYxxgwoIlLYm+Os+sgYY0yYJQVjjDFhlhSMMcaEWVIwxhgTZknBGGNMmCUFY4wxYZYUjDHGhMVcUthf34LbH4h2GMYY0y/FVFKod3tZV1pHdYsn2qEYY0y/FFNJoarFi0+V1PgBN5DbGGP6RIwlhWAJwR/QKEdijDH9U0wmhX31LVGOxBhj+qeYSQqtPj9NXj9gJQVjjOlKzCQFf0AZnZYYvK+WFIwxpjMxkxRS4uM4deQQ0uLj8FlJwRhjOhUzSaFNnEOspGCMMV2Iub6Z00Zk4BCJdhjGGNMvxVxSyEhwRTsEY4zpt2Ku+qii2U2RdUk1xphOxVxSKKxrYXNlQ7TDMMaYfinmkoJTxMYpGGNMFyKWFEQkUURWicgGEdkiInd3csx1IlIhIutDtxsiFU+bOIdYl1RjjOlCJBua3cA5qtooIi7gAxF5XVVXHHTcv1X1uxGMowNnqEuqqiLWC8kYYzqIWElBgxpDD12hW9R/ojtDicAKC8YYc6iItimIiFNE1gPlwJuqurKTwy4XkY0i8pyIjI5kPAD5GUmcNy4bhxUSjDHmEBFNCqrqV9UpQB4wU0ROOuiQV4B8VT0FeAt4vLPnEZGbRGSNiKypqKg4opgS4pykxcdZ1ZExxnSiT3ofqWot8B5wwUHbq1TVHXr4MDC9i/P/pqozVHVGdnb2EcXS6PGxo7oRt8+W5DTGmINFsvdRtohkhu4nAV8Ath90TG67h5cA2yIVT5t6t4/NFQ00+/yRfiljjBlwItn7KBd4XEScBJPPM6q6SETuAdao6svAbSJyCeADqoHrIhgPEOx9BLamgjHGdCZiSUFVNwJTO9n+83b3/wv4r0jF0Jm4tqRgM6UaY8whYnJEM2AD2IwxphMxlRQCGuCepT+ltKHASgrGGNOJmEoKu6t384cV99Lo/ohRqYnRDscYY/qdmEoKhXWFALT6msINzsYYYz4VU0mhoLYg9G8V5U3u7g82xpgYFFNJobA2WFIobqihotkT5WiMMab/iamkUFBXAIDb32QNzcYY04nYSgqh6qNWX5N1STXGmE7EVFJoqz5q9TYeUlJYX1bHntqmaIRljDH9RswkBa/fS3FDMQAtviZ8gY4T4hU1tFDX6otGaMYY02/ETFIoqi8ioG2JoIXTRg4J71NVPH5lb10zam0NxpgYFjNJoa09YVzmOBo9DR3WU2jfvuDxW1IwxsSumEkK1S3VJMUlcXLOyTS4G/m4tDY8U6qnXVWSTaltjIllMZMULj/hcpp+0sQpw0+hydvI3trmcAJo37zQ4rWkYIyJXTGTFABEhLSENAIawO1vCSeAtIQ4LpowHIAWKykYY2JYTCUFgLT4NABavI0dEkCC00FOSgIJcTH3JzHGmLBIrrzWL6UlhJKCr5HmUEnhQEMre+uaOTU3k3inJQVjTOyKuW/AtpKCBprDA9gaPD7Kmtw4RKxLqjEmpsVeUgiVFE7OdnFSdjoA3kAAh8DWygbe3FsRzfCMMSaqYi4ppManAtDk/XRKC48/gMvhQLAuqcaY2BZzSaGt+qigrorlxdXh0czxTgcupxBQCFgVkjEmRsVsQ3Ntaz0ljW5a/QES4xw4JA6XI5gjvf4ACXHOaIZpjDFREXtJIVRS8AeaAWj0+JiSkwHAvrrgNm9ASYhOeMYYE1UxV33U1qbg8QfbFBo9n7YhpCXEMTYjCYfY+s3GmNgUcyUFp8NJUlwSLd5GHBIsKSzdV0leWhIThqQwfUR8tEM0xpioibmSAgTbFZq8TQxLSkAEqlq8ePzBCZBU1cYqGGNiVmwmhfg0GjwNnDk6i5GpiQAkxjlp9vp5YUcphfUtUY7QGGOiI+aqjwCyU7I50HAACA5Yi3c6yEtPpK2A4PUHujnbGGMGr5gsKUwaOomdVTspbmihvNnDuIwkXA4HLkewgdkbsOojY0xsismkMDFrIsUNxaTE+TgpO43jhga7qYoIcQ7BG7CSgjEmNsVkUpg0dBIAhXV7mJSVitPxaRdUl0Pw2pKcxpgYFdNJYUfVjkP2jc9MISfFhq4ZY2JTTDY0H5N1DNB5Ujh2aGpfh2OMMf1GTJYUUuNTGZU2ih3VhyaFgGp4zIIxxsSaiCUFEUkUkVUiskFEtojI3Z0ckyAi/xaRXSKyUkTyIxXPwSYNndRpSWFtSS3vFlb2VRjGGNOvRLKk4AbOUdXJwBTgAhE57aBj5gM1qnoM8AfgtxGMp4OJWRPZVb3rkO1xTod1STXGxKyIJQUNagw9dIVuB3/bzgMeD91/DjhXpG9moxuZNpLK5kq8fm+H7cHeRwGb6sIYE5Mi2qYgIk4RWQ+UA2+q6sqDDhkF7AdQVR9QBwzt5HluEpE1IrKmouLoLJeZk5oDQHlTeYftLocDBaxXqjEmFvU6KYjIEBE5UUTGi0ivzlNVv6pOAfKAmSJy0sFP29lpnTzP31R1hqrOyM7O7m3I3cpJCSaFsqayDttdzrZRzdbYbIyJPd12SRWRDOA7wDeAeKACSARyRGQF8GdVfbenF1HVWhF5D7gA2NxuVxEwGigSkTggA6j+DNdx2NpKCmWNHZNCVmI8JwxLw2lrKhhjYlBP4xSeA54AzlLV2vY7RGQ6cLWIjFfVRw4+UUSyAW8oISQBX+DQhuSXgWuB5cAVwDvaR5X5XZUUMhNdZCa6+iIEY4zpd7pNCqp6Xjf71gJruzk9F3hcRJwEq6meUdVFInIPsEZVXwYeAZ4UkV0ESwhfP9wL+Ky6KimoKi0+P3EOB/HOmBzGYYyJYT1VH03rbr+qftzNvo3A1E62/7zd/VbgKz2HefSlxqeS7Eo+pKTQ6g+weE8FU4anM35ISjRCM8aYqOmp+uj+bvYpcM5RjKXPjUgdcUhSSAiVDlptVLMxJgb1VH30+b4KJBpyUnIOqT5yiBDvFNyWFIwxMahXE+KJiAu4FTg7tOk94K+q6u3ypAEgJzWHnVU7D9me4HTi9llSMMbEnt62pP4FmA78OXSbHto2oOWk5BxSfQTBKiS33x+FiIwxJrp6O3X2qaE5jNq8IyIbIhFQX8pJyaGquQpfwEec49M/xcSsFGyWC2NMLOptScEvIhPaHojIeGDA/5TOSc1BUSqaOk6dkZuayMi0xChFZYwx0dPbksKdwLsisofg1BRjgesjFlUfyU3NBaCksYTctNzwdrc/QKPHx5BEFw4b2WyMiSG9Sgqq+raITASOJZgUtquqO6KR9YGxmWMBKKwtZFrup0MyDjS0sq6sjgvGDyfZ5YxWeMYY0+d62/vICXwRyA+dc66IoKq/j2BsEZefmQ9AQW1Bh+1tYxXcfr8lBWNMTOlt9dErQCuwCRg0fTWHJA4hLT7tkKSQGBdKCtYt1RgTY3qbFPJU9ZSIRhIFIkJ+Zj4FdQXsr9tPYlwi2SnZNqrZGBOzetv76HUROT+ikURJfmY+BbUFfPGfX+TWV28FIDEuWGXU4h3wHayMMeaw9LaksAJ4IbS4jpdgY7OqanrEIusj+Zn5LN61GG/AS0CDJQOnQ5iZm2lTaBtjYk5vk8L9wOnApr5a76Cv5Gfm4w0EZ+vYU7MnPJAtLz0pypEZY0zf62310U5g82BLCPBpDyQAb8DLvrp9ADR6fBTVt0QpKmOMiY7elhRKgPdE5HUgPD5hoHdJhU+TwvCU4ZQ3lbOzaifjh4znQEMrmysbyElJwGWL7RhjYkRvv+32Am8TXKc5rd1twJswZALJrmS+PePbAOysDs6amhIfbGxutMZmY0wM6Wnltf8CFqvq3X0UT5/LSMyg4PsFDE0eyn3L7wtPpZ0aH/zTNIWmuzDGmFjQU/XRXuD7IjIZ2AC8DixR1ZqIR9aHslOyATgm65hPSwqu4J+m0euLWlzGGNPXelp57WngaQARmQpcADwfmvbiLYKliFURj7KPTMyayLrSdQDEOYTEOAeNHqs+MsbEjl63oKrqOlX9dWiJzrnAFuCGiEUWBeOHjKewtjA8XmFm7hAm5wz4oRjGGNNrPbUpfLmb3aqqNx3leKJqdPpovAEvFU0V5KTmMCw5PtohGWNMn+qpTeHibvYp8PxRjCXq8tLzANhfv5+c1BwAdlY3EudwMC4zOZqhGWNMn+ipTWHAL6RzONqSQlF9ETNGzgCgtMlNs9dPfkYSYgvuGGMGud4OXkNEvgScCITXqVTVeyIRVLS0TwptxqYnsaa0jgONbkalJRJQpdHjIz3BuqkaYwafXjU0i8hDwNeA7xGcDO8rBJfkHFSyU7JxOVwdkkJeehLJLicrD9RQ0tjK1soG1pXVRzFKY4yJnN72PjpDVa8BakID2U4HRkcurOhwiINR6aM6JAWHCCcO+3TwtirUtHrwBwbdNFDGGNPr6qO2meGaRWQkUAWMi0xI0TU6fXSHpBDclsSI0BxIAYWdNU3Uur0MTbLeScaYwaW3JYVFIpIJ3At8DBQQGtQ22OSl5x2SFIDwpHhDk4JtCVUtnj6Nyxhj+kKvkoKq/kJVa1V1IcG2hONU9a7IhhYdbUmhq1nCE+OcpLqclhSMMYNSbxuavxMqKaCqbsAhIt+OaGRRkpeeh9vvprK5sstjjh2aymhbhMcYMwj1tvroRlWtbXsQmhDvxsiEFF1jM4KdqnZU7QBAVfH4PQcdk0xemiUFY8zg09uk4JB2I7dCE+INylbWs8aehVOcLN61GIA7ltzBhAcnHFJyaPX52VxRjy8QiEaYxhgTEb1NCkuAZ0TkXBE5B3gKWNzdCSIyWkTeFZFtIrJFRL7fyTFzRKRORNaHbj8//Es4urKSspg9ZjaLdi5ic/lmHlj5AEX1Rdyx5I4OxzV7/eyobmJ7VWOUIjXGmKOvt0nhPwiuvHYr8J3Q/Tt7OMcH3KGqxwOnAd8RkRM6Oe59VZ0SuvWLEdJfmvgl1peu58qFV5KekM6tM27liQ1P8PaetwlogKUFS/mPN7/NtxedypxHj+ORj/8Z7ZCNMeao6O04hamq+hDwUNsGEbkYeKWrE1S1hODazqhqg4hsA0YBWz97uH1j7qS5/Odb/8ne2r08cekTXHDMBby5501uWnQTKa4UNpVvIjU+lQsmXMjHpdv4zms3MXHoSZw9dkq0QzfGmCPS25LCwyJyctsDEfkG8LPevoiI5ANTgZWd7D5dRDaIyOsicmJvnzOSTsg+gee+8hwbbtnAZcdfRpIriYe+9BB7avZQ2VzJE5c+QekdpTz71WdY8s3XSIxL5vqXvkmLt6XnJzfGmH5MuuqP3+EgkfHAc8BVwJnANcBcVa3rxbmpwFLgV6r6/EH70oGAqjaKyEXAA6o6sZPnuAm4CWDMmDHTCwsLe4w5EpbvX86xw44lKymrw/bnt73M5c/M43szv8eDFz4YldiMMaY7IrJWVWf0eFxvkkLoCScBLwL7gUtVtcefxSLiAhYBb6jq73txfAEwQ1W7HCQwY8YMXbNmTa9i7kvff/37PLjqQd655l0+P25OtMMxxpgOepsUuq0+EpFNIrJRRDYSLClkAfnAytC27s4V4BFgW1cJQURGtHV1FZGZoXiqegq6P7rn878iN3Us33r5Blp9rdEOxxhjPpOeGprnHsFzzwauBjaJyPrQtp8AYwBCDddXALeKiI/gpHtf194WXfqZjMRUfnjGvdy55Ks8ueFJbpw+KMf2GWMGuW6rj0QkVVW77Yjfm2OOpv5afQRQVN/CWY/OIDHOwdZvb7SV2owx/cZRqT4CXhKR+0XkbBFJaffk40Vkvoi8AVxwpMEOFqPSErn8+JvYXrmZ9wrei3Y4xhhz2LpNCqp6LsGBajcDW0Kjj6uAfwIjgGtV9bnIhzkwiAg3z7iaFFcGf1r9cLTDMcaYw9bj4DVVfQ14rQ9iGRSOGTKES4+7nBe2P02jp5HU+NRoh2SMMb3W28FrppdEhFtmXE+zt5kXtr0Q7XCMMeawWFKIgNmjZ5OXPpY/rnyEwMDsTGWMiVGWFCJARLj8+G+wvvR9NpcXRDscY4zptZ4Gr53T7v64g/Z9OVJBDQY3T7+OgAZ4bP3/RTsUY4zptZ5KCve1u7/woH29nhAvFh2ffSwnZE/jxe1PdbneszHG9Dc9JQXp4n5nj81Bvnrileyt3crK4k3RDsUYY3qlp6SgXdzv7LE5yDWnfBWARTue7+FIY4zpH3pKCuNF5GUReaXd/bbH43o4N+aNGzKa2aNn88oO65pqjBkYehq8Nq/d/fsO2nfwY9OJy4+/nB8u+SEbSrcxecTx0Q7HGGO61dM0F0vb34CPgHqC02Ev7ZMIB7gvTboUgCc2HtxOb4wx/U9PXVIfalsiU0QygA3AE8C60JKcpgeTho5jwpATWLL7NRvIZozp93pqUzhLVbeE7l8P7FDVk4HpwH9ENLJB5KJJc9lasYrtVaXRDsUYY7rVU1LwtLt/HsHlOFFV+3Y7DF8/8VIC6mfh1kXRDsUYY7rVU1KoFZG5IjKV4EpqiwFEJA5IinRwg8VpebMYkjiMt/ZYFZIxpn/rKSncDHwXeBT4QbsSwrnAq5EMbDBxiIO5k77ExrL38Ad80Q7HGGO61FPvox2qeoGqTlHVx9ptf0NV74h4dIPIpcddQm1rLR/u/zDaoRhjTJe6HacgIg92t19Vbzu64Qxe540/j3hnPP+75hlmjz4bl9MmqDXG9D89DV67BdgMPAMcwOY7+szSEtKYPfpsPtr3BiWNrYzJSI52SMYYc4iefq7mAn8DvghcDbiAl1X1cVV9PNLBDTaXHXcJBxr28FHR5miHYowxneqpTaFKVR9S1c8D1wGZwBYRubovghtsLjn2EgAW73oVjz8Q5WiMMeZQvarYFpFpwA+AbwKvA2sjGdRgNTZzLCdkn8Sa4jcpaWyNdjjGGHOInqa5uFtE1gI/BJYCM1R1vqpu7ZPoBqFLJs1le+Uq3L76aIdijDGH6KmkcBeQAUwGfg18LCIbRWSTiGyMeHSD0CXHXoJf/aw98G60QzHGmEP01PvI1kw4ymaOmkl2cjYvfvIy547/MsNTEqIdkjHGhPXU0FzY2Q0oAs7smxAHF6fDyZcmfYnXd77O8uJKfAGb9sIY03/01KaQLiL/JSJ/EpHzJeh7wB7gq30T4uAzd+JcGjx1bCxbSWFdc7TDMcaYsJ7aFJ4EjgU2ATcAS4ArgHmqOq+7E03Xzp9wPvHOeDaWvkVxg/VCMsb0Hz21KYwPrZ+AiPwdqATGqGpDxCMbxNIS0piTP4c1B96ksuUuWn1+EuOc0Q7LGGN6LCl42+6oqh/Yawnh6Lh40sUU1O6ipH43Na3enk8wxpg+0FNSmCwi9aFbA3BK230RsY72R2DupLkANHlWkpuaGOVojDEmqKfeR05VTQ/d0lQ1rt399O7OFZHRIvKuiGwTkS0i8v1OjhEReVBEdoXGP0w70gsaKPIz8zlp+Em8viu4Gpva4jvGmH4gkvM3+4A7VPV44DTgOyJywkHHXAhMDN1uAv4SwXj6nYsnXcz7he/z6s497K61XkjGmOiLWFJQ1RJV/Th0vwHYBow66LB5wBMatALIFJHcSMXU38ydNBe/+lle9Db7rGuqMaYf6JOVXkQkH5gKrDxo1yhgf7vHRRyaOAatWaNmMSx5GBtKl1Dr9lHd4ol2SMaYGBfxpCAiqcBCgms8H9w43dmiPYdUrovITSKyRkTWVFRURCLMqHA6nHztxK/xxq6XqG8tY2dNU7RDMsbEuIgmBRFxEUwI/6eqz3dySBEwut3jPIIrvHWgqn9T1RmqOiM7OzsywUbJ7afdjl/9vF/4OMUNrTR5fNEOyRgTwyKWFEREgEeAbar6+y4Oexm4JtQL6TSgTlVLIhVTfzQhawKXH385C7c+yqQsFwlxtnazMSZ6IvkNNJvgEp7niMj60O0iEblFRG4JHfMawXmUdgEPA9+OYDz91s3Tb6bOXccnFUuJc1hSMMZET0/TXHxmqvoBnbcZtD9Gge9EKoaBYk7+HEakjuBfm/7F9FEXgkJ+ZnK0wzLGxCD7WdoPtDU4v7rzVT6pLGNTRT1eW8PZGBMFlhT6ia+c8BXcfjclDSvxBpRd1hPJGBMFlhT6iVNHnUpiXCIbSleSm5rArpomPFZaMMb0MUsK/US8M55Zo2bxwf4POH5ompUWjDFRYUmhHzlzzJmsK1lHnMPNpKwUMhNd0Q7JGBNjLCn0I2eOORO/+llVvIqTstMZGZpS22ZQNcb0FUsK/cjpeafjEAdLC5aGt60pqWVtaV0UozLGxBJLCv1IRmIGs0bNYvHuxeFtcQ6hqKHFGp2NMX3CkkI/c+ExF7K6eDXlTeUAjM1IIqDw6q4ylu2rwhew5GCMiRxLCv3MRRMvQlHe2PUGAJkJLo7NSmVsRhL1bi8NHn+UIzTm6Gj1+dlR1UiL1z7T/YklhX5mau5UclJyeH57cFJZEeHE7DSmjcjki+OHM8R6JJlBwu0PsLmygQ+LqvFaCbjfsKTQzzjEwQ3TbuDF7S/yyievdNjncjrwB5R6tzdK0Rlz5AKh3nQZCS5Ozc2kweNjU3lDlKPq3xo9PrZWNvRJT0RLCv3QXWffxeScyXztua8x919zqWj6dGGh1SU1fFRcY91UzYC1oriG7VXBJDA6PYmxGUnsr28ZdPN9+QNKcUPLEa+ouLWygXcLK9lT20yrL/J/I0sK/VBCXAIvfv1FvnHSN3h156u8tvO18L4RqYk0e/1srmgYdP+JzOBX0eymtMmNQz6dQHlcZjJ+VfbVt/R4fqvP3+FLNqBKbauX6hZPuATSE48/wHuFlbT6ItOWUdvq5b3CSl7ZVcrKA7WUN7sP+zn8AQ1XqSU4HWQmupidl0WSy3m0wz2EJYV+Kj8zn79d/DfS4tNYVbwqvH10WhJj0pPYWdPEsv1VEftgG3O0qSpbKhpIjHMwITMlvD0zwcWMERmMTk/q9vwWn5/3CqvYUF6PqlLT6mHJ3greKazkvX1VvLarjE+qGntMDlsrG6hz+8Lz+q8vq2NXTVOvk0p3alu9vL+/imafn/GZKZw+agjHZqUCUN7kprSxtVevs7OmkTf3VuD2BZgwJIWzRg/ts/bEiK2nYI6c0+Hk1FGnsrJ4ZbttwozcTEanJ7GiuIZN5fWcOnJIFKM0g0VpUysprjjS4o/sa8Ht81Pr9uHxB8hIiCM9wRV6fjfVrV6m5mTgdHxaUhARxmR0v35IQJWPiqpp8fmZOiIDEaHe7UMVZuRm4hDYV9dCoyf4uolxnf+iLm9ys6e2mQmZySTEOQmoUt3iZU9tM2VNbmbmZuJyfvbfylUtHlwOB2eNySLF9enfMaDK+rI6Gr1+4p0OclISSI5zkOyKY1xo7ZTdobnOKprdHGh0MyotMSorMVpS6OdmjpzJfcvvo8XbQpLr019SOSkJnJM/jBTXpx9st99PdnIC8UfwoTaxqd7t5aOiGlwO4Yy8LIYmxR/W+Q0eH+VNbsZnJrOtqpE9tc0AJDodfD5/GIlOB9sqG0h1ORmb0XmJYF9dsM580tDUQ/Ztq2qkzu3jtJFDyElJAGBsRjKj0hLDqxXmpSWhqoh0vrbXiuIaSptaSYuP4/hhaQA4RDgnfxh7a5tZV1bH63vKGZ+ZzHFDUz/TKogThqSQn5HcIem1vc65+dmUNbkpamihoslNqz/AhFBCCKiysbweBeIdwnFDU5mUldLJK0SeJYV+blbeLHwBH+tL13P66NM77Gv7Rbe8uJqSxmC95Zj0JGbkZvZ5nGZg21rZSJwI8U4HK4pr+OL47F5/Kaoqa0tqafD4yEtL5KTsNNIT4nA5HHxcWseH+6s5I28IM0cOwe0PdGhPaK+i2UNxQytDk+M7JKXSxlY+qWpkTHoSI9MSO5xzcIwiQlWLB48/QG7qp8eqKm6/n9zURKbmZBzyw2lcZjKZiS52Vjeyv76F44am0uTxsbqkloQ4B9nJCeSkJIT/z5U0tlLS2IpThNzURFp8fvyqjM9MOSQhtHE6hJFpieFraJ/AHCJcPDEHb0BJdDq6TGx9wZJCPzdz1EwAPtr/0SFJoc2EzBTGpCdxoKGVooYWThmebqUF02sBVXwB5ZisFEakJFDr9nb5xd2Z3bXNVLd6mTEig4RQtc34UJtBvNPB9qoGEpxOnA7h0DLApyZlpVLW7GbpviqyEl3kpiYyPjOZ7OSEw/rlvLWygaoWD6eNGsLw5AQaPT7SE1x8bsywbs8bkuhi5sgheP0B4hwOHC7B5XRQ7/aFf3SNSElg8vB0alq94Ybx3aFSUVaii3EZyb3+Qj/4uDiHgyjUFh1CBlrXxhkzZuiaNWuiHUafmvLQFBRl/c3ru/3A1bZ6eaewklOGp3PMkOgUPc3AdXDVS4vPT1IXdfPtj1myp4JhyfGcMWpIp5/P7qp0DuYLBNhd00xxYysNbh9fOiaHuC5+eXfF4w/w/v4q6tw+EpwO3P4As/OywtVOn0Wz109hXTN7aps5My+LjEQXGkqmlaF2hKwk12El074mImtVdUZPx/WDvGR68t2Z32Vj2UY+3P9ht8dlJrqYkZvJmB56cXSmxednf/2R96k2A4fb52dHdSNNXh/Q8ZdrZbOHN/aUU9LY2u1zFNY141fllOHpXX7xH05VSJzDwbFDUzln7DAunDD8sBMCBEsnZ48eyonD0hiS6OL0UUMYnnx4bSQHS3Y5OX5YGheMH05GqBeQSLBsO7pyAAAfCklEQVQkkZuayLDk+H6dEA6HJYUB4MqTryQzMZPffPCbHgetjUlPIt7pOKzBbf6A8k5BJatLallVUnuk4ZoBwB9Qlu2vZnNFAxvK6g/5vGQkBnshfVxah7ubAVNDEuOZlJVyxD2WOnMkVaAuZzC5nJGXRW5q4lGro++qvWAwsaQwACS7kvnpWT/l1Z2vcv/y+3s8vrrFw9sFlRQ39DwYCKC82Y3bHyAvLZHhyQlHpb+26b8Cqmwor6fB4+P0UUM4Iy/rkC9Nl8PBjBGZeAMBNlfWd/lcOSkJnJSdHumQTR+ypDBA3HH6HVxxwhX8+K0f817Be3z12a9y74f3EtBDf8Ulu5zEOYSVB2pZV1bH7pom9oYaw1p8/kN+FRY3tOIKjX+YNiLjkGJwq8/P3tpmm1qjn/EFAhQ3BPvmH46VB2ooqGvmmCEpHXroHCwj0UV+RjL76lpo9nZ8jYAqWysbcNuo+kHHeh8NECLCwxc/zPL9yzn3iXMJaIBntz7Ltspt/GPePzocmxjn5KzRQ9lS2cCu0ICYIYkuclISeLewkglDkjluaFr4+GOGBHudOERQVTwBJaFd0X1XTRM7qptwOYS8z9BeYY6utoZbty/AygO1OAUmD88gNd5JsiuO5NBUCG5/AJdDcIS6aW6tbGBKTgYuh4NZIzPDy712Z1JWKvvqWqhq8ZLcbjDWJ1WNbK9qZEiol5AZPKz30QCzeNdivvzvL/PghQ/yccnHPPzxw5TcUcKw5M6727X9ikwJfVGsLqmlqKGV4ckJxDuFY7NSww1nAMv2VSECZ40eGt72dkEFdW4fyS4n5+Vnx0S9an+iqhTUtbCvvoUmr48Ul5PPjRmGP6DUub1sKK+npjU4c26C08EXxw/HKfB2QSWNXh85KQlUtwT3H874gzZefwCX04HXH6DW7aWk0c2umiby0hKZaaPpB4ze9j6yksIAc8ExF1D741rinfFsLNvIX9b8hWe2PMO3T/12p8enHtQAOHVEBh5/cCBPTaufZq+fz40ZGq5TTk+IC1UX+El2OXH7A9S7fYwIDdzxBgI4HZGflMsENXv9fFxaS3mzh/T4OHKSE3A5HQRUcTqErKR4zh49lJpWL35VWn1+fIEADqeDSVkpVLV6OdDQSnpCHCdnp3+mUbpt0z6UN7tZeSDYEWFsRhJTczKO6rWa/sFKCgPcKX85hZT4FJbPX37Y53oDAVq8/vDcNABVzR4+KKoGlJGpiYwf8mnPku56gxxOX/Sjqe3zG80RoJFU0ezmo6IaThmeTn5GUlSv0+0LUN3qISPBFa6iMgOHjVOIEddPuZ4VRSt4dsuzh32uy+HokBAAhibHc964bEanJ1Ha5KbB4yPe6SA+9Ou0otmNP9Dxh0Sz188beyrCjdkHi+QPj0+qG1lX1nXvmP4ooEpZk5tPqhq77O7Z9jfLTk7gwgnDGZfZ+5GykZIQF+yTbwlhcLPqowHuuzO/y7+3/JsbX7mRenc93zzlmyTEffaRmxDsvTRtRCZTcpT23+dlTW6WF9eQn5FEnMNBZkIcw5IT8AUCxMc52FBex+j0xA5VFPvrW9hS2RCRtghV5ZOqJnJSuh6YpKrsrm0mxeWMeoNoS6gX197a5nAjcNvI8+1Vjbh9fkalJQLCpop6RqcnccyQFJuyxPQp+7QNcC6ni6cuf4rRGaO54ZUbuGnRTUftuR0iHb7Ihycn4BShoK6FPbVNrCmto6bVQ3qCi5Oz0wgolDZ2XFBkV00TAnSVD1p9frZWNvBJVeNhx1cdqkcfmZZEq89PQyddM3dUN7GxvJ61pbX4jnAd4FafnwMNreyra6ai2X1Y4zkqmt28sac83GPn9FFDuGD88PDft8HtpaCumWX7q1m2v4qmUJvOQPKfb/4nJ/35JJ7c8GS0Q+lWq6+Vm165iQ/3dT9DQH/TWffzSLCSwiAwbsg4Nt6ykR+/9WN+99HvuHHajZw55syj/jpOh3D2mCwcIqTFx9Ho8YXnxhmWFE+C00FxY2u422qTx0dNq5cThqWiAAe1O/gCAd4trKQlVIWSl57YYQ76Nq0+P8UNrUw4aD6n4oZWHAK5KQm8XVhJWnwcs/OywvurWzxsqWxgaJKLqhYve2ubmZjV3ZRsh1JVAhq89qoWT7ihFYJTHOelJ3H80DQS4hyUN7k50Nga7gmUk5LAiJQEspLiGZIYz9iMZCYOSTmk8R/g1NBEbOXNHhQlOykhKnPpf1bbKrZx3/L7yEjI4JoXryE3LZcvjP9Cl8fvrt7NPzf+k03lm6h313PRxIu4bdZtOCTy1/yH5X/g4Y8f5pUdr7Dxlo1kp2RH/DV7q7SxlEfXPcq03Gl8YfwXwp06ShtLuer5q7h+yvV885RvRjSGgfOpM90SEX7+uZ+Tl57HlQuvZGnBUjz+oz+P0ZDEeDISghN/pSe4wj1TJDSFsNcfQFXxB5Tt1cFf/2nxcby2u5yK5o7xlDS6afEFmDEi2IuloK7jCGxVDVe3bCivp6zp01KI1x9gf30Lw0O9cUanJVHW5KbR46O21UuDJ7gAS3ZyPGfkZTEzNzO8mMnB2rd5tP/1X+/28m5hZXisR3ZyAnPGDOX8cdnMGjmE4SkJFNa10FZbVtbkprCuhbjQ2IDtVY2sLqlFVYlzCFNzMjpNCG1cTgej0hLJS0saUAnB6/dy+xu3kxqfyqZbNzE6fTQ/e+dnnbYl+QI+frTkRxz7p2O5e+ndbCrfREljCbe/cTtXv3D1Yb/26uLV7Kre1evjDzQc4Ffv/4rT806npqWGb7/WsdeeP+Bne+X2w3rOz2pX9S7Oefwc7v3wXtaVrONva//G5Icm85N3fsIF/3cBU/46hf9Z+T/89O2fMvmhySzfv7xvBpCq6oC6TZ8+XU3X1hSv0fw/5isL0KRfJunrO1/vs9eua/XonpomVVUtrG3ShdsP6Mclter1B/T1XWW6eHeZVrd4tLSxVVu8vvA5qqqfVDVoVbNbA4GA1rZ61O3z6+6aRl24/YAW1Dbp4t1l+vruMm32+rSu1aNNHp8u21epVc1uVVVtcHv1+e0HdGHoVtbY2mmM5U2tuqakRrdW1OvyompdvLtMX9tVqj5/QFVVVx+o0UU7S/XNPeW6cPsBfWVnqRbVt3R5zf5AIHzf5w9ooN3jBrdXG9zeI/iL9m9ljWX66/d/rZ9/7PPKAvRPK/+kqqoPr31YWYD+5K2fqMfnCR+/uWyznvWPs5QF6I0v36jF9cWqqhoIBPTHb/5YWYAuK1jW69dfV7JOE36RoMPvHa6FtYUd9m0u26z3fnivrixa2WH7tS9cq/G/iNddVbv0F0t/oSxA39v7nqqqenwenf3IbGUBmvyrZC2qKzrkev+w/A/a5Gnq/R+pC4FAQD//2Oc17p44ZQHh2/S/Tte1B9bqU5ue0nF/HKcsQB13O/S8J87TjaUbj+g1gTXai+/YiHVJFZF/AHOBclU9qZP9c4CXgL2hTc+r6j09Pa91Se1ZbWstL3/yMvd+dC+ljaVsvGUjuWm5h/UcvoCPOMdnr10sbmgh3hlcnASCs24u218V3j9rZCaj0g4dHV3W5ObDomriRPCrhn/p17R6WbYveP6Jw9I6XZ2rzu2lpLGVeKeDMelJnfbJL6hrZnN5PZ6AkhjnCFd7HTs0lcQ4J/vrW6hoDpZgshJdjMtM7nJpx1hW2VzJ5x77HFsrtgYnazz3N9w842Yg+NmZ//J8ntjwBNNzp/OTs37C0oKl/O/q/yU9IZ0HLniAqyd3LBU0e5sZ98A4JudMZsnVS8LbH177MA+sfIBHLnmEWXmzwtsb3A3MeHgGDe4GmrxNjB8yng+u/4CU+BRUldMfOT28jO2j8x4lMzGTtQfW8sv3f8l/zv5PfvOF39DibeHYPx1LvDOen571UzaVb+IPK/7Az876Gb/76HdcfcrV/P2SvwNQWFvI+f88nx1VO/j+rO/zxwv+2OPfSFW56927KKgt4IZpNzAnf05439/W/o2bF93MQ196iLPHns22ym2MTBvJrFGzwlWs/oCfyuZKEuMSyUg88jEhve2SGrFf9MDZwDRgcxf75wCLDvd5raTQe1vLt2ryr5L13MfPVX/A32Hfiv0rdMmuJeHHJQ0l+vj6x/Xdve/q5rLNmvjLRJ3z2BzdWr71qMVT3NCiBbVNWtns1lafv9NjvH6/7q1t0hXF1frB/ir1+j89bmd1o36wv0prWz2dnns4/IGOv+pN7wUCAT3/yfM18ZeJ+s6ed7o87rktz+nQ3w4N/9q9ddGtWtlU2eXx9354r7IAveudu7TZ06w/XPxDZQEa/4t4TfvvNH15+8taVFekK/av0CsXXqmOux26tGCpvrbjNXXc7dDLnr5MK5sqdVnBMmUB+uv3f63nPn6uygIJ/xLP+32e1rfWh1/z7T1v6zEPHhPe/43nvqGqqrcvvl0ddzv01+//Wl/f+brm/T5PM36doXP/NVdlgehfVv9FW7wt6vP7dEflDvX5fR2uxev36l3v3BUudbAA/fOqP2sgENAlu5ao6x6XnvfEeYf8v4wkellSiGhVD5BvSSG6/r7278oC9OZXbg5/wa8vWa8pv0pR591OfW3Ha1rWWKZZv81SFqApv0rRi/91sSb9Mkkzfp2hlz59aZSvwETKn1f9WX/y1k/Cj3dV7dKKpgpVDVZDjrx/pN78ys3hbW0Wbl2oLEAfWPFAj69R1VylH+77UA/UH+jxWK/fq9968VvKAsJf5N977XtaUFOgp/zllA7VLCxAf7H0F+Fzf//R75UFqPNup6b+d6oO+90wbfI0aUVThX7xyS/q/R/dr/Wt9drqPbRa0R/w68bSjbp45+Jw1VBtS61e8tQl4dfKuTdH15es1wZ3g856eFa4enb4vcOVBeiI+0bo3e/drW/veVt/9vbPwlW4175wrTZ5mnTuv+YqC9C0/05TFqATH5yo1c3VPf5NjqbeJoWIjmgWkfzQF39X1UcLgSLgAPAjVd3SxfPcBNwEMGbMmOmFhYURinjwUVVueuUm/r7u7zjEwS3Tb2HhtoU4HU6GpwxnR9UOpudOZ3nRcp687EmueeEavAEvt864Fa/fyzNbn6HqP6qOqCrJ9D9F9UUc8+AxuP1uFl+1mPzMfKb+dSoA3zjpG7y9920aPA3Uu+s5deSpLLt+GXGOOFYXr+bipy5meMpwPr7546P+uVBVXtj+AiuLVnLG6DOYd9w8ADx+D39e/Wdafa0cN+w46t31XHXyVR2mXFlfup4Xtr3AzuqdXHrcpXz1xK8ecTxrD6ylsrmSabnTwr2UVJX3Ct7jxe0vUt5czuzRs3lj9xss2rEIAIc4mJM/h+/N/B6XHHsJDnHg8Xt4fP3jrC1Zy8nDT+aqU64iM7Fv11LvbfVRNJNCOhBQ1UYRuQh4QFUn9vSc1qbw2RxoOMAdS+7g6c1PM2XEFJ687EmGJQ9j3tPzWFW8ijtOv4P7zr+P2xffzoOrHmTLt7ewqWwTX33uq3z0ra7XhzYD062LbuWRdY8wMm0kAQ2QlpBGaWMpF0+6mBe3v4jb72bpdUvZVb2Lq56/iu/P+j4nDz+Z77z2HXLTcln0jUWcOPzEaF9Gv7K+dD3lTeVMz53O0OShPZ/Qx/p9Uujk2AJghqpWdnecJYXPTlVZX7qek3NODv/Ca/W18sK2F7j0uEtJciXh9XvZUbWDE4efSHVLNcN+N4wzRp9BanwqT1/xNJmJmcFiJtonfcrN0VfSUMLYP45l/tT5XD35am565SZ2VO3gqcuf4vITLscf8NPoaQw3bn7rpW/x6PpHAZiTP4dnv/Jsl7Pymv6r3ycFERkBlKmqishM4DlgrPYQkCWFvjXr77NYVbwKCM6z9I95/+DaF69lY9lGll23jLSEtA7Hqyqv7XyNs8eefcg+c2Q2lm3kr2v+SmlTKTdMvYHctFwWbl3IutJ1OMTBtNxpzBo1i5mjZpIQl8CG0g1sqdiCP+DnwokXMjJtJJXNlfx59Z/57/f/m53f28mErAlA9xMaqiori1eys2onXz/p67icrk6PM/1b1KfOFpGnCDYmDxORIuD/AS4AVX0IuAK4VUR8QAvw9Z4Sgul7f7rwT+yp2cP60vX85sPfkBSXxBMbngBg/svz+fcV/+7wZfLmnjeZ+9Rczht/Hq9d9Zq1RRwlBbUFzHlsDm6/mxRXCs9vex4AQTgl5xS8AS+LdixC0fD2tvsA6QnpjEgdwY6qHcQ54rj0uEvDCQG6n2VWRDgt7zROyzstQldn+hObOtv0Squvlcv+fRmLdy1mbMZYrp18Lfcsu4eXvv4Slxx7Sfi4KxdeycJtC/H4PfzsrJ/xi3N+EcWoB6b3Ct7j+GHHk5OaA8C+un3Me3oee2r2sPamtYxOH82S3Uto8bUwa9QsxmaOBYJ999eWrGVl0Uo8fg9Tc6dy8vCTafI2Mf/l+VQ2V/Ll477Mop2LeOLSJ5g+cno0L9P0sX5RfRQJlhSixxfw8cCKB5g9ZjbTc6dzykOnENAAz33lORLiEghogKl/ncr8qfOpaqli0Y5FfPStj1hVvIrrplyH0+HkttdvY23JWl6/6nXSEw5d8L2ovojrX7qe88efz52z7/xMca4oWsHWiq18a+q3jvSSI6qyuZLVxauZOWomQ5OH4va5+cHiH/DQ2ofITs7mezO/R2FdIc9ufZaABnjmime4cOKFn+m12v6fR3v6bRM9lhRMxC3asYiLn7r4kO2rblhFkiuJk/8SbND2BXzce969DE0ayrdeDn5RXzTxIl75xisdGqsrmiqY/NBkShpLSHGlsO/2fWQlZXV47q0VW/njij/ymy/8hqykLPwBPx6/hyRXcHT0prJNzP7HbBo8DSy7bhlnjT0rgn8BaPG2UNlcSV563iFfuOVN5Wwp38KErAmMyRgT3r63Zi8/f+/n/GvTvwhogNHpo/n5537OI+seYUXRCr576ndZtm8ZG8s2khafxtxJc/nlOb9k/JDxEb0WM7hZUjB9YnXxagrrCnH73LT6WslKyuKy4y8D4GvPfY139r7Didkn8v6+9wlogM+N/RyXH385ty2+jQcueIBbZtzC23veJt4Zz6s7X+WBlQ/w6LxHufbFa7lnzj3c9bm7wq/V7G3m1IdPZWvFVuZOmstLX3+J+S/P56XtL7HwqwuZPWY2J/35JJq8TTjEwbDkYSy7bhnVLdVkJGZ85n7hXr+XV3a8whu73iA9IZ0ZI2dw7LBjeXrz09y//H58AR9XnnwlT172ZDjJ7avbx6kPn0p5UzmCcOlxlzI8ZTiNnkae3/Y8IsIt02/h9NGnc/sbt1NUX0RqfCqPznuUK064AlWlxddCYlyi9fIyR4UlBRN1Hr8n3L1x/svzOWvMWdx66q2kuFKY+9Rc3tn7DklxSdS01gDBQT/XTL6GR+c9ytx/zWVp4VIeuOABHv74YW6cdiNLdi/h31v+zdWnXM2TG59k3rHzeOmTl0h2JePxe7j0uEt5butzvHrlq7R4W7ji2StIi0+jwdPAccOOY/n85b1KDAENsPbAWrZVbqO6pZqH1jzEJ1WfkJGQgdsfTH5trpl8DRkJGfzPqv/hpOEnkRofnJOpsLaQJm8Tj857lFXFq3h8w+OoKolxiZyWdxr3nX8feel5QDDZFdcXk5uWGz7fmKPNkoLp14rri7n035dy/LDj+coJX+GDfR/w1OanWHb9MvIz8ymqL+LMf5xJYV0hLocLbyC4RsFvv/Bb7jzjTu58807uX34/uam5rL5xdXCQ3f6POGfcObx19VuICCuLVvKn1X8iJyWHB1c+yISsCZyQfQL76/ZzWt5p3DT9Jk4afhKqSnFDMauLV7OscBkvbH+BwrpPR81PGjqJX5/763CD+toDaznQcIDRGaOZMXIGqsq9H93L23vfDp8T54jjzjPu7DAJmjHRZEnBDHi7qnfxj3X/4LZZt/HHFX9kYtZE5k+bH97/zJZnyM/MZ+aomTS4G/jV+79i/tT5TBx66MD4F7a9wP3L76eqpYqclByWFy3H4/cwLnMcTd4mypvKAUiMS2RO/hy+efI3mTFyBllJWQxNHmpVOGbAs6RgTDcqmyt5bP1jrDmwhtT4VCbnTObUUacyZcQUEuOiu5azMZEQ9cFrxvRnw5KH8aMzfhTtMIzpd6xMbIwxJsySgjHGmDBLCsYYY8IsKRhjjAmzpGCMMSbMkoIxxpgwSwrGGGPCLCkYY4wJG3AjmkWkAijs8cDODQO6XQN6ELNrjz2xet1g197ZtY9V1eyeTh5wSeFIiMia3gzzHozs2mPv2mP1usGu/Uiu3aqPjDHGhFlSMMYYExZrSeFv0Q4giuzaY0+sXjfYtX9mMdWmYIwxpnuxVlIwxhjTjZhJCiJygYh8IiK7ROTH0Y4n0kSkQEQ2ich6EVkT2pYlIm+KyM7Qv0OiHeeREpF/iEi5iGxut63T65SgB0OfgY0iMi16kR+5Lq59gYgUh9739SJyUbt9/xW69k9E5IvRifrIichoEXlXRLaJyBYR+X5o+6B/37u59qP3vqvqoL8BTmA3MB6IBzYAJ0Q7rghfcwEw7KBtvwN+HLr/Y+C30Y7zKFzn2cA0YHNP1wlcBLwOCHAasDLa8Ufg2hcAP+rk2BNCn/sEYFzo/4Mz2tfwGa87F5gWup8G7Ahd36B/37u59qP2vsdKSWEmsEtV96iqB3gamBflmKJhHvB46P7jwKVRjOWoUNVlQPVBm7u6znnAExq0AsgUkdy+ifTo6+LauzIPeFpV3aq6F9hF8P/FgKOqJar6ceh+A7ANGEUMvO/dXHtXDvt9j5WkMArY3+5xEd3/IQcDBZaIyFoRuSm0LUdVSyD44QKGRy26yOrqOmPlc/DdUDXJP9pVEQ7KaxeRfGAqsJIYe98PunY4Su97rCQF6WTbYO92NVtVpwEXAt8RkbOjHVA/EAufg78AE4ApQAlwf2j7oLt2EUkFFgI/UNX67g7tZNtgu/aj9r7HSlIoAka3e5wHHIhSLH1CVQ+E/i0HXiBYZCxrKzaH/i2PXoQR1dV1DvrPgaqWqapfVQPAw3xaVTCorl1EXAS/FP9PVZ8PbY6J972zaz+a73usJIXVwEQRGSci8cDXgZejHFPEiEiKiKS13QfOBzYTvOZrQ4ddC7wUnQgjrqvrfBm4JtQb5TSgrq26YbA4qK78MoLvOwSv/esikiAi44CJwKq+ju9oEBEBHgG2qerv2+0a9O97V9d+VN/3aLem92Gr/UUEW+p3Az+NdjwRvtbxBHscbAC2tF0vMBR4G9gZ+jcr2rEehWt9imBx2UvwV9H8rq6TYFH6f0OfgU3AjGjHH4FrfzJ0bRtDXwi57Y7/aejaPwEujHb8R3DdZxKsAtkIrA/dLoqF972baz9q77uNaDbGGBMWK9VHxhhjesGSgjHGmDBLCsYYY8IsKRhjjAmzpGCMMSbMkoIZlETkOhH5UwSe9x4R+ULo/g9EJPkoPvelInJCZ69lTF+xLqlmUBKR6wj2R/9uBF+jIPQalYdxjlNV/V3sewxYpKrPHZ0IjTl8VlIwA46IvBia6G9Lu8n+EJHrRWSHiCwFZrfbfrGIrBSRdSLylojkhLYvEJHHRWSJBNef+LKI/E6C61AsDk0ncPBrPyYiV4jIbcBI4F0ReTe073wRWS4iH4vIs6H5adrWtvi5iHwAfEVEbhSR1SKyQUQWikiyiJwBXALcG5oPf0Lba4We49xQ/JtCE54ltHvuu0OvuUlEjgtt/1y7ufXXtY1wN6YnlhTMQPQtVZ0OzABuE5GhoWH+dxNMBucRnEe+zQfAaao6leC06f/Rbt8E4EsEpxj+J/Cuqp4MtIS2d0pVHyQ4h8znVfXzIjIM+BnwBQ1ORLgG+GG7U1pV9UxVfRp4XlVPVdXJBKc+nq+qHxEciXqnqk5R1d1tJ4pIIvAY8LVQbHHAre2euzL0mn8BfhTa9iPgO6o6BTgrdD3G9MiSghmIbhORDcAKgpN9TQRmAe+paoUG18z4d7vj84A3RGQTcCdwYrt9r6uql+AUAU5gcWj7JiD/MGI6jWAi+lBE1hOce2dsu/3t4zlJRN4PxXPVQfF05lhgr6ruCD1+nOACO23aJoRb2y7mD4Hfh0o0marqO4xrMTHMkoIZUERkDvAF4PTQL+11QGJod1cNZP8D/Cn0K/vmdscDuAE0OLukVz9tZAsQ/EXe69CAN0O/8qeo6gmqOr/d/qZ29x8DvhuK5+6D4unqubvjDv3rb4tZVX8D3AAkASvaqpWM6YklBTPQZAA1qtoc+qI7LbR9JTAnVJXkAr5y0DnFofvXcvQ0EFwSEYKlltkicgxAqJ1gUhfnpQEloTiv6uL52tsO5Lc9N3A1sLS7wERkgqpuUtXfEqzKsqRgesWSghloFgNxIrIR+AXBL2M0OBXyAmA58BbwcbtzFgDPisj7QK97CvXC34DXReRdVa0ArgOeCsW2gq6/iO8imMTeJPiF3+Zp4M5Qw/CEto2q2gpcH7qGTQRLMQ/1ENsPRGRzqJqtheAaxcb0yLqkGmOMCbOSgjHGmDBLCsYYY8IsKRhjjAmzpGCMMSbMkoIxxpgwSwrGGGPCLCkYY4wJs6RgjDEm7P8D8eJrE5KUpcwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_rmses, c=train_color, linestyle=train_style)\n",
    "plt.plot(test_rmses, c=test_color, linestyle=test_style)\n",
    "plt.ylabel('RMSE (kcal/mol)')\n",
    "plt.xlabel('adam iterations')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:12.654234Z",
     "start_time": "2019-03-07T17:10:12.647930Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-25.757173958164927, 4.985307878846929)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minval, maxval = np.min(prediction_traj[50:]), np.max(prediction_traj[50:])\n",
    "minval, maxval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:13.396050Z",
     "start_time": "2019-03-07T17:10:13.391820Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-15.5, 5.5)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minval, maxval = -15.5, 5.5\n",
    "minval, maxval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:13.980994Z",
     "start_time": "2019-03-07T17:10:13.976664Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-14.21, 3.4300000000000006)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(expt_means), np.max(expt_means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:14.450286Z",
     "start_time": "2019-03-07T17:10:14.447915Z"
    }
   },
   "outputs": [],
   "source": [
    "# color by group?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:14.818893Z",
     "start_time": "2019-03-07T17:10:14.816127Z"
    }
   },
   "outputs": [],
   "source": [
    "train_color = 'lightblue'\n",
    "test_color = 'green'\n",
    "train_style = '--'\n",
    "test_style = '-'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:15.174402Z",
     "start_time": "2019-03-07T17:10:15.167726Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(11433, 43)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs = np.vstack(computed_features)\n",
    "inputs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:15.620418Z",
     "start_time": "2019-03-07T17:10:15.615716Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[False, True]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(set(inputs[:,-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:10:16.395457Z",
     "start_time": "2019-03-07T17:10:16.392838Z"
    }
   },
   "outputs": [],
   "source": [
    "s = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-07T17:12:25.994451Z",
     "start_time": "2019-03-07T17:10:34.416557Z"
    }
   },
   "outputs": [],
   "source": [
    "from bayes_implicit_solvent.utils import remove_top_right_spines\n",
    "diag = np.arange(minval, maxval)\n",
    "\n",
    "for t in range(len(prediction_traj)):\n",
    "    plt.figure(figsize=(6*1.5,3*1.5))\n",
    "    ax = plt.subplot(1,2,1)\n",
    "    remove_top_right_spines(ax)\n",
    "    \n",
    "    plt.plot(train_rmses[:t], c=train_color, linestyle=train_style, label='train (n={})'.format(len(train_inds)))\n",
    "    plt.plot(test_rmses[:t], c=test_color, linestyle=test_style, label='test (n={})'.format(len(test_inds)))\n",
    "    plt.hlines(2.425900278028874, 0, len(train_rmses), linestyles='-', label='OBC2 (n=631)')\n",
    "    plt.hlines(1.642, 0, len(train_rmses), linestyles='--', label='SMIRNOFF+TIP3P (n=642)')\n",
    "    plt.hlines(0.98, 0, len(train_rmses), linestyles='dotted', label='OPLS3e (n=418)')\n",
    "    \n",
    "    \n",
    "    plt.ylabel('RMSE (kcal/mol)')\n",
    "    plt.xlabel('adam iterations')\n",
    "    plt.legend(loc='upper right')\n",
    "    \n",
    "    plt.xlim(0, len(train_rmses))\n",
    "    plt.ylim(0, 4)\n",
    "    \n",
    "    plt.title('RMSE trace')\n",
    "    \n",
    "    ax = plt.subplot(1,2,2)\n",
    "    remove_top_right_spines(ax)\n",
    "    \n",
    "    plt.plot(diag, diag, linestyle='--', c='grey')\n",
    "    plt.scatter(prediction_traj[t][train_inds], expt_means[train_inds], c=train_color, alpha=0.8, s=s)\n",
    "    plt.scatter(prediction_traj[t][test_inds], expt_means[test_inds], c=test_color, alpha=0.8, s=s)\n",
    "    plt.xlim(minval, maxval)\n",
    "    plt.ylim(minval, maxval)\n",
    "    \n",
    "    plt.xticks([-15,-10,-5,0,5])\n",
    "    plt.yticks([-15,-10,-5,0,5])\n",
    "    \n",
    "    plt.xlabel(r'predicted $\\Delta G$ (kcal/mol)')\n",
    "    plt.ylabel(r'measured $\\Delta G$ (kcal/mol)')\n",
    "    \n",
    "    plt.title('train RMSE: {:.2f}\\ntest RMSE: {:.2f}'.format(train_rmses[t], test_rmses[t]))\n",
    "    plt.tight_layout()\n",
    "    plt.savefig('linear_parameterizer_student_t_loss/{:04}.png'.format(t), bbox_inches='tight')\n",
    "    \n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}