StudentPegasus_test.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "StudentPegasus_test.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNlrE4PnRITSCtwRsZ/YmlN",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hadywalied/DistillPegasus/blob/main/StudentPegasus_test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KXvJyxy1HWyX",
        "outputId": "741c4b7e-9a4e-46b4-e025-4725d80c59bf"
      },
      "source": [
        "!pip install torch transformers datasets sentencepiece\n",
        "!pip install tensorboard scikit-learn psutil sacrebleu rouge-score tensorflow_datasets pytorch-lightning matplotlib git-python faiss-cpu streamlit elasticsearch nltk pandas datasets fire pytest conllu sentencepiece protobuf\n",
        "!pip install jax jaxlib\n",
        "!#pip install torch-lr-finder\n",
        "!pip install wandb"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.1+cu101)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.6.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.7.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.95)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: huggingface-hub==0.0.8 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.8)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.45)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (4.0.1)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.7/dist-packages (from datasets) (2021.5.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: tensorboard in /usr/local/lib/python3.7/dist-packages (2.5.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (0.22.2.post1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.7/dist-packages (5.4.8)\n",
            "Requirement already satisfied: sacrebleu in /usr/local/lib/python3.7/dist-packages (1.5.1)\n",
            "Requirement already satisfied: rouge-score in /usr/local/lib/python3.7/dist-packages (0.0.4)\n",
            "Requirement already satisfied: tensorflow_datasets in /usr/local/lib/python3.7/dist-packages (4.0.1)\n",
            "Requirement already satisfied: pytorch-lightning in /usr/local/lib/python3.7/dist-packages (1.3.3)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (3.2.2)\n",
            "Requirement already satisfied: git-python in /usr/local/lib/python3.7/dist-packages (1.0.3)\n",
            "Requirement already satisfied: faiss-cpu in /usr/local/lib/python3.7/dist-packages (1.7.1)\n",
            "Requirement already satisfied: streamlit in /usr/local/lib/python3.7/dist-packages (0.82.0)\n",
            "Requirement already satisfied: elasticsearch in /usr/local/lib/python3.7/dist-packages (7.13.0)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.7/dist-packages (3.2.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (1.1.5)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.7.0)\n",
            "Requirement already satisfied: fire in /usr/local/lib/python3.7/dist-packages (0.4.0)\n",
            "Requirement already satisfied: pytest in /usr/local/lib/python3.7/dist-packages (3.6.4)\n",
            "Requirement already satisfied: conllu in /usr/local/lib/python3.7/dist-packages (4.4)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (0.1.95)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.7/dist-packages (3.12.4)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (1.19.5)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (0.12.0)\n",
            "Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (1.34.1)\n",
            "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.7/dist-packages (from tensorboard) (0.36.2)\n",
            "Requirement already satisfied: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (1.30.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (0.6.1)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (2.23.0)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (56.1.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (0.4.4)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (3.3.4)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard) (1.8.0)\n",
            "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn) (1.4.1)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn) (1.0.1)\n",
            "Requirement already satisfied: portalocker==2.0.0 in /usr/local/lib/python3.7/dist-packages (from sacrebleu) (2.0.0)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from rouge-score) (1.15.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.3.3)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (4.41.1)\n",
            "Requirement already satisfied: tensorflow-metadata in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.0.0)\n",
            "Requirement already satisfied: promise in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (2.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.16.0)\n",
            "Requirement already satisfied: dm-tree in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (0.1.6)\n",
            "Requirement already satisfied: importlib-resources; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (5.1.3)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (1.1.0)\n",
            "Requirement already satisfied: attrs>=18.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_datasets) (21.2.0)\n",
            "Requirement already satisfied: torchmetrics>=0.2.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (0.3.2)\n",
            "Requirement already satisfied: PyYAML<=5.4.1,>=5.1 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (5.4.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (20.9)\n",
            "Requirement already satisfied: torch>=1.4 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (1.8.1+cu101)\n",
            "Requirement already satisfied: fsspec[http]>=2021.4.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (2021.5.0)\n",
            "Requirement already satisfied: pyDeprecate==0.3.0 in /usr/local/lib/python3.7/dist-packages (from pytorch-lightning) (0.3.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib) (0.10.0)\n",
            "Requirement already satisfied: gitpython in /usr/local/lib/python3.7/dist-packages (from git-python) (3.1.17)\n",
            "Requirement already satisfied: astor in /usr/local/lib/python3.7/dist-packages (from streamlit) (0.8.1)\n",
            "Requirement already satisfied: watchdog; platform_system != \"Darwin\" in /usr/local/lib/python3.7/dist-packages (from streamlit) (2.1.2)\n",
            "Requirement already satisfied: toml in /usr/local/lib/python3.7/dist-packages (from streamlit) (0.10.2)\n",
            "Requirement already satisfied: cachetools>=4.0 in /usr/local/lib/python3.7/dist-packages (from streamlit) (4.2.2)\n",
            "Requirement already satisfied: tzlocal in /usr/local/lib/python3.7/dist-packages (from streamlit) (1.5.1)\n",
            "Requirement already satisfied: pyarrow; python_version < \"3.9\" in /usr/local/lib/python3.7/dist-packages (from streamlit) (3.0.0)\n",
            "Requirement already satisfied: altair>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from streamlit) (4.1.0)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from streamlit) (7.1.2)\n",
            "Requirement already satisfied: blinker in /usr/local/lib/python3.7/dist-packages (from streamlit) (1.4)\n",
            "Requirement already satisfied: validators in /usr/local/lib/python3.7/dist-packages (from streamlit) (0.18.2)\n",
            "Requirement already satisfied: base58 in /usr/local/lib/python3.7/dist-packages (from streamlit) (2.1.0)\n",
            "Requirement already satisfied: tornado>=5.0 in /usr/local/lib/python3.7/dist-packages (from streamlit) (5.1.1)\n",
            "Requirement already satisfied: click<8.0,>=7.0 in /usr/local/lib/python3.7/dist-packages (from streamlit) (7.1.2)\n",
            "Requirement already satisfied: pydeck>=0.1.dev5 in /usr/local/lib/python3.7/dist-packages (from streamlit) (0.6.2)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from elasticsearch) (2020.12.5)\n",
            "Requirement already satisfied: urllib3<2,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from elasticsearch) (1.24.3)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from datasets) (4.0.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.11.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.0.8)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.7/dist-packages (from pytest) (1.10.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytest) (8.7.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.7/dist-packages (from pytest) (1.4.0)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.7/dist-packages (from pytest) (0.7.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard) (4.7.2)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard) (2.10)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (1.3.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2,>=1.52.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow-metadata->tensorflow_datasets) (1.53.0)\n",
            "Requirement already satisfied: zipp>=0.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-resources; python_version < \"3.9\"->tensorflow_datasets) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.4->pytorch-lightning) (3.7.4.3)\n",
            "Requirement already satisfied: aiohttp; extra == \"http\" in /usr/local/lib/python3.7/dist-packages (from fsspec[http]>=2021.4.0->pytorch-lightning) (3.7.4.post0)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from gitpython->git-python) (4.0.7)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from altair>=3.2.0->streamlit) (2.11.3)\n",
            "Requirement already satisfied: toolz in /usr/local/lib/python3.7/dist-packages (from altair>=3.2.0->streamlit) (0.11.1)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.7/dist-packages (from altair>=3.2.0->streamlit) (2.6.0)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.7/dist-packages (from altair>=3.2.0->streamlit) (0.3)\n",
            "Requirement already satisfied: decorator>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from validators->streamlit) (4.4.2)\n",
            "Requirement already satisfied: traitlets>=4.3.2 in /usr/local/lib/python3.7/dist-packages (from pydeck>=0.1.dev5->streamlit) (5.0.5)\n",
            "Requirement already satisfied: ipykernel>=5.1.2; python_version >= \"3.4\" in /usr/local/lib/python3.7/dist-packages (from pydeck>=0.1.dev5->streamlit) (5.5.5)\n",
            "Requirement already satisfied: ipywidgets>=7.0.0 in /usr/local/lib/python3.7/dist-packages (from pydeck>=0.1.dev5->streamlit) (7.6.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard) (0.4.8)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (3.1.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp; extra == \"http\"->fsspec[http]>=2021.4.0->pytorch-lightning) (1.6.3)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp; extra == \"http\"->fsspec[http]>=2021.4.0->pytorch-lightning) (5.1.0)\n",
            "Requirement already satisfied: async-timeout<4.0,>=3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp; extra == \"http\"->fsspec[http]>=2021.4.0->pytorch-lightning) (3.0.1)\n",
            "Requirement already satisfied: smmap<5,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->gitpython->git-python) (4.0.0)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->altair>=3.2.0->streamlit) (2.0.1)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.3.2->pydeck>=0.1.dev5->streamlit) (0.2.0)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.7/dist-packages (from ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (5.3.5)\n",
            "Requirement already satisfied: ipython>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (5.5.0)\n",
            "Requirement already satisfied: jupyterlab-widgets>=1.0.0; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (1.0.0)\n",
            "Requirement already satisfied: nbformat>=4.2.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (5.1.3)\n",
            "Requirement already satisfied: widgetsnbextension~=3.5.0 in /usr/local/lib/python3.7/dist-packages (from ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (3.5.1)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (22.0.3)\n",
            "Requirement already satisfied: jupyter-core>=4.6.0 in /usr/local/lib/python3.7/dist-packages (from jupyter-client->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (4.7.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (0.8.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (1.0.18)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (2.6.1)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (4.8.0)\n",
            "Requirement already satisfied: notebook>=4.4.1 in /usr/local/lib/python3.7/dist-packages (from widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (5.3.1)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (0.2.5)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect; sys_platform != \"win32\"->ipython>=5.0.0->ipykernel>=5.1.2; python_version >= \"3.4\"->pydeck>=0.1.dev5->streamlit) (0.7.0)\n",
            "Requirement already satisfied: nbconvert in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (5.6.1)\n",
            "Requirement already satisfied: terminado>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (0.10.0)\n",
            "Requirement already satisfied: Send2Trash in /usr/local/lib/python3.7/dist-packages (from notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (1.5.0)\n",
            "Requirement already satisfied: mistune<2,>=0.8.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (0.8.4)\n",
            "Requirement already satisfied: testpath in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (0.5.0)\n",
            "Requirement already satisfied: bleach in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (3.3.0)\n",
            "Requirement already satisfied: defusedxml in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (0.7.1)\n",
            "Requirement already satisfied: pandocfilters>=1.4.1 in /usr/local/lib/python3.7/dist-packages (from nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (1.4.3)\n",
            "Requirement already satisfied: webencodings in /usr/local/lib/python3.7/dist-packages (from bleach->nbconvert->notebook>=4.4.1->widgetsnbextension~=3.5.0->ipywidgets>=7.0.0->pydeck>=0.1.dev5->streamlit) (0.5.1)\n",
            "Requirement already satisfied: jax in /usr/local/lib/python3.7/dist-packages (0.2.13)\n",
            "Requirement already satisfied: jaxlib in /usr/local/lib/python3.7/dist-packages (0.1.66+cuda110)\n",
            "Requirement already satisfied: opt-einsum in /usr/local/lib/python3.7/dist-packages (from jax) (3.3.0)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.7/dist-packages (from jax) (0.12.0)\n",
            "Requirement already satisfied: numpy>=1.12 in /usr/local/lib/python3.7/dist-packages (from jax) (1.19.5)\n",
            "Requirement already satisfied: flatbuffers in /usr/local/lib/python3.7/dist-packages (from jaxlib) (1.12)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from jaxlib) (1.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py->jax) (1.15.0)\n",
            "Requirement already satisfied: wandb in /usr/local/lib/python3.7/dist-packages (0.10.31)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (0.4.0)\n",
            "Requirement already satisfied: PyYAML in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.1)\n",
            "Requirement already satisfied: subprocess32>=3.5.3 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.5.4)\n",
            "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.8.1)\n",
            "Requirement already satisfied: protobuf>=3.12.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.12.4)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.7/dist-packages (from wandb) (0.1.2)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.3)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.4.8)\n",
            "Requirement already satisfied: sentry-sdk>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.1.0)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.15.0)\n",
            "Requirement already satisfied: configparser>=3.8.1 in /usr/local/lib/python3.7/dist-packages (from wandb) (5.0.2)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (1.0.1)\n",
            "Requirement already satisfied: requests<3,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (2.23.0)\n",
            "Requirement already satisfied: Click>=7.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (7.1.2)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from wandb) (3.1.17)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from protobuf>=3.12.0->wandb) (56.1.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (2020.12.5)\n",
            "Requirement already satisfied: urllib3>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from sentry-sdk>=0.4.0->wandb) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.0.0->wandb) (2.10)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (4.0.7)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.0; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from GitPython>=1.0.0->wandb) (3.7.4.3)\n",
            "Requirement already satisfied: smmap<5,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb) (4.0.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o3_0cgehNiPQ"
      },
      "source": [
        "# *Imports* *and* *drive storage*"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_B8QMlIHZSE"
      },
      "source": [
        "# imports \n",
        "import logging\n",
        "from transformers import PegasusTokenizerFast, PegasusForConditionalGeneration,PegasusConfig\n",
        "from transformers import AutoTokenizer,AutoModelForSeq2SeqLM, Trainer, TrainingArguments, TrainerCallback, TrainerState, TrainerControl, EarlyStoppingCallback\n",
        "import datasets\n",
        "\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, random_split\n",
        "from torch.nn import functional as F\n",
        "\n",
        "from typing import Callable, Dict, Iterable, List, Tuple, Union\n",
        "from transformers import EvalPrediction, PreTrainedTokenizer\n",
        "from transformers import TrainingArguments, Trainer\n",
        "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
        "\n",
        "from transformers import AdamW\n",
        "import wandb\n",
        "import gc\n",
        "\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3O0RDBsUHxOB"
      },
      "source": [
        "logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s -   %(message)s',\n",
        "                    datefmt='%m/%d/%Y %H:%M:%S',\n",
        "                    level=logging.INFO)\n",
        "logger = logging.getLogger(__name__)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwUHQNn6SEbg"
      },
      "source": [
        "student_decoders = 4"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sepzo0h1HznV",
        "outputId": "4964b17c-607b-427c-afb6-b7e9a31db492"
      },
      "source": [
        "from google.colab import drive\n",
        "import os\n",
        "\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "directory = f'/content/drive/MyDrive/GP/dataset'\n",
        "if not os.path.exists(directory):\n",
        "    os.makedirs(directory)\n",
        "\n",
        "PATH = f'/content/drive/MyDrive/GP/student_{student_decoders}'\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qNxFShgZIQ8A"
      },
      "source": [
        "data_name = 'gigaword'\n",
        "model_name = f'google/pegasus-{data_name}'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PLZFdPafNeVf"
      },
      "source": [
        "# Preparing Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jBX5GHE9NWkN"
      },
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yTQRjGMdS6I3"
      },
      "source": [
        "# Preparing the Student"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zHbo5QZmS5O4",
        "outputId": "14dd6436-cd0e-416a-9ccd-582c3907457a"
      },
      "source": [
        "#Student configuration\n",
        "\n",
        "import warnings\n",
        "import torch\n",
        "from torch import nn\n",
        "from typing import Optional, Tuple, List, Union\n",
        "from transformers import PegasusModel, PegasusConfig, PegasusForConditionalGeneration\n",
        "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, PreTrainedModel\n",
        "from transformers import SummarizationPipeline\n",
        "\n",
        "students_config_book = {\n",
        "    '2': PegasusConfig(encoder_layers=2, decoder_layers=2),\n",
        "    '4': PegasusConfig(encoder_layers=4, decoder_layers=4),\n",
        "    '6': PegasusConfig(encoder_layers=6, decoder_layers=6),\n",
        "    '8': PegasusConfig(encoder_layers=8, decoder_layers=8),\n",
        "    '10': PegasusConfig(encoder_layers=10, decoder_layers=10),\n",
        "    '12': PegasusConfig(encoder_layers=12, decoder_layers=12),\n",
        "    '16': PegasusConfig(encoder_layers=16, decoder_layers=16)\n",
        "}\n",
        "\n",
        "\n",
        "LAYERS_TO_COPY = {   \n",
        "    4:{\n",
        "        1: [0],\n",
        "        2: [0, 3],\n",
        "        3: [0, 1, 3],\n",
        "        4: [0, 1, 2, 3],\n",
        "    },\n",
        "    8:{\n",
        "        1: [0],\n",
        "        2: [0, 7],\n",
        "        3: [0, 4, 7],\n",
        "        4: [0, 3, 6, 7],\n",
        "        6: [0, 2, 3, 5, 6, 7],\n",
        "        8: list(range(8)),  \n",
        "    },    \n",
        "    12: {\n",
        "        1: [0],\n",
        "        2: [0, 11],\n",
        "        3: [0, 6, 11],\n",
        "        4: [0, 4, 9, 11],\n",
        "        6: [0, 2, 5, 8, 10, 11],\n",
        "        8: [0, 1, 3, 5, 7, 9, 10, 11],\n",
        "        12: list(range(12)),  \n",
        "    },\n",
        "    16: {  # maps  num layers in student -> which teacher layers to copy\n",
        "        1: [0],\n",
        "        2: [0, 15],\n",
        "        3: [0, 8, 15],\n",
        "        4: [0, 5, 10, 15],\n",
        "        6: [0, 3, 6, 9, 12, 15],\n",
        "        8: [0, 2, 4, 6, 8, 10, 12, 15],\n",
        "        9: [0, 1, 3, 5, 7, 9, 11, 13, 15],\n",
        "        12: [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 13, 15],\n",
        "        16: list(range(16)),\n",
        "    },}\n",
        "LAYERS_TO_SUPERVISE = {\n",
        "    # maps  num layers in student -> which teacher layers to copy.\n",
        "    8: {1: [5], 2: [3, 5], 3: [1, 4, 5], 4: [1, 2, 4, 5]},\n",
        "    12: {1: [11], 2: [5, 11], 3: [3, 7, 11], 4:[1, 3, 7, 11],6: [1, 3, 5, 8, 10, 11], 8:[1,2,3,5,7,8,9,11] },\n",
        "    16: {1: [15], 4: [4, 9, 12, 15], 8: [1, 3, 5, 7, 9, 11, 13, 15], 12:[1,2,3,5,7,8,9,11,12,13,14,15]},\n",
        "}\n",
        "\n",
        "\n",
        "def copy_layers(src_layers: nn.ModuleList, dest_layers: nn.ModuleList, layers_to_copy) -> None:\n",
        "    layers_to_copy = nn.ModuleList([src_layers[i] for i in layers_to_copy])\n",
        "    assert len(dest_layers) == len(\n",
        "        layers_to_copy), f\"{len(dest_layers)} != {len(layers_to_copy)}\"\n",
        "    dest_layers.load_state_dict(layers_to_copy.state_dict())\n",
        "\n",
        "# Copied from transformers.models.bart.modeling_bart.shift_tokens_right\n",
        "\n",
        "\n",
        "def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int):\n",
        "    \"\"\"\n",
        "    Shift input ids one token to the right.\n",
        "    \"\"\"\n",
        "    shifted_input_ids = input_ids.new_zeros(input_ids.shape)\n",
        "    shifted_input_ids[:, 1:] = input_ids[:, :-1].clone()\n",
        "    shifted_input_ids[:, 0] = decoder_start_token_id\n",
        "\n",
        "    assert pad_token_id is not None, \"self.model.config.pad_token_id has to be defined.\"\n",
        "    # replace possible -100 values in labels by `pad_token_id`\n",
        "    shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id)\n",
        "\n",
        "    return shifted_input_ids\n",
        "\n",
        "\n",
        "def pick_layers_to_copy(n_student, n_teacher):\n",
        "    try:\n",
        "        val = LAYERS_TO_COPY[n_teacher][n_student]\n",
        "        return val\n",
        "    except KeyError:\n",
        "        if n_student != n_teacher:\n",
        "            warnings.warn(\n",
        "                f\"no hardcoded layers to copy for teacher {n_teacher} -> student {n_student}, defaulting to first {n_student}\"\n",
        "            )\n",
        "        return list(range(n_student))\n",
        "\n",
        "def get_layers_to_supervise(n_student, n_teacher) -> List[int]:\n",
        "    \"\"\"Used or the --supervise_forward kwarg\"\"\"\n",
        "    if n_student > n_teacher:\n",
        "        raise ValueError(f\"Cannot perform intermediate supervision for student {n_student} > teacher {n_teacher}\")\n",
        "    elif n_teacher == n_student:\n",
        "        return list(range(n_teacher))\n",
        "    elif n_student == 1:\n",
        "        return [n_teacher - 1]\n",
        "    else:\n",
        "        return LAYERS_TO_SUPERVISE[n_teacher][n_student]\n",
        "\n",
        "def create_student_with_configuration(teacher,\n",
        "                                      e=None,\n",
        "                                      d=None,\n",
        "                                      copy_first_teacher_layers = False,\n",
        "                                      save_path='./student'):\n",
        "\n",
        "    teacher.eval()\n",
        "    teacher_e, teacher_d = teacher.config.encoder_layers, teacher.config.decoder_layers\n",
        "    init_kwargs = teacher.config.to_diff_dict()\n",
        "    if e is None:\n",
        "        e = teacher_e\n",
        "    if d is None:\n",
        "        d = teacher_d\n",
        "    init_kwargs.update({\"encoder_layers\": e, \"decoder_layers\": d})\n",
        "    student_cfg = teacher.config_class(**init_kwargs)\n",
        "    student = AutoModelForSeq2SeqLM.from_config(student_cfg)\n",
        "    # Start by copying the full teacher state dict this will copy the first N teacher layers to the student.\n",
        "    info = student.load_state_dict(teacher.state_dict(), strict=False)\n",
        "    # every student key should have a teacher keys.\n",
        "    assert info.missing_keys == [], info.missing_keys\n",
        "\n",
        "    if copy_first_teacher_layers:  # Our copying is done. We just log and save\n",
        "        e_layers_to_copy, d_layers_to_copy = list(range(e)), list(range(d))\n",
        "        #student.save_pretrained(save_path)\n",
        "        return student, e_layers_to_copy, d_layers_to_copy\n",
        "\n",
        "    # Decide which layers of the teacher to copy. Not exactly alternating -- we try to keep first and last layer.\n",
        "    e_layers_to_copy: List[int] = pick_layers_to_copy(e, teacher_e)\n",
        "    d_layers_to_copy: List[int] = pick_layers_to_copy(d, teacher_d)\n",
        "\n",
        "    copy_layers(teacher.model.encoder.layers,\n",
        "                student.model.encoder.layers, e_layers_to_copy)\n",
        "    copy_layers(teacher.model.decoder.layers,\n",
        "                student.model.decoder.layers, d_layers_to_copy)\n",
        "\n",
        "    student.config.init_metadata = dict(\n",
        "        teacher_type=teacher.config.model_type,\n",
        "        copied_encoder_layers=e_layers_to_copy,\n",
        "        copied_decoder_layers=d_layers_to_copy,\n",
        "    )\n",
        "    #student.save_pretrained(save_path)\n",
        "    # Save information about copying for easier reproducibility\n",
        "\n",
        "    return student, e_layers_to_copy, d_layers_to_copy\n",
        "#student = create_student_with_configuration(teacher,\n",
        "#                                      e=4,\n",
        "#                                      d=4,\n",
        "#                                      copy_first_teacher_layers = False,\n",
        "#                                      save_path='./student')\n",
        "#import gc\n",
        "#del copy_teacher\n",
        "gc.collect()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eJ6-gxvoTDkv"
      },
      "source": [
        "teacher = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n",
        "\n",
        "student, e_layers_list, d_layers_list = create_student_with_configuration(\n",
        "                                      teacher,\n",
        "                                      e=16,\n",
        "                                      d=student_decoders,\n",
        "                                      copy_first_teacher_layers = False,\n",
        "                                      save_path=PATH)\n",
        "student.to('cuda')\n",
        "\n",
        "\n",
        "del teacher\n",
        "gc.collect()\n",
        "torch.cuda.empty_cache()\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xOrJfFt5Ipf6"
      },
      "source": [
        "for param in student.model.shared.parameters():\n",
        "  param.requires_grad = False\n",
        "for param in student.model.encoder.embed_tokens.parameters():\n",
        "  param.requires_grad = False\n",
        "for param in student.model.encoder.embed_positions.parameters():\n",
        "  param.requires_grad = False\n",
        "for param in student.model.decoder.embed_tokens.parameters():\n",
        "  param.requires_grad = False\n",
        "for param in student.model.decoder.embed_positions.parameters():\n",
        "  param.requires_grad = False"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C8IGlyEJfokx"
      },
      "source": [
        "#Data Preparation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "clean-biography"
      },
      "source": [
        "class PegasusDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels['input_ids'][idx])  # torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "    def __len__(self):\n",
        "        return len(self.labels[\"input_ids\"])\n",
        "\n",
        "\n",
        "def prepare_data(model_name, \n",
        "                 train_texts, train_labels, \n",
        "                 val_texts=None, val_labels=None, \n",
        "                 test_texts=None, test_labels=None):\n",
        "  \"\"\"\n",
        "  Prepare input data for model fine-tuning\n",
        "  \"\"\"\n",
        "  tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
        "\n",
        "  prepare_val = False if val_texts is None or val_labels is None else True\n",
        "  prepare_test = False if test_texts is None or test_labels is None else True\n",
        "\n",
        "  def tokenize_data(texts, labels):\n",
        "    encodings = tokenizer(texts, truncation=True, padding='longest')\n",
        "    decodings = tokenizer(labels, truncation=True, padding='longest')\n",
        "    dataset_tokenized = PegasusDataset(encodings, decodings)\n",
        "    return dataset_tokenized\n",
        "\n",
        "  train_dataset = tokenize_data(train_texts, train_labels)\n",
        "  val_dataset = tokenize_data(val_texts, val_labels) if prepare_val else None\n",
        "  test_dataset = tokenize_data(test_texts, test_labels) if prepare_test else None\n",
        "\n",
        "  return train_dataset, val_dataset, test_dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IMHO85SOgDqo",
        "outputId": "58b9b31b-75f0-47ec-9e83-e478c4579270"
      },
      "source": [
        "dataset = datasets.load_dataset(data_name)\n",
        "train_texts, train_labels = dataset['train']['document'][:50000], dataset['train']['summary'][:50000]\n",
        "valid_texts, valid_labels = dataset['validation']['document'][:1000], dataset['validation']['summary'][:1000]\n",
        "test_texts, test_labels = dataset['test']['document'][:1000], dataset['test']['summary'][:1000]\n",
        "train_dataset, valid_dataset, test_dataset = prepare_data(model_name, train_texts, train_labels,valid_texts, valid_labels,test_texts, test_labels)\n",
        "\n",
        "del dataset \n",
        "del (test_texts, test_labels, valid_texts, valid_labels, train_texts, train_labels)\n",
        "gc.collect()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "06/01/2021 05:02:26 - WARNING - datasets.builder -   Using custom data configuration default\n",
            "06/01/2021 05:02:26 - WARNING - datasets.builder -   Reusing dataset gigaword (/root/.cache/huggingface/datasets/gigaword/default/1.2.0/ea83a8b819190acac5f2dae011fad51dccf269a0604ec5dd24795b64efb424b6)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XuTaVdcFfpAb"
      },
      "source": [
        "#Trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8gDJRB9nrh5R"
      },
      "source": [
        "class MyCallback(TrainerCallback):\n",
        "    \"\"\"\n",
        "    A :class:`~transformers.TrainerCallback` that handles the default flow of the training loop for logs, evaluation\n",
        "    and checkpoints.\n",
        "    \"\"\"\n",
        "\n",
        "    def on_step_end(self, args: TrainingArguments, state: TrainerState, control: TrainerControl, **kwargs):\n",
        "        gc.collect()\n",
        "        torch.cuda.empty_cache()\n",
        "\n",
        "class MyTrainer(Trainer):\n",
        "    \n",
        "    def shift_tokens_right(self, input_ids, pad_token_id):\n",
        "      \"\"\"Shift input ids one token to the right, and wrap the last non pad token (usually <eos>).\"\"\"\n",
        "      prev_output_tokens = input_ids.clone()\n",
        "      #print(pad_token_id, input_ids)\n",
        "      x= (input_ids.ne(pad_token_id).sum(dim=1) - 1)\n",
        "      index_of_eos = x.unsqueeze(-1)\n",
        "      prev_output_tokens[:, 0] = input_ids.gather(1, index_of_eos).squeeze()\n",
        "      prev_output_tokens[:, 1:] = input_ids[:, :-1]\n",
        "      return prev_output_tokens\n",
        "    \n",
        "    \n",
        "    def compute_loss(self, model, inputs, return_outputs=False):\n",
        "        labels = inputs[\"labels\"]\n",
        "        pad_token_id = tokenizer.pad_token_id\n",
        "        decoder_input_ids = self.shift_tokens_right(labels, pad_token_id)\n",
        "        input_ids = inputs['input_ids']\n",
        "        attention_mask = inputs['attention_mask']\n",
        "        #print(f\"input: {input_ids}, masks: {attention_mask}\")\n",
        "        outputs = model(input_ids,\n",
        "                        attention_mask=attention_mask,\n",
        "                        decoder_input_ids=decoder_input_ids,\n",
        "                        output_hidden_states=False,\n",
        "                        output_attentions=False,\n",
        "                        use_cache=False)\n",
        "        logits = outputs[\"logits\"]\n",
        "        #print(f'logits: {logits}, labels: {labels}')\n",
        "        loss_fct = torch.nn.CrossEntropyLoss()\n",
        "        loss = loss_fct(logits.view(-1, logits.shape[-1]),labels.view(-1))\n",
        "        #print(loss)\n",
        "        return (loss, outputs) if return_outputs else loss\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ggM1GrLauUq"
      },
      "source": [
        "metric = datasets.load_metric('rouge')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGmj0bgeQpM-"
      },
      "source": [
        "training_args = TrainingArguments(\n",
        "    output_dir=f'{PATH}/output',          # output directory\n",
        "    num_train_epochs=5,              # total number of training epochs\n",
        "    per_device_train_batch_size=16,  # batch size per device during training\n",
        "    per_device_eval_batch_size=16,   # batch size for evaluation\n",
        "    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "    weight_decay=0.01,               # strength of weight decay\n",
        "    logging_dir=f'{PATH}/logs',            # directory for storing logs\n",
        "    logging_steps=10,\n",
        "    #load_best_model_at_end  = True,\n",
        "    #evaluation_strategy = 'epoch',\n",
        "    #fp16 = True,\n",
        "    #fp16_full_eval = True,\n",
        "    #resume_from_checkpoint = f'{PATH}/output/checkpoint-{}'\n",
        ")\n",
        "\n",
        "\n",
        "trainer = MyTrainer(\n",
        "    model=student,                         # the instantiated 🤗 Transformers model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    train_dataset=train_dataset,         # training dataset\n",
        "    eval_dataset=valid_dataset ,            # evaluation dataset\n",
        "    compute_metrics = metric,\n",
        "    tokenizer = tokenizer,\n",
        "    callbacks = [MyCallback]\n",
        ")\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "YDPraLtGXh9K",
        "outputId": "2ca98c59-4ebd-4c16-eb14-82589111cc84"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mhadywalied\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "                Tracking run with wandb version 0.10.31<br/>\n",
              "                Syncing run <strong style=\"color:#cdcd00\">/content/drive/MyDrive/GP/student_4/output</strong> to <a href=\"https://wandb.ai\" target=\"_blank\">Weights & Biases</a> <a href=\"https://docs.wandb.com/integrations/jupyter.html\" target=\"_blank\">(Documentation)</a>.<br/>\n",
              "                Project page: <a href=\"https://wandb.ai/hadywalied/huggingface\" target=\"_blank\">https://wandb.ai/hadywalied/huggingface</a><br/>\n",
              "                Run page: <a href=\"https://wandb.ai/hadywalied/huggingface/runs/33ufkcbx\" target=\"_blank\">https://wandb.ai/hadywalied/huggingface/runs/33ufkcbx</a><br/>\n",
              "                Run data is saved locally in <code>/content/wandb/run-20210601_050321-33ufkcbx</code><br/><br/>\n",
              "            "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3305' max='15625' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 3305/15625 1:30:22 < 5:37:04, 0.61 it/s, Epoch 1.06/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>10.975600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>10.938200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>30</td>\n",
              "      <td>10.944900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>40</td>\n",
              "      <td>10.897800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>50</td>\n",
              "      <td>10.898900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>60</td>\n",
              "      <td>10.890300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>70</td>\n",
              "      <td>10.845500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>80</td>\n",
              "      <td>10.806600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>90</td>\n",
              "      <td>10.758200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>10.733100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>110</td>\n",
              "      <td>10.685300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>120</td>\n",
              "      <td>10.633100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>130</td>\n",
              "      <td>10.575600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>140</td>\n",
              "      <td>10.514000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>150</td>\n",
              "      <td>10.454400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>160</td>\n",
              "      <td>10.389600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>170</td>\n",
              "      <td>10.355100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>180</td>\n",
              "      <td>10.307300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>190</td>\n",
              "      <td>10.222700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>10.167800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>210</td>\n",
              "      <td>10.114500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>220</td>\n",
              "      <td>9.996900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>230</td>\n",
              "      <td>9.930300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>240</td>\n",
              "      <td>9.859100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>250</td>\n",
              "      <td>9.728200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>260</td>\n",
              "      <td>9.659000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>270</td>\n",
              "      <td>9.565400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>280</td>\n",
              "      <td>9.414500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>290</td>\n",
              "      <td>9.288800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>9.153300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>310</td>\n",
              "      <td>8.925700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>320</td>\n",
              "      <td>8.715300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>330</td>\n",
              "      <td>8.484400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>340</td>\n",
              "      <td>8.201700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>350</td>\n",
              "      <td>7.797300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>360</td>\n",
              "      <td>7.396500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>370</td>\n",
              "      <td>6.861600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>380</td>\n",
              "      <td>6.328200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>390</td>\n",
              "      <td>5.787700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>5.287100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>410</td>\n",
              "      <td>4.716100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>420</td>\n",
              "      <td>4.128200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>430</td>\n",
              "      <td>3.543000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>440</td>\n",
              "      <td>3.022900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>450</td>\n",
              "      <td>2.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>460</td>\n",
              "      <td>2.247600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>470</td>\n",
              "      <td>2.027000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>480</td>\n",
              "      <td>1.755400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>490</td>\n",
              "      <td>1.607700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>1.466400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>510</td>\n",
              "      <td>1.388100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>520</td>\n",
              "      <td>1.254900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>530</td>\n",
              "      <td>1.186600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>540</td>\n",
              "      <td>1.090000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>550</td>\n",
              "      <td>1.010500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>560</td>\n",
              "      <td>0.995000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>570</td>\n",
              "      <td>0.966300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>580</td>\n",
              "      <td>0.966500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>590</td>\n",
              "      <td>0.937400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>0.891200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>610</td>\n",
              "      <td>0.835200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>620</td>\n",
              "      <td>0.840000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>630</td>\n",
              "      <td>0.821700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>640</td>\n",
              "      <td>0.780100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>650</td>\n",
              "      <td>0.788800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>660</td>\n",
              "      <td>0.761900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>670</td>\n",
              "      <td>0.817700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>680</td>\n",
              "      <td>0.748300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>690</td>\n",
              "      <td>0.779500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>0.785900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>710</td>\n",
              "      <td>0.731800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>720</td>\n",
              "      <td>0.707400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>730</td>\n",
              "      <td>0.748400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>740</td>\n",
              "      <td>0.735300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>750</td>\n",
              "      <td>0.710200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>760</td>\n",
              "      <td>0.684000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>770</td>\n",
              "      <td>0.671200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>780</td>\n",
              "      <td>0.726400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>790</td>\n",
              "      <td>0.682400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.661700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>810</td>\n",
              "      <td>0.730500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>820</td>\n",
              "      <td>0.708500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>830</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>840</td>\n",
              "      <td>0.657300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>850</td>\n",
              "      <td>0.686500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>860</td>\n",
              "      <td>0.656200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>870</td>\n",
              "      <td>0.643300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>880</td>\n",
              "      <td>0.654900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>890</td>\n",
              "      <td>0.651900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>900</td>\n",
              "      <td>0.632700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>910</td>\n",
              "      <td>0.621200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>920</td>\n",
              "      <td>0.614000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>930</td>\n",
              "      <td>0.657100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>940</td>\n",
              "      <td>0.641100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>950</td>\n",
              "      <td>0.642800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>960</td>\n",
              "      <td>0.638800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>970</td>\n",
              "      <td>0.576400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>980</td>\n",
              "      <td>0.588900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>990</td>\n",
              "      <td>0.578200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.585700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1010</td>\n",
              "      <td>0.646400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1020</td>\n",
              "      <td>0.600200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1030</td>\n",
              "      <td>0.647400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1040</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1050</td>\n",
              "      <td>0.587400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1060</td>\n",
              "      <td>0.574300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1070</td>\n",
              "      <td>0.633000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1080</td>\n",
              "      <td>0.562100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1090</td>\n",
              "      <td>0.609200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1100</td>\n",
              "      <td>0.567200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1110</td>\n",
              "      <td>0.609200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1120</td>\n",
              "      <td>0.652100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1130</td>\n",
              "      <td>0.578600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1140</td>\n",
              "      <td>0.583600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1150</td>\n",
              "      <td>0.564700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1160</td>\n",
              "      <td>0.583800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1170</td>\n",
              "      <td>0.552300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1180</td>\n",
              "      <td>0.589400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1190</td>\n",
              "      <td>0.593700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1200</td>\n",
              "      <td>0.557900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1210</td>\n",
              "      <td>0.536000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1220</td>\n",
              "      <td>0.561700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1230</td>\n",
              "      <td>0.587300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1240</td>\n",
              "      <td>0.605000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1250</td>\n",
              "      <td>0.568900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1260</td>\n",
              "      <td>0.598100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1270</td>\n",
              "      <td>0.569200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1280</td>\n",
              "      <td>0.575600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1290</td>\n",
              "      <td>0.572100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1300</td>\n",
              "      <td>0.575100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1310</td>\n",
              "      <td>0.552100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1320</td>\n",
              "      <td>0.553700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1330</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1340</td>\n",
              "      <td>0.550800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1350</td>\n",
              "      <td>0.572400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1360</td>\n",
              "      <td>0.565300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1370</td>\n",
              "      <td>0.547000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1380</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1390</td>\n",
              "      <td>0.552800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1400</td>\n",
              "      <td>0.562600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1410</td>\n",
              "      <td>0.578100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1420</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1430</td>\n",
              "      <td>0.563700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1440</td>\n",
              "      <td>0.564600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1450</td>\n",
              "      <td>0.562300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1460</td>\n",
              "      <td>0.596300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1470</td>\n",
              "      <td>0.561300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1480</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1490</td>\n",
              "      <td>0.572000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.543400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1510</td>\n",
              "      <td>0.591500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1520</td>\n",
              "      <td>0.567400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1530</td>\n",
              "      <td>0.532500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1540</td>\n",
              "      <td>0.580100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1550</td>\n",
              "      <td>0.543600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1560</td>\n",
              "      <td>0.535400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1570</td>\n",
              "      <td>0.514800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1580</td>\n",
              "      <td>0.559400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1590</td>\n",
              "      <td>0.595300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1600</td>\n",
              "      <td>0.512300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1610</td>\n",
              "      <td>0.565200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1620</td>\n",
              "      <td>0.524500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1630</td>\n",
              "      <td>0.524400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1640</td>\n",
              "      <td>0.556700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1650</td>\n",
              "      <td>0.553900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1660</td>\n",
              "      <td>0.568800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1670</td>\n",
              "      <td>0.529700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1680</td>\n",
              "      <td>0.522000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1690</td>\n",
              "      <td>0.495500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1700</td>\n",
              "      <td>0.519300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1710</td>\n",
              "      <td>0.496400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1720</td>\n",
              "      <td>0.516000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1730</td>\n",
              "      <td>0.546900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1740</td>\n",
              "      <td>0.554400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1750</td>\n",
              "      <td>0.526700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1760</td>\n",
              "      <td>0.493500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1770</td>\n",
              "      <td>0.532100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1780</td>\n",
              "      <td>0.526300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1790</td>\n",
              "      <td>0.528600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1800</td>\n",
              "      <td>0.556600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1810</td>\n",
              "      <td>0.579100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1820</td>\n",
              "      <td>0.506800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1830</td>\n",
              "      <td>0.541200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1840</td>\n",
              "      <td>0.533900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1850</td>\n",
              "      <td>0.482400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1860</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1870</td>\n",
              "      <td>0.530000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1880</td>\n",
              "      <td>0.505700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1890</td>\n",
              "      <td>0.523300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1900</td>\n",
              "      <td>0.528900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1910</td>\n",
              "      <td>0.501600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1920</td>\n",
              "      <td>0.530000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1930</td>\n",
              "      <td>0.521000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1940</td>\n",
              "      <td>0.577600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1950</td>\n",
              "      <td>0.527400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1960</td>\n",
              "      <td>0.479400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1970</td>\n",
              "      <td>0.551600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1980</td>\n",
              "      <td>0.516100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1990</td>\n",
              "      <td>0.524100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.492100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2010</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2020</td>\n",
              "      <td>0.523000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2030</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2040</td>\n",
              "      <td>0.500300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2050</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2060</td>\n",
              "      <td>0.507700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2070</td>\n",
              "      <td>0.517000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2080</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2090</td>\n",
              "      <td>0.527600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2100</td>\n",
              "      <td>0.513500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2110</td>\n",
              "      <td>0.534600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2120</td>\n",
              "      <td>0.516900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2130</td>\n",
              "      <td>0.538200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2140</td>\n",
              "      <td>0.501600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2150</td>\n",
              "      <td>0.486200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2160</td>\n",
              "      <td>0.520200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2170</td>\n",
              "      <td>0.452500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2180</td>\n",
              "      <td>0.541700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2190</td>\n",
              "      <td>0.555900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2200</td>\n",
              "      <td>0.515500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2210</td>\n",
              "      <td>0.477600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2220</td>\n",
              "      <td>0.511800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2230</td>\n",
              "      <td>0.501200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2240</td>\n",
              "      <td>0.502200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2250</td>\n",
              "      <td>0.534300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2260</td>\n",
              "      <td>0.505400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2270</td>\n",
              "      <td>0.519100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2280</td>\n",
              "      <td>0.479300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2290</td>\n",
              "      <td>0.495400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2300</td>\n",
              "      <td>0.501400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2310</td>\n",
              "      <td>0.532200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2320</td>\n",
              "      <td>0.496600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2330</td>\n",
              "      <td>0.512600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2340</td>\n",
              "      <td>0.459500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2350</td>\n",
              "      <td>0.467200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2360</td>\n",
              "      <td>0.476800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2370</td>\n",
              "      <td>0.500300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2380</td>\n",
              "      <td>0.480800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2390</td>\n",
              "      <td>0.506300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2400</td>\n",
              "      <td>0.475500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2410</td>\n",
              "      <td>0.492600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2420</td>\n",
              "      <td>0.481300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2430</td>\n",
              "      <td>0.518400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2440</td>\n",
              "      <td>0.494900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2450</td>\n",
              "      <td>0.514500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2460</td>\n",
              "      <td>0.473900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2470</td>\n",
              "      <td>0.521000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2480</td>\n",
              "      <td>0.457900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2490</td>\n",
              "      <td>0.526300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.504900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2510</td>\n",
              "      <td>0.523000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2520</td>\n",
              "      <td>0.466400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2530</td>\n",
              "      <td>0.478100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2540</td>\n",
              "      <td>0.508600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2550</td>\n",
              "      <td>0.485700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2560</td>\n",
              "      <td>0.529200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2570</td>\n",
              "      <td>0.449100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2580</td>\n",
              "      <td>0.491000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2590</td>\n",
              "      <td>0.514600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2600</td>\n",
              "      <td>0.484900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2610</td>\n",
              "      <td>0.471800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2620</td>\n",
              "      <td>0.515300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2630</td>\n",
              "      <td>0.459900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2640</td>\n",
              "      <td>0.471700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2650</td>\n",
              "      <td>0.522800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2660</td>\n",
              "      <td>0.479400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2670</td>\n",
              "      <td>0.499800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2680</td>\n",
              "      <td>0.528900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2690</td>\n",
              "      <td>0.471100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2700</td>\n",
              "      <td>0.473500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2710</td>\n",
              "      <td>0.510300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2720</td>\n",
              "      <td>0.512100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2730</td>\n",
              "      <td>0.457800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2740</td>\n",
              "      <td>0.478000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2750</td>\n",
              "      <td>0.455300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2760</td>\n",
              "      <td>0.519800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2770</td>\n",
              "      <td>0.500400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2780</td>\n",
              "      <td>0.484200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2790</td>\n",
              "      <td>0.510300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2800</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2810</td>\n",
              "      <td>0.517100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2820</td>\n",
              "      <td>0.516300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2830</td>\n",
              "      <td>0.468300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2840</td>\n",
              "      <td>0.525800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2850</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2860</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2870</td>\n",
              "      <td>0.486000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2880</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2890</td>\n",
              "      <td>0.475100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2900</td>\n",
              "      <td>0.493500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2910</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2920</td>\n",
              "      <td>0.506100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2930</td>\n",
              "      <td>0.499500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2940</td>\n",
              "      <td>0.476400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2950</td>\n",
              "      <td>0.524400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2960</td>\n",
              "      <td>0.507900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2970</td>\n",
              "      <td>0.501500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2980</td>\n",
              "      <td>0.465600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2990</td>\n",
              "      <td>0.449700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.481500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3010</td>\n",
              "      <td>0.494700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3020</td>\n",
              "      <td>0.466700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3030</td>\n",
              "      <td>0.504300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3040</td>\n",
              "      <td>0.496400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3050</td>\n",
              "      <td>0.470700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3060</td>\n",
              "      <td>0.479100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3070</td>\n",
              "      <td>0.463000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3080</td>\n",
              "      <td>0.461800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3090</td>\n",
              "      <td>0.485600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3100</td>\n",
              "      <td>0.510700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3110</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3120</td>\n",
              "      <td>0.476700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3130</td>\n",
              "      <td>0.482900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3140</td>\n",
              "      <td>0.453300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3150</td>\n",
              "      <td>0.470600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3160</td>\n",
              "      <td>0.482700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3170</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3180</td>\n",
              "      <td>0.476700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3190</td>\n",
              "      <td>0.521500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3200</td>\n",
              "      <td>0.470600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3210</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3220</td>\n",
              "      <td>0.493000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3230</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3240</td>\n",
              "      <td>0.439100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3250</td>\n",
              "      <td>0.520300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3260</td>\n",
              "      <td>0.485900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3270</td>\n",
              "      <td>0.475600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3280</td>\n",
              "      <td>0.452200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3290</td>\n",
              "      <td>0.500400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3300</td>\n",
              "      <td>0.490100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='3673' max='15625' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 3673/15625 1:40:48 < 5:28:12, 0.61 it/s, Epoch 1.18/5]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>10.975600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>10.938200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>30</td>\n",
              "      <td>10.944900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>40</td>\n",
              "      <td>10.897800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>50</td>\n",
              "      <td>10.898900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>60</td>\n",
              "      <td>10.890300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>70</td>\n",
              "      <td>10.845500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>80</td>\n",
              "      <td>10.806600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>90</td>\n",
              "      <td>10.758200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>100</td>\n",
              "      <td>10.733100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>110</td>\n",
              "      <td>10.685300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>120</td>\n",
              "      <td>10.633100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>130</td>\n",
              "      <td>10.575600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>140</td>\n",
              "      <td>10.514000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>150</td>\n",
              "      <td>10.454400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>160</td>\n",
              "      <td>10.389600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>170</td>\n",
              "      <td>10.355100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>180</td>\n",
              "      <td>10.307300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>190</td>\n",
              "      <td>10.222700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>200</td>\n",
              "      <td>10.167800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>210</td>\n",
              "      <td>10.114500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>220</td>\n",
              "      <td>9.996900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>230</td>\n",
              "      <td>9.930300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>240</td>\n",
              "      <td>9.859100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>250</td>\n",
              "      <td>9.728200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>260</td>\n",
              "      <td>9.659000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>270</td>\n",
              "      <td>9.565400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>280</td>\n",
              "      <td>9.414500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>290</td>\n",
              "      <td>9.288800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>300</td>\n",
              "      <td>9.153300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>310</td>\n",
              "      <td>8.925700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>320</td>\n",
              "      <td>8.715300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>330</td>\n",
              "      <td>8.484400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>340</td>\n",
              "      <td>8.201700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>350</td>\n",
              "      <td>7.797300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>360</td>\n",
              "      <td>7.396500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>370</td>\n",
              "      <td>6.861600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>380</td>\n",
              "      <td>6.328200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>390</td>\n",
              "      <td>5.787700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>400</td>\n",
              "      <td>5.287100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>410</td>\n",
              "      <td>4.716100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>420</td>\n",
              "      <td>4.128200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>430</td>\n",
              "      <td>3.543000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>440</td>\n",
              "      <td>3.022900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>450</td>\n",
              "      <td>2.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>460</td>\n",
              "      <td>2.247600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>470</td>\n",
              "      <td>2.027000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>480</td>\n",
              "      <td>1.755400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>490</td>\n",
              "      <td>1.607700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>1.466400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>510</td>\n",
              "      <td>1.388100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>520</td>\n",
              "      <td>1.254900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>530</td>\n",
              "      <td>1.186600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>540</td>\n",
              "      <td>1.090000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>550</td>\n",
              "      <td>1.010500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>560</td>\n",
              "      <td>0.995000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>570</td>\n",
              "      <td>0.966300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>580</td>\n",
              "      <td>0.966500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>590</td>\n",
              "      <td>0.937400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>600</td>\n",
              "      <td>0.891200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>610</td>\n",
              "      <td>0.835200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>620</td>\n",
              "      <td>0.840000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>630</td>\n",
              "      <td>0.821700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>640</td>\n",
              "      <td>0.780100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>650</td>\n",
              "      <td>0.788800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>660</td>\n",
              "      <td>0.761900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>670</td>\n",
              "      <td>0.817700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>680</td>\n",
              "      <td>0.748300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>690</td>\n",
              "      <td>0.779500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>700</td>\n",
              "      <td>0.785900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>710</td>\n",
              "      <td>0.731800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>720</td>\n",
              "      <td>0.707400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>730</td>\n",
              "      <td>0.748400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>740</td>\n",
              "      <td>0.735300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>750</td>\n",
              "      <td>0.710200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>760</td>\n",
              "      <td>0.684000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>770</td>\n",
              "      <td>0.671200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>780</td>\n",
              "      <td>0.726400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>790</td>\n",
              "      <td>0.682400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>800</td>\n",
              "      <td>0.661700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>810</td>\n",
              "      <td>0.730500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>820</td>\n",
              "      <td>0.708500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>830</td>\n",
              "      <td>0.638400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>840</td>\n",
              "      <td>0.657300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>850</td>\n",
              "      <td>0.686500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>860</td>\n",
              "      <td>0.656200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>870</td>\n",
              "      <td>0.643300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>880</td>\n",
              "      <td>0.654900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>890</td>\n",
              "      <td>0.651900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>900</td>\n",
              "      <td>0.632700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>910</td>\n",
              "      <td>0.621200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>920</td>\n",
              "      <td>0.614000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>930</td>\n",
              "      <td>0.657100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>940</td>\n",
              "      <td>0.641100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>950</td>\n",
              "      <td>0.642800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>960</td>\n",
              "      <td>0.638800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>970</td>\n",
              "      <td>0.576400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>980</td>\n",
              "      <td>0.588900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>990</td>\n",
              "      <td>0.578200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.585700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1010</td>\n",
              "      <td>0.646400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1020</td>\n",
              "      <td>0.600200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1030</td>\n",
              "      <td>0.647400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1040</td>\n",
              "      <td>0.643200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1050</td>\n",
              "      <td>0.587400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1060</td>\n",
              "      <td>0.574300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1070</td>\n",
              "      <td>0.633000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1080</td>\n",
              "      <td>0.562100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1090</td>\n",
              "      <td>0.609200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1100</td>\n",
              "      <td>0.567200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1110</td>\n",
              "      <td>0.609200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1120</td>\n",
              "      <td>0.652100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1130</td>\n",
              "      <td>0.578600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1140</td>\n",
              "      <td>0.583600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1150</td>\n",
              "      <td>0.564700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1160</td>\n",
              "      <td>0.583800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1170</td>\n",
              "      <td>0.552300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1180</td>\n",
              "      <td>0.589400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1190</td>\n",
              "      <td>0.593700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1200</td>\n",
              "      <td>0.557900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1210</td>\n",
              "      <td>0.536000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1220</td>\n",
              "      <td>0.561700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1230</td>\n",
              "      <td>0.587300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1240</td>\n",
              "      <td>0.605000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1250</td>\n",
              "      <td>0.568900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1260</td>\n",
              "      <td>0.598100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1270</td>\n",
              "      <td>0.569200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1280</td>\n",
              "      <td>0.575600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1290</td>\n",
              "      <td>0.572100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1300</td>\n",
              "      <td>0.575100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1310</td>\n",
              "      <td>0.552100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1320</td>\n",
              "      <td>0.553700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1330</td>\n",
              "      <td>0.537600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1340</td>\n",
              "      <td>0.550800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1350</td>\n",
              "      <td>0.572400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1360</td>\n",
              "      <td>0.565300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1370</td>\n",
              "      <td>0.547000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1380</td>\n",
              "      <td>0.558400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1390</td>\n",
              "      <td>0.552800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1400</td>\n",
              "      <td>0.562600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1410</td>\n",
              "      <td>0.578100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1420</td>\n",
              "      <td>0.539200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1430</td>\n",
              "      <td>0.563700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1440</td>\n",
              "      <td>0.564600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1450</td>\n",
              "      <td>0.562300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1460</td>\n",
              "      <td>0.596300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1470</td>\n",
              "      <td>0.561300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1480</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1490</td>\n",
              "      <td>0.572000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.543400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1510</td>\n",
              "      <td>0.591500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1520</td>\n",
              "      <td>0.567400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1530</td>\n",
              "      <td>0.532500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1540</td>\n",
              "      <td>0.580100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1550</td>\n",
              "      <td>0.543600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1560</td>\n",
              "      <td>0.535400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1570</td>\n",
              "      <td>0.514800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1580</td>\n",
              "      <td>0.559400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1590</td>\n",
              "      <td>0.595300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1600</td>\n",
              "      <td>0.512300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1610</td>\n",
              "      <td>0.565200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1620</td>\n",
              "      <td>0.524500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1630</td>\n",
              "      <td>0.524400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1640</td>\n",
              "      <td>0.556700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1650</td>\n",
              "      <td>0.553900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1660</td>\n",
              "      <td>0.568800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1670</td>\n",
              "      <td>0.529700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1680</td>\n",
              "      <td>0.522000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1690</td>\n",
              "      <td>0.495500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1700</td>\n",
              "      <td>0.519300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1710</td>\n",
              "      <td>0.496400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1720</td>\n",
              "      <td>0.516000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1730</td>\n",
              "      <td>0.546900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1740</td>\n",
              "      <td>0.554400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1750</td>\n",
              "      <td>0.526700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1760</td>\n",
              "      <td>0.493500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1770</td>\n",
              "      <td>0.532100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1780</td>\n",
              "      <td>0.526300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1790</td>\n",
              "      <td>0.528600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1800</td>\n",
              "      <td>0.556600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1810</td>\n",
              "      <td>0.579100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1820</td>\n",
              "      <td>0.506800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1830</td>\n",
              "      <td>0.541200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1840</td>\n",
              "      <td>0.533900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1850</td>\n",
              "      <td>0.482400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1860</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1870</td>\n",
              "      <td>0.530000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1880</td>\n",
              "      <td>0.505700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1890</td>\n",
              "      <td>0.523300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1900</td>\n",
              "      <td>0.528900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1910</td>\n",
              "      <td>0.501600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1920</td>\n",
              "      <td>0.530000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1930</td>\n",
              "      <td>0.521000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1940</td>\n",
              "      <td>0.577600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1950</td>\n",
              "      <td>0.527400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1960</td>\n",
              "      <td>0.479400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1970</td>\n",
              "      <td>0.551600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1980</td>\n",
              "      <td>0.516100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1990</td>\n",
              "      <td>0.524100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2000</td>\n",
              "      <td>0.492100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2010</td>\n",
              "      <td>0.505600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2020</td>\n",
              "      <td>0.523000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2030</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2040</td>\n",
              "      <td>0.500300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2050</td>\n",
              "      <td>0.528000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2060</td>\n",
              "      <td>0.507700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2070</td>\n",
              "      <td>0.517000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2080</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2090</td>\n",
              "      <td>0.527600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2100</td>\n",
              "      <td>0.513500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2110</td>\n",
              "      <td>0.534600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2120</td>\n",
              "      <td>0.516900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2130</td>\n",
              "      <td>0.538200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2140</td>\n",
              "      <td>0.501600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2150</td>\n",
              "      <td>0.486200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2160</td>\n",
              "      <td>0.520200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2170</td>\n",
              "      <td>0.452500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2180</td>\n",
              "      <td>0.541700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2190</td>\n",
              "      <td>0.555900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2200</td>\n",
              "      <td>0.515500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2210</td>\n",
              "      <td>0.477600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2220</td>\n",
              "      <td>0.511800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2230</td>\n",
              "      <td>0.501200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2240</td>\n",
              "      <td>0.502200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2250</td>\n",
              "      <td>0.534300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2260</td>\n",
              "      <td>0.505400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2270</td>\n",
              "      <td>0.519100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2280</td>\n",
              "      <td>0.479300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2290</td>\n",
              "      <td>0.495400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2300</td>\n",
              "      <td>0.501400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2310</td>\n",
              "      <td>0.532200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2320</td>\n",
              "      <td>0.496600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2330</td>\n",
              "      <td>0.512600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2340</td>\n",
              "      <td>0.459500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2350</td>\n",
              "      <td>0.467200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2360</td>\n",
              "      <td>0.476800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2370</td>\n",
              "      <td>0.500300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2380</td>\n",
              "      <td>0.480800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2390</td>\n",
              "      <td>0.506300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2400</td>\n",
              "      <td>0.475500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2410</td>\n",
              "      <td>0.492600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2420</td>\n",
              "      <td>0.481300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2430</td>\n",
              "      <td>0.518400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2440</td>\n",
              "      <td>0.494900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2450</td>\n",
              "      <td>0.514500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2460</td>\n",
              "      <td>0.473900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2470</td>\n",
              "      <td>0.521000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2480</td>\n",
              "      <td>0.457900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2490</td>\n",
              "      <td>0.526300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2500</td>\n",
              "      <td>0.504900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2510</td>\n",
              "      <td>0.523000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2520</td>\n",
              "      <td>0.466400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2530</td>\n",
              "      <td>0.478100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2540</td>\n",
              "      <td>0.508600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2550</td>\n",
              "      <td>0.485700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2560</td>\n",
              "      <td>0.529200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2570</td>\n",
              "      <td>0.449100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2580</td>\n",
              "      <td>0.491000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2590</td>\n",
              "      <td>0.514600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2600</td>\n",
              "      <td>0.484900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2610</td>\n",
              "      <td>0.471800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2620</td>\n",
              "      <td>0.515300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2630</td>\n",
              "      <td>0.459900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2640</td>\n",
              "      <td>0.471700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2650</td>\n",
              "      <td>0.522800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2660</td>\n",
              "      <td>0.479400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2670</td>\n",
              "      <td>0.499800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2680</td>\n",
              "      <td>0.528900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2690</td>\n",
              "      <td>0.471100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2700</td>\n",
              "      <td>0.473500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2710</td>\n",
              "      <td>0.510300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2720</td>\n",
              "      <td>0.512100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2730</td>\n",
              "      <td>0.457800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2740</td>\n",
              "      <td>0.478000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2750</td>\n",
              "      <td>0.455300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2760</td>\n",
              "      <td>0.519800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2770</td>\n",
              "      <td>0.500400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2780</td>\n",
              "      <td>0.484200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2790</td>\n",
              "      <td>0.510300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2800</td>\n",
              "      <td>0.481600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2810</td>\n",
              "      <td>0.517100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2820</td>\n",
              "      <td>0.516300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2830</td>\n",
              "      <td>0.468300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2840</td>\n",
              "      <td>0.525800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2850</td>\n",
              "      <td>0.519700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2860</td>\n",
              "      <td>0.520000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2870</td>\n",
              "      <td>0.486000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2880</td>\n",
              "      <td>0.475200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2890</td>\n",
              "      <td>0.475100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2900</td>\n",
              "      <td>0.493500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2910</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2920</td>\n",
              "      <td>0.506100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2930</td>\n",
              "      <td>0.499500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2940</td>\n",
              "      <td>0.476400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2950</td>\n",
              "      <td>0.524400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2960</td>\n",
              "      <td>0.507900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2970</td>\n",
              "      <td>0.501500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2980</td>\n",
              "      <td>0.465600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2990</td>\n",
              "      <td>0.449700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3000</td>\n",
              "      <td>0.481500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3010</td>\n",
              "      <td>0.494700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3020</td>\n",
              "      <td>0.466700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3030</td>\n",
              "      <td>0.504300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3040</td>\n",
              "      <td>0.496400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3050</td>\n",
              "      <td>0.470700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3060</td>\n",
              "      <td>0.479100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3070</td>\n",
              "      <td>0.463000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3080</td>\n",
              "      <td>0.461800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3090</td>\n",
              "      <td>0.485600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3100</td>\n",
              "      <td>0.510700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3110</td>\n",
              "      <td>0.492800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3120</td>\n",
              "      <td>0.476700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3130</td>\n",
              "      <td>0.482900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3140</td>\n",
              "      <td>0.453300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3150</td>\n",
              "      <td>0.470600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3160</td>\n",
              "      <td>0.482700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3170</td>\n",
              "      <td>0.491200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3180</td>\n",
              "      <td>0.476700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3190</td>\n",
              "      <td>0.521500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3200</td>\n",
              "      <td>0.470600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3210</td>\n",
              "      <td>0.478400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3220</td>\n",
              "      <td>0.493000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3230</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3240</td>\n",
              "      <td>0.439100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3250</td>\n",
              "      <td>0.520300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3260</td>\n",
              "      <td>0.485900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3270</td>\n",
              "      <td>0.475600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3280</td>\n",
              "      <td>0.452200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3290</td>\n",
              "      <td>0.500400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3300</td>\n",
              "      <td>0.490100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3310</td>\n",
              "      <td>0.470200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3320</td>\n",
              "      <td>0.460700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3330</td>\n",
              "      <td>0.477100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3340</td>\n",
              "      <td>0.441600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3350</td>\n",
              "      <td>0.483700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3360</td>\n",
              "      <td>0.468800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3370</td>\n",
              "      <td>0.479100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3380</td>\n",
              "      <td>0.437600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3390</td>\n",
              "      <td>0.475100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3400</td>\n",
              "      <td>0.472400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3410</td>\n",
              "      <td>0.453200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3420</td>\n",
              "      <td>0.480100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3430</td>\n",
              "      <td>0.471700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3440</td>\n",
              "      <td>0.469400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3450</td>\n",
              "      <td>0.449000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3460</td>\n",
              "      <td>0.472700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3470</td>\n",
              "      <td>0.448600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3480</td>\n",
              "      <td>0.466100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3490</td>\n",
              "      <td>0.471700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3500</td>\n",
              "      <td>0.437200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3510</td>\n",
              "      <td>0.447900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3520</td>\n",
              "      <td>0.434700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3530</td>\n",
              "      <td>0.486100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3540</td>\n",
              "      <td>0.442400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3550</td>\n",
              "      <td>0.472200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3560</td>\n",
              "      <td>0.523400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3570</td>\n",
              "      <td>0.456400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3580</td>\n",
              "      <td>0.421400</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3590</td>\n",
              "      <td>0.429600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3600</td>\n",
              "      <td>0.460100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3610</td>\n",
              "      <td>0.482100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3620</td>\n",
              "      <td>0.427600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3630</td>\n",
              "      <td>0.464600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3640</td>\n",
              "      <td>0.443900</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3650</td>\n",
              "      <td>0.467700</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3660</td>\n",
              "      <td>0.470000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3670</td>\n",
              "      <td>0.443100</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zyJ6Zm7MLSLK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OzqEs84M5MtJ"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}