Cells_are_Transformers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Cells are Transformers",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMrQN2O7xmcAn9knlHMTrjq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "618fb955a1934bd98bfee388b7de7573": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_561f1c1441984c2b8159bdfae42f5a2f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1e8eebdabaa94b60b328b4071aafc206",
              "IPY_MODEL_1046c2b5a6f2448891f56eaeb50da86d"
            ]
          }
        },
        "561f1c1441984c2b8159bdfae42f5a2f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1e8eebdabaa94b60b328b4071aafc206": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4df71db955694e5abb25801e800e2ec6",
            "_dom_classes": [],
            "description": "update step: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 30,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 30,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1b5f5b209dfe4057b2f95ba6fd718fac"
          }
        },
        "1046c2b5a6f2448891f56eaeb50da86d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_186cf5ec3d8743cfac21d4c5c6e5bd17",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 30/30 [00:43&lt;00:00,  1.44s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f15b116e51843d2acabb245f9052414"
          }
        },
        "4df71db955694e5abb25801e800e2ec6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1b5f5b209dfe4057b2f95ba6fd718fac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "186cf5ec3d8743cfac21d4c5c6e5bd17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f15b116e51843d2acabb245f9052414": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3cbb439813724ec198206de71e9cd3c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_1bce3133cc82463f83c5b29ac377a7f5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_749f8705da9747f4b6fa7fc4417c8c10",
              "IPY_MODEL_c670451e984f49e18367a99ba3168092"
            ]
          }
        },
        "1bce3133cc82463f83c5b29ac377a7f5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "749f8705da9747f4b6fa7fc4417c8c10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_14e8f57c270f4dc8b3a3a1ebb42a8b00",
            "_dom_classes": [],
            "description": "evaluation step: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d24aae09ad1a499e9906322797ba4770"
          }
        },
        "c670451e984f49e18367a99ba3168092": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_340d94472be34298b5866f2c2171c5d6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1/1 [00:01&lt;00:00,  1.06s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_80d3deecad2947868d5fb2141de20d8f"
          }
        },
        "14e8f57c270f4dc8b3a3a1ebb42a8b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d24aae09ad1a499e9906322797ba4770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "340d94472be34298b5866f2c2171c5d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "80d3deecad2947868d5fb2141de20d8f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fb3a14ac179d43198bc911ef4c968366": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2c2d737038934d2aae872c2f66325dd6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_983b182454ae4317847529fd34d3accc",
              "IPY_MODEL_e184d2e3719043d488e45f540df922ca"
            ]
          }
        },
        "2c2d737038934d2aae872c2f66325dd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "983b182454ae4317847529fd34d3accc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_acbf53b6a93149eebe828a92773739cb",
            "_dom_classes": [],
            "description": "update step:  33%",
            "_model_name": "FloatProgressModel",
            "bar_style": "danger",
            "max": 30,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 10,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_067c88d62d0d416f83331e521a9b08f7"
          }
        },
        "e184d2e3719043d488e45f540df922ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_eba54b5d72f44ba5a0c3c264a5ebc483",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 10/30 [00:05&lt;00:10,  1.97it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5cc9837967ff4c88af135db879fa33d9"
          }
        },
        "acbf53b6a93149eebe828a92773739cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "067c88d62d0d416f83331e521a9b08f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "eba54b5d72f44ba5a0c3c264a5ebc483": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5cc9837967ff4c88af135db879fa33d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Mikcl/CellTransformer/blob/main/Cells_are_Transformers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g305-z3wlXTQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f8f9603c-7bfa-4e86-8e4f-431f834a1da5"
      },
      "source": [
        "!pip install linear-attention-transformer\n",
        "!pip install einops"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting linear-attention-transformer\n",
            "  Downloading https://files.pythonhosted.org/packages/ec/e3/65952551818f1cf2bb7be07d43c02ec5a17d1744ccaa98e759a966b4ba17/linear_attention_transformer-0.15.3-py3-none-any.whl\n",
            "Collecting product-key-memory>=0.1.5\n",
            "  Downloading https://files.pythonhosted.org/packages/31/3b/c1f8977e4b04f047acc7b23c7424d1e2e624ed7031e699a2ac2287af4c1f/product_key_memory-0.1.10.tar.gz\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (from linear-attention-transformer) (1.7.0+cu101)\n",
            "Collecting local-attention\n",
            "  Downloading https://files.pythonhosted.org/packages/a0/86/f1df73868c1c433a9184d94e86cdd970951ecf14d8b556b41302febb9a12/local_attention-1.2.2-py3-none-any.whl\n",
            "Collecting axial-positional-embedding\n",
            "  Downloading https://files.pythonhosted.org/packages/7a/27/ad886f872b15153905d957a70670efe7521a07c70d324ff224f998e52492/axial_positional_embedding-0.2.1.tar.gz\n",
            "Collecting linformer>=0.1.0\n",
            "  Downloading https://files.pythonhosted.org/packages/c4/5f/10f3ebeb144982a0170b5ec9dcf80e6ae82a6e6e9afddd21c9bf985e62c5/linformer-0.2.1-py3-none-any.whl\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch->linear-attention-transformer) (1.19.5)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch->linear-attention-transformer) (0.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch->linear-attention-transformer) (3.7.4.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch->linear-attention-transformer) (0.16.0)\n",
            "Building wheels for collected packages: product-key-memory, axial-positional-embedding\n",
            "  Building wheel for product-key-memory (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for product-key-memory: filename=product_key_memory-0.1.10-cp36-none-any.whl size=3072 sha256=84e5885f782ce25479f53cf19b3e48a378fa26eb4cda3aa4712ffef279097eb7\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/e0/3b/fd3111a4fac652ed014ccfd4757754f006132723985e229419\n",
            "  Building wheel for axial-positional-embedding (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for axial-positional-embedding: filename=axial_positional_embedding-0.2.1-cp36-none-any.whl size=2904 sha256=44ba1088153f4523598beed1f2bf677616580e047a4159796736ff9a777a63b8\n",
            "  Stored in directory: /root/.cache/pip/wheels/cd/f8/93/25b60e319a481e8f324dcb1871aff818eb0c8143ed20b732b4\n",
            "Successfully built product-key-memory axial-positional-embedding\n",
            "Installing collected packages: product-key-memory, local-attention, axial-positional-embedding, linformer, linear-attention-transformer\n",
            "Successfully installed axial-positional-embedding-0.2.1 linear-attention-transformer-0.15.3 linformer-0.2.1 local-attention-1.2.2 product-key-memory-0.1.10\n",
            "Collecting einops\n",
            "  Downloading https://files.pythonhosted.org/packages/5d/a0/9935e030634bf60ecd572c775f64ace82ceddf2f504a5fd3902438f07090/einops-0.3.0-py2.py3-none-any.whl\n",
            "Installing collected packages: einops\n",
            "Successfully installed einops-0.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Awl8-4fPlAZ8"
      },
      "source": [
        "import os\n",
        "import base64\n",
        "import numpy as np\n",
        "import torch\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "from torch import nn\n",
        "from IPython.display import Image, HTML, clear_output\n",
        "import tqdm\n",
        "\n",
        "import io\n",
        "import PIL.Image, PIL.ImageDraw\n",
        "import base64\n",
        "import zipfile\n",
        "import json\n",
        "import requests\n",
        "import matplotlib.pylab as plt\n",
        "import glob\n",
        "\n",
        "import torch.optim as optim\n",
        "\n",
        "from linear_attention_transformer import LinearAttentionTransformerLM\n",
        "from linear_attention_transformer.autoregressive_wrapper import AutoregressiveWrapper\n",
        "\n",
        "from einops import rearrange\n",
        "\n",
        "clear_output()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "90lPXBTtlaph"
      },
      "source": [
        "#@title Cellular Automata Parameters\n",
        "CHANNEL_N = 4        # Number of CA state channels\n",
        "TARGET_PADDING = 16   # Number of pixels used to pad the target image border\n",
        "TARGET_SIZE = 40\n",
        "\n",
        "# 👁, 🦎\n",
        "TARGET_EMOJI = \"🦎\" #@param {type:\"string\"}\n",
        "\n",
        "EXPERIMENT_TYPE = \"Growing\" #@param [\"Growing\"] # \"Persistent\", \"Regenerating\"\n",
        "EXPERIMENT_MAP = {\"Growing\":0, \"Persistent\":1, \"Regenerating\":2}\n",
        "EXPERIMENT_N = EXPERIMENT_MAP[EXPERIMENT_TYPE]\n",
        "\n",
        "USE_PATTERN_POOL = [0, 1, 1][EXPERIMENT_N]\n",
        "DAMAGE_N = [0, 0, 3][EXPERIMENT_N]  # Number of patterns to damage in a batch"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qTMuZlP6m4Z6"
      },
      "source": [
        "#@title Transformer Token Details\n",
        "TOKEN_OFFSET = 2\n",
        "NUM_TOKENS = 257 + TOKEN_OFFSET\n",
        "ENC_SEQ_LEN = 9 * CHANNEL_N\n",
        "DEC_SEQ_LEN = ENC_SEQ_LEN + CHANNEL_N"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iBCiVj1WmnAE"
      },
      "source": [
        "#@title Tranformer Hyperparameters \n",
        "\n",
        "EPOCHS = 10 #int(1e3)\n",
        "BATCH_SIZE = 16\n",
        "LEARNING_RATE = 1e-4\n",
        "# GENERATE_EVERY  = 100   # For visual results during training. . "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXPB73jkn1cB"
      },
      "source": [
        "#@title Helpers \n",
        "\n",
        "def to_rgba(x):\n",
        "  return x[:,:4,:,:]\n",
        "\n",
        "def to_rgb(x):\n",
        "  return x[:,:3,:,:]\n",
        "\n",
        "def make_seed(size, n=1):\n",
        "  x = np.zeros([n, size, size, CHANNEL_N], np.float32)\n",
        "  x[:, size//2, size//2, 3:] = 1.0\n",
        "  return x\n",
        "\n",
        "def plot_loss(loss_log):\n",
        "  plt.figure(figsize=(10, 4))\n",
        "  plt.title('Loss history (log10)')\n",
        "  plt.plot(np.log10(loss_log), '.', alpha=0.1)\n",
        "  plt.show()\n",
        "\n",
        "def np2pil(a):\n",
        "  if a.dtype in [np.float32, np.float64]:\n",
        "    a = np.uint8(np.clip(a, 0, 1)*255)\n",
        "  return PIL.Image.fromarray(a)\n",
        "\n",
        "def imwrite(f, a, fmt=None):\n",
        "  a = np.asarray(a)\n",
        "  if isinstance(f, str):\n",
        "    fmt = f.rsplit('.', 1)[-1].lower()\n",
        "    if fmt == 'jpg':\n",
        "      fmt = 'jpeg'\n",
        "    f = open(f, 'wb')\n",
        "  np2pil(a).save(f, fmt, quality=95)\n",
        "\n",
        "def imencode(a, fmt='jpeg'):\n",
        "  a = np.asarray(a)\n",
        "  if len(a.shape) == 3 and a.shape[-1] == 4:\n",
        "    fmt = 'png'\n",
        "  f = io.BytesIO()\n",
        "  imwrite(f, a, fmt)\n",
        "  return f.getvalue()\n",
        "\n",
        "def im2url(a, fmt='jpeg'):\n",
        "  encoded = imencode(a, fmt)\n",
        "  base64_byte_string = base64.b64encode(encoded).decode('ascii')\n",
        "  return 'data:image/' + fmt.upper() + ';base64,' + base64_byte_string\n"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fup4z40WpClo"
      },
      "source": [
        "#@title Load image utils. \n",
        "\n",
        "# Returns h,w,c image where c=RGBA\n",
        "def load_image(url, max_size=TARGET_SIZE):\n",
        "  r = requests.get(url)\n",
        "  img = PIL.Image.open(io.BytesIO(r.content))\n",
        "  img.thumbnail((max_size, max_size), PIL.Image.ANTIALIAS)\n",
        "  img = np.float32(img)\n",
        "  return img\n",
        "\n",
        "def load_emoji(emoji):\n",
        "  code = hex(ord(emoji))[2:].lower()\n",
        "  url = 'https://github.com/googlefonts/noto-emoji/raw/master/png/128/emoji_u%s.png'%code\n",
        "  return load_image(url)"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rKmByqYpcE6"
      },
      "source": [
        "#@title To Device\n",
        "if torch.cuda.is_available():  \n",
        "  dev = \"cuda:0\" \n",
        "else:  \n",
        "  dev = \"cpu\"\n",
        "device = torch.device(dev)  "
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dpswGT_xsezf"
      },
      "source": [
        "#@title Load Emoji\n",
        "target_img = load_emoji(TARGET_EMOJI)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zK7C4qZRsGnt"
      },
      "source": [
        "p = TARGET_PADDING\n",
        "target_img_tensor = torch.from_numpy(target_img)\n",
        "\n",
        "# (72,72,4)\n",
        "pad_target = torch.nn.functional.pad(target_img_tensor, pad=(0,0,p,p,p,p), mode='constant', value=0)\n",
        "h, w = pad_target.shape[:2]\n",
        "\n",
        "# (CHANNEL_N, 72, 72)\n",
        "seed = np.zeros([CHANNEL_N, h, w], np.float32)\n",
        "seed[3:, h//2, w//2] = 255\n",
        "\n",
        "pad_target = pad_target.permute(2, 0, 1)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YM2KOzGus8Hv"
      },
      "source": [
        "pad_target = pad_target.to(device)\n",
        "\n",
        "def loss_f(x):\n",
        "    x = x[:,:4,:,:]\n",
        "    return torch.mean(torch.square(x-pad_target), [-1,-2,-3])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FP3syDMHy-Jv"
      },
      "source": [
        "#@title Masking Utils\n",
        "IMAGE_BATCH_SIZE = 1 #number of images to concurrently train on. \n",
        "LIVE_MIN = 0\n",
        "\n",
        "# Applied on default/non-tokenized values\n",
        "def get_living_mask(x):\n",
        "  alpha = x[:, 3:4, :, :]\n",
        "  return torch.nn.functional.max_pool2d(input=alpha, kernel_size=3, stride=1, padding=1) > LIVE_MIN\n",
        "\n",
        "# 3d tensor as boolean. \n",
        "alive_target_mask = get_living_mask(pad_target[None,...])\n",
        "\n",
        "def get_dilatied_mask(x):\n",
        "  # Returns x dilated mask as FloatType \n",
        "  output = x.sum(-3) > 0\n",
        "  output = output.type(torch.FloatTensor)\n",
        "  kernel = torch.tensor([[1,1,1],[1,1,1],[1,1,1]]).type(torch.FloatTensor)\n",
        "  dilated = torch.nn.functional.conv2d(output[None,...], kernel[None, None, ...].expand(1, -1, -1, -1), groups=1, padding=1)\n",
        "  return dilated\n",
        "\n",
        "def get_living_target(x):\n",
        "  living_mask = get_living_mask(x)\n",
        "\n",
        "  # print('SUM OF 1s', torch.sum(torch.flatten(living_mask)))\n",
        "  living_target_mask = (living_mask & alive_target_mask)\n",
        "  living_target = pad_target[None,...] * living_target_mask.float()\n",
        "  living_target = living_target[0]\n",
        "  return living_target\n",
        "\n",
        "def mask_target(mask):\n",
        "  '''\n",
        "    mask - tensor: 2D - Float\n",
        "\n",
        "    returns - tensor: 4D - T\n",
        "  '''\n",
        "  # returns 4d tensor. \n",
        "  target_batch = pad_target[None, ...] * mask\n",
        "  return target_batch\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T1fglUUQteEd"
      },
      "source": [
        "#@title Define Model\n",
        "\n",
        "def tokenize(x):\n",
        "  return (x + 2).long()\n",
        "\n",
        "def detokenize(x):\n",
        "  return (x - 2).long()\n",
        "\n",
        "def percieve(x):\n",
        "  # x - 4d tensor - N, C, H, W\n",
        "  _,_,h,w = x.shape\n",
        "  padded_x = torch.nn.functional.pad(x, pad=(1,1,1,1), mode='constant', value=0)\n",
        "  stacked_image = torch.cat(\n",
        "      [padded_x[:,:,i:i+h, j:j+w]  for j in range(3) for i in range(3)]\n",
        "      ,1\n",
        "  )\n",
        "  return stacked_image\n",
        "\n",
        "\n",
        "# instantiate model\n",
        "enc = LinearAttentionTransformerLM(\n",
        "    num_tokens = NUM_TOKENS,\n",
        "    dim = 512,\n",
        "    heads = 8,\n",
        "    depth = 1,\n",
        "    max_seq_len = ENC_SEQ_LEN,\n",
        "    one_kv_head = True,\n",
        "    return_embeddings = True\n",
        ").to(device)\n",
        "\n",
        "dec = LinearAttentionTransformerLM(\n",
        "    num_tokens = NUM_TOKENS,\n",
        "    dim = 512,\n",
        "    heads = 8,\n",
        "    depth = 3,\n",
        "    causal = True,\n",
        "    blindspot_size = 2,             # a small blindspot greatly saves on memory\n",
        "    max_seq_len = DEC_SEQ_LEN,\n",
        "    one_kv_head = True,\n",
        "    receives_context = True\n",
        ").to(device)\n",
        "\n",
        "dec = AutoregressiveWrapper(dec)\n",
        "\n",
        "# optimizer\n",
        "\n",
        "optim = torch.optim.Adam([*enc.parameters(), *dec.parameters()], lr=LEARNING_RATE)\n"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OIz10RkPL14v"
      },
      "source": [
        "#@title Dataset Generation Helpers. \n",
        "SENTENCES = int(alive_target_mask.shape[2] * alive_target_mask.shape[3])\n",
        "\n",
        "class CellTraining(Dataset):\n",
        "    def __init__(self, X, Y):\n",
        "        self.src = X\n",
        "        self.tgt = Y\n",
        "        assert X.shape[0] == Y.shape[0], 'Dataset X and Y not of same length'\n",
        "\n",
        "        b = X.shape[0]\n",
        "        self.SRC_MASK = torch.ones(b, ENC_SEQ_LEN).bool()\n",
        "        self.TGT_MASK = torch.ones(b, DEC_SEQ_LEN).bool()\n",
        "\n",
        "        self.length = b\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        src = self.src[idx]\n",
        "        tgt = self.tgt[idx]\n",
        "        src_mask = self.SRC_MASK[idx]\n",
        "        tgt_mask = self.TGT_MASK[idx] \n",
        "        return src.to(device), tgt.to(device), src_mask.to(device), tgt_mask.to(device)\n",
        "\n",
        "\n",
        "class CellEvaluating(Dataset):\n",
        "    def __init__(self, X):\n",
        "        self.src = X\n",
        "        b = X.shape[0]\n",
        "        self.SRC_MASK = torch.ones(b, ENC_SEQ_LEN).bool()\n",
        "        self.length = b\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.length\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        src = self.src[idx]\n",
        "        src_mask = self.SRC_MASK[idx]\n",
        "        return src.to(device), src_mask.to(device)\n",
        "\n",
        "\n",
        "\n",
        "def get_start_tokens(number_of_sentences):\n",
        "    return torch.ones((number_of_sentences, 1)).long().to(device)\n",
        "\n",
        "def image_to_sentences(x):\n",
        "  '''\n",
        "    ## maps a 3D tensor of shape (c, h, w) to a 2D tensor of (h*w, c)\n",
        "    these are known as sentences or 'patches' of size one\n",
        "  '''\n",
        "  return rearrange(x,'c h w -> (h w) c')\n",
        "\n",
        "\n",
        "def sentences_to_image(x, height, width):\n",
        "  '''\n",
        "    ## maps a 2D tesnor of shape (h*w, c) to image/3D tensor shape of (c, h, w)\n",
        "    inverse of image_to_sentence\n",
        "    height: int - image dimension\n",
        "    width: int - image dimension\n",
        "  '''\n",
        "  return rearrange(x, '(h w) (p1 p2 c) -> c (h p1) (w p2)', p1 =1, p2 =1, h=height, w=width)\n",
        "\n",
        "\n",
        "def visualize_batched(x, c=4):\n",
        "  x = x[:,:c,:,:].cpu().numpy()\n",
        "  vis1 = np.hstack(np.transpose(x, (0,2,3,1)))\n",
        "  plt.imshow(vis1)\n",
        "  plt.show()\n",
        "\n",
        "\n"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eF1yyIPht6vl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "618fb955a1934bd98bfee388b7de7573",
            "561f1c1441984c2b8159bdfae42f5a2f",
            "1e8eebdabaa94b60b328b4071aafc206",
            "1046c2b5a6f2448891f56eaeb50da86d",
            "4df71db955694e5abb25801e800e2ec6",
            "1b5f5b209dfe4057b2f95ba6fd718fac",
            "186cf5ec3d8743cfac21d4c5c6e5bd17",
            "8f15b116e51843d2acabb245f9052414",
            "3cbb439813724ec198206de71e9cd3c7",
            "1bce3133cc82463f83c5b29ac377a7f5",
            "749f8705da9747f4b6fa7fc4417c8c10",
            "c670451e984f49e18367a99ba3168092",
            "14e8f57c270f4dc8b3a3a1ebb42a8b00",
            "d24aae09ad1a499e9906322797ba4770",
            "340d94472be34298b5866f2c2171c5d6",
            "80d3deecad2947868d5fb2141de20d8f",
            "fb3a14ac179d43198bc911ef4c968366",
            "2c2d737038934d2aae872c2f66325dd6",
            "983b182454ae4317847529fd34d3accc",
            "e184d2e3719043d488e45f540df922ca",
            "acbf53b6a93149eebe828a92773739cb",
            "067c88d62d0d416f83331e521a9b08f7",
            "eba54b5d72f44ba5a0c3c264a5ebc483",
            "5cc9837967ff4c88af135db879fa33d9"
          ]
        },
        "outputId": "cb1f9d17-a899-47cc-ca68-6963bf9d00c3"
      },
      "source": [
        "#@title Training { vertical-output: true}\n",
        "\n",
        "# TODO add the Zero training input output? --> check NUM_TOKENS amount\n",
        "\n",
        "# training\n",
        "log_loss = []\n",
        "UPDATE_STEPS = 20\n",
        "SENTENCE_BATCH = 32\n",
        "\n",
        "EVAL_STEPS = 1\n",
        "\n",
        "# X - 4d tensor - maybe move this\n",
        "def get_input_values(x):\n",
        "  step_output_float_mask = get_dilatied_mask(x).to(device)\n",
        "\n",
        "  # X values\n",
        "  percieved_images = percieve(x)\n",
        "  X_img = percieved_images[0]\n",
        "\n",
        "  X_seq_un_clean = image_to_sentences(X_img)\n",
        "  valid_instances = (torch.tensor(X_seq_un_clean).bool().float().sum(dim=-1) != 0).nonzero().flatten().long()\n",
        "\n",
        "  X_seq_cleaned = X_seq_un_clean[valid_instances]\n",
        "\n",
        "  X_seq = tokenize(X_seq_cleaned)\n",
        "\n",
        "  return step_output_float_mask, valid_instances, X_seq\n",
        "\n",
        "\n",
        "next_token_loss = [['EPOCH', 'STEP', 'LOSS']]\n",
        "epoch_losses = []\n",
        "\n",
        "for i in tqdm.tqdm(range(EPOCHS), mininterval=10., desc='training epochs'):\n",
        "  # update_steps = i+1\n",
        "  # repeats = (20 - i)**2\n",
        "  # for(r)\n",
        "\n",
        "    x0 = np.repeat(seed[None,...], IMAGE_BATCH_SIZE, 0)\n",
        "    x0 = torch.Tensor(x0).to(device)\n",
        "    x = x0\n",
        "    step_loss = []\n",
        "    step_size = []\n",
        "    \n",
        "    for step in tqdm.auto.tqdm(range(UPDATE_STEPS), mininterval=1., desc='update step'):\n",
        "      update_step_loss = []\n",
        "      step_output_float_mask, valid_instances, X_seq = get_input_values(x)\n",
        "      step_size.append(len(valid_instances))\n",
        "      # target \n",
        "      y_target = mask_target(step_output_float_mask.bool().float())[0]\n",
        "      # Extend y _ target channels here.\n",
        "\n",
        "      # Prefix\n",
        "      prefix = get_start_tokens(X_seq.shape[0])\n",
        "\n",
        "      # Y valyes from target\n",
        "      y_seq_un_cleaned = image_to_sentences(y_target)\n",
        "      y_seq_cleaned = y_seq_un_cleaned[valid_instances]\n",
        "      y_seq = tokenize(y_seq_cleaned)\n",
        "      Y_seq = torch.cat([prefix, X_seq, y_seq], 1)\n",
        "\n",
        "      # Derive dataset for training.\n",
        "      dataset = CellTraining(X_seq, Y_seq)\n",
        "\n",
        "      data_loader = DataLoader(dataset=dataset, batch_size=SENTENCE_BATCH, shuffle=True)\n",
        "            \n",
        "      enc.train(), dec.train()\n",
        "      for batch_i , (src, tgt, src_mask, tgt_mask) in enumerate(data_loader):\n",
        "        context = enc(src, input_mask = src_mask)\n",
        "        loss = dec(tgt, context = context, input_mask = tgt_mask, context_mask = src_mask, return_loss = True)\n",
        "        loss.backward()\n",
        "        loss_value = loss.item()\n",
        "        next_token_loss.append([i, step, loss_value])\n",
        "        update_step_loss.append(loss_value)\n",
        "        optim.step()\n",
        "        optim.zero_grad()\n",
        "\n",
        "      # update what x is (the target dilated & alive tgt mask)\n",
        "      x = mask_target(step_output_float_mask.bool().float()).to(device)\n",
        "      step_loss.append(sum(update_step_loss)/len(update_step_loss))\n",
        "\n",
        "    epoch_losses.append(step_loss)\n",
        "    # Generate:\n",
        "\n",
        "    enc.eval(), dec.eval()\n",
        "    for step in tqdm.auto.tqdm(range(EVAL_STEPS), mininterval=1., desc='evaluation step'):\n",
        "      output_seq = torch.zeros(SENTENCES, CHANNEL_N).long().to(device) # image_to_sentences(x[0]).long()\n",
        "\n",
        "      step_output_float_mask, update_instances, X_seq = get_input_values(x)\n",
        "\n",
        "      dataset = CellEvaluating(X_seq)\n",
        "      data_loader = DataLoader(dataset=dataset, batch_size=SENTENCE_BATCH, shuffle=False)\n",
        "      \n",
        "      generated= torch.zeros(X_seq.shape[0], CHANNEL_N).long().to(device)\n",
        "\n",
        "      # if (step == 0 or step == EVAL_STEPS-1):\n",
        "      #   plt.figure()\n",
        "      #   plt.imshow(np.hstack(np.squeeze(np.transpose(step_output_float_mask.bool().float().long().cpu().numpy(), (0,2,3,1)), axis=-1)), cmap='gray',  vmin = 0, vmax = 1)\n",
        "      #   plt.show()\n",
        "\n",
        "      for batch_i , (src, src_mask) in enumerate(data_loader):\n",
        "        start_tokens = (torch.ones((src.shape[0], 1)) * 1).long().to(device)\n",
        "\n",
        "        context = enc(src)\n",
        "        \n",
        "        sample = dec.generate(start_tokens, CHANNEL_N , context = context)\n",
        "        predicted_cells = sample[:, - CHANNEL_N: ]\n",
        "        # if(batch_i ==0):\n",
        "        #   print(predicted_cells[0], src[0])\n",
        "        s = batch_i*SENTENCE_BATCH\n",
        "        \n",
        "        locations = torch.arange(s,s+predicted_cells.shape[0])\n",
        "\n",
        "        generated[locations] = predicted_cells\n",
        "\n",
        "      output_seq[update_instances] = detokenize(generated)\n",
        "      # print('instance zero', output_seq[update_instances[0]])\n",
        "      output_img = sentences_to_image(output_seq, x.shape[2], x.shape[3])\n",
        "      \n",
        "      x = output_img[None,...]\n",
        "    visualize_batched(x.detach().clone())\n",
        "\n",
        "    plt.figure()\n",
        "    plt.plot(step_size)\n",
        "    plt.xlabel(\"Step\")\n",
        "    plt.ylabel(\"Size\")\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "    \n",
        "    plt.figure()\n",
        "    for ep, l in enumerate(epoch_losses):\n",
        "      plt.plot(l, label=f'EPOCH-{ep}')\n",
        "\n",
        "    plt.xlabel(\"Step\")\n",
        "    plt.ylabel(\"Loss\")\n",
        "\n",
        "    plt.legend()\n",
        "    plt.show()\n",
        "\n",
        "\n",
        "      \n",
        "    # TODO - produced x - calculate l2 loss and add to log loss - need alignment first"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "training epochs:   0%|          | 0/10 [00:00<?, ?it/s]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "618fb955a1934bd98bfee388b7de7573",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='update step', max=30.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:22: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3cbb439813724ec198206de71e9cd3c7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='evaluation step', max=1.0, style=ProgressStyle(descriptio…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAaPUlEQVR4nO2de3hU5bXG35XJhUtCwl0QQhAQi6KgsRWl1opYtYraWqttbeuxtae2faTq8dbTU7X2VHu37emFapVztFWhatUqahGqVURuKki4X8MtAcwNCLmt88eM3/piZ2BCkslkvvf3PDy8s2fvb397Jmv22muvvZaoKgghmU9WV0+AEJIaaOyEBAKNnZBAoLETEgg0dkICgcZOSCC0y9hF5DwRWS0i60Tk1o6aFCGk45Ejvc8uIhEAawBMBVAOYBGAK1V1ZcdNjxDSUWS3Y9sPA1inqhsAQEQeBXAxgITGPmDAAC0pKWnHLgkhh2LJkiW7VXVgvPfaY+xHA9jqvS4H8JFDbVBSUoLFixe3Y5eEkEMhIpsTvdfpAToRuVZEFovI4srKys7eHSEkAe0x9m0Ahnuvh8WWtUJVZ6hqqaqWDhwY17sghKSA9hj7IgBjRGSkiOQCuALA0x0zLUJIR3PE1+yq2iQi3wTwAoAIgD+q6rsdNjNCSIfSngAdVPU5AM910FwIIZ0IM+gICQQaOyGBQGMnJBBo7IQEAo2dkECgsRMSCDR2QgKBxk5IINDYCQkEGjshgUBjJyQQaOyEBAKNnZBAoLETEgg0dkICgcZOSCDQ2AkJBBo7IYFAYyckEGjshAQCjZ2QQDissYvIH0WkQkRWeMv6ichLIrI29n/fzp0mIaS9JHNmfwjAeR9YdiuAuao6BsDc2GtCSBpzWGNX1VcA7P3A4osBzIzpmQAu6eB5EUI6mCO9Zh+sqjtieieAwR00H0JIJ9HuAJ2qKgBN9D67uBKSHhypse8SkSEAEPu/ItGK7OJKSHpwpMb+NIAvxfSXAPy1Y6ZDCOkskrn19mcACwCMFZFyEbkGwD0AporIWgDnxF4TQtKYw3ZxVdUrE7w1pYPnQgjpRJhBR0gg0NgJCQQaOyGBQGMnJBBo7IQEAo2dkECgsRMSCDR2QgKBxk5IINDYCQkEGjshgUBjJyQQaOyEBAKNnZBAOOwjrqTriFb8iiIiHb4+CQue2QkJBBo7IYFANz7NUMR3xfc21Dh976pZTo/oaUU8rxszLe44vqR7Hy48sxMSCDR2QgKBxk5IIPCaPQ3wr69bvNtnexuqnL7k1Tudfl032MZNtv47VRud/t2p1zvdjBans3j9HizJ1I0fLiLzRGSliLwrItfHlrNtMyHdiGTc+CYAN6rqOACnAfiGiIwD2zYT0q1IpknEDgA7YrpWRMoAHI1o2+azYqvNBDAfwC2dMssMx3fdI2K/v79e84zTrzeudzqv1tzyFu8S4Pf6d6eHvt3P6f866Sqnm9W2jYBufEi0KUAnIiUAJgJYiCTbNrOLKyHpQdLGLiL5AP4CYLqq1vjvHaptM7u4EpIeJBWNF5EcRA39EVV9IrZ4l4gMUdUdh2vbTP6VFi9C7rvuK2s2OX3PmsdtneZmpxs07u8qsmtNf2/zo06f2G+U05cMP93pVi698C5sppNMNF4APACgTFV/5r3Fts2EdCOSObOfAeAqAMtF5K3YstsRbdP8eKyF82YAl3fOFAkhHUEy0fh/AgnDtmzbfIS08sS9T/dHK2c73VAYcTp7n7ftwca4Y7bY6siqsnWuW/Irpz82+ESni3J625j+AziM0mckvFAjJBBo7IQEAnPjU4jvKvvR75oG89Gfr1zsdPbyPU77wXIZk28v8uyNyD8tj6H3H9Y4vedTI51+YPyLTt806lNON7Y0OZ3l7WxtbbnTPbJynV5evcnpTy/6gdMNl1giEC8N0gue2QkJBBo7IYFANz6FJHJrNzea+71nc5nTvb6z1rbNtd9l/VWpDTq6wMnInoM2fvV+p7MqbfkzW950+sZjLnU6Jyv+n8Lp875t8/Hc+B0Ry+D59+Hnxd22dc4/3fiuhmd2QgKBxk5IINCNTyGJEmnq6uudbim0hwdbTjD3XnWG09mF95neb8kzzZ84yun9xRaxbxhmrv7GHdu9MW1ClQetKs79m+c4fUrRaKfX1tm2J0SGOf1/W+zR2n55fZz+/glftLkxD7/L4adOSCDQ2AkJBLrxKUQSuPGDcryIes8cp/d9d7zTORHLb+9Rd8Dpnr0sv1162KBVJ1ulGq22HT9/3l1OZ2XZb/0be1c5/d2qvzhdvK+X0+MLS5x+bvcSp0v7mqt/9iDLvW919yHBsZPUwTM7IYFAYyckEOjGpxC/TrsfCR+ebxH40bkWUV/dtMNptHjR7CJz3QvybMw+/iOutZbrvkPtN/3cp7/j9IYvPOj01MGnOD1bpzt9V+3DTj9fvczm3MtKjD07yWraD8yziuIt3pz9S4ZkSFRLX+IX6Wnz+CHCT4iQQKCxExIIdONTiO/GNzWbm50bsa/huXPvdvqY575qGzeY/3ogYvrYOhvn7Fxzm5cVWh7747YYA7It8p+Xbev487n06MlOj88f4fSYmVOdHlEy1GnfdW+VPNMO191/dqBVXj0j+UcMz+yEBAKNnZBAoBvfRUQiFjr3o9aDs80lPl7NVV6lVjFGG+1rO3NDndNfbrY8+UHn2jhlB8w9Lm96z+l1VTbm6CLLdff56IOXOP3ND33O6VllVpGmutEq7RS2sYhlooj9xn12J8LvTntsvs2zocWO98S+x8QdnxVyjGTqxvcQkTdF5O1YF9c7Y8tHishCEVknIo+JSO7hxiKEdB3JuPEHAZytqicBmADgPBE5DcC9AH6uqqMBvAfgms6bJiGkvSRTN14BvO8r5sT+KYCzAbzv180EcAeA33b8FDMT37303d1e2T2cfuys252e/MzNTle1mNt8d7FF11f0N5f4M1k25kX9Tf86yyrMTHrpRqdvPM6KT27bvtrpvfv22rbvPOD0UdkDnG7xIvma7WW9JOFB+667f0fgc2/c6/SbXgfbwoPmQOaIXQrtvOgx263E/2xDd+mTCtCJSCTWDaYCwEsA1gOoUtX3v51yRNs4x9uWXVwJSQOSMnZVbVbVCQCGAfgwgOOS3QG7uBKSHrQpGq+qVSIyD8AkAEUikh07uw8DsK0zJhgCvivb0NLg9PH9S5yef/4Pnb5w0fecLq+x6Po/rOANTmw0l/7CrfZIbN5B29fvB5qLe+eW/3X6YL3nlh81yOkTsu3H+skLrFpO3x6FTjd59eezxf68/Lr0fn77wr12yTC36h2nT+prte6l2tZvzrZutjWNVlTzmDlfdvqtc37jdJ8ce0Q3dJc+mWj8QBEpiumeAKYCKAMwD8BlsdXYxZWQNCeZM/sQADNFJILoj8PjqvqsiKwE8KiI3A1gGaJtnQkhaUoy0fh3AEyMs3wDotfvpAPJzYqfrjCmv8U/5022SPXkOTc5vafJika+c8Dc1KuWmrt7wQgbf/eYPKffrTEXd4vYHYHKPHPdqyIW+b9/16tO39LHXP2+ebaOH13PicT/Uxuab+PPXrfA6d715nRWH7Q7CP7din1q9fD9uvfNaq5+orseIcJ0WUICgcZOSCAwN76L8CvVNHmPhd66/H6ns72kkae2v+502Xm2zpUlZzr9yz1/c3pzo7mvD3+yyOliryjlJG+/Q/PM9V2da+53uXnK2AiL/M+YYDXhn53zNacfmmiXFaX9xzr9wvp/OP3KtoU2z/3WqXb6UfZo7R17nnJ6e6OF47O9Oxc/OeErTl838iKnE1WtCTEC78MzOyGBQGMnJBDoxqeQRNHgRu9RzQW7Vzpd0WDua8T7Xa5rsCSZq0af4/Rvdjzn9F4xF31Ds7mveRHTH1ll45zzpkXsn77U3P51+XYpMX63ufdrn7zA6aVeQcundlpEvabB8ucv/NOXnG4q8Nxp7yN5ZMWfnZZBw02PsCKc4xuHOP3NURcjHv4lkrB7rINndkICgcZOSCDQjU8h2mLupR8xvmyBFZnMz7NKL1VNVoVmvVrUesGeMqc/MaTU6VN7WnHILRGr7lJu6fYoOGDu/aAiazXVWGo55D2LzHUfa547PjnfknaWDbLknHtPs9z42RsfdfrHq03Db1NVa0kyg72kmuIiSxzqOdDc+J497XLgW8fa5UML7Fj8DrnsEhsffiqEBAKNnZBAoBufShL8tJ4zaILTD2+f7/TqfdttJa8O/MKqtTZknVWt6V1nbn9NlvnfG3taRHq05+K+UWTLx/Sx5We8ZmNuPdFy0Veenu90dQ9b/5Y37K7Bzz7cx9aJ2DqVQ614pm61S4xHptmjsmePsqSaNsOg+2HhmZ2QQKCxExIIdONTiB8x9t3OyQNPcPrmVdZZ9aLB9gTxompz3X+62pJP9r9r1V2a1KsG08seNd0y0qL0z1v9SByVZ7/1w3dayP7tUfboK/bZpKuOseXDt9n6Gz9u7v1nN1q5nA/V20HOGNTT6fqDxU4fN2CU034yTHOL95iqd+fCz5HJ4rmqTfDTIiQQaOyEBALd+BSSlSBkfFyBJZA8cep/Oj1t6CSnHy1/xenPvXqr0+q1fEKBReyfu/R3tryHJcxc9uZdTr/XYG5/faFF3cc0mAvds7+dDz66yZ53LZ1rEfjqz/d3elStzae4ypJeXiu28d8eYPN5atcip68rnIZ4MEmmY+CnSEgg0NgJCQRRTV0RvtLSUl28eHHK9tfd8QsnRryqNXk/tI6l+fkWdb/jtOlOf+vUq+OO+fedS5y+9u1fOb09a7fTRU223+Fe/vmxEftbGe91X+3Xz84ZA73TR7WtgsVqV4x/r7Zxqg9YXn3Z+X9wujDHIvz+1U/o1WYOh4gsUdXSeO8lfWaPtYBaJiLPxl6ziysh3Yi2uPHXI9oc4n3YxZWQbkRSbryIDEO0U+sPANwA4CIAlQCOUtUmEZkE4A5V/cShxqEbHx//O2jxSrdkeRkkvvva4rnQ/rcX8ZJPEo3pR7Z319sjq/dvftH0+ued3tFY4fSALNvv8B425vE5tnxcb5vn2H0W7Z+TZ258Q5Vt+1iOOYR3j7DClV8fYwUkW7WUyuINpEPREW78LwDcDLgLuP5IsosrISQ9SKbX24UAKlR1yeHWTbA9WzYTkgYk4xOdAWCaiFwAoAeAPgDuQ5JdXFV1BoAZQNSN75BZZxh+UcRIEtHmRHXRZy2Y5fRnJn0m7pi+SzyghxWWnFZ4otPfeeeXTmu+VZXZWmTJM3uaLUmmwovSb/em36PWlt/2siXlPzvZ9rvCC+vO2jrPad+NZw58x3DYT1FVb1PVYapaAuAKAC+r6ufBLq6EdCva85N5C4AbRGQdotfw7OJKSBrTptCmqs4HMD+m2cW1i2h1B8Vzm33XvdX6fszek/sarFb8mY9c4XRLruW9Z+31IuFV5tI3DTze6bX9bNBdTaYPFFhBy5aL+zld4NWun+jl0r+wf73T2722UEN72eWDfyxMsGkbvBgiJBBo7IQEAjMUuiFtbWnku7vZkey4+qVzn3H6grmXOL0zYrdLTx13ltO/OcO6tS7cu8npa9/6tdOLs8zlPlYs3/7yA1blpq7IwvFlBebSr6gxl95341vUTxCiG98WeGYnJBBo7IQEAt34DCJRpHrXAUtomb78905P7jfO6T6F9qhsQ19LehEvB36h7HL6xjUPO/3S5P92et17Vuv+RxWWerFmj0X1j95o3WNrP2ZFLMd69e3L6zZ7R2Y3fVS952ZZwaZN8NMiJBBo7IQEAt34DCJRpHpbvSWozC5/1emle6wW/br6HTZQP6vxHmk03dLDxny5cpnTG/fZtned+kWnH5o11+nbp1iRzIeGft/pk3NsTCtDCVTUl4N0LDyzExIINHZCAoFufAbhV6HxI/Mn9x3j9LdHf8rpWbvMpT+2l3VZ3bLPEmnqYYku04+2bb8y4lyni3sNjjuHfxt9jtNTFt7j9GAvoF5vKfN4LdcSbyarVdHxyUpwjP4ljHj5/4keBw4RfhKEBAKNnZBAoBufQSR69NXnHxVvO53jff1fK7nA6fvWPeX0oDxLsPnp+K/GHbPZT3Tx+OLoqbbtRhsTXpPYhfu9vH31imHmWKupVviH6Ff4kVbF5R1+cc7QXfqwj56QgKCxExIIdOMzCN+tbRWp9lo4PXX6HU4vr97o9PlLbfm5BVZ8ck2tJbesq7Oaosf0GmL79ebQ4rn0H+pb7PRH+1se/oKGFU6XN9nW+TWWP7+zoM7GTOCK7/Sq2Syr3eD00bkDnD6x70in/cuctj4mnAnwzE5IINDYCQkEuvEZSutijKaH+FVfxNzjrwyxzl0TC61L7BPbLPGmd8RqxfvutN8MoLnZXHG/3vvVXmT+lXfNja++3OZQ86iNU1dr+fYHmi0yX1NvRTInzP260xV59bbf/Tbmf5Rc6vQ9E6wdoX9p8MHjyVSSMnYR2QSgFkAzgCZVLRWRfgAeA1ACYBOAy1X1vc6ZJiGkvbTl5+zjqjrBaxp3K4C5qjoGwNzYa0JImpJsF9dNAEpVdbe3bDWAs1R1h4gMATBfVcceahx2ce16OrvueqIa9XWN5n4f/6Il51Q0Ww681wwWjV7izZxSq4Tzkf7HOf1WlUXgl+5d5/T05TOcbsq2Sbx2+o+cPn2A1b0HWt+x6M7tpjqii6sCeFFElojItbFlg1X1/QurnQAGx9+UEJIOJBugm6yq20RkEICXRGSV/6aqqojEdRFiPw7XAkBxcXG8VQghKSApY1fVbbH/K0TkSUQrAO4SkSGeG1+RYFt2cU0jfNc94SOirbJkvG29NxIlpfjjN3sbF+T2dvqGYz/t9E2bHnI6x/JoWrnVP14z2+m/Tb7b6Uleoo6vB3vdaa949ydOL9trtej/xY1v9tz4SPd14w9FMv3Ze4tIwfsawLkAVgB4GtHurQC7uBKS9iRzZh8M4MnYL3k2gD+p6hwRWQTgcRG5BsBmAJd33jQJIe3lsMYe69Z6UpzlewBM6YxJkdTgu9wJWym1w6PN8sb3k1i+Puoipx/ZMs/ppT0sVz/PSstjTtNSpz/7urnxlw47w+lntr3h9GM7LRGo2esY67v3H0QCSKrJ/CMkhACgsRMSDMyNJ51Gq0duvWh/XsQ6t76919o8DSuw6pNbe1nmdVadFaJ8vPI1T79uO2vxbvRk2zlscIu1tZpy1Mlx5wMAWQE88sozOyGBQGMnJBDoxpOU4D9C6kfmGy5/xun1tdYB9oYVf3D66QMLbCDPRW/1bG22ueHF2VapZuaEG5zum5sfdw5A63r0mUrmHyEhBACNnZBgoLETEgi8ZicpJ9H1+6gC6zf310nfc3rJe9Zaer7X5KKm2dLsRnr95i4eMsnpvnl26y30hhHhHTEhgUJjJyQQ6MaTLiWRS+9zitdy2tfJELrr7hP20RMSEDR2QgKBbjxJGzrDzQ7ddffhJ0FIINDYCQkEGjshgUBjJyQQaOyEBEJSxi4iRSIyW0RWiUiZiEwSkX4i8pKIrI3937ezJ0sIOXKSPbPfB2COqh6HaFnpMrCLKyHdimQ6whQCOBPAAwCgqg2qWgXgYgAzY6vNBHBJZ02SENJ+kjmzjwRQCeBBEVkmIvfH2kCxiysh3YhkjD0bwMkAfquqEwHswwdcdo3W5U3YxVVEFovI4srKyvbOlxByhCRj7OUAylV1Yez1bESNf1eseysO18VVVUtVtXTgwIEdMWdCyBFwWGNX1Z0AtorI2NiiKQBWgl1cCelWJPsgzLcAPCIiuQA2ALga0R8KdnElpJuQlLGr6lsASuO8xS6uhHQTmEFHSCDQ2AkJBBo7IYFAYyckEGjshAQCjZ2QQKCxExIINHZCAoHGTkgg0NgJCQQaOyGBQGMnJBBo7IQEAo2dkECgsRMSCDR2QgKBxk5IINDYCQkEGjshgUBjJyQQaOyEBAKNnZBASKax41gRecv7VyMi09mymZDuRTIdYVar6gRVnQDgFAD7ATwJtmwmpFvRVjd+CoD1qroZbNlMSLeircZ+BYA/x3RSLZvZxZWQ9CBpY4/1eZsGYNYH3ztUy2Z2cSUkPWjLmf18AEtVdVfsdVItmwkh6UFbjP1KmAsPsGUzId2KpIxdRHoDmArgCW/xPQCmishaAOfEXhNC0hSJXm6naGcilQD2Adidsp2mBwMQ1jGHdrxA+hzzCFWNGxxLqbEDgIgsVtV4vd4zltCOObTjBbrHMTNdlpBAoLETEghdYewzumCfXU1oxxza8QLd4JhTfs1OCOka6MYTEggpNXYROU9EVovIOhHJuKfkRGS4iMwTkZUi8q6IXB9bntGPA4tIRESWicizsdcjRWRh7Ht+LJZqnTGISJGIzBaRVSJSJiKTusN3nDJjF5EIgP9BNO12HIArRWRcqvafIpoA3Kiq4wCcBuAbsWPM9MeBrwdQ5r2+F8DPVXU0gPcAXNMls+o87gMwR1WPA3ASosee/t+xqqbkH4BJAF7wXt8G4LZU7b8r/iGaQjwVwGoAQ2LLhgBY3dVz68BjHIboH/fZAJ4FIIgml2TH+967+z8AhQA2Ihbv8pan/XecSjf+aABbvdflsWUZiYiUAJgIYCGSfBy4m/ILADcDaIm97g+gSlWbYq8z7XseCaASwIOxS5f7Y+nkaf8dM0DXCYhIPoC/AJiuqjX+exr96c+IWyAiciGAClVd0tVzSSHZAE4G8FtVnYho+ncrlz1dv+NUGvs2AMO918NiyzIKEclB1NAfUdX3HxzK1MeBzwAwTUQ2AXgUUVf+PgBFIpIdWyfTvudyAOWqujD2ejaixp/233EqjX0RgDGxSG0uolVvnk7h/jsdEREADwAoU9WfeW9l5OPAqnqbqg5T1RJEv8+XVfXzAOYBuCy2WsYcLwCo6k4AW0VkbGzRFAAr0Q2+41Q/9XYBotd4EQB/VNUfpGznKUBEJgN4FcBy2DXs7Yhetz8OoBjAZgCXq+reLplkJyEiZwG4SVUvFJFjED3T9wOwDMAXVPVgV86vIxGRCQDuB5ALYAOAqxE9cab1d8wMOkICgQE6QgKBxk5IINDYCQkEGjshgUBjJyQQaOyEBAKNnZBAoLETEgj/DzPEteV+c3giAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "No handles with labels found to put in legend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEGCAYAAAB2EqL0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXgV5dnH8e/NGvYlRLagQRbZZJHI8roLWndwo/pWxYqirdUi2orVSlv7tta6VGtrSxVFRRZFhVqrUkRrrSIBSdgxyBbWsCSsgSTnfv84Q00VSAJJ5pyT3+e6uM6cZ+ac3tNJ8nOemXkec3dERERKUyPsAkREJD4oMEREpEwUGCIiUiYKDBERKRMFhoiIlEmtsAuoDC1atPC0tLSwyxARiSvz5s3b6u4ph1ufkIGRlpZGRkZG2GWIiMQVM1tzpPXqkhIRkTJRYIiISJkoMEREpEwS8hqGiEh1V1hYSE5ODgUFBd9Yl5SURGpqKrVr1y7XdyowREQSUE5ODo0aNSItLQ0z+0+7u7Nt2zZycnJo3759ub5TXVIiIgmooKCA5OTk/woLADMjOTn5kGcepVFgiIgkqK+HRWntpVGXlIgcE3cnf18hG/IK2LRzHxvyCsjdtR9NnRCOzq0acUnPNpXy3ZUWGGY2HrgE2OLuPYK25sAUIA1YDQxz9x0WjbsngYuAvcCN7j4/+Mxw4IHga3/p7hMqq2YRObxlm3Yyf00eG/P3sTG/IPqaV8DG/AL2FRZ/Y/uj/I9YOUaX9GwTf4EBvAA8DbxYom0MMMvdHzazMcH7e4ELgU7Bv/7AM0D/IGDGAumAA/PMbIa776jEukWkhOwtu3l85nLeXrgJgBoGxzVKonXTJLq2bsw5XY6jdZMk2jSt95/XFg3rUrOGEiNs7n7I7qejPfurtMBw93+aWdrXmocAZwfLE4APiAbGEOBFj+7Fp2bW1MxaB9vOdPftAGY2E7gAmFRZdYtIVM6OvTz5jy+YNj+HerVrcuegTlzdN5XWTZKoVVOXP2NdUlIS27Zt+8aF74N3SSUlJZX7O6v6GkZLd98YLG8CWgbLbYF1JbbLCdoO1/4NZjYSGAlw/PHHV2DJItVL7q79/GF2Nq/MWQsGN53Wnu+d3YHkhnXDLk3KITU1lZycHHJzc7+x7uBzGOUV2kVvd3czq7CrYu4+DhgHkJ6erqttIuWUv7eQcR+tZPy/VnOgOMKw9FTuOLcTbZrWC7s0OQq1a9cu93MWpanqwNhsZq3dfWPQ5bQlaF8PtCuxXWrQtp6vurAOtn9QBXWKVBt7DxTx/Mer+fOHK9lZUMRlvdpw13mdad+iQdilSYyp6sCYAQwHHg5ep5do/4GZTSZ60Ts/CJV3gV+ZWbNgu/OB+6q4ZpGE9dfMDfz8r0vYuns/g7ocx93nn0S3No3DLktiVGXeVjuJ6NlBCzPLIXq308PAVDMbAawBhgWbv030ltpsorfVfhfA3beb2UPA3GC7Xxy8AC4iR2/b7v08OH0xf1u4kV6pTfjz9afQ94TmYZclMc4S8eGa9PR01wRKIof2zqKN3P/GInYWFDJqcGduPfNE3fUkAJjZPHdPP9x6PektUk3k7T3A2BmLmb5gA93bNGbiLf3p0krdT1J2CgyRamDW0s2MeX0hO/YcYNTgTtx+Tkdq66xCykmBIZLA8vcV8tBbS3htXg5dWjXi+RtPpUfbJmGXJXFKgSGSoD5ckcuYaVls2bWfH5zTkTsGdaRurZphlyVxTIEhkmD27C/il39byqTP1tLxuIa8fl1ferVrGnZZkgAUGCIJZN6a7Yyemsna7XsZeeaJjD6vM0m1dVYhFUOBIZIADhRFeHLWCp75YCVtmtZj8i0D6H9icthlSYJRYIjEuRWbd3HXlAUs3rCTYemp/PSSbjRKqh12WZKAFBgicSoSccZ/vIpH3l1Oo7q1GHd9X87v3irssiSBKTBE4tD6vH3cMzWTT77cxuCuLXn4ypNpoeHHpZIpMETiiLvzxufrGTt9MRF3HrmyJ1enpx5yVjWRiqbAEIkTuwoKuXdaFm8v3MSpac147OreHJ9cP+yypBpRYIjEgX0HihnxQgbz1+5gzIVduOWMEzVntlQ5BYZIjNtfVMzIlzLIWLOdJ6/pw6W92oRdklRTCgyRGFZYHOEHr3zOR19s5ZGreiosJFQarlIkRhVHnHtezWTmks38/LLuDEtvV/qHRCqRAkMkBrk797+xkOkLNnDvBV0Y/j9pYZckosAQiTXuzi/eWsLkueu449yOfO/sDmGXJAIoMERizuMzV/D8x6u56bT2jD6vc9jliPyHAkMkhvzxg2x+/3421/Zrx08v6aoH8iSmKDBEYsSEf6/mkXeWM7R3G3459GSFhcQcBYZIDJiasY6xMxbzre4tefTqXnooT2KSAkMkZH/N3MCYaVmc2TmFp67tQ62a+rWU2KSfTJEQvbNoE6OmLCA9rTl/vq6v5tyWmKbAEAnJ+8s2c8ek+fRKbcL4G0+lXh2FhcQ2BYZICP65IpfbXp5P19aNeeGmfjSsq1F6JPYpMESq2CcrtzHypQw6pDTkxZv60VjTqUqcUGCIVKGM1dsZMWEu7ZrV5+UR/Whav07YJYmUmQJDpIosWJfHjc/PpVXjJCbe0p9kTakqcUaBIVIFFq3P54bn5tC8QR1euWUAxzVKCrskkXILJTDM7C4zW2xmi8xskpklmVl7M5tjZtlmNsXM6gTb1g3eZwfr08KoWeRoLdu0k+ufm0OjpNq8ckt/WjVRWEh8qvLAMLO2wJ1Aurv3AGoC1wC/AZ5w947ADmBE8JERwI6g/YlgO5G4kL1lN9c9O4e6tWryyi39SW2mObglfoXVJVULqGdmtYD6wEbgXOC1YP0EYGiwPCR4T7B+kGmQHYkDq7bu4X//8ilgTLylPyckNwi7JJFjUuWB4e7rgUeBtUSDIh+YB+S5e1GwWQ7QNlhuC6wLPlsUbJ/89e81s5FmlmFmGbm5uZW7EyKl2Ji/j+/85VOKIs4rt/SnQ0rDsEsSOWZhdEk1I3rW0B5oAzQALjjW73X3ce6e7u7pKSkpx/p1Ikft4Dzc+fsKeWlEPzq3bBR2SSIVIowuqcHAKnfPdfdC4HXgNKBp0EUFkAqsD5bXA+0AgvVNgG1VW7JI2f3m78uYt2YHD1/Zk+5tmoRdjkiFCSMw1gIDzKx+cC1iELAEmA1cFWwzHJgeLM8I3hOsf9/dvQrrFSmzdxZt5Nl/rWL4wBO4tFebsMsRqVBhXMOYQ/Ti9XxgYVDDOOBeYLSZZRO9RvFc8JHngOSgfTQwpqprFimL1Vv38KNXs+iV2oSfXNw17HJEKlwoI565+1hg7NeavwT6HWLbAuDqqqhL5GgVFBbz/YnzqVHD+MN3TtEw5ZKQNESmSAX42YzFLNm4k/E3putZC0lYGhpE5BhNm5fD5LnruP2cDpzbpWXY5YhUGgWGyDFYtmkn97+5kAEnNueuwZ3DLkekUikwRI7S7v1FfH/ifBol1dZc3FIt6Cdc5Ci4O2OmZbF66x5+f20fjT4r1YICQ+QovPTpGt7K2sg93zqJASd+Y6QakYSkwBAppwXr8njorSUM6nIct53ZIexyRKqMAkOkHPL2HuD2ifM5rlESjw3rRY0aGjhZqg89hyFSRpGIM3pqJrm79vPqbQM1H7dUOzrDECmjZz5cyfvLtvDAJV3p1a5p2OWIVDkFhkgZfLJyG4+9t5xLe7Xh+gEnhF2OSCgUGCKl2LKzgDsmfU5aiwb8+oqT0YSPUl3pGobIERQVR7hj0ufs3l/IxJv707CufmWk+tJPv8gRPD5zBXNWbefxYb04qZVmzpPqTV1SIofx/rLN/PGDlVzbrx1XnJIadjkioVNgiBzCuu17uWtKJt1aN2bspd3DLkckJigwRL5mf1ExP3hlPpGI88x1p5BUW5MhiYCuYYh8w6/+tpTMnHz+dF1fTkhuEHY5IjFDZxgiJfw1cwMTPlnDzae354IercIuRySmKDBEAitzdzNmWhZ9T2jGvRd2CbsckZijwBAB9h0o5vsvz6du7Zo8/b99qK3JkES+QdcwpNpzd+5/cyErtuzixZv60bpJvbBLEolJ+s8oqfbeXLCe1+ev585zO3FGp5SwyxGJWQoMqdY25Rcwdvpi+p7QjDsHdQq7HJGYpsCQasvdGfN6FgeKIzx6dS9qajIkkSNSYEi1NWXuOj5YnsuYC7rQvoWetxApjQJDqqWcHXv55d+WMvDEZG4YmBZ2OSJxQYEh1U4k4vz4tSzcnUeu6ql5uUXKSIEh1c7Lc9bw75XbeOCSbrRrXj/sckTihgJDqpXVW/fw67eXcVbnFK45tV3Y5YjEFQWGVBvFEedHr2VSq6bx8JWaalWkvEIJDDNramavmdkyM1tqZgPNrLmZzTSzL4LXZsG2ZmZPmVm2mWWZ2Slh1Czxb/y/VjF39Q5+dml3Pc0tchTCOsN4EnjH3bsAvYClwBhglrt3AmYF7wEuBDoF/0YCz1R9uRLvsrfs4rfvLWdw15ZccUrbsMsRiUtVHhhm1gQ4E3gOwN0PuHseMASYEGw2ARgaLA8BXvSoT4GmZta6isuWOFZUHOHuqZnUr1OTX13RQ11RIkcpjDOM9kAu8LyZfW5mz5pZA6Clu28MttkEtAyW2wLrSnw+J2j7L2Y20swyzCwjNze3EsuXePPnf35JZk4+Dw3pwXGNksIuRyRuhREYtYBTgGfcvQ+wh6+6nwBwdwe8PF/q7uPcPd3d01NSNICcRC3duJPf/WMFF/dszaW92oRdjkhcCyMwcoAcd58TvH+NaIBsPtjVFLxuCdavB0re/5gatIkc0YGiaFdUk3q1eWhIj7DLEYl7VR4Y7r4JWGdmJwVNg4AlwAxgeNA2HJgeLM8AbgjulhoA5JfouhI5rKdnZ7Nk405+dfnJNG9QJ+xyROJeWBMo3QFMNLM6wJfAd4mG11QzGwGsAYYF274NXARkA3uDbUWOaPmmXfxxdjaX92nL+d01N7dIRQglMNx9AZB+iFWDDrGtA7dXelGSMIoj0WHLG9erzU8v6RZ2OSIJQ096S8J5+dM1fL42j59e0lVdUSIVSIEhCWVD3j4eeWcZZ3RqwdDeekBPpCIpMCRhuDsPTl9MsTv/N1RjRYlUNAWGJIy/L9rEP5ZuZvR5nTk+WcOWi1S0MgWGmbU0s+fM7O/B+27B3UwiMSF/byFjZyymR9vG3HRa+7DLEUlIZT3DeAF4Fzj4qOwKYFRlFCRyNB5+Zxnbdu/n4St6UqumTpxFKkNZf7NauPtUIALg7kVAcaVVJVIOc77cxqTP1jLi9Pb0aNsk7HJEElZZA2OPmSUTjO908InrSqtKpIwKCou5742FpDarx13ndQ67HJGEVtYH9+4mOkRHBzP7GEgBrq60qkTK6I+zs/kydw8v3tSP+nXCGrhApHoo02+Yu88zs7OAkwADlrt7YaVWJlKKFZt38cyHK7m8T1vO7KwRikUqW1nvkloJ3Ozui919kbsXmtlblVybyGFFIs59ry+kYd1aPHBx17DLEakWynoNoxA4x8yeDwYMhENMYiRSVSbOWcO8NTt44OJuJDesG3Y5ItVCWQNjr7t/m+jc2x+Z2fGUc4IjkYqyMX8fv3lnOWd0aqH5uUWqUFmvEhqAuz9iZvOB94DmlVaVyBGMnb6YokhEw3+IVLGynmE8eHDB3f8BfAt4ulIqEjmCt7I28N6Szdw1WMN/iFS1I55hmFkXd18GrDezU762Whe9pUpt2VnAA28uole7pow4XcN/iFS10rqkRgMjgceC91+/bnFuhVckcgjuzr3Tsth3oJjHh/XS8B8iISjtt+5ZM2vl7ue4+znABGA3sAi4qtKrEwlMmbuO2ctzGXNhFzqkNAy7HJFqqbTA+BNwAMDMzgR+TTQ08oFxlVuaSNS67Xt56K0lDDwxmeED08IuR6TaKq1Lqqa7bw+Wvw2Mc/dpwDQzW1C5pYlEH9C7+9VMapjx6LBe1Kihu6JEwlLaGUZNMzsYKoOA90us08A9UunGf7yKz1Zt58FLu9G2ab2wyxGp1kr7oz8J+NDMtgL7gI8AzKwjGq1WKtkXm3fxyLvLGdy1JVf1TQ27HJFq74iB4e7/Z2azgNbAe+5+8C6pGsAdlV2cVF+FxRFGT82kYd1a/PoKPaAnEgtK7VZy908P0baicsoRiXr6/WwWrs/nT9edQkojjRUlEgt0M7vEnKycPJ6enc3lfdpyQY/WYZcjIgEFhsSUgsJiRk/NJKVhXX52WfewyxGREnSnk8SU3767nOwtu3lpRD+a1KsddjkiUoLOMCRmfPrlNsZ/vIrrB5zAGZ00g55IrFFgSEzYVVDIPa9mckLz+tx3UZewyxGRQ1CXlMSEh95awoa8fbx620Dq19GPpUgsCu0Mw8xqmtnnB+cGN7P2ZjbHzLLNbMrBqWDNrG7wPjtYnxZWzVI5/pa1kakZOXzv7A70PUHzconEqjC7pH5IdMrXg34DPOHuHYEdwIigfQSwI2h/IthOEsT6vH3c93oWvds1ZdTgzmGXIyJHEEpgmFkqcDHwbPDeiM6t8VqwyQRgaLA8JHhPsH6Q6bHfhFBUHGHU5M+JODx1TR9qa44LkZgW1m/o74AfA5HgfTKQ5+5FwfscoG2w3BZYBxCszw+2/y9mNtLMMswsIzc3tzJrlwry9Oxs5q7ewUNDu2u6VZE4UOWBYWaXAFvcfV5Ffq+7j3P3dHdPT0nRLZmxbu7q7Tw16wsu79OWy/toYEGReBDG7SinAZeZ2UVAEtAYeBJoama1grOIVGB9sP16oB2QEwy13gTYVvVlS0XJ31fIqMkLSG1Wn18M0dPcIvGiys8w3P0+d0919zTgGuB9d/8OMJuvpn0dDkwPlmcE7wnWv19i1FyJM+7OT95YyOadBTx1bR8aJelpbpF4EUtXGe8FRptZNtFrFM8F7c8ByUH7aGBMSPVJBXg1I4e/ZW3krvM607td07DLEZFyCPUJKXf/APggWP4S6HeIbQqAq6u0MKkUK3N3M3bGYgaemMxtZ3UIuxwRKadYOsOQBLa/qJg7J31OUu0aPPHt3tTU3NwicUdjMEiVePTd5SzesJNx1/elVZOksMsRkaOgMwypdB+uyOUvH63iugHHc373VmGXIyJHSYEhlWrr7v3cPTWTzi0b8sDF3cIuR0SOgbqkpNK4O/e8msnOgkJevrkfSbVrhl2SiBwDnWFIpfnD7Gw+WJ7L/Rd1pUurxmGXIyLHSIEhleK9xZt49L0VDO3dhhsGnhB2OSJSARQYUuGWbdrJqCkL6JXahIev7IkGFxZJDAoMqVDb9xzg5gkZNKxbi3E3pOu6hUgC0UVvqTAHiiJ87+V5bNm1n6m3DqRlYz1vIZJIdIYhFebnf13MnFXbeeTKnhonSiQBKTCkQrz06RomzlnLbWd1YGiftqV/QETijgJDjtm/V27lZzMWc26X4/jRt04KuxwRqSQKDDkma7ft5fsT59O+RQOevEaDCookMgWGHLVdBYXc/OJc3OHZG9I1GZJIgtNdUnJUIhHnrikLWJm7hxdv6kdaiwZhlyQilUxnGHJUHpu5nH8s3cKDl3TjtI4twi5HRKqAAkPKbfqC9fxh9kqu7ddOw36IVCMKDCmXD1fkcs+rmfRr35yfX9ZDw36IVCMKDCmzjNXbufWlDDod14i/3JBOnVr68RGpTvQbL2WyeEM+331hLm2a1OPFEf1oUk93RIlUNwoMKdWXubu54bnPaFS3Fi/d3J8WDeuGXZKIhECBIUe0Pm8f1z07B4CXb+5P26b1Qq5IRMKi5zDksHJ37ef6Z+ewa38Rk0cO4MSUhmGXJCIh0hmGHFL+vkJuGP8ZG/L38fyNp9K9TZOwSxKRkCkw5Bv2Hijiphfmkr1lF3++Pp30tOZhlyQiMUCBIf9lf1Ext740j8/X7uDJa/pwVueUsEsSkRihaxjyH0XFEUZNXsBHX2zlkSt7ctHJrcMuSURiiM4wBIgOJnjf6wv5+6JNPHBxV4ad2i7skkQkxigwhEjEeWD6Il6dl8Od53bk5jNODLskEYlBVR4YZtbOzGab2RIzW2xmPwzam5vZTDP7InhtFrSbmT1lZtlmlmVmp1R1zYksEnHuf3Mhr8xZy/fO7sBd53UOuyQRiVFhnGEUAXe7ezdgAHC7mXUDxgCz3L0TMCt4D3Ah0Cn4NxJ4pupLTkwHu6EmfbaO28/pwI+/dZIGExSRw6rywHD3je4+P1jeBSwF2gJDgAnBZhOAocHyEOBFj/oUaGpmuhp7jCIR595pWUzJWMcd53bknvMVFiJyZKFewzCzNKAPMAdo6e4bg1WbgJbBcltgXYmP5QRtX/+ukWaWYWYZubm5lVZzIiiOOD+elhW9ZjGoE6PP66ywEJFShRYYZtYQmAaMcvedJde5uwNenu9z93Hunu7u6SkpenbgcIojzo9ey+S1eTmMGqywEJGyCyUwzKw20bCY6O6vB82bD3Y1Ba9bgvb1QMl7PFODNimn4ohzz6uZvD5/PXcN7syowbrALSJlF8ZdUgY8Byx198dLrJoBDA+WhwPTS7TfENwtNQDIL9F1JWVUVBxh9NQFvPH5eu4+rzM/HNwp7JJEJM6E8aT3acD1wEIzWxC0/QR4GJhqZiOANcCwYN3bwEVANrAX+G7Vlhv/omGRyYzMDfzoWydx+zkdwy5JROJQlQeGu/8LOFyn+aBDbO/A7ZVaVAIrKo4wasoC3srayL0XdOF7Z3cIuyQRiVMaSyqBFRQWc9eUBfx90Sbuu7ALt56lsBCRo6fASFD5+woZ+WIGc1Zt56eXdGPE6e3DLklE4pwCIwFtyi/gxuc/Y2Xubp68pjdDen/jsRURkXJTYCSY7C27GD5+Lnl7D/D8jf04vVOLsEsSkQShwEgg89bsYMSEudSqUYMptw6kR1tNqyoiFUeBkSBmLd3M7a/Mp1XjJF68qT/HJ9cPuyQRSTAKjAQwZe5afvLGIrq3acz4G0+lRcO6YZckIglIgRHH3J2n38/msZkrOLNzCs985xQa1NUhFZHKob8ucao44jw4fRET56zlij5t+c1VPaldUxMoikjlUWDEoT37ixg9dQHvLt7MbWd14N4LNJeFiFQ+BUacyVyXx6gpC1i9bQ8PXtKNm/RAnohUEQVGnCiOOH/6cCVPzFxBSqO6vHLzAAZ2SA67LBGpRhQYcWB93j7umrKAz1Zt5+KTW/Ory0+mSf3aYZclItWMAiPGvZW1gZ+8vpDiiPPbq3pyVd9UXa8QkVAoMGLU7v1FjJ2+mGnzc+jdrim/+3Zv0lo0CLssEanGFBgxaP7aHYyavICcHXu589yO3DGok26ZFZHQKTBiSFFxhD/MXslT739Bq8ZJTLl1IKemNQ+7LBERQIERM7Jy8vjpm4vIzMlnSO82PDS0B42TdGFbRGKHAiNkO/Yc4LfvLWfSZ2tJblBX81eISMxSYIQkEnGmZKzjN+8sY1dBEd/9n/aMOq+TzipEJGYpMEKQuS6PB6dHu5/6pTXnF0O706VV47DLEhE5IgVGFdqx5wCPvLucyXPX0qJhXX737d4M6d1Gz1WISFxQYFSB4ogzee5afvvucnYVFHHTae0ZNbgTjdT9JCJxRIFRiSIRZ9ayLfz+/S/Iysmnf/vm/GJID05q1Sjs0kREyk2BUQn2HShm2vwcxv9rFV9u3UPbpvV48preXNZL3U8iEr8UGBUod9d+XvpkNS99uoYdewvpmdqEp67tw0U9WlFLT2qLSJxTYFSALzbv4tmPVvHGgvUUFkcY1KUlt5zRnn7tm+uMQkQShgLjKLk7/165jb989CUfLM+lbq0aXN03lRGnt+fElIZhlyciUuEUGOVQWBxh7urtzFq6hVlLN7N6215aNKzD6PM6c92AE2jeoE7YJYqIVBoFRiny9h7gg+W5/GPpZj5ckcuugiLq1KzBgA7JfP/sjlzWuw1JtWuGXaaISKVTYHyNu7Mydw+zlm5m1tItZKzZTsShRcM6XNC9FYO6tuSMTi1oUFf/14lI9RI3f/XM7ALgSaAm8Ky7P1zR/xtZOXncOelzVm/bC0DX1o35/tkdGdT1OHqlNqVGDV3AFpHqKy4Cw8xqAn8AzgNygLlmNsPdl1Tk/07bpvVIa9GAEae359yuLWnbtF5Ffr2ISFyLi8AA+gHZ7v4lgJlNBoYAFRoYyQ3r8sJ3+1XkV4qIJIx4eZqsLbCuxPucoO0/zGykmWWYWUZubm6VFiciUh3ES2CUyt3HuXu6u6enpKSEXY6ISMKJl8BYD7Qr8T41aBMRkSoSL4ExF+hkZu3NrA5wDTAj5JpERKqVuLjo7e5FZvYD4F2it9WOd/fFIZclIlKtxEVgALj728DbYdchIlJdxUuXlIiIhEyBISIiZWLuHnYNFc7McoE1x/AVLYCtFVROLND+xL5E26dE2x9IvH061P6c4O6HfS4hIQPjWJlZhrunh11HRdH+xL5E26dE2x9IvH06mv1Rl5SIiJSJAkNERMpEgXFo48IuoIJpf2Jfou1Tou0PJN4+lXt/dA1DRETKRGcYIiJSJgoMEREpEwVGCWZ2gZktN7NsMxsTdj0VwcxWm9lCM1tgZhlh11NeZjbezLaY2aISbc3NbKaZfRG8NguzxvI6zD79zMzWB8dpgZldFGaN5WFm7cxstpktMbPFZvbDoD0uj9MR9ieej1GSmX1mZpnBPv08aG9vZnOCv3lTgsFdD/89uoYRFUwDu4IS08AC11b0NLBVzcxWA+nuHpcPHJnZmcBu4EV37xG0PQJsd/eHg2Bv5u73hllneRxmn34G7Hb3R8Os7WiYWWugtbvPN7NGwDxgKHAjcXicjrA/w4jfY2RAA3ffbWa1gX8BPwRGA6+7+2Qz+xOQ6e7PHO57dIbxlf9MA+vuB4CD08BKiNz9n8D2rzUPASYEyxOI/jLHjcPsU9xy943uPj9Y3gUsJTojZlwepyPsT9zyqN3B29rBPwfOBV4L2ks9RgqMr5Q6DWyccuA9M5tnZiPDLqaCtHT3jcHyJqBlmMVUoB+YWVbQZRUX3TdfZ2ZpQB9gDglwnL62PxDHx8jMaprZAmALMDukoGkAAAMfSURBVBNYCeS5e1GwSal/8xQYie90dz8FuBC4PegOSRge7VNNhH7VZ4AOQG9gI/BYuOWUn5k1BKYBo9x9Z8l18XicDrE/cX2M3L3Y3XsTnbG0H9ClvN+hwPhKQk4D6+7rg9ctwBtEf1Di3eagn/lgf/OWkOs5Zu6+OfiFjgB/Ic6OU9AvPg2Y6O6vB81xe5wOtT/xfowOcvc8YDYwEGhqZgfnRSr1b54C4ysJNw2smTUILtphZg2A84FFR/5UXJgBDA+WhwPTQ6ylQhz8wxq4nDg6TsEF1eeApe7+eIlVcXmcDrc/cX6MUsysabBcj+jNPUuJBsdVwWalHiPdJVVCcJvc7/hqGtj/C7mkY2JmJxI9q4Do7IqvxNs+mdkk4GyiQzFvBsYCbwJTgeOJDmM/zN3j5iLyYfbpbKJdHQ6sBm4t0f8f08zsdOAjYCEQCZp/QrTfP+6O0xH251ri9xj1JHpRuybRE4Wp7v6L4G/EZKA58DlwnbvvP+z3KDBERKQs1CUlIiJlosAQEZEyUWCIiEiZKDBERKRMFBgiIlImCgyRCmJm9wcjgWYFo5n2N7NRZlY/7NpEKoJuqxWpAGY2EHgcONvd95tZC6AO8G/ieLRgkZJ0hiFSMVoDWw8+9BQExFVAG2C2mc0GMLPzzewTM5tvZq8G4xUdnLfkkWDuks/MrGNYOyJyOAoMkYrxHtDOzFaY2R/N7Cx3fwrYAJzj7ucEZx0PAIODASEziM5HcFC+u58MPE10xAGRmFKr9E1EpDTBxDR9gTOAc4Ap9s1ZGwcA3YCPo8MVUQf4pMT6SSVen6jcikXKT4EhUkHcvRj4APjAzBby1cB7Bxkw092vPdxXHGZZJCaoS0qkApjZSWbWqURTb6ID7u0CGgVtnwKnHbw+EYwm3LnEZ75d4rXkmYdITNAZhkjFaAj8PhhCugjIBkYSHeH0HTPbEFzHuBGYZGZ1g889QHQueYBmZpYF7A8+JxJTdFutSAwws9Xo9luJceqSEhGRMtEZhoiIlInOMEREpEwUGCIiUiYKDBERKRMFhoiIlIkCQ0REyuT/Aas/WpNIo6kpAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEGCAYAAACHGfl5AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV1b3//9cn80zIxJQAgYQhyCRhdAKsilZxKFKoUm1VisOt/Xqrop20/bW16u21daiiYsWr4GzBqjihIHNA5jGBAGHKACGBkPnz+yMHGjCQgZzsM3yej0ceydlnn3M+m0POO2utvdcSVcUYY4x/C3C6AGOMMc6zMDDGGGNhYIwxxsLAGGMMFgbGGGOAIKcLaK6EhATt3r2702UYY4xXWbVqVaGqJp7pfq8Lg+7du5OVleV0GcYY41VEZNfZ7rduImOMMRYGxhhjLAyMMcbghWMGxhjfV1VVRV5eHuXl5U6X4nXCwsJITk4mODi4WY+zMDDGeJy8vDyio6Pp3r07IuJ0OV5DVSkqKiIvL4/U1NRmPda6iYwxHqe8vJz4+HgLgmYSEeLj41vUorIwMMZ4JAuClmnpv5t1E5lmKymvYvmOQ2zPLyWlfQRpSVGkJkQSFhzodGnGmBayMDCNKq+qYdWuwyzJKWRxdhHr8oqpPW0ZjACBlLgI0hKjSEuKomdS3fe0pChiwpo3kGWMJwgMDKR///4nb0+aNInp06czevRo9u/fT1hYGFFRUcycOZPevXtTWVnJAw88wIcffoiIkJGRwbPPPktycjIABw4c4Be/+AUrV64kNjaWDh068NRTTxESEsLVV1/Nhg0bTr7WI488QlRUFL/85S+/U9cnn3zCvffeS01NDbfffjvTp09vleO1MDDfUV1Ty7q9R1iaU8Ti7EKydh2msrqWwABhUEos94xJY1RaAhmdY9h7+DjZ+UfrvgqOkpN/lEXbC6msqT35fEnRoQzuGsulfTowuk8iSdFhDh6dMU0THh7OmjVrGrzv9ddfJzMzkxkzZnD//fczd+5cHn74YUpLS9m6dSuBgYG88sor3HDDDSxfvhyA66+/nltuuYU5c+YAsHbtWg4ePEhKSkqTa6qpqeHuu+/ms88+Izk5maFDhzJ+/HgyMjLO+XgtDMxJZZXVPDp3Ex+t309pRTUAfTvFMGVENy5Ii2dYajxRoaf+l4npFEzfTjGnbKuuqSXvREgUHGXbwVKW5hQxf+NBAAYkt2NM7yQu7ZvEeZ3bERBgfcPGO1188cU89dRTlJWV8corr7Bz504CA+u6S3/yk58wc+ZMvvzyS0SE4OBgpk2bdvKxAwcOBCA3N7fJr7dixQrS0tLo0aMHUNda+de//mVhYFrPvuLj3P5qFlsOlDBhSDIX90pkZI944qNCm/1cQYEBdE+IpHtCJN+jA1B3ytvm/aUs2JrPF5sP8vcvt/O3L7aTGB3KmN6JjO2TxIXpid8JG2MenbeRTftKWvU5MzrH8Ltr+p11n+PHjzNo0KCTtx966CF++MMfnrLPvHnz6N+/P9nZ2XTt2pWYmFP/MMrMzGTjxo0ADBky5IyvlZOTc8prHThwoMEuor17957SkkhOTj7Z8jhX9ptnWLXrMD97bRUVVTW8fMtQxvRJavXXEBEyOseQ0TmGu8ekcehYJV9vy+eLzfl8suEAb2XlERwojOyZwJQR3bi0T5K1GIyjztZNdNNNNxEeHk737t15+umnOXz48Dm9Vs+ePU95rUceeeScnq8lLAz83Hur85j+7no6xYYx+47hpHeIbpPXjYsM4frByVw/OJmqmlpW7TrMgi35fLhuP3fMyiI1IZKfXpjKhPOTCQ+xs5T8WWN/wTvhxJjBCXFxcezevZvS0lKio//zO7Rq1SquvvpqAN55551mv86ePXu45pprAJg2bRoDBw5kz549J+/Py8ujS5cuLT2MU9h1Bn6qplZ57OMt3PfWWoZ0a88Hd13QZkFwuuDAAEb0iOehq/ry9f2jeXryYGLCgvjNBxsY+dgXPDl/K/klNi2B8VyRkZHccsst3HfffdTU1AAwa9YsysrKGDt2LGPHjqWiooIZM2acfMy6detYtGjRWZ83JSWFNWvWsGbNGqZNm8bQoUPZvn07O3fupLKykjlz5jB+/PhWOQYLAz9UWl7F1FlZPP91DjcN78qs24bRPjLE6bKAuvGGawZ25oO7L+DtaSMZnhrHs19lc+FfFvDLt9eyeX/z+o5VlZrTz4M1pglOjBmc+GrsFM4///nPhIWF0atXL9LT03n77bd5//33ERFEhPfff5/PP/+cnj170q9fPx566CE6duzYrJqCgoJ45plnuOKKK+jbty8TJ06kX7/WaTmJqnf9omRmZqotbtNyu4vKuH3WSnIKjvHINRlMGdnd6ZIalVt4jFcW7+StrDyOV9VwYVoCt1+USu+O0RSUVpBfUkHB0RPfy+u2lVZQ4PoKDBB+eXlvbh3V3cYhvMTmzZvp27ev02V4rYb+/URklapmnuEhFgb+ZNmOIu78v1XU1CrP3TSEC9MTnC6pWYrLKnljxW5eXZLLwZKKBvdpHxFMYnQoidGhJEWHkRgdytYDpXy9rYBh3eN4fMIAuidEtnHlprksDM5NS8LABpD9xOwVu/nNBxvoGh/By7cMJdULPxBjI0K4a3Qat1/Yg/kbD1BaXu360K/78E+ICiUk6Ls9n6rKu6v38ui8jYz720IeHNeHW0ZaK8GY+iwMfFxtrfLYJ1uYsXAHF/dK5OnJg2kX7t3TQ4QE1Y0rNJWIMGFIMhemJfDQe+t4dN4mPt5wgCcmDKBbvPeFor9QVZusrgVa2ttjA8g+rKK6hnvfXMOMhTuYMqIbM2/J9PogOBcd24Ux89ahPDFhAJv3lzDuqUX8c/FOam2A2eOEhYVRVFTU4g82f3ViPYOwsOZP+eK2MQMRmQlcDeSr6nkN3H8T8CAgQClwp6qubex5bcygaY4crztjaPnOQ0y/sg8/u7iH/ZVVz/4jx3novfV8tbWA4alxPDFhIF3jI5wuy7jYSmctd6aVzhwbQBaRi4GjwKwzhMEoYLOqHhaRK4FHVHV4Y89rYdC4vcXH+ckrK9hZeIwnbxzItYNa56IUX6OqvL0qjz/M20R1rTL9yj5MGdHNxhKMT2osDNzWTaSqC4FDZ7l/iaqeuIZ7GZDsrlr8yaZ9Jdzw3GL2F5fz6k+HWRCchYgwMTOFT++7mGGpcfxu7kbuen01ldW1jT/YGB/jKWMGtwEfn+lOEZkqIlkiklVQUNCGZXmXb7YXMvGFpQjC23eOZFRP7zp11Cmd2oXzz58M5dff78snGw9w1+urqKiucbosY9qU42EgImOoC4MHz7SPqs5Q1UxVzUxMTGy74rzIe6vzuPWVFXSJDef9u0fRp2NM4w8yJ4kIt1/Ugz9c24/PN+cz7bVVlFdZIBj/4WgYiMgA4CXgWlUtcrIWb6WqPLsgm/veWsvQ7nG8fedIOrULd7osrzVlZHf+dH1/FmwtYKoFgvEjjoWBiHQF3gOmqOo2p+rwZtU1tfz6gw08MX8r1w7qzKs/HWZLTLaCHw3vyuM/GMCi7QXc/moWxystEIzvc9tFZyIyGxgNJIhIHvA7IBhAVZ8HfgvEA8+5TnmsPttItznVvuLjTH9vPQu3FXDn6J7cf3lvOwumFU0cmlI3p9E7a/npP1fy8q2ZRITYNZrGd9ncRF6mplaZtTSXJ+dvpUaV31ydwU3Duzldls/64Nu93PfWGjK7xzHz1qG2EpvxWjY3kQ/ZtK+Eh95bx9q8I1zcK5E/XnceKXF2oZQ7XTe4C4EBwi/eXMOtM1fwyk+GEm1dccYHWRh4geOVNfzti+28uGgHseHB/G3SIMYP7GxXFLeRawZ2JjBA+Pnsb/nxzBU2NmN8koWBh1u0vYBfvb+B3YfKmJiZzMNX9SU2wjMWovEnV/XvRIAI97yxmikvLWfWbcP9ep4n43scv87ANKzoaAX3vbmGKS+vIDBAeOOO4Tw+YaAFgYPGndeRf9w8hE37S7j5peV22qnxKRYGHkZVeXdVHt/769fMW7ePn49N4+N7L7KriT3EZRkdeHryYNbvPcKrS3KdLseYVmNh4GHe/3Yv//32WnomRvHRzy/ivst7ExYc6HRZpp5x53VidO9EnlmQzeFjlU6XY0yrsDDwIKrKy9/spFeHKN762UjSO0Q7XZI5g4eu7MuximqeWZDtdCnGtAoLAw+yencxG/eV8GNbktHj9e4YzYQhycxamsvuojKnyzHmnFkYeJBZS3OJDg3i+sE27bQ3uO+y3gQGCE98utXpUow5ZxYGHqKgtIKP1u/nB0OSibSrXL1Cx3Zh3H5hD+at3cfaPcVOl2PMObEw8BBzVuymqkaZMtKmlvAmP7ukB/GRIfzpo822Xq/xahYGHqC6ppbXl+/movQEeiZGOV2OaYbosGDu/V46y3ce4ovN+U6XY0yLWRh4gM82HeRASTk/Htnd6VJMC0we1pXUhEge+2QL1TW2ZKbxThYGHuDVpbl0iQ1nbJ8kp0sxLRAcGMCD43qTnX+Ut7LynC7HmBaxMHDY1gOlLNtxiCkjuxFop5N6rSv6dWRIt/b87+fbOFZR7XQ5xjSbhYHDXluWS0hQABMzU5wuxZwDEeHhq/pQUFrBi4t2OF2OMc1mYeCgkvIq3lu9l/EDOxMXaRPQebsh3eK48ryOzFi4g/zScqfLMaZZLAwc9O6qPMoqa/ixnU7qMx4Y14fK6lqe+ny706UY0ywWBg6prVVeW7qLQSmxDEiOdboc00pSEyK5aXhX3ly5h+z8UqfLMabJLAwcsjinkB2Fx7hllLUKfM3PL00nPDiQxz62aSqM97AwcMirS3YRHxnCVf07OV2KaWXxUaHcObonn28+yPIdRU6XY0yTWBg4YM+hMr7ccpBJw1IIDbK1CnzRTy9IpWNMGH/6eItNU2G8gtvCQERmiki+iGw4w/0iIn8XkWwRWSci57urFk/z+vLdANw03LqIfFV4SCD3Xd6LtXuK+XDdfqfLMaZR7mwZ/BMYd5b7rwTSXV9TgX+4sRaPUV5Vw5srd3NZRgc6x4Y7XY5xox+cn0yfjtE8+elWKqttmgrj2dwWBqq6EDh0ll2uBWZpnWVArIj4fAf6h+v2c7isiltsHiKfFxggPDiuD7uKypi9YrfT5RhzVk6OGXQB9tS7nefa9h0iMlVEskQkq6CgoE2KcwdV5dUluaQlRTGyZ7zT5Zg2MLp3IsNT4/j7F9s5atNUGA/mFQPIqjpDVTNVNTMxMdHpclpszZ5i1u89wo9HdkPE5iHyByLCQ1f1pehYJTMW2jQVxnM5GQZ7gfoT8iS7tvms15buIio0iBvOT3a6FNOGBqXEclX/jry0yKapMJ7LyTCYC/zYdVbRCOCIqvrsaReFRyv4cN1+bji/C1G2rKXfuf+KPlRU1/L0F9lOl2JMg9x5aulsYCnQW0TyROQ2EZkmItNcu3wE7ACygReBu9xViyd4c+UeKmtqbR4iP5WaEMnkYSnMXrGbnYXHnC7HmO9w25+oqjq5kfsVuNtdr+9p3lmVx8ge8aQlRTtdinHIzy9N573Ve3ly/laevclvLqsxXsIrBpC9Xd7hMnYWHuPyfh2cLsU4KCk6jNsv6sG/1+9nzZ5ip8sx5hQWBm1gSU7d/DSjeiY4XIlx2tSLexAfGcKfP9ps01QYj2Jh0AaWZBeSEBVCrw5RTpdiHBYVGsTPL01n+c5DfLXVe6+ZMb7HwsDNVJUlOUWM7Jlg1xYYACYP60q3+Aj+8skWamqtdWA8g4WBm+UUHCW/tIIL7Ipj4xISFMAvL+/NlgOlfPCtT19aY7yIhYGb2XiBacj3+3diQHI7/vrZNsqrapwuxxgLA3dbnF1IcvtwusZHOF2K8SABAcL0cX3YW3yc15bucrocYywM3KmmVlm24xCjrIvINGBUWgKX9ErkmQXZHCmrcroc4+csDNxo074Sjhyv4oI06yIyDXtwXB9Kyqv4x9c5Tpdi/JyFgRstySkEYGQPaxmYhmV0juH6QV14ZfFO9hUfd7oc48csDNxocU4R6UlRJMWEOV2K8WD/77JeqMJTn29zuhTjxywM3KSyupaVO228wDQuJS6CH4/sxjur8tiw94jT5Rg/ZWHgJmvzijleVcMoGy8wTfBfl6YTFxnKw++vtwvRjCMsDNxkcXYhIjAi1VoGpnHtwoP57TUZrMs7wmtLc50ux/ghCwM3WZJTxHmd29EuItjpUoyXuGZAJy5KT+DJT7dx4IitiGbaloWBG5RVVvPt7sOMSrNWgWk6EeH/u+48qmpq+f2HG50ux/gZCwM3yMo9TFWN2hQUptm6xUfy80vT+Wj9Ab7cctDpcowfsTBwg8U5hQQHCkO7t3e6FOOF7rioB+lJUfzmg42UVVY7XY7xExYGbrA0p4jBKe2JCLGF703zhQQF8Kcb+rO3+Dh/+3y70+UYP2Fh0MqOlFWxfu8RGy8w52Ro9zgmDU3hpW92snl/idPlGD9gYdDKlu0sQtWmrDbnbvqVfYgND+bh99dTa9ceGDezMGhlS7ILCQ8OZFBKrNOlGC8XGxHCr77fl293F/PGit1Ol2N8nFvDQETGichWEckWkekN3N9VRBaIyLcisk5ErnJnPW1hSU4RQ1PjCAmynDXn7vrBXRjVM56/fLKF/FK79sC4j9s+sUQkEHgWuBLIACaLSMZpu/0aeEtVBwOTgOfcVU9byC8pZ3v+UVvi0rSaE9ceVFTV8ocPNztdjvFh7vzzdRiQrao7VLUSmANce9o+CsS4fm4H7HNjPW63dIctcWlaX4/EKO4ek8a8tfv4eluB0+UYH+XOMOgC7Kl3O8+1rb5HgJtFJA/4CPivhp5IRKaKSJaIZBUUeO4vw+LsQtqFB5PROabxnY1phmmje9AjMZLffLDB1kw2buF0x/Zk4J+qmgxcBbwmIt+pSVVnqGqmqmYmJia2eZFNtSSniBE94ggMEKdLMT4mNCiQP17Xn92Hynj6S7v2wLQ+d4bBXiCl3u1k17b6bgPeAlDVpUAY4JV9LLuLysg7fNyWuDRuM7JnPD84P5kXvt7BtoOlTpdjfIw7w2AlkC4iqSISQt0A8dzT9tkNXAogIn2pCwPH+4HW7ikmO795v2wnlri0xWyMO/3q+32JCgvi0XkbUbVrD0zrcVsYqGo1cA8wH9hM3VlDG0Xk9yIy3rXbfwN3iMhaYDZwqzr8P7yyupYfz1zBtc8sZtWuw01+3JKcIpKiQ+mZGOXG6oy/i4sM4d5L01mcXWSDyaZVuXXMQFU/UtVeqtpTVf/o2vZbVZ3r+nmTql6gqgNVdZCqfurOeppi0fYCjhyvIiBAuHXmCtbsKW70MarKkpwiRvWMR8TGC4x73TS8G93iI3js4y22KpppNU4PIHucuWv3ERsRzL//6yJiI4OZ8vJy1uedfV3a7flHKTxaYUtcmjYREhTAA1f0YcuBUt5bned0OcZHWBjUc7yyhs82HeTK8zrSNT6C2XeMICYsmJtfXn7WhcoXZ9t4gWlbV/XvyMCUWP7n020cr7RTTc25szCo58st+ZRV1nDNgM4AJLePYM7UEUSGBDLl5eVnnD1ySU4R3eIjSG4f0ZblGj8mIvzqqr4cKCln5uKdTpdjfICFQT3z1u4jMTqU4T3+8xd+SlwEs6eOIDQokJteWs7WA6eeZVRdU8uyHUXWKjBtblhqHJdldOAfX+VQdLTC6XKMl7MwcCktr+LLrfl8v3+n71w01i0+kjfuGE5QgHDTS8tOOe10474SSsurGWlTUBgHPDiuD8eranj6y2ynSzFezsLA5dONB6msruWagZ0bvL9HYhRv3DECECa/uJycgqNA3RKXACN7WMvAtL20pCgmDU3h/5btIrfwmNPlGC9mYeAyb90+usSGc37XM69DkJYUxew7hlNbq/zoxWXkFh5jaU4RvTtEkxgd2obVGvMf934vnZCgAB6fv8XpUowXa1IYiEjkiTmDRKSXiIwXkWD3ltZ2Dh2r5JvthVwzsHOj1wmkd4jmjTtGUFWjTH5xGStzD9kSl8ZRSdFhTL24Bx+tP8Dq3U2/UNKY+praMlgIhIlIF+BTYArwT3cV1dY+2XCA6lrlmoGdmrR/747R/N9twzleVUN5Va1NWW0cd8dFPUiMDuVP/95s01SYFmlqGIiqlgE3AM+p6o1AP/eV1bbmrd1Hj8RIMjo1ferpjM4xvH77cG4d1Z2L0i0MjLMiQ4P4f9/rRdauw3y66aDT5Rgv1OQwEJGRwE3Av13bAt1TUts6WFLOsp1FXDOg8S6i0/Xr3I5HxvcjLNgn/imMl5uYmUxaUhR/+XgLVTW1TpdjvExTw+AXwEPA+67J5noAC9xXVtv597r9qHLGs4iM8RZBgQFMH9eHHYXHmLNyT+MPMKaeJoWBqn6tquNV9S+ugeRCVf25m2trE/PW7SOjUwxpSTbbqPF+l/ZNYlhqHH/7fBtHK6qdLsd4kaaeTfSGiMSISCSwAdgkIve7tzT323OojG93F1urwPgMEeHhq/pSeLSSGQt3OF2O8SJN7SbKUNUS4DrgYyCVujOKvNqH6/YDcPWApp1FZIw3GJQSy9UDOvHiwh0cLCl3uhzjJZoaBsGu6wquA+aqahXg9eevzV27j8FdY0mJswnmjG954Io+VNfW8tTn25wuxXiJpobBC0AuEAksFJFuQMNTeHqJ7PxSNu8vOTlDqTG+pGt8BFNGdOfNlXvYadNUmCZo6gDy31W1i6pepXV2AWPcXJtbzVu7HxHrIjK+687RPQkODOCFr3OcLsV4gaYOILcTkb+KSJbr63+oayV4JVVl3rp9jEiNJykmzOlyjHGLxOhQJmam8O7qPA4csbEDc3ZN7SaaCZQCE11fJcAr7irK3TbtL2FHwTE7i8j4vKkX96BW4cVFdmaRObumhkFPVf2dqu5wfT0K9HBnYe40d+0+ggKEced1dLoUY9wqJS6Cawd2ZvaK3Rw+Vul0OcaDNTUMjovIhSduiMgFwPHGHiQi40Rkq4hki8j0M+wzUUQ2ichGEXmjifW0mKry4dr9XJieQFxkiLtfzhjHTRvdk7LKGv65JNfpUowHa2oYTAOeFZFcEckFngF+drYHiEgg8CxwJZABTBaRjNP2SadumosLVLUfddNeuNXq3cXsLT7OeOsiMn6iV4doLs/owD+X5NpVyeaMmno20VpVHQgMAAao6mBgbCMPGwZku7qVKoE5wLWn7XMH8KyqHna9Tn6zqm+BeWv3ERIUwGUZHdz9UsZ4jLvGpHHkeBWzl+92uhTjoZq10pmqlriuRAa4r5HduwD1Z8vKc22rrxfQS0QWi8gyERnXnHqaq6ZW+ff6/YztnUR0mM+szWNMowalxDKqZzwvLtpBRXWN0+UYD3Quy142b77nhgUB6cBoYDLwooh8Z91JEZl64rTWgoKCFr/Y8h1FFJRW2FlExi/dPSaN/NIK3l211+lSjAc6lzBobDqKvUBKvdvJrm315eGa3kJVdwLbqAuHU19IdYaqZqpqZmJiYosLnrduH5EhgYztk9Ti5zDGW43qGc/A5Ha8sDCHalvvwJzmrGEgIqUiUtLAVynQ2J/XK4F0EUkVkRBgEjD3tH0+oK5VgIgkUNdt5JYToiura/l4wwEuy+hAeIgtRmP8j4hw5+g0dhWV8dGGA06XYzzMWcNAVaNVNaaBr2hVDWrksdXAPcB8YDPwlmthnN+LyHjXbvOBIhHZRN1iOferatG5H9Z3Lc4upLisyrqIjF+7PKMDaUlRPLcg29ZKNqc4l26iRqnqR6raS1V7quofXdt+q6pzXT+rqt6nqhmq2l9V57irlpS4cG67MJWL0lvezWSMtwsIEO68pCdbDpSyYKvbT94zXsStYeBJ0pKi+c3VGYQE+c0hG9Og8YM60yU2nGcX5FjrwJxkn4zG+JngwAB+dkkPVu06zIqdh5wux3gICwNj/NDEzBQSokJ47iub3trUsTAwxg+FBQfykwtS+XpbARv2HnG6HOMBLAyM8VNTRnYjOjSIf1jrwGBhYIzfigkLZsrIbny0YT87Co46XY5xmIWBMX7spxemEhIYwPO2NKbfszAwxo8lRIUyaWgK73+7l33FjS5RYnyYhYExfu6Oi3ugCjMW2tKY/szCwBg/l9w+ghvO78IbK3ZzsKTc6XKMQywMjDHcMyad2lq1M4v8mIWBMYau8RH84Pxkax34MQsDYwxQt/iNtQ78l4WBMQY4tXVw4Ii1DvyNhYEx5qR7xta1Duy6A/9jYWCMOSklLoIJQ6x14I8sDIwxp/jP2EG206WYNmRhYIw5xYnWwewVe6x14EcsDIwx33H3mDRq1VoH/sTCwBjzHSlxEdyYWdc62H/E5izyBxYGxpgG3TX6ROvAzizyBxYGxpgGnWgdzLHWgV9waxiIyDgR2Soi2SIy/Sz7/UBEVEQy3VmPMaZ5TowdPLfAWge+zm1hICKBwLPAlUAGMFlEMhrYLxq4F1jurlqMMS2T3D6CGzNTeHPlHlvvwMe5s2UwDMhW1R2qWgnMAa5tYL8/AH8B7Bw2YzzQ3WN6otjYga9zZxh0AfbUu53n2naSiJwPpKjqv8/2RCIyVUSyRCSroKCg9Ss1xpyRtQ78g2MDyCISAPwV+O/G9lXVGaqaqaqZiYmJ7i/OGHOKu0bXtQ6es+sOfJY7w2AvkFLvdrJr2wnRwHnAVyKSC4wA5togsjGex1oHvs+dYbASSBeRVBEJASYBc0/cqapHVDVBVburandgGTBeVbPcWJMxpoXuHpMGYK0DH+W2MFDVauAeYD6wGXhLVTeKyO9FZLy7XtcY4x5dYsOZ6GodZOeXOl2OaWWiqk7X0CyZmZmalWWNB2OcUFBaweX/+zXd4iN5985RBAaI0yWZJhKRVap6xm54uwLZGNNkidGhPHrteazZU8zL3+xwuhzTiiwMjDHNcs2ATlzRrwNPfrqN7PyjTpdjWomFgTGmWUSEP1x3HhEhgTzwzlpqar2rq9k0zMLAGNNsSdFhPDq+H6t3FzPzm51Ol2NagYWBMaZFxg/szGUZHXjy063kFFh3kbezMDDGtIiI8PUYOJQAAA2xSURBVMfrziMsOJAH3lln3UVezsLAGNNiSTFhPDI+g1W7DvPKYusu8mYWBsaYc3LdoC58r28ST8zfys7CY06XY1rIwsAYc05EhD9e35/QoADuf9vOLvJWFgbGmHPWISaM313Tj6xdh3l1Sa7T5ZgWsDAwxrSKG87vwtg+STw+fwu51l3kdSwMjDGtQkT40/X9CQ4M4IF31lFr3UVexcLAGNNqOrYL47dXZ7Ai9xCzluY6XY5pBgsDY0yrmjAkmdG9E/nLJ1vZVWTdRd7CwsAY06pEhD/f0J+gQGHqrFUcKatyuiTTBBYGxphW16ldOM/fPISdhce47dWVHK+scbok0wgLA2OMW1yQlsBTkwaxavdh7np9FVU1tU6XZM7CwsAY4zZX9e/EH6/rz4KtBXaGkYcLcroAY4xv+9Hwrhw6VsGTn26jfUQIv7m6LyK2XKansTAwxrjd3WPSKDpWyczFO4mPCuHuMWlOl2ROY2FgjHE7EeE338+guKyKJ+ZvpX1ECD8a3tXpskw9FgbGmDYRECA8PmEAxWWV/PqD9bSPCObK/p2cLsu4uHUAWUTGichWEckWkekN3H+fiGwSkXUi8oWIdHNnPcYYZwUHBvDcTUMY3LU9985Zw5LsQqdLMi5uCwMRCQSeBa4EMoDJIpJx2m7fApmqOgB4B3jcXfUYYzxDeEggM28ZSmpCJHfMymJdXrHTJRnc2zIYBmSr6g5VrQTmANfW30FVF6hqmevmMiDZjfUYYzxEu4hgZt02jPaRIdz6ykpbQ9kDuDMMugB76t3Oc207k9uAjxu6Q0SmikiWiGQVFBS0YonGGKd0iAnjtduGEyAw5aXltkqawzziojMRuRnIBJ5o6H5VnaGqmaqamZiY2LbFGWPcJjUhkld/Oozy6lpufH4JG/Yecbokv+XOMNgLpNS7nezadgoR+R7wK2C8qla4sR5jjAfq17kd70wbSWhQIJNmLGNJjg0qO8GdYbASSBeRVBEJASYBc+vvICKDgReoC4J8N9ZijPFgPRKjePfOUXSODePWmSv5ZMN+p0vyO24LA1WtBu4B5gObgbdUdaOI/F5Exrt2ewKIAt4WkTUiMvcMT2eM8XEd24Xx1s9G0j+5HXe9vpo3lu92uiS/IqreNXFUZmamZmVlOV2GMcZNjlfWcPcbq/lySz7/fVkv7hmbZnMZtQIRWaWqmWe63yMGkI0x5oTwkEBemDKEGwZ34X8+28aj8zbZbKdtwKajMMZ4nODAAJ68cSBxkSG89M1ODh2r5MkbBxISZH+/uouFgTHGIwUECL/6fl8SokN57OMtHC6r5PmbhxAZah9b7mAxa4zxWCLCtEt68vgPBrA4u5AfvbScoqN2Bro7WBgYYzzexKEpPH/zEDbvL+HSv37NrKW5VNsymq3KwsAY4xUu79eRefdcSEanGH77r41c9fdFfLPdLlBrLRYGxhiv0btjNK/fPpwXpgyhvKqWm19ezu2vZpFr8xqdMwsDY4xXERGu6NeRz+67mAfH9WFpTiGX/e/X/PmjzZSWVzldnteyMDDGeKXQoEDuHN2TBb8czXWDuvDCwh2MefIr3ly5mxq7LqHZLAyMMV4tKSaMJ24cyNx7LqBbfCQPvruea5/9huU7ipwuzatYGBhjfMKA5FjemTaSv00aRNHRSn44Yxk3Pr+E+RsPWEuhCWxuImOMzymrrGbOij3MXLyTvMPHSU2I5KcXpjLh/GTCQwKdLs8Rjc1NZGFgjPFZ1TW1fLLxAC8u3MHavCO0jwhmyohuTBnZncToUKfLa1MWBsYYv6eqrMw9zIuLdvD55oMEBwZww+Au3H5RKmlJ0U6X1yYaCwOb5MMY4/NEhGGpcQxLjWNHwVFe/mYn76zKY87KPYztk8SEIclckJZAu/Bgp0t1jLUMjDF+qehoBf+3bDezluZSdKySwABhSNf2XNI7kUt6JdKvc4xPraNg3UTGGHMWVTW1fLu7mK+35fPV1gI27isBIDE6lEt61QXDRekJxEaEOFzpubEwMMaYZsgvKWfh9kK+2prPou2FHDleRYDAoJRYLkxPpG/HaHp1jKZbXARBgd5zdr6FgTHGtFB1TS1r84r5emsBX20rYP3eI5z4yAwJCqBnYhS9OkTRq0O06yuKlPYRBAR4XveShYExxrSSsspqsvOPsu3gUbYfLGXrwVK2HzzK3uLjJ/cJDw4kLSmKrnERdIgJo1O7MDq0q/veMSaMpJhQQoPa/loHO5vIGGNaSURIEAOSYxmQHHvK9tLyKrbnH2XbgdK6oMgvZfP+Er7cks/xqprvPE98ZMjJoEiKCaN9RDDtI0Jo5/rePiKYWNf3duHBbdIdZWFgjDHnKDosmPO7tuf8ru1P2a6qlJRXc7CknANHXF8lrq8j5ew/Us7avCMUl1VSfZYpM2LCgmgfGcKUEd24/aIebjkGt4aBiIwD/gYEAi+p6mOn3R8KzAKGAEXAD1U11501GWNMWxER2oXX/XXfq8OZL25TVY5WVFNcVsXhskoOl1VRXFbJ4WP1fi6rcutV024LAxEJBJ4FLgPygJUiMldVN9Xb7TbgsKqmicgk4C/AD91VkzHGeCIRITosmOiwYFLiIhypwZ0dUcOAbFXdoaqVwBzg2tP2uRZ41fXzO8Cl4ktXeRhjjJdwZxh0AfbUu53n2tbgPqpaDRwB4k9/IhGZKiJZIpJVUFDgpnKNMcZ/ecUVE6o6Q1UzVTUzMTHR6XKMMcbnuDMM9gIp9W4nu7Y1uI+IBAHtqBtINsYY04bcGQYrgXQRSRWREGASMPe0feYCt7h+ngB8qd52FZwxxvgAt51NpKrVInIPMJ+6U0tnqupGEfk9kKWqc4GXgddEJBs4RF1gGGOMaWNuvc5AVT8CPjpt22/r/VwO3OjOGowxxjTOKwaQjTHGuJfXTVQnIgXArhY+PAEobMVyPIGvHZOvHQ/43jH52vGA7x1TQ8fTTVXPeDqm14XBuRCRrLPN2ueNfO2YfO14wPeOydeOB3zvmFpyPNZNZIwxxsLAGGOM/4XBDKcLcANfOyZfOx7wvWPyteMB3zumZh+PX40ZGGOMaZi/tQyMMcY0wMLAGGOM/4SBiIwTka0iki0i052upzWISK6IrBeRNSKS5XQ9zSUiM0UkX0Q21NsWJyKfich21/f2Z3sOT3OGY3pERPa63qc1InKVkzU2h4ikiMgCEdkkIhtF5F7Xdq98n85yPN78HoWJyAoRWes6pkdd21NFZLnrM+9N1xxxZ34efxgzcK26to16q64Bk09bdc3riEgukKmqXnmxjIhcDBwFZqnqea5tjwOHVPUxV2i3V9UHnayzOc5wTI8AR1X1SSdrawkR6QR0UtXVIhINrAKuA27FC9+nsxzPRLz3PRIgUlWPikgw8A1wL3Af8J6qzhGR54G1qvqPMz2Pv7QMmrLqmmljqrqQugkK66u/+t2r1P2ieo0zHJPXUtX9qrra9XMpsJm6Ram88n06y/F4La1z1HUz2PWlwFjqVpCEJrxH/hIGTVl1zRsp8KmIrBKRqU4X00o6qOp+188HgA5OFtOK7hGRda5uJK/oUjmdiHQHBgPL8YH36bTjAS9+j0QkUETWAPnAZ0AOUOxaQRKa8JnnL2Hgqy5U1fOBK4G7XV0UPsO1toUv9GP+A+gJDAL2A//jbDnNJyJRwLvAL1S1pP593vg+NXA8Xv0eqWqNqg6ibhGxYUCf5j6Hv4RBU1Zd8zqqutf1PR94n7r/BN7uoKtf90T/br7D9ZwzVT3o+mWtBV7Ey94nVz/0u8Drqvqea7PXvk8NHY+3v0cnqGoxsAAYCcS6VpCEJnzm+UsYNGXVNa8iIpGuATBEJBK4HNhw9kd5hfqr390C/MvBWlrFiQ9Nl+vxovfJNTj5MrBZVf9a7y6vfJ/OdDxe/h4likis6+dw6k6U2UxdKExw7dboe+QXZxMBuE4Ve4r/rLr2R4dLOici0oO61gDULVL0hrcdk4jMBkZTN93uQeB3wAfAW0BX6qYqn6iqXjMge4ZjGk1d94MCucDP6vW3ezQRuRBYBKwHal2bH6aun93r3qezHM9kvPc9GkDdAHEgdX/gv6Wqv3d9RswB4oBvgZtVteKMz+MvYWCMMebM/KWbyBhjzFlYGBhjjLEwMMYYY2FgjDEGCwNjjDFYGBjTJCLyK9eMkOtcs1oOF5FfiEiE07UZ0xrs1FJjGiEiI4G/AqNVtUJEEoAQYAlePGusMfVZy8CYxnUCCk9csOP68J8AdAYWiMgCABG5XESWishqEXnbNf/NiXUnHnetPbFCRNKcOhBjzsTCwJjGfQqkiMg2EXlORC5R1b8D+4AxqjrG1Vr4NfA91+SBWdTNJ3/CEVXtDzxD3ZXwxniUoMZ3Mca/uRYNGQJcBIwB3pTvrpY3AsgAFtdNf0MIsLTe/bPrff9f91ZsTPNZGBjTBKpaA3wFfCUi6/nPJG0nCPCZqk4+01Oc4WdjPIJ1ExnTCBHpLSLp9TYNom5ytlIg2rVtGXDBifEA16yyveo95of1vtdvMRjjEaxlYEzjooCnXdMEVwPZwFTqZrr8RET2ucYNbgVmi0io63G/pm7tbYD2IrIOqHA9zhiPYqeWGuNmIpKLnYJqPJx1ExljjLGWgTHGGGsZGGOMwcLAGGMMFgbGGGOwMDDGGIOFgTHGGOD/BzRLQ1wJr5vtAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "\n",
            "training epochs:  10%|█         | 1/10 [00:44<06:42, 44.71s/it]\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A\u001b[A"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fb3a14ac179d43198bc911ef4c968366",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='update step', max=30.0, style=ProgressStyle(description_w…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-49-6e3ad432a6b8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     69\u001b[0m         \u001b[0mnext_token_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss_value\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mupdate_step_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss_value\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m         \u001b[0moptim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     24\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 26\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     27\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    117\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    118\u001b[0m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'weight_decay'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 119\u001b[0;31m                    \u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'eps'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    120\u001b[0m                    )\n\u001b[1;32m    121\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/optim/functional.py\u001b[0m in \u001b[0;36madam\u001b[0;34m(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, amsgrad, beta1, beta2, lr, weight_decay, eps)\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m         \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m         \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    }
  ]
}