垃圾邮件分类.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyP4+PNUF/QFaJyFMd7JdRyC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7278d9aa63804e548d7894c63a8dd0c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_374b6b2919244908a23034b108f6d4a6",
              "IPY_MODEL_e4b328a2be534c43bc6e76ad43e2b6d1",
              "IPY_MODEL_0b050688682542a9bfa0a1f9bfe083e5"
            ],
            "layout": "IPY_MODEL_16e6ec9bc6a14c51b81cdd2f719adfdc"
          }
        },
        "374b6b2919244908a23034b108f6d4a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c9c00b58fb9442b88b5f18a1b69f5f7",
            "placeholder": "​",
            "style": "IPY_MODEL_ec69d8fb085744c5bd500055f1d4295c",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "e4b328a2be534c43bc6e76ad43e2b6d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_099d4cf3b84442eaa2f07966f1f0537a",
            "max": 28,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9422ed1dbfc54024883f19b7bf26cdf6",
            "value": 28
          }
        },
        "0b050688682542a9bfa0a1f9bfe083e5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8995739765d4360bc0beb254a0fd0aa",
            "placeholder": "​",
            "style": "IPY_MODEL_ffe58d14e0a64e3ebf30005a8edf43e2",
            "value": " 28.0/28.0 [00:00&lt;00:00, 1.71kB/s]"
          }
        },
        "16e6ec9bc6a14c51b81cdd2f719adfdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c9c00b58fb9442b88b5f18a1b69f5f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec69d8fb085744c5bd500055f1d4295c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "099d4cf3b84442eaa2f07966f1f0537a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9422ed1dbfc54024883f19b7bf26cdf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b8995739765d4360bc0beb254a0fd0aa": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ffe58d14e0a64e3ebf30005a8edf43e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "be964dddb6cf4063bddad92e6c663ba3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_01a057a42b2c4ad4af863f37e6e4968e",
              "IPY_MODEL_b91be76a74344b7687e6436c20026dbe",
              "IPY_MODEL_16c4267493ca437aae5a78ce21772b19"
            ],
            "layout": "IPY_MODEL_31d7ae70ce4c49b8b774e4b2bca90ed1"
          }
        },
        "01a057a42b2c4ad4af863f37e6e4968e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6d1379ac77ff4b639980cfb3ce67c3a0",
            "placeholder": "​",
            "style": "IPY_MODEL_d494391f42b64a72a4d01a56fa073096",
            "value": "vocab.txt: 100%"
          }
        },
        "b91be76a74344b7687e6436c20026dbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f6e7f165d5048daa29e0936c93b6b13",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_76921243f530491c9e2fb701afc98067",
            "value": 231508
          }
        },
        "16c4267493ca437aae5a78ce21772b19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f4464e1617db4f789cb16b3ffacc8cf1",
            "placeholder": "​",
            "style": "IPY_MODEL_b859b67c48334d81831f5e87d10d87eb",
            "value": " 232k/232k [00:00&lt;00:00, 1.72MB/s]"
          }
        },
        "31d7ae70ce4c49b8b774e4b2bca90ed1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d1379ac77ff4b639980cfb3ce67c3a0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d494391f42b64a72a4d01a56fa073096": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6f6e7f165d5048daa29e0936c93b6b13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76921243f530491c9e2fb701afc98067": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f4464e1617db4f789cb16b3ffacc8cf1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b859b67c48334d81831f5e87d10d87eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "16dd491baf3f4ec29e50fcbb14c3bf0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6d935fca0c204f1898744d3e17a90117",
              "IPY_MODEL_b9fc1015b7194af09aabd6d55e688b5d",
              "IPY_MODEL_bf702ef8fddd42d4bd16f557f6b2d900"
            ],
            "layout": "IPY_MODEL_30a83db3596748a1a7b730997e329483"
          }
        },
        "6d935fca0c204f1898744d3e17a90117": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e259b4f2974742b981cf1721fa8ecad6",
            "placeholder": "​",
            "style": "IPY_MODEL_47cbbb4e0d8745a99c2e249573d7120c",
            "value": "tokenizer.json: 100%"
          }
        },
        "b9fc1015b7194af09aabd6d55e688b5d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2ef8c2586a9b45489ad6c09a1b7f6dc1",
            "max": 466062,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1138e3682c3b4aa78174f50ffa72f213",
            "value": 466062
          }
        },
        "bf702ef8fddd42d4bd16f557f6b2d900": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b69d4112a847417b88c94b2eb20c0a6d",
            "placeholder": "​",
            "style": "IPY_MODEL_10ffc5538aa04eea9399af8b5287e967",
            "value": " 466k/466k [00:00&lt;00:00, 3.59MB/s]"
          }
        },
        "30a83db3596748a1a7b730997e329483": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e259b4f2974742b981cf1721fa8ecad6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47cbbb4e0d8745a99c2e249573d7120c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2ef8c2586a9b45489ad6c09a1b7f6dc1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1138e3682c3b4aa78174f50ffa72f213": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b69d4112a847417b88c94b2eb20c0a6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10ffc5538aa04eea9399af8b5287e967": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9280fd1b75449b5872433dcbf31d287": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d8babce170ad450b95eb63103e240321",
              "IPY_MODEL_0845e1569e2c4b69b13f06412444cdf1",
              "IPY_MODEL_ece0edb2cdb44a5cb25c26e4ee061c4c"
            ],
            "layout": "IPY_MODEL_64bfb3b588c44fe6868f50645e7dae39"
          }
        },
        "d8babce170ad450b95eb63103e240321": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_54491bac97b14c5f9abf356bf25e1f54",
            "placeholder": "​",
            "style": "IPY_MODEL_3a3ad210d9b44bfb91fb85d1e70a549e",
            "value": "config.json: 100%"
          }
        },
        "0845e1569e2c4b69b13f06412444cdf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_60fddad207bf4b4ea1d12f9d400ef876",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_74d4b2b4dd5e4eefad12b16f5db5da65",
            "value": 570
          }
        },
        "ece0edb2cdb44a5cb25c26e4ee061c4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7979854df1f41f4ad3cec9586ac95e8",
            "placeholder": "​",
            "style": "IPY_MODEL_36e2f45a93f44bc78ec8efa578366317",
            "value": " 570/570 [00:00&lt;00:00, 46.1kB/s]"
          }
        },
        "64bfb3b588c44fe6868f50645e7dae39": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54491bac97b14c5f9abf356bf25e1f54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a3ad210d9b44bfb91fb85d1e70a549e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "60fddad207bf4b4ea1d12f9d400ef876": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "74d4b2b4dd5e4eefad12b16f5db5da65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f7979854df1f41f4ad3cec9586ac95e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "36e2f45a93f44bc78ec8efa578366317": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd53f6fc7db24e828f69a2b04507103b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ad6de96f498b4c46ba4f9526f5487151",
              "IPY_MODEL_7cf14cee1b2b46119695a4f49d8969da",
              "IPY_MODEL_c276bf2b5fd342b3bf0fa1e71e53abf9"
            ],
            "layout": "IPY_MODEL_741bb00c00654d659527987618fcf6e8"
          }
        },
        "ad6de96f498b4c46ba4f9526f5487151": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a1e3409367e9432e90018f1305c703e9",
            "placeholder": "​",
            "style": "IPY_MODEL_7292aad4f7204a298ba9dcac7eb5e9ad",
            "value": "model.safetensors: 100%"
          }
        },
        "7cf14cee1b2b46119695a4f49d8969da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7aaf3ebd5ef49d5985dd2c35c08b103",
            "max": 440449768,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_25e085779daa41f58d7cf7d4feb0ac4c",
            "value": 440449768
          }
        },
        "c276bf2b5fd342b3bf0fa1e71e53abf9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8ae65c2c0f0048eab469987f1fb76e52",
            "placeholder": "​",
            "style": "IPY_MODEL_11bc9b99cd1140e48f2ef5c9cf34a23b",
            "value": " 440M/440M [00:01&lt;00:00, 226MB/s]"
          }
        },
        "741bb00c00654d659527987618fcf6e8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a1e3409367e9432e90018f1305c703e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7292aad4f7204a298ba9dcac7eb5e9ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f7aaf3ebd5ef49d5985dd2c35c08b103": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25e085779daa41f58d7cf7d4feb0ac4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8ae65c2c0f0048eab469987f1fb76e52": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "11bc9b99cd1140e48f2ef5c9cf34a23b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Aikoin/Fine_Tuning_BERT_for_Text_Classification/blob/main/Fine_Tuning_BERT_for_Text_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 1. Introduction\n",
        "BERT(Bidirectional Encoder Representations from Transformers) is a Machine Learning model based on transformers, i.e. attention components able to learn contextual relations between words.  \n",
        "The Natural Language Processing (NLP) community can leverage powerful tools like BERT in (at least) two ways:  \n",
        "\n",
        "1. **Feature-based approac**h  \n",
        "  1.1 Download a pre-trained BERT model.   \n",
        "  1.2 Use BERT to turn natural language sentences into a vector representation.  \n",
        "  1.3 Feed the pre-trained vector representations into a model for a downstream task (such as text classification).\n",
        "  \n",
        "2. **Perform fine-tuning**  \n",
        "  2.1 Download a pre-trained BERT model.  \n",
        "  2.2 Update the model weights on the downstream task.  \n",
        "  \n",
        "  \n",
        "  In this post, we will follow the fine-tuning approach on binary text classification example."
      ],
      "metadata": {
        "id": "MsyAH55HEOU7"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 2. Environment setup"
      ],
      "metadata": {
        "id": "YCsbA743NodK"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TxDg7J9DD54t",
        "outputId": "67f7faf3-3752-45bc-c6f2-348bad79bff3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.16.4->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.11.17)\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from tabulate import tabulate\n",
        "from tqdm import trange\n",
        "import random"
      ],
      "metadata": {
        "id": "-0NoJoQBFPvI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 3. Dataset\n",
        "We use the public SMS Spam Collection Data Set⁵ from the UCI Machine Learning Repository⁶. The data consists of a text file with a set of SMS messages labeled as either spam or ham."
      ],
      "metadata": {
        "id": "dC72WUPVNvKw"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!wget 'https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip'"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qbHdkqKINtds",
        "outputId": "a57f7d66-a5a9-4a02-8dd6-7c99d499dcd5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "--2023-12-22 10:16:07--  https://archive.ics.uci.edu/ml/machine-learning-databases/00228/smsspamcollection.zip\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: unspecified\n",
            "Saving to: ‘smsspamcollection.zip’\n",
            "\n",
            "\rsmsspamcollection.z     [<=>                 ]       0  --.-KB/s               \rsmsspamcollection.z     [ <=>                ] 198.65K  --.-KB/s    in 0.08s   \n",
            "\n",
            "2023-12-22 10:16:07 (2.43 MB/s) - ‘smsspamcollection.zip’ saved [203415]\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!unzip -o smsspamcollection.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5AKJw2GCOFKj",
        "outputId": "5225f4d5-3ae5-4fd2-ace3-3ab52c35b2d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Archive:  smsspamcollection.zip\n",
            "  inflating: SMSSpamCollection       \n",
            "  inflating: readme                  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head -10  SMSSpamCollection"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0SUF8pVtOUfO",
        "outputId": "f64e011e-7133-4fa6-9b38-9f177e5511d6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ham\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\n",
            "ham\tOk lar... Joking wif u oni...\n",
            "spam\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\n",
            "ham\tU dun say so early hor... U c already then say...\n",
            "ham\tNah I don't think he goes to usf, he lives around here though\n",
            "spam\tFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you up for it still? Tb ok! XxX std chgs to send, £1.50 to rcv\n",
            "ham\tEven my brother is not like to speak with me. They treat me like aids patent.\n",
            "ham\tAs per your request 'Melle Melle (Oru Minnaminunginte Nurungu Vettam)' has been set as your callertune for all Callers. Press *9 to copy your friends Callertune\n",
            "spam\tWINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
            "spam\tHad your mobile 11 months or more? U R entitled to Update to the latest colour mobiles with camera for Free! Call The Mobile Update Co FREE on 08002986030\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "For each line, the label is followed by a tab space and the raw text message. We choose to process the file to get a pandas.DataFrame object, as it is a common starting point in Data Science experiments:"
      ],
      "metadata": {
        "id": "jx646IOfPEb4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "file_path = '/content/SMSSpamCollection'\n",
        "df = pd.DataFrame({'label':int(), 'text':str()}, index = [])\n",
        "with open(file_path) as f:\n",
        "  for line in f.readlines():\n",
        "    split = line.split('\\t')\n",
        "    new_row = {'label': 1 if split[0] == 'spam' else 0, 'text': split[1]}\n",
        "    df = pd.concat([df, pd.DataFrame([new_row])], ignore_index=True)\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "GdOaKs40PEqp",
        "outputId": "72e8eaa8-ab33-4881-a8a9-b1ebf49138ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   label                                               text\n",
              "0      0  Go until jurong point, crazy.. Available only ...\n",
              "1      0                    Ok lar... Joking wif u oni...\\n\n",
              "2      1  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3      0  U dun say so early hor... U c already then say...\n",
              "4      0  Nah I don't think he goes to usf, he lives aro..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-dc9afaba-984e-46b9-ac0b-d5cde8b530dd\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Ok lar... Joking wif u oni...\\n</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-dc9afaba-984e-46b9-ac0b-d5cde8b530dd')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-dc9afaba-984e-46b9-ac0b-d5cde8b530dd button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-dc9afaba-984e-46b9-ac0b-d5cde8b530dd');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-0c1f6867-9c0d-42f3-9516-02b3bf0c97d4\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0c1f6867-9c0d-42f3-9516-02b3bf0c97d4')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-0c1f6867-9c0d-42f3-9516-02b3bf0c97d4 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "text = df.text.values\n",
        "labels = df.label.values"
      ],
      "metadata": {
        "id": "KVeGDfS1V-7U"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 4. Preprocessing\n",
        "We need to preprocess the text source before feeding it to BERT. To do so, we download the BertTokenizer:"
      ],
      "metadata": {
        "id": "4LS9_H8yYXMo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased',\n",
        "                                          do_lower_case = True) # Construct a BERT tokenizer. Based on WordPiece."
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 201,
          "referenced_widgets": [
            "7278d9aa63804e548d7894c63a8dd0c8",
            "374b6b2919244908a23034b108f6d4a6",
            "e4b328a2be534c43bc6e76ad43e2b6d1",
            "0b050688682542a9bfa0a1f9bfe083e5",
            "16e6ec9bc6a14c51b81cdd2f719adfdc",
            "3c9c00b58fb9442b88b5f18a1b69f5f7",
            "ec69d8fb085744c5bd500055f1d4295c",
            "099d4cf3b84442eaa2f07966f1f0537a",
            "9422ed1dbfc54024883f19b7bf26cdf6",
            "b8995739765d4360bc0beb254a0fd0aa",
            "ffe58d14e0a64e3ebf30005a8edf43e2",
            "be964dddb6cf4063bddad92e6c663ba3",
            "01a057a42b2c4ad4af863f37e6e4968e",
            "b91be76a74344b7687e6436c20026dbe",
            "16c4267493ca437aae5a78ce21772b19",
            "31d7ae70ce4c49b8b774e4b2bca90ed1",
            "6d1379ac77ff4b639980cfb3ce67c3a0",
            "d494391f42b64a72a4d01a56fa073096",
            "6f6e7f165d5048daa29e0936c93b6b13",
            "76921243f530491c9e2fb701afc98067",
            "f4464e1617db4f789cb16b3ffacc8cf1",
            "b859b67c48334d81831f5e87d10d87eb",
            "16dd491baf3f4ec29e50fcbb14c3bf0d",
            "6d935fca0c204f1898744d3e17a90117",
            "b9fc1015b7194af09aabd6d55e688b5d",
            "bf702ef8fddd42d4bd16f557f6b2d900",
            "30a83db3596748a1a7b730997e329483",
            "e259b4f2974742b981cf1721fa8ecad6",
            "47cbbb4e0d8745a99c2e249573d7120c",
            "2ef8c2586a9b45489ad6c09a1b7f6dc1",
            "1138e3682c3b4aa78174f50ffa72f213",
            "b69d4112a847417b88c94b2eb20c0a6d",
            "10ffc5538aa04eea9399af8b5287e967",
            "f9280fd1b75449b5872433dcbf31d287",
            "d8babce170ad450b95eb63103e240321",
            "0845e1569e2c4b69b13f06412444cdf1",
            "ece0edb2cdb44a5cb25c26e4ee061c4c",
            "64bfb3b588c44fe6868f50645e7dae39",
            "54491bac97b14c5f9abf356bf25e1f54",
            "3a3ad210d9b44bfb91fb85d1e70a549e",
            "60fddad207bf4b4ea1d12f9d400ef876",
            "74d4b2b4dd5e4eefad12b16f5db5da65",
            "f7979854df1f41f4ad3cec9586ac95e8",
            "36e2f45a93f44bc78ec8efa578366317"
          ]
        },
        "id": "a3ke2W4lYVGw",
        "outputId": "0dce6034-1002-4210-cb84-290f541d0cb8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7278d9aa63804e548d7894c63a8dd0c8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "be964dddb6cf4063bddad92e6c663ba3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "16dd491baf3f4ec29e50fcbb14c3bf0d"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f9280fd1b75449b5872433dcbf31d287"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let us observe how the tokenizer can split a random sentence into word-level tokens and map them to their respective IDs in the BERT vocabulary:"
      ],
      "metadata": {
        "id": "10Dcz2t6ocuj"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def print_rand_sentence():\n",
        "  '''Displays the tokens and respective IDs of a random text sample'''\n",
        "  index = random.randint(0, len(text) -1)\n",
        "  print(text[index])\n",
        "  tokens = tokenizer.tokenize(text[index])\n",
        "  table = np.array([tokens,\n",
        "                    tokenizer.convert_tokens_to_ids(tokens)]).T\n",
        "  print(tabulate(table,\n",
        "                 headers = ['Tokens', 'Token IDs'],\n",
        "                 tablefmt = 'fancy_grid'))\n",
        "\n",
        "print_rand_sentence()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "srQjzGxzn5bx",
        "outputId": "9e428470-98db-46e8-c14a-95e0897ec841"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Buy one egg for me da..please:)\n",
            "\n",
            "╒══════════╤═════════════╕\n",
            "│ Tokens   │   Token IDs │\n",
            "╞══════════╪═════════════╡\n",
            "│ buy      │        4965 │\n",
            "├──────────┼─────────────┤\n",
            "│ one      │        2028 │\n",
            "├──────────┼─────────────┤\n",
            "│ egg      │        8288 │\n",
            "├──────────┼─────────────┤\n",
            "│ for      │        2005 │\n",
            "├──────────┼─────────────┤\n",
            "│ me       │        2033 │\n",
            "├──────────┼─────────────┤\n",
            "│ da       │        4830 │\n",
            "├──────────┼─────────────┤\n",
            "│ .        │        1012 │\n",
            "├──────────┼─────────────┤\n",
            "│ .        │        1012 │\n",
            "├──────────┼─────────────┤\n",
            "│ please   │        3531 │\n",
            "├──────────┼─────────────┤\n",
            "│ :        │        1024 │\n",
            "├──────────┼─────────────┤\n",
            "│ )        │        1007 │\n",
            "╘══════════╧═════════════╛\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "BERT requires the following preprocessing steps:\n",
        "1. Add **special tokens**:\n",
        "- [CLS]: at the beginning of each sentence (ID 101)\n",
        "- [SEP]: at the end of each sentence (ID 102)\n",
        "2. Make sentences of the **same length**:\n",
        "- This is achieved by *padding*, i.e. adding values of convenience to shorter sequences to match the desired length. Longer sequences are truncated.\n",
        "- The padding ([PAD]) tokens have ID 0.\n",
        "- The maximum sequence length allowed is of 512 tokens.\n",
        "3. Create an **attention mask**:\n",
        "- List of 0/1 indicating whether the model should consider the tokens or not when learning their contextual representation. We expect [PAD] tokens to have value 0.\n",
        "\n",
        "The process can be represented as follows:\n",
        "![image.png](https://miro.medium.com/v2/resize:fit:1400/format:webp/1*vaw98m1VVncgKxNFWI0d2Q.png)"
      ],
      "metadata": {
        "id": "wCgqWIqpEqkW"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can perform all the needed steps by using the `tokenizer.encode_plus` method. When called, it returns a `transformers.tokenization.tokenization-utils_base.BatchEncoding` object with the following fields:\n",
        "- input_ids: list of token IDs.\n",
        "- token_type_ids: list of token type IDs.\n",
        "- attention_mask: list of 0/1 indicating which tokens should be considered by the model (return_attention_mask = True)."
      ],
      "metadata": {
        "id": "phRMUaSIJy41"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "token_id = []\n",
        "attention_masks = []\n",
        "\n",
        "def preprocessing(input_text, tokenizer):\n",
        "  '''\n",
        "  Returns <class transformers.tokenization_utils_base.BatchEncoding> with the following fields:\n",
        "    - input_ids: list of token ids\n",
        "    - token_type_ids: list of token type ids\n",
        "    - attention_mask: list of indices (0,1) specifying which tokens should considered by the model (return_attention_mask = True).\n",
        "  '''\n",
        "  return tokenizer.encode_plus(\n",
        "      input_text,\n",
        "      add_special_tokens = True,\n",
        "      max_length = 32,\n",
        "      padding = 'max_length',\n",
        "      truncation=True,\n",
        "      return_attention_mask = True,\n",
        "      return_tensors = 'pt'\n",
        "  )\n",
        "\n",
        "for sample in text:\n",
        "  encoding_dict = preprocessing(sample, tokenizer)\n",
        "  token_id.append(encoding_dict['input_ids'])\n",
        "  attention_masks.append(encoding_dict['attention_mask'])\n",
        "\n",
        "token_id = torch.cat(token_id, dim = 0)\n",
        "attention_masks = torch.cat(attention_masks, dim = 0)\n",
        "labels = torch.tensor(labels)"
      ],
      "metadata": {
        "id": "m6N2pQHvGcuK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Note: BERT is a model with absolute position embeddings, so it is usually advised to pad the inputs on the right (end of the sequence) rather than the left (beginning of the sequence)."
      ],
      "metadata": {
        "id": "ETVl-4_eWxij"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 5. Data split\n",
        "We split the dataset into train (80%) and validation (20%) sets, and wrap them around a `torch.utils.data.DataLoader` object. With its intuitive syntax, `DataLoader` provides an iterable over the given dataset."
      ],
      "metadata": {
        "id": "liftWpqzWd5H"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "val_ratio = 0.2\n",
        "# Recommended batch size: 16, 32. See: https://arxiv.org/pdf/1810.04805.pdf\n",
        "batch_size = 16\n",
        "\n",
        "# Indices of the train and validation splits stratified by labels\n",
        "train_idx, val_idx = train_test_split(\n",
        "    np.arange(len(labels)),\n",
        "    test_size = val_ratio,\n",
        "    shuffle = True,\n",
        "    stratify = labels)\n",
        "\n",
        "# Train and validation sets\n",
        "train_set = TensorDataset(token_id[train_idx],\n",
        "                          attention_masks[train_idx],\n",
        "                          labels[train_idx])\n",
        "\n",
        "val_set = TensorDataset(token_id[val_idx],\n",
        "                        attention_masks[val_idx],\n",
        "                        labels[val_idx])\n",
        "\n",
        "# Prepare DataLoader\n",
        "train_dataloader = DataLoader(\n",
        "    train_set,\n",
        "    sampler = RandomSampler(train_set),\n",
        "    batch_size = batch_size\n",
        ")\n",
        "\n",
        "validation_dataloader = DataLoader(\n",
        "    val_set,\n",
        "    sampler = SequentialSampler(val_set),\n",
        "    batch_size = batch_size\n",
        ")"
      ],
      "metadata": {
        "id": "eJZ_7kqfVrpm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 6. Train\n",
        "It is time for the fine-tuning task:\n",
        "Select hyperparameters based on the recommendations from the BERT paper:\n",
        ">The optimal hyperparameter values are task-specific, but we found the following range of possible values to work well across all tasks:\n",
        "- Batch size: 16, 32\n",
        "- Learning rate (Adam): 5e-5, 3e-5, 2e-5\n",
        "- Number of epochs: 2, 3, 4\n",
        "\n",
        "Define some functions to assess validation metrics (accuracy, precision, recall and specificity) during the training process:\n",
        "![](https://upload.cc/i1/2023/12/22/tOex6S.png)"
      ],
      "metadata": {
        "id": "nrwsADmfcOu2"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def b_tp(preds, labels):\n",
        "  return sum([preds == labels and preds == 1 for preds, labels in zip(preds, labels)])\n",
        "\n",
        "def b_fp(preds, labels):\n",
        "  return sum([preds != labels and preds == 1 for preds, labels in zip(preds, labels)])\n",
        "\n",
        "def b_tn(preds, labels):\n",
        "  return sum([preds == labels and preds == 0 for preds, labels in zip(preds, labels)])\n",
        "\n",
        "def b_fn(preds, labels):\n",
        "  return sum([preds != labels and preds == 0 for preds, labels in zip(preds, labels)])\n",
        "\n",
        "def b_metrics(preds, labels):\n",
        "  '''\n",
        "  Returns the following metrics:\n",
        "    - accuracy    = (TP + TN) / N\n",
        "    - precision   = TP / (TP + FP)\n",
        "    - recall      = TP / (TP + FN)\n",
        "    - specificity = TN / (TN + FP)\n",
        "  '''\n",
        "  preds = np.argmax(preds, axis = 1).flatten()\n",
        "  labels = labels.flatten()\n",
        "  tp = b_tp(preds, labels)\n",
        "  tn = b_tn(preds, labels)\n",
        "  fp = b_fp(preds, labels)\n",
        "  fn = b_fn(preds, labels)\n",
        "  b_accuracy = (tp + tn) / len(labels)\n",
        "  b_precision = tp / (tp + fp) if (tp + fp) > 0 else 'nan'\n",
        "  b_recall = tp / (tp + fn) if (tp + fn) > 0 else 'nan'\n",
        "  b_specificity = tn / (tn + fp) if (tn + fp) > 0 else 'nan'\n",
        "  return b_accuracy, b_precision, b_recall, b_specificity"
      ],
      "metadata": {
        "id": "lHk85H1-cNMu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Download `transformers.BertForSequenceClassification`, which is a BERT model with a linear layer for sentence classification (or regression) on top of the pooled output:"
      ],
      "metadata": {
        "id": "Gve2ua14moaz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the BertForSequenceClassification model\n",
        "model = BertForSequenceClassification.from_pretrained(\n",
        "    'bert-base-uncased',\n",
        "    num_labels = 2,\n",
        "    output_attentions = False,\n",
        "    output_hidden_states = False,\n",
        ")\n",
        "\n",
        "# Recommended learning rates (Adam): 5e-5, 3e-5, 2e-5\n",
        "optimizer = torch.optim.AdamW(model.parameters(),\n",
        "                              lr = 5e-5,\n",
        "                              eps = 1e-08)\n",
        "\n",
        "# Run on GPU\n",
        "model.cuda()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 981,
          "referenced_widgets": [
            "bd53f6fc7db24e828f69a2b04507103b",
            "ad6de96f498b4c46ba4f9526f5487151",
            "7cf14cee1b2b46119695a4f49d8969da",
            "c276bf2b5fd342b3bf0fa1e71e53abf9",
            "741bb00c00654d659527987618fcf6e8",
            "a1e3409367e9432e90018f1305c703e9",
            "7292aad4f7204a298ba9dcac7eb5e9ad",
            "f7aaf3ebd5ef49d5985dd2c35c08b103",
            "25e085779daa41f58d7cf7d4feb0ac4c",
            "8ae65c2c0f0048eab469987f1fb76e52",
            "11bc9b99cd1140e48f2ef5c9cf34a23b"
          ]
        },
        "id": "Tp5yhYgmmGiM",
        "outputId": "7e7232a6-be77-4538-819c-7f308d1c1983"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/440M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bd53f6fc7db24e828f69a2b04507103b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0-11): 12 x BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Perform the training procedure:"
      ],
      "metadata": {
        "id": "7CX43F3xqyxt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "\n",
        "# Recommended number of epochs: 2, 3, 4\n",
        "epochs = 2\n",
        "\n",
        "for _ in trange(epochs, desc = 'Epoch'):\n",
        "\n",
        "  # ========== Training ==========\n",
        "\n",
        "  # Set model to training mode\n",
        "  model.train()\n",
        "\n",
        "  # Tracking variables\n",
        "  tr_loss = 0\n",
        "  nb_tr_examples, nb_tr_steps = 0, 0\n",
        "\n",
        "  for step, batch in enumerate(train_dataloader):\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    optimizer.zero_grad()\n",
        "    # Forward pass\n",
        "    train_output = model(b_input_ids,\n",
        "                         token_type_ids = None,\n",
        "                         attention_mask = b_input_mask,\n",
        "                         labels = b_labels)\n",
        "    # Backward pass\n",
        "    train_output.loss.backward()\n",
        "    optimizer.step()\n",
        "    # Update tracking variables\n",
        "    tr_loss += train_output.loss.item()\n",
        "    nb_tr_examples += b_input_ids.size(0)\n",
        "    nb_tr_steps += 1\n",
        "\n",
        "  # ========== Validation ==========\n",
        "\n",
        "  # Set model to evaluation mode\n",
        "  model.eval()\n",
        "\n",
        "  # Tracking variables\n",
        "  val_accuracy = []\n",
        "  val_precision = []\n",
        "  val_recall = []\n",
        "  val_specificity = []\n",
        "\n",
        "  for batch in validation_dataloader:\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    with torch.no_grad():\n",
        "      # Forward pass\n",
        "      eval_output = model(b_input_ids,\n",
        "                          token_type_ids = None,\n",
        "                          attention_mask = b_input_mask)\n",
        "    logits = eval_output.logits.detach().cpu().numpy()\n",
        "    label_ids = b_labels.to('cpu').numpy()\n",
        "    # Calculate validation metrics\n",
        "    b_accuracy, b_precision, b_recall, b_specificity = b_metrics(logits, label_ids)\n",
        "    val_accuracy.append(b_accuracy)\n",
        "    # Update precision only when (tp + fp) !=0; ignore nan\n",
        "    if b_precision != 'nan': val_precision.append(b_precision)\n",
        "    # Update recall only when (tp + fn) !=0; ignore nan\n",
        "    if b_recall != 'nan': val_recall.append(b_recall)\n",
        "    # Update specificity only when (tn + fp) !=0; ignore nan\n",
        "    if b_specificity != 'nan': val_specificity.append(b_specificity)\n",
        "\n",
        "  print('\\n\\t - Train loss: {:.4f}'.format(tr_loss / nb_tr_steps))\n",
        "  print('\\t - Validation Accuracy: {:.4f}'.format(sum(val_accuracy)/len(val_accuracy)))\n",
        "  print('\\t - Validation Precision: {:.4f}'.format(sum(val_precision)/len(val_precision)) if len(val_precision)>0 else '\\t - Validation Precision: NaN')\n",
        "  print('\\t - Validation Recall: {:.4f}'.format(sum(val_recall)/len(val_recall)) if len(val_recall)>0 else '\\t - Validation Recall: NaN')\n",
        "  print('\\t - Validation Specificity: {:.4f}\\n'.format(sum(val_specificity)/len(val_specificity)) if len(val_specificity)>0 else '\\t - Validation Specificity: NaN')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7fYiPmqOpRYY",
        "outputId": "a9e3533b-75a4-40f6-da1b-8ce161f4539b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch:  50%|█████     | 1/2 [00:32<00:32, 32.64s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\t - Train loss: 0.0875\n",
            "\t - Validation Accuracy: 0.9830\n",
            "\t - Validation Precision: 0.9299\n",
            "\t - Validation Recall: 0.9358\n",
            "\t - Validation Specificity: 0.9909\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Epoch: 100%|██████████| 2/2 [01:04<00:00, 32.43s/it]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\t - Train loss: 0.0271\n",
            "\t - Validation Accuracy: 0.9786\n",
            "\t - Validation Precision: 0.8939\n",
            "\t - Validation Recall: 0.9672\n",
            "\t - Validation Specificity: 0.9816\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 7. Predict"
      ],
      "metadata": {
        "id": "BeozHEHS5wCf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "new_sentence = 'WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.'\n",
        "\n",
        "# We need Token IDs and Attention Mask for inference on the new sentence\n",
        "test_ids = []\n",
        "test_attention_mask = []\n",
        "\n",
        "# Apply the tokenizer\n",
        "encoding = preprocessing(new_sentence, tokenizer)\n",
        "\n",
        "# Extract IDs and Attention Mask\n",
        "test_ids.append(encoding['input_ids'])\n",
        "test_attention_mask.append(encoding['attention_mask'])\n",
        "test_ids = torch.cat(test_ids, dim = 0)\n",
        "test_attention_mask = torch.cat(test_attention_mask, dim = 0)\n",
        "\n",
        "# Forward pass, calculate logit predictions\n",
        "with torch.no_grad():\n",
        "  output = model(test_ids.to(device), token_type_ids = None, attention_mask = test_attention_mask.to(device))\n",
        "\n",
        "prediction = 'Spam' if np.argmax(output.logits.cpu().numpy()).flatten().item() == 1 else 'Ham'\n",
        "\n",
        "print('Input Sentence: ', new_sentence)\n",
        "print('Predicted Class: ', prediction)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "20Zk3urn5gAb",
        "outputId": "baf8cc90-ae18-4544-856e-87dbf5eb68d0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input Sentence:  WINNER!! As a valued network customer you have been selected to receivea £900 prize reward! To claim call 09061701461. Claim code KL341. Valid 12 hours only.\n",
            "Predicted Class:  Spam\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 8. Conclusions\n"
      ],
      "metadata": {
        "id": "HYplvhWaEDXR"
      }
    }
  ]
}