Lesson_7_BERT_on_外賣資料集_(non_verbos).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lesson 7 BERT on 外賣資料集 (non-verbos).ipynb",
      "provenance": [],
      "collapsed_sections": [
        "MYd9orp08ZIm",
        "N2rG8eFm8Zxk",
        "e5BHDZ15sUxU",
        "nyJSOSeWlbEd",
        "st9l91IBrhh7",
        "GlLMYeYqye8D",
        "tDIxmVxm1aMD",
        "DEw7j4pR5X12",
        "ncHHb6HRAuNg",
        "90dxdDZMCAzu",
        "7has_SKG5akR",
        "DMYvn8yI5cQY"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a1a5947b611b429dac50052023b8018c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9deb1cdef32544358a939eed59967066",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ee376086a8c64e68a27abbc812454150",
              "IPY_MODEL_641fbfb2d30e43caa470285c3cff1cca"
            ]
          }
        },
        "9deb1cdef32544358a939eed59967066": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ee376086a8c64e68a27abbc812454150": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_48265f5dfc214c09bad7e6433a080c64",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 109540,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 109540,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_bd77fff048da481a89be24e48c196c05"
          }
        },
        "641fbfb2d30e43caa470285c3cff1cca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f18d06d3883847beb0404714c526117e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 110k/110k [00:00&lt;00:00, 195kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_30849ad86b0a4d849921c31f523c3af6"
          }
        },
        "48265f5dfc214c09bad7e6433a080c64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "bd77fff048da481a89be24e48c196c05": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f18d06d3883847beb0404714c526117e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "30849ad86b0a4d849921c31f523c3af6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "42139180f73e4c609adb68255b7d00c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f5c905b396404025be647540d97ea8cd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_95b8b2bfe221489ca071534b49bc5c4b",
              "IPY_MODEL_31d422f9766349c1aa6260f1d6fc3330"
            ]
          }
        },
        "f5c905b396404025be647540d97ea8cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "95b8b2bfe221489ca071534b49bc5c4b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a851ee9a608645d894aba41067193d6e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 268943,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 268943,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a15700016fa24eaf96543a60d0b971d8"
          }
        },
        "31d422f9766349c1aa6260f1d6fc3330": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_56bf1dd1c0a943f280d2cb9de0e2a896",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 269k/269k [00:00&lt;00:00, 646kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0579d6134b74a01a88f38c8ffb69b5f"
          }
        },
        "a851ee9a608645d894aba41067193d6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a15700016fa24eaf96543a60d0b971d8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "56bf1dd1c0a943f280d2cb9de0e2a896": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0579d6134b74a01a88f38c8ffb69b5f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e1b7a52c688c49eb84c7403894f6c32c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_368b5a6b51cb4696bfde004554b02003",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ca065f1ed4384a6d9ef58ec7f31d0def",
              "IPY_MODEL_be43783e42d54b149a416179c5db2e9e"
            ]
          }
        },
        "368b5a6b51cb4696bfde004554b02003": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ca065f1ed4384a6d9ef58ec7f31d0def": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_18384cc9c20d444bb060e77b5aaf9683",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 624,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 624,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a371bc69fd2041b49a5842f2fc2adfc3"
          }
        },
        "be43783e42d54b149a416179c5db2e9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4b9f37a59cb54b1fb4468bed5c03dddb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 624/624 [00:00&lt;00:00, 3.74kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_35e93c354d864a48bb94a2306cdf5a17"
          }
        },
        "18384cc9c20d444bb060e77b5aaf9683": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a371bc69fd2041b49a5842f2fc2adfc3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4b9f37a59cb54b1fb4468bed5c03dddb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "35e93c354d864a48bb94a2306cdf5a17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "af074c574ea74d2baee3e2508e84072a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_b510d95533b34d18be18fecb7e8c8a79",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_07d1ec10ec3049039a4b9ed493cf377d",
              "IPY_MODEL_cca410a3fc974cd08ab3426267c3d750"
            ]
          }
        },
        "b510d95533b34d18be18fecb7e8c8a79": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "07d1ec10ec3049039a4b9ed493cf377d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2d4320ef89814e94b179f895e965c06e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 411577189,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 411577189,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ffb2d34d66354856be773c31da73db2c"
          }
        },
        "cca410a3fc974cd08ab3426267c3d750": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d29abcbcc35a461fae7e5987c12022a3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 412M/412M [00:06&lt;00:00, 66.8MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1ff51ed8c8249dfb5b727d71102e279"
          }
        },
        "2d4320ef89814e94b179f895e965c06e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ffb2d34d66354856be773c31da73db2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d29abcbcc35a461fae7e5987c12022a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1ff51ed8c8249dfb5b727d71102e279": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/UDICatNCHU/PyTorch-Tutorial/blob/master/Lesson_7_BERT_on_%E5%A4%96%E8%B3%A3%E8%B3%87%E6%96%99%E9%9B%86_(non_verbos).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MYd9orp08ZIm"
      },
      "source": [
        "### 確認 GPU"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yiM1leq_8Yz5",
        "outputId": "451d2dc7-2959-4bcd-f87a-1fa4afe1e780",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!nvidia-smi -L"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU 0: Tesla T4 (UUID: GPU-7aa91c2c-cf39-b858-5c8d-908748c6f8a8)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N2rG8eFm8Zxk"
      },
      "source": [
        "### Mount 雲端硬碟"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NWEwUDmgIgk5",
        "outputId": "6ffaf393-b355-42c8-8f9d-92624e48613f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive', force_remount=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e5BHDZ15sUxU"
      },
      "source": [
        "### 實作 BERT 情緒分類 "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tMNKsADVi5G8"
      },
      "source": [
        "\n",
        "\n",
        "dataset 來源: https://github.com/SophonPlus/ChineseNlpCorpus"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nyJSOSeWlbEd"
      },
      "source": [
        "#### 下載資料集"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LZWa_bZqZSXZ",
        "outputId": "55e27877-9d4d-424b-87d1-e56a585900d4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!mkdir /content/drive/My\\ Drive/Colab\\ Notebooks/waimai\n",
        "%cd /content/drive/My\\ Drive/Colab\\ Notebooks/waimai"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "mkdir: cannot create directory ‘/content/drive/My Drive/Colab Notebooks/waimai’: File exists\n",
            "/content/drive/My Drive/Colab Notebooks/waimai\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qyEzersBffhU",
        "outputId": "e9af0e7d-e934-4f07-a868-4e5fe2944405",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!wget -O  waimai_10k_zh_tw.csv \"https://www.dropbox.com/s/opundqbrxock8fi/waimai_10k_zh_tw.csv?dl=0\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-09 10:17:09--  https://www.dropbox.com/s/opundqbrxock8fi/waimai_10k_zh_tw.csv?dl=0\n",
            "Resolving www.dropbox.com (www.dropbox.com)... 162.125.6.1, 2620:100:601c:1::a27d:601\n",
            "Connecting to www.dropbox.com (www.dropbox.com)|162.125.6.1|:443... connected.\n",
            "HTTP request sent, awaiting response... 301 Moved Permanently\n",
            "Location: /s/raw/opundqbrxock8fi/waimai_10k_zh_tw.csv [following]\n",
            "--2020-11-09 10:17:09--  https://www.dropbox.com/s/raw/opundqbrxock8fi/waimai_10k_zh_tw.csv\n",
            "Reusing existing connection to www.dropbox.com:443.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com/cd/0/inline/BC3Xyty-dgpL4kLMCP80nbjGhgiIZgp0MjF6Njow2t26XXdEH55fmW6GCnZBtfnAmczvCxoV_-sp6QoMlX8osSDfzlCTV9l8-e_6sRGz3k_JoF4PbYzmQGUqHKgf_twzakU/file# [following]\n",
            "--2020-11-09 10:17:10--  https://uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com/cd/0/inline/BC3Xyty-dgpL4kLMCP80nbjGhgiIZgp0MjF6Njow2t26XXdEH55fmW6GCnZBtfnAmczvCxoV_-sp6QoMlX8osSDfzlCTV9l8-e_6sRGz3k_JoF4PbYzmQGUqHKgf_twzakU/file\n",
            "Resolving uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com (uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com)... 162.125.4.15, 2620:100:601c:15::a27d:60f\n",
            "Connecting to uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com (uc7aa3a168d9a0e5361f8f1316c7.dl.dropboxusercontent.com)|162.125.4.15|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 919380 (898K) [text/plain]\n",
            "Saving to: ‘waimai_10k_zh_tw.csv’\n",
            "\n",
            "waimai_10k_zh_tw.cs 100%[===================>] 897.83K  --.-KB/s    in 0.04s   \n",
            "\n",
            "2020-11-09 10:17:10 (20.7 MB/s) - ‘waimai_10k_zh_tw.csv’ saved [919380/919380]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bh-PM_ZraBcE",
        "outputId": "6b782085-1a0f-4b87-b376-5d9012122694",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'BERT on 外賣資料集.ipynb'   waimai_10k_zh_tw.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "st9l91IBrhh7"
      },
      "source": [
        "#### 外賣資料集格式\n",
        "label | review |\n",
        ":--- | :--- |\n",
        "`1`代表正面<br><br>`0`代表負面 |  評論內容 |"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OuEO815zrdy3",
        "outputId": "75273e76-62d8-407c-cb94-b9edb4a553ef",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import pandas as pd\n",
        "pd_all = pd.read_csv('./waimai_10k_zh_tw.csv')\n",
        "\n",
        "print('評論總數：%d' % pd_all.shape[0])\n",
        "print('正面評論：%d' % pd_all[pd_all.label==1].shape[0])\n",
        "print('負面評論：%d' % pd_all[pd_all.label==0].shape[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "評論總數：11987\n",
            "正面評論：4000\n",
            "負面評論：7987\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GwW49PzEueeW",
        "outputId": "f2ba6aba-b8b3-4356-e61b-df1289f05e57",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 343
        }
      },
      "source": [
        "pd_all.sample(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>review</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>11451</th>\n",
              "      <td>0</td>\n",
              "      <td>烤腸忘給了</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3097</th>\n",
              "      <td>1</td>\n",
              "      <td>味道不錯，快遞師傅贊贊噠</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>604</th>\n",
              "      <td>1</td>\n",
              "      <td>比預定時間快了20分鐘～很好～</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8397</th>\n",
              "      <td>0</td>\n",
              "      <td>今日菜有些鹹</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2414</th>\n",
              "      <td>1</td>\n",
              "      <td>上次說錯了，培根味道一般、美式密肉的好吃，真的鮮榨果汁！</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2396</th>\n",
              "      <td>1</td>\n",
              "      <td>不錯，下次還會再買的</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>715</th>\n",
              "      <td>1</td>\n",
              "      <td>送餐超快,面不坨,味道好</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5822</th>\n",
              "      <td>0</td>\n",
              "      <td>慢，早餐吃成午餐</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1984</th>\n",
              "      <td>1</td>\n",
              "      <td>很好吃，謝謝</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11109</th>\n",
              "      <td>0</td>\n",
              "      <td>速度慢，1個多小時才到，而且，我的面竟然沒有湯</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       label                        review\n",
              "11451      0                         烤腸忘給了\n",
              "3097       1                  味道不錯，快遞師傅贊贊噠\n",
              "604        1               比預定時間快了20分鐘～很好～\n",
              "8397       0                        今日菜有些鹹\n",
              "2414       1  上次說錯了，培根味道一般、美式密肉的好吃，真的鮮榨果汁！\n",
              "2396       1                    不錯，下次還會再買的\n",
              "715        1                  送餐超快,面不坨,味道好\n",
              "5822       0                      慢，早餐吃成午餐\n",
              "1984       1                        很好吃，謝謝\n",
              "11109      0       速度慢，1個多小時才到，而且，我的面竟然沒有湯"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GlLMYeYqye8D"
      },
      "source": [
        "#### 切 training data 及 testing data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NNVHVfbaygor"
      },
      "source": [
        "# 將要訓練的句子存起來\n",
        "def save_sentence(filepath, sent_list):\n",
        "    f = open(filepath, 'w', encoding='utf-8')\n",
        "    for sent in sent_list:\n",
        "        f.write(sent + '\\n')\n",
        "    f.close()\n",
        "\n",
        "# 分割 train_data 和 test_data\n",
        "def split_data(filename):\n",
        "\n",
        "    fp = open(filename, 'r', encoding='utf-8')\n",
        "    line = fp.readline() # 第一行是label, review\n",
        "    line = fp.readline()\n",
        "\n",
        "    train_sent_num = 3000\n",
        "    test_sent_num = 1000\n",
        "    # 計算目前資料筆數\n",
        "    train_positive_num = 0 \n",
        "    train_negative_num = 0\n",
        "    test_positive_num = 0\n",
        "    test_negative_num = 0\n",
        "    train_data = []\n",
        "    test_data = []\n",
        "\n",
        "    # 用 while 逐行讀取檔案內容，直至檔案結尾\n",
        "    while line:\n",
        "        sent = ''\n",
        "        sent = line.replace('\\n', '')\n",
        "\n",
        "        if line[:2] == '1,':\n",
        "            if train_positive_num < train_sent_num:\n",
        "                train_data.append(sent)\n",
        "                train_positive_num += 1\n",
        "            elif test_positive_num < test_sent_num:\n",
        "                test_data.append(sent)\n",
        "                test_positive_num += 1\n",
        "        else:\n",
        "            if train_negative_num < train_sent_num:\n",
        "                train_data.append(sent)\n",
        "                train_negative_num += 1\n",
        "            elif test_negative_num < test_sent_num:\n",
        "                test_data.append(sent)\n",
        "                test_negative_num += 1\n",
        "        \n",
        "        line = fp.readline()\n",
        "    \n",
        "    fp.close()\n",
        "\n",
        "    save_sentence('./train_data.txt', train_data)\n",
        "    save_sentence('./test_data.txt', test_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sjhg0oWo1Bg6"
      },
      "source": [
        "split_data('waimai_10k_zh_tw.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XTa-Zz6K1WTN",
        "outputId": "381cf148-3913-4327-a9ae-935357c4ff55",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "'BERT on 外賣資料集.ipynb'   train_data.txt\n",
            " test_data.txt\t\t     waimai_10k_zh_tw.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tDIxmVxm1aMD"
      },
      "source": [
        "#### 載入相關套件"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I03a6Hxm1bcA",
        "outputId": "478689de-106c-49ac-e5b2-81a162ac329c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/4e/4f1ede0fd7a36278844a277f8d53c21f88f37f3754abf76a5d6224f76d4a/transformers-3.4.0-py3-none-any.whl (1.3MB)\n",
            "\r\u001b[K     |▎                               | 10kB 23.3MB/s eta 0:00:01\r\u001b[K     |▌                               | 20kB 30.6MB/s eta 0:00:01\r\u001b[K     |▉                               | 30kB 35.7MB/s eta 0:00:01\r\u001b[K     |█                               | 40kB 22.8MB/s eta 0:00:01\r\u001b[K     |█▎                              | 51kB 15.2MB/s eta 0:00:01\r\u001b[K     |█▋                              | 61kB 14.2MB/s eta 0:00:01\r\u001b[K     |█▉                              | 71kB 13.1MB/s eta 0:00:01\r\u001b[K     |██                              | 81kB 12.9MB/s eta 0:00:01\r\u001b[K     |██▍                             | 92kB 12.9MB/s eta 0:00:01\r\u001b[K     |██▋                             | 102kB 13.2MB/s eta 0:00:01\r\u001b[K     |██▉                             | 112kB 13.2MB/s eta 0:00:01\r\u001b[K     |███▏                            | 122kB 13.2MB/s eta 0:00:01\r\u001b[K     |███▍                            | 133kB 13.2MB/s eta 0:00:01\r\u001b[K     |███▋                            | 143kB 13.2MB/s eta 0:00:01\r\u001b[K     |████                            | 153kB 13.2MB/s eta 0:00:01\r\u001b[K     |████▏                           | 163kB 13.2MB/s eta 0:00:01\r\u001b[K     |████▍                           | 174kB 13.2MB/s eta 0:00:01\r\u001b[K     |████▊                           | 184kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████                           | 194kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 204kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 215kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 225kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████                          | 235kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 245kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 256kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 266kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████                         | 276kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 286kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 296kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 307kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████                        | 317kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 327kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 337kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 348kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 358kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 368kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████▋                      | 378kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████                      | 389kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 399kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 409kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 419kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████                     | 430kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 440kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 450kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 460kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████                    | 471kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 481kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 491kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████▊                   | 501kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 512kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 522kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 532kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 542kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 552kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 563kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████▋                 | 573kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 583kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 593kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 604kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 614kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████                | 624kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 634kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 645kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 655kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 665kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 675kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 686kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 696kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 706kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 716kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 727kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 737kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 747kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 757kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 768kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 778kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 788kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▎           | 798kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▋           | 808kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 819kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 829kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 839kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 849kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 860kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 870kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▍         | 880kB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████▊         | 890kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 901kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 911kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 921kB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 931kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 942kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 952kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 962kB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 972kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 983kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 993kB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▌      | 1.0MB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 1.0MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.0MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 1.0MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.0MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▍   | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.1MB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.2MB 13.2MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.3MB 13.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Collecting tokenizers==0.9.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/a5/78be1a55b2ac8d6a956f0a211d372726e2b1dd2666bb537fea9b03abd62c/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 51.5MB/s \n",
            "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 47.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers) (3.12.4)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 50.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers) (50.3.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.17.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=c8102c8c3307024dbb8c5949da20cbed2b578cd5a2557d46811f7f3c9ad56150\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.9.2 transformers-3.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ThY3sx4a1d_r"
      },
      "source": [
        "import torch\n",
        "from transformers import (AdamW, BertForSequenceClassification, BertTokenizerFast,\n",
        "                          Trainer, TrainingArguments)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DEw7j4pR5X12"
      },
      "source": [
        "#### 讀取資料（整理出資料中的 sentence、label）"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GBd-IsUc5ZbE"
      },
      "source": [
        "def read_waimai(path):\n",
        "    with open(path,'r',encoding='utf-8') as f:\n",
        "        data = f.read()\n",
        "    LS_pairs = data.split(\"\\n\")\n",
        "    \n",
        "    texts = []\n",
        "    labels = []\n",
        "    for LS_pair in LS_pairs:\n",
        "        if LS_pair != \"\":\n",
        "            try:\n",
        "                L = LS_pair[:1]\n",
        "                S = LS_pair[2:]\n",
        "                texts.append(S)\n",
        "                labels.append(int(L))\n",
        "            except:\n",
        "                continue\n",
        "    return texts, labels\n",
        "\n",
        "train_texts, train_labels = read_waimai('./train_data.txt')\n",
        "test_texts, test_labels = read_waimai('./test_data.txt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KZIIwRgOAcJ2",
        "outputId": "227f8d47-66c8-441c-a859-ad5551965b16",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(train_texts[0], train_labels[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "很快，好吃，味道足，量大 1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ncHHb6HRAuNg"
      },
      "source": [
        "#### Tokenize（將資料轉換成 token id 、tpye_id 與 attention_mask）\n",
        "`truncation=True` 和 `padding=True`\n",
        "\n",
        "確保輸入不超過模型的最大限制以及讓每一筆資料長度都一樣"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C9Rp8I0rAv7y",
        "outputId": "c364d1d4-4060-4ea6-f05d-4f15391eecd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114,
          "referenced_widgets": [
            "a1a5947b611b429dac50052023b8018c",
            "9deb1cdef32544358a939eed59967066",
            "ee376086a8c64e68a27abbc812454150",
            "641fbfb2d30e43caa470285c3cff1cca",
            "48265f5dfc214c09bad7e6433a080c64",
            "bd77fff048da481a89be24e48c196c05",
            "f18d06d3883847beb0404714c526117e",
            "30849ad86b0a4d849921c31f523c3af6",
            "42139180f73e4c609adb68255b7d00c2",
            "f5c905b396404025be647540d97ea8cd",
            "95b8b2bfe221489ca071534b49bc5c4b",
            "31d422f9766349c1aa6260f1d6fc3330",
            "a851ee9a608645d894aba41067193d6e",
            "a15700016fa24eaf96543a60d0b971d8",
            "56bf1dd1c0a943f280d2cb9de0e2a896",
            "b0579d6134b74a01a88f38c8ffb69b5f"
          ]
        }
      },
      "source": [
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-chinese')\n",
        "\n",
        "train_encodings = tokenizer(train_texts, truncation=True, padding=True)\n",
        "test_encodings = tokenizer(test_texts, truncation=True, padding=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a1a5947b611b429dac50052023b8018c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=109540.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "42139180f73e4c609adb68255b7d00c2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=268943.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Tv4IdCbA_t1",
        "outputId": "b8728194-a3fd-49c5-8606-e4e57a3fcf12",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(train_encodings[0].ids)\n",
        "print(tokenizer.decode(train_encodings[0].ids))\n",
        "\n",
        "print(train_encodings[0].type_ids)\n",
        "print(train_encodings[0].attention_mask)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[101, 2523, 2571, 8024, 1962, 1391, 8024, 1456, 6887, 6639, 8024, 7030, 1920, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "[CLS] 很 快 ， 好 吃 ， 味 道 足 ， 量 大 [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\n",
            "[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
            "[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "90dxdDZMCAzu"
      },
      "source": [
        "#### 定義 Dataset，並轉換成 tensor 格式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rixWLPihCGIf"
      },
      "source": [
        "class WaimaiDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings, labels):\n",
        "        self.encodings = encodings\n",
        "        self.labels = labels\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "        item['labels'] = torch.tensor(self.labels[idx])\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)\n",
        "\n",
        "train_dataset = WaimaiDataset(train_encodings, train_labels)\n",
        "test_dataset = WaimaiDataset(test_encodings, test_labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KySRUEGpCUpE",
        "outputId": "58731654-7bfa-417b-af3b-3041b70d885f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "print(train_dataset[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': tensor([ 101, 2523, 2571, 8024, 1962, 1391, 8024, 1456, 6887, 6639, 8024, 7030,\n",
            "        1920,  102,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0]), 'token_type_ids': tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0]), 'attention_mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
            "        0, 0, 0, 0]), 'labels': tensor(1)}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7has_SKG5akR"
      },
      "source": [
        "#### Train"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vn4QrG1x5b3G",
        "outputId": "c9bc1ff3-cb44-445b-f606-bb7d632b4242",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 218,
          "referenced_widgets": [
            "e1b7a52c688c49eb84c7403894f6c32c",
            "368b5a6b51cb4696bfde004554b02003",
            "ca065f1ed4384a6d9ef58ec7f31d0def",
            "be43783e42d54b149a416179c5db2e9e",
            "18384cc9c20d444bb060e77b5aaf9683",
            "a371bc69fd2041b49a5842f2fc2adfc3",
            "4b9f37a59cb54b1fb4468bed5c03dddb",
            "35e93c354d864a48bb94a2306cdf5a17",
            "af074c574ea74d2baee3e2508e84072a",
            "b510d95533b34d18be18fecb7e8c8a79",
            "07d1ec10ec3049039a4b9ed493cf377d",
            "cca410a3fc974cd08ab3426267c3d750",
            "2d4320ef89814e94b179f895e965c06e",
            "ffb2d34d66354856be773c31da73db2c",
            "d29abcbcc35a461fae7e5987c12022a3",
            "c1ff51ed8c8249dfb5b727d71102e279"
          ]
        }
      },
      "source": [
        "# 設定訓練參數\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./trained_model',\n",
        "    num_train_epochs=2,\n",
        "    per_device_train_batch_size=8,\n",
        "    per_device_eval_batch_size=8,\n",
        ")\n",
        "\n",
        "# 使用 bert-base-chinese 預訓練模型\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-chinese\")\n",
        "\n",
        "# 將參數與資料丟入 trainer \n",
        "trainer = Trainer(\n",
        "    model=model,                 \n",
        "    args=training_args,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=test_dataset\n",
        ")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e1b7a52c688c49eb84c7403894f6c32c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=624.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "af074c574ea74d2baee3e2508e84072a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=411577189.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-chinese were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-chinese and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "giYmp8pV7s41",
        "outputId": "443d7629-496e-483a-b3ea-989a5f83852d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 213
        }
      },
      "source": [
        "%%time\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='1500' max='1500' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1500/1500 17:57, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>500</td>\n",
              "      <td>0.410871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1000</td>\n",
              "      <td>0.346714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1500</td>\n",
              "      <td>0.300215</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "CPU times: user 10min 23s, sys: 7min 21s, total: 17min 44s\n",
            "Wall time: 17min 57s\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1500, training_loss=0.3526003011067708)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mff-sKvI7s0t"
      },
      "source": [
        "# 儲存模型\n",
        "trainer.save_model(\"./trained_model\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DMYvn8yI5cQY"
      },
      "source": [
        "#### Inference"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Op5ACwrSGJTW",
        "outputId": "fc602513-1b3f-44b7-9cd7-a8952b235b45",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "from transformers import BertTokenizer, BertConfig, BertForSequenceClassification\n",
        "import torch\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-chinese\")\n",
        "config = BertConfig.from_pretrained(\"./trained_model/config.json\") \n",
        "model = BertForSequenceClassification.from_pretrained(\"./trained_model/pytorch_model.bin\", config=config)\n",
        "\n",
        "sentence = \"這間餐廳的東西很好吃。\"\n",
        "\n",
        "inputs = tokenizer(sentence, return_tensors=\"pt\")\n",
        "outputs = model(**inputs)\n",
        "predicts = outputs[0]\n",
        "max_val = torch.max(predicts)\n",
        "predict_label = (predicts == max_val).nonzero().numpy()[0][1]\n",
        "\n",
        "if str(predict_label) == '1':\n",
        "    print('正面')\n",
        "else:\n",
        "    print('負面')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "正面\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}