sentiment_BERT_nsmc.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "analyze_BERT_nsmc.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "b0bb472bcd084162854679a275641be2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4ad25a84890444b2a1dbac91ca295426",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_06c488b2bcc947bb87ce87c343755938",
              "IPY_MODEL_f725a24b36cd4be2849931a6361e5e73"
            ]
          }
        },
        "4ad25a84890444b2a1dbac91ca295426": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "06c488b2bcc947bb87ce87c343755938": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_ef4cb1177f1c4c9296e41bea9df25316",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 995526,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 995526,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_562436db4dd34af5952bc548d5032a3c"
          }
        },
        "f725a24b36cd4be2849931a6361e5e73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6f248cfb47e8432381979427bb2b0b78",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 996k/996k [00:00&lt;00:00, 2.06MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6282ad3999d4b19b88c8fdfa10ff319"
          }
        },
        "ef4cb1177f1c4c9296e41bea9df25316": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "562436db4dd34af5952bc548d5032a3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f248cfb47e8432381979427bb2b0b78": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6282ad3999d4b19b88c8fdfa10ff319": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "63c7f6afb3c5431f8e117743ab97be87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_543471167b9d45259931a46d06bf2f13",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_721814f33b9e4846a7bc346c7e2356b5",
              "IPY_MODEL_5ea669a93cc546ee909026e029a8460d"
            ]
          }
        },
        "543471167b9d45259931a46d06bf2f13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "721814f33b9e4846a7bc346c7e2356b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_b8c66a9ead254104b141bf9e3df08249",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 625,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 625,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fb36743a112e4e788786526efb662878"
          }
        },
        "5ea669a93cc546ee909026e029a8460d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_26269a1e9fdd4d02b446a82421810718",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 625/625 [00:00&lt;00:00, 2.33kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c1264e4d4c4546aaa5afab6b038257c0"
          }
        },
        "b8c66a9ead254104b141bf9e3df08249": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fb36743a112e4e788786526efb662878": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "26269a1e9fdd4d02b446a82421810718": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c1264e4d4c4546aaa5afab6b038257c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fa007a9486f24fb6aa04ee32fc9e6851": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ad0a81cb10684e1db3cb5b51fb69406a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b5d1c3d882134dbaa5e65561ccaec075",
              "IPY_MODEL_107947515f5d4eabb79ae37eef9f5257"
            ]
          }
        },
        "ad0a81cb10684e1db3cb5b51fb69406a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b5d1c3d882134dbaa5e65561ccaec075": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1e2a57008d9e4fc095817c538d9287c2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 714314041,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 714314041,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7df1bea6ce74fe3aafe4a7af5509df0"
          }
        },
        "107947515f5d4eabb79ae37eef9f5257": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d3e70ed57f9848cf9ee377619a70c3ea",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 714M/714M [00:38&lt;00:00, 18.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_78011db17dfb489291516704783c05e3"
          }
        },
        "1e2a57008d9e4fc095817c538d9287c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7df1bea6ce74fe3aafe4a7af5509df0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d3e70ed57f9848cf9ee377619a70c3ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "78011db17dfb489291516704783c05e3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/snoop2head/yonsei-exchange-program/blob/master/analyze_BERT_nsmc.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P58qy4--s5_x",
        "colab_type": "text"
      },
      "source": [
        "# 네이버 영화리뷰 감정분석 with Hugging Face BERT\n",
        "### 코드 참조: https://github.com/deepseasw/bert-naver-movie-review\n",
        "\n",
        "BERT(Bidirectional Encoder Representations from Transformers)는 구글이 개발한 사전훈련(pre-training) 모델입니다. 위키피디아 같은 텍스트 코퍼스를 사용해서 미리 학습을 하면, 언어의 기본적인 패턴을 이해한 모델이 만들어집니다. 이를 기반으로 새로운 문제에 적용하는 전이학습(transfer learning)을 수행합니다. 좀 더 적은 데이터로 보다 빠르게 학습이 가능하다는 장점이 있습니다. 그래서 최근 자연어처리의 핵심 기법으로 떠오르고 있습니다.\n",
        "\n",
        "이 예제에서는 한글 NLP의 Hello world라고 할 수 있는 네이버 영화리뷰 감정분석을 구현해보겠습니다. 가장 유명한 모델 중 하나인 Hugging Face의 PyTorch BERT를 사용하였습니다. 아래의 Chris McCormick의 블로그를 참조하여 한글에 맞게 수정하였음을 미리 알려드립니다.\n",
        "\n",
        "[BERT Fine-Tuning Tutorial with PyTorch](https://mccormickml.com/2019/07/22/BERT-fine-tuning)\n",
        "<br>\n",
        "<br>\n",
        "<br>\n",
        "BERT에 대해서 좀 더 자세한 설명은 박상길님과 Jay Alammar의 블로그를 참조하시기 바랍니다.\n",
        "\n",
        "[BERT 톺아보기](http://docs.likejazz.com/bert/)\n",
        "\n",
        "[The Illustrated BERT, ELMo, and co. (How NLP Cracked Transfer Learning)](http://jalammar.github.io/illustrated-bert)\n",
        "<br>\n",
        "<br>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i45d7E0L8bZ_",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **준비 사항**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WkAHQrj2Vjbl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 605
        },
        "outputId": "bfe0d439-d8f7-4baf-d009-f41cd1082884"
      },
      "source": [
        "# Hugging Face의 트랜스포머 모델을 설치\n",
        "!pip install transformers"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
            "\r\u001b[K     |▍                               | 10kB 26.7MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 3.4MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 4.4MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 4.9MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 3.8MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 4.4MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 4.9MB/s eta 0:00:01\r\u001b[K     |███▍                            | 81kB 5.3MB/s eta 0:00:01\r\u001b[K     |███▉                            | 92kB 5.7MB/s eta 0:00:01\r\u001b[K     |████▎                           | 102kB 5.4MB/s eta 0:00:01\r\u001b[K     |████▊                           | 112kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 122kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 133kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████                          | 143kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 153kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 163kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 174kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 184kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████                        | 194kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 204kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████                       | 215kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 225kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 235kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 245kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 256kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████                     | 266kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 276kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████                    | 286kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 296kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 307kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 317kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 327kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 337kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 348kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 358kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 368kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 378kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 389kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 399kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 409kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 419kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 430kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 440kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 450kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 460kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 471kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 481kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 491kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 501kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 512kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 522kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 532kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 542kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 552kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 563kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 573kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 583kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 593kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 604kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 614kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 624kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 634kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 645kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 655kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 665kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 675kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 686kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 696kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 706kB 5.4MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 716kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 727kB 5.4MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 737kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 747kB 5.4MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 757kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 768kB 5.4MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 778kB 5.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 36.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers==0.8.1.rc1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 41.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 58.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=397d0e479243fe24fe6eab28541c93074286192e385b909fc0eaec7c73c1a669\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sentencepiece, tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "75dIz2fNWG8F",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "import torch\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from transformers import BertForSequenceClassification, AdamW, BertConfig\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import random\n",
        "import time\n",
        "import datetime"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h_U3uMySBCIV",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **데이터 로드**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ImBtAkSyTW1r",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "1bddcced-caa5-4ed0-bfdd-aefb3038c0f4"
      },
      "source": [
        "# 네이버 영화리뷰 감정분석 데이터 다운로드\n",
        "!git clone https://github.com/e9t/nsmc.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'nsmc'...\n",
            "remote: Enumerating objects: 14763, done.\u001b[K\n",
            "remote: Total 14763 (delta 0), reused 0 (delta 0), pack-reused 14763\u001b[K\n",
            "Receiving objects: 100% (14763/14763), 56.19 MiB | 23.19 MiB/s, done.\n",
            "Resolving deltas: 100% (1749/1749), done.\n",
            "Checking out files: 100% (14737/14737), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EbWW6zqQA030",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 155
        },
        "outputId": "346411a6-3f56-46a5-a3a6-5234df0436cc"
      },
      "source": [
        "# get yonsei exchange dataset && codes\n",
        "!git clone https://github.com/snoop2head/yonsei-exchange-program.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'yonsei-exchange-life'...\n",
            "remote: Enumerating objects: 972, done.\u001b[K\n",
            "remote: Counting objects:   0% (1/972)\u001b[K\rremote: Counting objects:   1% (10/972)\u001b[K\rremote: Counting objects:   2% (20/972)\u001b[K\rremote: Counting objects:   3% (30/972)\u001b[K\rremote: Counting objects:   4% (39/972)\u001b[K\rremote: Counting objects:   5% (49/972)\u001b[K\rremote: Counting objects:   6% (59/972)\u001b[K\rremote: Counting objects:   7% (69/972)\u001b[K\rremote: Counting objects:   8% (78/972)\u001b[K\rremote: Counting objects:   9% (88/972)\u001b[K\rremote: Counting objects:  10% (98/972)\u001b[K\rremote: Counting objects:  11% (107/972)\u001b[K\rremote: Counting objects:  12% (117/972)\u001b[K\rremote: Counting objects:  13% (127/972)\u001b[K\rremote: Counting objects:  14% (137/972)\u001b[K\rremote: Counting objects:  15% (146/972)\u001b[K\rremote: Counting objects:  16% (156/972)\u001b[K\rremote: Counting objects:  17% (166/972)\u001b[K\rremote: Counting objects:  18% (175/972)\u001b[K\rremote: Counting objects:  19% (185/972)\u001b[K\rremote: Counting objects:  20% (195/972)\u001b[K\rremote: Counting objects:  21% (205/972)\u001b[K\rremote: Counting objects:  22% (214/972)\u001b[K\rremote: Counting objects:  23% (224/972)\u001b[K\rremote: Counting objects:  24% (234/972)\u001b[K\rremote: Counting objects:  25% (243/972)\u001b[K\rremote: Counting objects:  26% (253/972)\u001b[K\rremote: Counting objects:  27% (263/972)\u001b[K\rremote: Counting objects:  28% (273/972)\u001b[K\rremote: Counting objects:  29% (282/972)\u001b[K\rremote: Counting objects:  30% (292/972)\u001b[K\rremote: Counting objects:  31% (302/972)\u001b[K\rremote: Counting objects:  32% (312/972)\u001b[K\rremote: Counting objects:  33% (321/972)\u001b[K\rremote: Counting objects:  34% (331/972)\u001b[K\rremote: Counting objects:  35% (341/972)\u001b[K\rremote: Counting objects:  36% (350/972)\u001b[K\rremote: Counting objects:  37% (360/972)\u001b[K\rremote: Counting objects:  38% (370/972)\u001b[K\rremote: Counting objects:  39% (380/972)\u001b[K\rremote: Counting objects:  40% (389/972)\u001b[K\rremote: Counting objects:  41% (399/972)\u001b[K\rremote: Counting objects:  42% (409/972)\u001b[K\rremote: Counting objects:  43% (418/972)\u001b[K\rremote: Counting objects:  44% (428/972)\u001b[K\rremote: Counting objects:  45% (438/972)\u001b[K\rremote: Counting objects:  46% (448/972)\u001b[K\rremote: Counting objects:  47% (457/972)\u001b[K\rremote: Counting objects:  48% (467/972)\u001b[K\rremote: Counting objects:  49% (477/972)\u001b[K\rremote: Counting objects:  50% (486/972)\u001b[K\rremote: Counting objects:  51% (496/972)\u001b[K\rremote: Counting objects:  52% (506/972)\u001b[K\rremote: Counting objects:  53% (516/972)\u001b[K\rremote: Counting objects:  54% (525/972)\u001b[K\rremote: Counting objects:  55% (535/972)\u001b[K\rremote: Counting objects:  56% (545/972)\u001b[K\rremote: Counting objects:  57% (555/972)\u001b[K\rremote: Counting objects:  58% (564/972)\u001b[K\rremote: Counting objects:  59% (574/972)\u001b[K\rremote: Counting objects:  60% (584/972)\u001b[K\rremote: Counting objects:  61% (593/972)\u001b[K\rremote: Counting objects:  62% (603/972)\u001b[K\rremote: Counting objects:  63% (613/972)\u001b[K\rremote: Counting objects:  64% (623/972)\u001b[K\rremote: Counting objects:  65% (632/972)\u001b[K\rremote: Counting objects:  66% (642/972)\u001b[K\rremote: Counting objects:  67% (652/972)\u001b[K\rremote: Counting objects:  68% (661/972)\u001b[K\rremote: Counting objects:  69% (671/972)\u001b[K\rremote: Counting objects:  70% (681/972)\u001b[K\rremote: Counting objects:  71% (691/972)\u001b[K\rremote: Counting objects:  72% (700/972)\u001b[K\rremote: Counting objects:  73% (710/972)\u001b[K\rremote: Counting objects:  74% (720/972)\u001b[K\rremote: Counting objects:  75% (729/972)\u001b[K\rremote: Counting objects:  76% (739/972)\u001b[K\rremote: Counting objects:  77% (749/972)\u001b[K\rremote: Counting objects:  78% (759/972)\u001b[K\rremote: Counting objects:  79% (768/972)\u001b[K\rremote: Counting objects:  80% (778/972)\u001b[K\rremote: Counting objects:  81% (788/972)\u001b[K\rremote: Counting objects:  82% (798/972)\u001b[K\rremote: Counting objects:  83% (807/972)\u001b[K\rremote: Counting objects:  84% (817/972)\u001b[K\rremote: Counting objects:  85% (827/972)\u001b[K\rremote: Counting objects:  86% (836/972)\u001b[K\rremote: Counting objects:  87% (846/972)\u001b[K\rremote: Counting objects:  88% (856/972)\u001b[K\rremote: Counting objects:  89% (866/972)\u001b[K\rremote: Counting objects:  90% (875/972)\u001b[K\rremote: Counting objects:  91% (885/972)\u001b[K\rremote: Counting objects:  92% (895/972)\u001b[K\rremote: Counting objects:  93% (904/972)\u001b[K\rremote: Counting objects:  94% (914/972)\u001b[K\rremote: Counting objects:  95% (924/972)\u001b[K\rremote: Counting objects:  96% (934/972)\u001b[K\rremote: Counting objects:  97% (943/972)\u001b[K\rremote: Counting objects:  98% (953/972)\u001b[K\rremote: Counting objects:  99% (963/972)\u001b[K\rremote: Counting objects: 100% (972/972)\u001b[K\rremote: Counting objects: 100% (972/972), done.\u001b[K\n",
            "remote: Compressing objects: 100% (963/963), done.\u001b[K\n",
            "remote: Total 2315 (delta 13), reused 961 (delta 9), pack-reused 1343\u001b[K\n",
            "Receiving objects: 100% (2315/2315), 71.66 MiB | 29.55 MiB/s, done.\n",
            "Resolving deltas: 100% (135/135), done.\n",
            "Checking out files: 100% (2845/2845), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RNWLrmTw_9Mr",
        "colab_type": "text"
      },
      "source": [
        "박은정님의 네이버 영화리뷰 감정분석 데이터를 Github에서 다운로드 합니다. 아래와 같이 nsmc 디렉토리에 있는 ratings_train.txt와 ratings_test.txt를 사용하겠습니다. \n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nCKSDHcXTiKn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        },
        "outputId": "ce78e8c8-d7f8-413c-8a57-6b04fed67aa8"
      },
      "source": [
        "# 디렉토리의 파일 목록\n",
        "!ls nsmc -la"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 38660\n",
            "drwxr-xr-x 5 root root     4096 Aug 15 20:58 .\n",
            "drwxr-xr-x 1 root root     4096 Aug 15 20:59 ..\n",
            "drwxr-xr-x 2 root root     4096 Aug 15 20:58 code\n",
            "drwxr-xr-x 8 root root     4096 Aug 15 20:58 .git\n",
            "-rw-r--r-- 1 root root  4893335 Aug 15 20:58 ratings_test.txt\n",
            "-rw-r--r-- 1 root root 14628807 Aug 15 20:58 ratings_train.txt\n",
            "-rw-r--r-- 1 root root 19515078 Aug 15 20:58 ratings.txt\n",
            "drwxr-xr-x 2 root root   483328 Aug 15 20:58 raw\n",
            "-rw-r--r-- 1 root root     2596 Aug 15 20:58 README.md\n",
            "-rw-r--r-- 1 root root    36746 Aug 15 20:58 synopses.json\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UZIpl8JmBJFh",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "outputId": "7fb744a5-701c-4188-a3fa-04ad0ba695df"
      },
      "source": [
        "# 디렉토리의 파일 목록\n",
        "!ls yonsei-exchange-program -la"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "total 23580\n",
            "drwxr-xr-x 7 root root     4096 Aug 15 20:59 .\n",
            "drwxr-xr-x 1 root root     4096 Aug 15 20:59 ..\n",
            "-rw-r--r-- 1 root root   154837 Aug 15 20:59 analyze_BERT_nsmc_corpus.ipynb\n",
            "-rw-r--r-- 1 root root   138879 Aug 15 20:59 analyze_RNN.ipynb\n",
            "drwxr-xr-x 3 root root     4096 Aug 15 20:59 _archived\n",
            "-rw-r--r-- 1 root root 16504680 Aug 15 20:59 best_model.h5\n",
            "-rw-r--r-- 1 root root   160660 Aug 15 20:59 cluster_departments.ipynb\n",
            "-rw-r--r-- 1 root root    37016 Aug 15 20:59 collect_reviews.ipynb\n",
            "drwxr-xr-x 5 root root     4096 Aug 15 20:59 data\n",
            "drwxr-xr-x 4 root root     4096 Aug 15 20:59 data_sentiment\n",
            "-rw-r--r-- 1 root root    23075 Aug 15 20:59 extract_keyword_tfidf.ipynb\n",
            "-rw-r--r-- 1 root root   698692 Aug 15 20:59 generate_wordcloud.ipynb\n",
            "-rw-r--r-- 1 root root  4134325 Aug 15 20:59 generate_wordcloud_konlpy.ipynb\n",
            "-rw-r--r-- 1 root root    30228 Aug 15 20:59 generate_wordcloud_tfidf.ipynb\n",
            "drwxr-xr-x 8 root root     4096 Aug 15 20:59 .git\n",
            "-rw-r--r-- 1 root root      113 Aug 15 20:59 .gitignore\n",
            "drwxr-xr-x 2 root root     4096 Aug 15 20:59 img\n",
            "-rw-r--r-- 1 root root     1070 Aug 15 20:59 LICENSE\n",
            "-rw-r--r-- 1 root root     2535 Aug 15 20:59 README.md\n",
            "-rw-r--r-- 1 root root  2215022 Aug 15 20:59 README.pdf\n",
            "-rw-r--r-- 1 root root     2382 Aug 15 20:59 requirements.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0LPEdb2tWfIU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "51a99f2e-6820-4f9c-b113-9d07c7f4a0ee"
      },
      "source": [
        "# 판다스로 훈련셋과 테스트셋 데이터 로드\n",
        "train = pd.read_csv(\"nsmc/ratings_train.txt\", sep='\\t')\n",
        "test = pd.read_csv(\"nsmc/ratings_test.txt\", sep='\\t')\n",
        "\n",
        "print(train.shape)\n",
        "print(test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(150000, 3)\n",
            "(50000, 3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Cl0j7TZBoeL",
        "colab_type": "text"
      },
      "source": [
        "훈련셋 150,000개와 테스트셋 50,000개의 데이터가 존재합니다.\n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tejY9ZhABYWl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 348
        },
        "outputId": "66a51aff-3844-4155-91e5-a5da101954cb"
      },
      "source": [
        "# 훈련셋을 10개 랜덤으로 추출\n",
        "train.sample(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>document</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>30194</th>\n",
              "      <td>9432200</td>\n",
              "      <td>영상의 구성과 배우들의 연기, 배경음악까지 어느것 하나 빠지지 않는 작품</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33441</th>\n",
              "      <td>1108351</td>\n",
              "      <td>약간은허술한시나리오로 상상력으로 거침없이 나아간다 굳!</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>125175</th>\n",
              "      <td>5233569</td>\n",
              "      <td>주온 말아먹기 . -_-</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44637</th>\n",
              "      <td>7790670</td>\n",
              "      <td>이 곳에서 한국인들의 별점 수준이 드러난다.</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>130984</th>\n",
              "      <td>10250063</td>\n",
              "      <td>이하루.. 에로계의 배모양이라더니.. ㅠ</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>141301</th>\n",
              "      <td>8727289</td>\n",
              "      <td>영화 꼬라지 하고는~ㅉㅉ</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32862</th>\n",
              "      <td>1074898</td>\n",
              "      <td>코미디의 극</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>77699</th>\n",
              "      <td>2556224</td>\n",
              "      <td>흠... 초반이라 그런가 아직 몰입이 안되는데 ;;; 그래도 님들 믿고 달려볼께요</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9439</th>\n",
              "      <td>10068061</td>\n",
              "      <td>여운이 길어요 왜 그렇게 눈물이 나던지..</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30049</th>\n",
              "      <td>7792951</td>\n",
              "      <td>안봤지만 달려보자</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "              id                                       document  label\n",
              "30194    9432200       영상의 구성과 배우들의 연기, 배경음악까지 어느것 하나 빠지지 않는 작품      1\n",
              "33441    1108351                 약간은허술한시나리오로 상상력으로 거침없이 나아간다 굳!      1\n",
              "125175   5233569                                  주온 말아먹기 . -_-      0\n",
              "44637    7790670                       이 곳에서 한국인들의 별점 수준이 드러난다.      1\n",
              "130984  10250063                         이하루.. 에로계의 배모양이라더니.. ㅠ      0\n",
              "141301   8727289                                  영화 꼬라지 하고는~ㅉㅉ      0\n",
              "32862    1074898                                         코미디의 극      1\n",
              "77699    2556224  흠... 초반이라 그런가 아직 몰입이 안되는데 ;;; 그래도 님들 믿고 달려볼께요      1\n",
              "9439    10068061                        여운이 길어요 왜 그렇게 눈물이 나던지..      1\n",
              "30049    7792951                                      안봤지만 달려보자      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hZ12coTLB7H3",
        "colab_type": "text"
      },
      "source": [
        "id는 회원정보, document는 리뷰 문장입니다. label이 0이면 부정, 1이면 긍정으로 분류됩니다. id는 사용하지 않기 때문에 document와 label만 추출하겠습니다. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XgjMzosCDD35",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **전처리 - 훈련셋**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2GoESQ0jbybJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        },
        "outputId": "35c845db-a4ff-4606-b095-1e991fdead21"
      },
      "source": [
        "# 리뷰 문장 추출\n",
        "sentences = train['document']\n",
        "sentences[:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                                  아 더빙.. 진짜 짜증나네요 목소리\n",
              "1                    흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나\n",
              "2                                    너무재밓었다그래서보는것을추천한다\n",
              "3                        교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정\n",
              "4    사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...\n",
              "5        막 걸음마 뗀 3세부터 초등학교 1학년생인 8살용영화.ㅋㅋㅋ...별반개도 아까움.\n",
              "6                                원작의 긴장감을 제대로 살려내지못했다.\n",
              "7    별 반개도 아깝다 욕나온다 이응경 길용우 연기생활이몇년인지..정말 발로해도 그것보단...\n",
              "8                               액션이 없는데도 재미 있는 몇안되는 영화\n",
              "9        왜케 평점이 낮은건데? 꽤 볼만한데.. 헐리우드식 화려함에만 너무 길들여져 있나?\n",
              "Name: document, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KkJZvhccRUJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        },
        "outputId": "2386679f-0a67-4c58-f7a5-660e4f5ae066"
      },
      "source": [
        "# BERT의 입력 형식에 맞게 변환\n",
        "sentences = [\"[CLS] \" + str(sentence) + \" [SEP]\" for sentence in sentences]\n",
        "sentences[:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS] 아 더빙.. 진짜 짜증나네요 목소리 [SEP]',\n",
              " '[CLS] 흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나 [SEP]',\n",
              " '[CLS] 너무재밓었다그래서보는것을추천한다 [SEP]',\n",
              " '[CLS] 교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정 [SEP]',\n",
              " '[CLS] 사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 던스트가 너무나도 이뻐보였다 [SEP]',\n",
              " '[CLS] 막 걸음마 뗀 3세부터 초등학교 1학년생인 8살용영화.ㅋㅋㅋ...별반개도 아까움. [SEP]',\n",
              " '[CLS] 원작의 긴장감을 제대로 살려내지못했다. [SEP]',\n",
              " '[CLS] 별 반개도 아깝다 욕나온다 이응경 길용우 연기생활이몇년인지..정말 발로해도 그것보단 낫겟다 납치.감금만반복반복..이드라마는 가족도없다 연기못하는사람만모엿네 [SEP]',\n",
              " '[CLS] 액션이 없는데도 재미 있는 몇안되는 영화 [SEP]',\n",
              " '[CLS] 왜케 평점이 낮은건데? 꽤 볼만한데.. 헐리우드식 화려함에만 너무 길들여져 있나? [SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NeXrqpRaBjOX",
        "colab_type": "text"
      },
      "source": [
        "![대체 텍스트](https://mino-park7.github.io/images/2019/02/bert-input-representation.png)\n",
        "\n",
        "BERT의 입력은 위의 그림과 같은 형식입니다. Classification을 뜻하는 [CLS] 심볼이 제일 앞에 삽입됩니다. 파인튜닝시 출력에서 이 위치의 값을 사용하여 분류를 합니다. [SEP]은 Seperation을 가리키는데, 두 문장를 구분하는 역할을 합니다. 이 예제에서는 문장이 하나이므로 [SEP]도 하나만 넣습니다.\n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hBblIVQcXJR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "fd23f1d8-a8df-4c9d-c919-c5627dd45710"
      },
      "source": [
        "# 라벨 추출\n",
        "labels = train['label'].values\n",
        "labels"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0, 1, 0, ..., 0, 1, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PwEplfDvcnZG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 101,
          "referenced_widgets": [
            "b0bb472bcd084162854679a275641be2",
            "4ad25a84890444b2a1dbac91ca295426",
            "06c488b2bcc947bb87ce87c343755938",
            "f725a24b36cd4be2849931a6361e5e73",
            "ef4cb1177f1c4c9296e41bea9df25316",
            "562436db4dd34af5952bc548d5032a3c",
            "6f248cfb47e8432381979427bb2b0b78",
            "f6282ad3999d4b19b88c8fdfa10ff319"
          ]
        },
        "outputId": "4c728fbb-b069-403d-c6aa-959783d914c4"
      },
      "source": [
        "# BERT의 토크나이저로 문장을 토큰으로 분리\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)\n",
        "tokenized_texts = [tokenizer.tokenize(sent) for sent in sentences]\n",
        "\n",
        "print (sentences[0])\n",
        "print (tokenized_texts[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b0bb472bcd084162854679a275641be2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=995526.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "[CLS] 아 더빙.. 진짜 짜증나네요 목소리 [SEP]\n",
            "['[CLS]', '아', '더', '##빙', '.', '.', '진', '##짜', '짜', '##증', '##나', '##네', '##요', '목', '##소', '##리', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gV6SRXmlTMjr",
        "colab_type": "text"
      },
      "source": [
        "BERT는 형태소분석으로 토큰을 분리하지 않습니다. WordPiece라는 통계적인 방식을 사용합니다. 한 단어내에서 자주 나오는 글자들을 붙여서 하나의 토큰으로 만듭니다. 이렇게 하면 언어에 상관없이 토큰을 생성할 수 있다는 장점이 있습니다. 또한 신조어 같이 사전에 없는 단어를 처리하기도 좋습니다. \n",
        "\n",
        "위의 결과에서 ## 기호는 앞 토큰과 이어진다는 표시입니다. 토크나이저는 여러 언어의 데이터를 기반으로 만든 'bert-base-multilingual-cased'를 사용합니다. 그래서 한글도 처리가 가능합니다.\n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VJ76KiP_dLn-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 294
        },
        "outputId": "7ab18ad3-c207-4163-88f1-b79bc248475a"
      },
      "source": [
        "# 입력 토큰의 최대 시퀀스 길이\n",
        "MAX_LEN = 128\n",
        "\n",
        "# 토큰을 숫자 인덱스로 변환\n",
        "input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "\n",
        "# 문장을 MAX_LEN 길이에 맞게 자르고, 모자란 부분을 패딩 0으로 채움\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "input_ids[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([   101,   9519,   9074, 119005,    119,    119,   9708, 119235,\n",
              "         9715, 119230,  16439,  77884,  48549,   9284,  22333,  12692,\n",
              "          102,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0,\n",
              "            0,      0,      0,      0,      0,      0,      0,      0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kHI53Jt8VrY8",
        "colab_type": "text"
      },
      "source": [
        "보통 딥러닝 모델에는 토큰 자체를 입력으로 넣을 수 없습니다. 임베딩 레이어에는 토큰을 숫자로 된 인덱스로 변환하여 사용합니다. BERT의 토크나이저는 {단어토큰:인덱스}로 구성된 단어사전을 가지고 있습니다. 이를 참조하여 토큰을 인덱스로 바꿔줍니다.\n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pKfL8SotdVaW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "728eef72-0ca5-4b4e-f1e4-b908f00d1d95"
      },
      "source": [
        "# 어텐션 마스크 초기화\n",
        "attention_masks = []\n",
        "\n",
        "# 어텐션 마스크를 패딩이 아니면 1, 패딩이면 0으로 설정\n",
        "# 패딩 부분은 BERT 모델에서 어텐션을 수행하지 않아 속도 향상\n",
        "for seq in input_ids:\n",
        "    seq_mask = [float(i>0) for i in seq]\n",
        "    attention_masks.append(seq_mask)\n",
        "\n",
        "print(attention_masks[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1f5Vq3-7eNKH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 846
        },
        "outputId": "6d959957-67aa-4d73-9bde-bb46fcc57dea"
      },
      "source": [
        "# 훈련셋과 검증셋으로 분리\n",
        "train_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids,\n",
        "                                                                                    labels, \n",
        "                                                                                    random_state=2018, \n",
        "                                                                                    test_size=0.1)\n",
        "\n",
        "# 어텐션 마스크를 훈련셋과 검증셋으로 분리\n",
        "train_masks, validation_masks, _, _ = train_test_split(attention_masks, \n",
        "                                                       input_ids,\n",
        "                                                       random_state=2018, \n",
        "                                                       test_size=0.1)\n",
        "\n",
        "# 데이터를 파이토치의 텐서로 변환\n",
        "train_inputs = torch.tensor(train_inputs)\n",
        "train_labels = torch.tensor(train_labels)\n",
        "train_masks = torch.tensor(train_masks)\n",
        "validation_inputs = torch.tensor(validation_inputs)\n",
        "validation_labels = torch.tensor(validation_labels)\n",
        "validation_masks = torch.tensor(validation_masks)\t\t\t\t\n",
        "\n",
        "print(train_inputs[0])\n",
        "print(train_labels[0])\n",
        "print(train_masks[0])\n",
        "print(validation_inputs[0])\n",
        "print(validation_labels[0])\n",
        "print(validation_masks[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([   101,   9711,  11489,   9364,  41850,   9004,  32537,   9491,  35506,\n",
            "         17360,  48549,    119,    119,   9477,  26444,  12692,   9665,  21789,\n",
            "         11287,   9708, 119235,   9659,  22458, 119136,  12965,  48549,    119,\n",
            "           119,   9532,  22879,   9685,  16985,  14523,  48549,    119,    119,\n",
            "          9596, 118728,    119,    119,   9178, 106065, 118916,    119,    119,\n",
            "          8903,  11664,  11513,   9960,  14423,  25503, 118671,  48549,    119,\n",
            "           119,  21890,   9546,  37819,  22879,   9356,  14867,   9715, 119230,\n",
            "        118716,  48345,    119,   9663,  23321,  10954,   9638,  35506, 106320,\n",
            "         10739,  20173,   9359,  19105,  11102,  42428,  17196,  48549,    119,\n",
            "           119,    100,    117,   9947,  12945,   9532,  25503,   8932,  14423,\n",
            "         35506, 119050,  11903,  14867,  10003,  14863,  33188,  48345,    119,\n",
            "           102,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0])\n",
            "tensor(0)\n",
            "tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
            "        1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0.])\n",
            "tensor([   101,   1871, 111754, 111754, 111754, 111754,    102,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0,      0,      0,      0,      0,      0,      0,      0,\n",
            "             0,      0])\n",
            "tensor(1)\n",
            "tensor([1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0.])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I3vlyUJuVRo5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 배치 사이즈\n",
        "batch_size = 32\n",
        "\n",
        "# 파이토치의 DataLoader로 입력, 마스크, 라벨을 묶어 데이터 설정\n",
        "# 학습시 배치 사이즈 만큼 데이터를 가져옴\n",
        "train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "train_sampler = RandomSampler(train_data)\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "validation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\n",
        "validation_sampler = SequentialSampler(validation_data)\n",
        "validation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkqUHx51dffp",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **전처리 - 테스트셋**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xgrsNuArd4pj",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 207
        },
        "outputId": "d6bbe239-0555-481b-c865-9ac66cadab24"
      },
      "source": [
        "# 리뷰 문장 추출\n",
        "sentences = test['document']\n",
        "sentences[:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                                                  굳 ㅋ\n",
              "1                                 GDNTOPCLASSINTHECLUB\n",
              "2               뭐야 이 평점들은.... 나쁘진 않지만 10점 짜리는 더더욱 아니잖아\n",
              "3                     지루하지는 않은데 완전 막장임... 돈주고 보기에는....\n",
              "4    3D만 아니었어도 별 다섯 개 줬을텐데.. 왜 3D로 나와서 제 심기를 불편하게 하죠??\n",
              "5                                   음악이 주가 된, 최고의 음악영화\n",
              "6                                              진정한 쓰레기\n",
              "7             마치 미국애니에서 튀어나온듯한 창의력없는 로봇디자인부터가,고개를 젖게한다\n",
              "8    갈수록 개판되가는 중국영화 유치하고 내용없음 폼잡다 끝남 말도안되는 무기에 유치한c...\n",
              "9       이별의 아픔뒤에 찾아오는 새로운 인연의 기쁨 But, 모든 사람이 그렇지는 않네..\n",
              "Name: document, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Gtz3QZt9d4pz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 190
        },
        "outputId": "f014ebe8-d832-4006-a24d-4089e275a767"
      },
      "source": [
        "# BERT의 입력 형식에 맞게 변환\n",
        "sentences = [\"[CLS] \" + str(sentence) + \" [SEP]\" for sentence in sentences]\n",
        "sentences[:10]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS] 굳 ㅋ [SEP]',\n",
              " '[CLS] GDNTOPCLASSINTHECLUB [SEP]',\n",
              " '[CLS] 뭐야 이 평점들은.... 나쁘진 않지만 10점 짜리는 더더욱 아니잖아 [SEP]',\n",
              " '[CLS] 지루하지는 않은데 완전 막장임... 돈주고 보기에는.... [SEP]',\n",
              " '[CLS] 3D만 아니었어도 별 다섯 개 줬을텐데.. 왜 3D로 나와서 제 심기를 불편하게 하죠?? [SEP]',\n",
              " '[CLS] 음악이 주가 된, 최고의 음악영화 [SEP]',\n",
              " '[CLS] 진정한 쓰레기 [SEP]',\n",
              " '[CLS] 마치 미국애니에서 튀어나온듯한 창의력없는 로봇디자인부터가,고개를 젖게한다 [SEP]',\n",
              " '[CLS] 갈수록 개판되가는 중국영화 유치하고 내용없음 폼잡다 끝남 말도안되는 무기에 유치한cg남무 아 그립다 동사서독같은 영화가 이건 3류아류작이다 [SEP]',\n",
              " '[CLS] 이별의 아픔뒤에 찾아오는 새로운 인연의 기쁨 But, 모든 사람이 그렇지는 않네.. [SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "li8oRajbd4p3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "64e46a76-2148-4a8c-c4ca-e6ec6c88231e"
      },
      "source": [
        "# 라벨 추출\n",
        "labels = test['label'].values\n",
        "labels"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1, 0, 0, ..., 0, 0, 0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "lvpQ49nEd4p6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "ba64a750-17ab-4d0b-e40e-ef7502b1dd30"
      },
      "source": [
        "# BERT의 토크나이저로 문장을 토큰으로 분리\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-multilingual-cased', do_lower_case=False)\n",
        "tokenized_texts = [tokenizer.tokenize(sent) for sent in sentences]\n",
        "\n",
        "print (sentences[0])\n",
        "print (tokenized_texts[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[CLS] 굳 ㅋ [SEP]\n",
            "['[CLS]', '굳', '[UNK]', '[SEP]']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "HI9viuAvd4p_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 225
        },
        "outputId": "f5004a6a-87d1-4ed7-e2f6-ab5f1166f56c"
      },
      "source": [
        "# 입력 토큰의 최대 시퀀스 길이\n",
        "MAX_LEN = 128\n",
        "\n",
        "# 토큰을 숫자 인덱스로 변환\n",
        "input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "\n",
        "# 문장을 MAX_LEN 길이에 맞게 자르고, 모자란 부분을 패딩 0으로 채움\n",
        "input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "input_ids[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 101, 8911,  100,  102,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
              "          0,    0,    0,    0,    0,    0,    0])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "v1NKmP0Fd4qD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "ca2c84df-a8a1-423e-b91c-c07f0f8164b1"
      },
      "source": [
        "# 어텐션 마스크 초기화\n",
        "attention_masks = []\n",
        "\n",
        "# 어텐션 마스크를 패딩이 아니면 1, 패딩이면 0으로 설정\n",
        "# 패딩 부분은 BERT 모델에서 어텐션을 수행하지 않아 속도 향상\n",
        "for seq in input_ids:\n",
        "    seq_mask = [float(i>0) for i in seq]\n",
        "    attention_masks.append(seq_mask)\n",
        "\n",
        "print(attention_masks[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RIkaYCGbd4qG",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "outputId": "429a5eda-87d8-4bf3-8133-e4f054903782"
      },
      "source": [
        "# 데이터를 파이토치의 텐서로 변환\n",
        "test_inputs = torch.tensor(input_ids)\n",
        "test_labels = torch.tensor(labels)\n",
        "test_masks = torch.tensor(attention_masks)\n",
        "\n",
        "print(test_inputs[0])\n",
        "print(test_labels[0])\n",
        "print(test_masks[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 101, 8911,  100,  102,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
            "           0,    0,    0,    0,    0,    0,    0,    0])\n",
            "tensor(1)\n",
            "tensor([1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0.])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7gwdYv1Ad4qK",
        "colab": {}
      },
      "source": [
        "# 배치 사이즈\n",
        "batch_size = 32\n",
        "\n",
        "# 파이토치의 DataLoader로 입력, 마스크, 라벨을 묶어 데이터 설정\n",
        "# 학습시 배치 사이즈 만큼 데이터를 가져옴\n",
        "test_data = TensorDataset(test_inputs, test_masks, test_labels)\n",
        "test_sampler = RandomSampler(test_data)\n",
        "test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=batch_size)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FBvpU-Hfgcth",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **모델 생성**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "heToD1ev0mOg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "01d804b5-9183-4540-9f9e-a57c4a260f7e"
      },
      "source": [
        "# GPU 디바이스 이름 구함\n",
        "device_name = tf.test.gpu_device_name()\n",
        "\n",
        "# GPU 디바이스 이름 검사\n",
        "if device_name == '/device:GPU:0':\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "else:\n",
        "    raise SystemError('GPU device not found')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found GPU at: /device:GPU:0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f6enIxvt1FB2",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "451f2fc8-bd4b-46b9-c66d-8033f827987e"
      },
      "source": [
        "# 디바이스 설정\n",
        "if torch.cuda.is_available():    \n",
        "    device = torch.device(\"cuda\")\n",
        "    print('There are %d GPU(s) available.' % torch.cuda.device_count())\n",
        "    print('We will use the GPU:', torch.cuda.get_device_name(0))\n",
        "else:\n",
        "    device = torch.device(\"cpu\")\n",
        "    print('No GPU available, using the CPU instead.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "We will use the GPU: Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MS2MXSiLg5zC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000,
          "referenced_widgets": [
            "63c7f6afb3c5431f8e117743ab97be87",
            "543471167b9d45259931a46d06bf2f13",
            "721814f33b9e4846a7bc346c7e2356b5",
            "5ea669a93cc546ee909026e029a8460d",
            "b8c66a9ead254104b141bf9e3df08249",
            "fb36743a112e4e788786526efb662878",
            "26269a1e9fdd4d02b446a82421810718",
            "c1264e4d4c4546aaa5afab6b038257c0",
            "fa007a9486f24fb6aa04ee32fc9e6851",
            "ad0a81cb10684e1db3cb5b51fb69406a",
            "b5d1c3d882134dbaa5e65561ccaec075",
            "107947515f5d4eabb79ae37eef9f5257",
            "1e2a57008d9e4fc095817c538d9287c2",
            "b7df1bea6ce74fe3aafe4a7af5509df0",
            "d3e70ed57f9848cf9ee377619a70c3ea",
            "78011db17dfb489291516704783c05e3"
          ]
        },
        "outputId": "2c8ccac6-49ef-4bde-e8e9-827b2579aa87"
      },
      "source": [
        "# 분류를 위한 BERT 모델 생성\n",
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-multilingual-cased\", num_labels=2)\n",
        "model.cuda()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "63c7f6afb3c5431f8e117743ab97be87",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=625.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fa007a9486f24fb6aa04ee32fc9e6851",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=714314041.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-multilingual-cased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-multilingual-cased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RlszxDr6h8GP",
        "colab_type": "text"
      },
      "source": [
        "![대체 텍스트](http://www.mccormickml.com/assets/BERT/padding_and_mask.png)\n",
        "\n",
        "사전훈련된 BERT는 다양한 문제로 전이학습이 가능합니다. 여기서는 위의 그림과 같이 한 문장을 분류하는 방법을 사용합니다. 영화리뷰 문장이 입력으로 들어가면, 긍정/부정으로 구분합니다. 모델의 출력에서 [CLS] 위치인 첫 번째 토큰에 새로운 레이어를 붙여서 파인튜닝을 합니다. Huggning Face는 BertForSequenceClassification() 함수를 제공하기 때문에 쉽게 구현할 수 있습니다.\n",
        "<br>\n",
        "<br>\n",
        "<br>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIdfbLTuWmxk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 옵티마이저 설정\n",
        "optimizer = AdamW(model.parameters(),\n",
        "                  lr = 2e-5, # 학습률\n",
        "                  eps = 1e-8 # 0으로 나누는 것을 방지하기 위한 epsilon 값\n",
        "                )\n",
        "\n",
        "# 에폭수\n",
        "epochs = 4\n",
        "\n",
        "# 총 훈련 스텝 : 배치반복 횟수 * 에폭\n",
        "total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "# 학습률을 조금씩 감소시키는 스케줄러 생성\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, \n",
        "                                            num_warmup_steps = 0,\n",
        "                                            num_training_steps = total_steps)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gzCHV_ghj7DM",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **모델 학습**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S0-p6pPVXCRe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 정확도 계산 함수\n",
        "def flat_accuracy(preds, labels):\n",
        "    \n",
        "    pred_flat = np.argmax(preds, axis=1).flatten()\n",
        "    labels_flat = labels.flatten()\n",
        "\n",
        "    return np.sum(pred_flat == labels_flat) / len(labels_flat)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJXISnJzCdLM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 시간 표시 함수\n",
        "def format_time(elapsed):\n",
        "\n",
        "    # 반올림\n",
        "    elapsed_rounded = int(round((elapsed)))\n",
        "    \n",
        "    # hh:mm:ss으로 형태 변경\n",
        "    return str(datetime.timedelta(seconds=elapsed_rounded))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "muU2kS2GCh4y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "fa91e5d1-a165-4574-83e7-bb210b4d9c5e"
      },
      "source": [
        "# 재현을 위해 랜덤시드 고정\n",
        "seed_val = 42\n",
        "random.seed(seed_val)\n",
        "np.random.seed(seed_val)\n",
        "torch.manual_seed(seed_val)\n",
        "torch.cuda.manual_seed_all(seed_val)\n",
        "\n",
        "# 그래디언트 초기화\n",
        "model.zero_grad()\n",
        "\n",
        "# 에폭만큼 반복\n",
        "for epoch_i in range(0, epochs):\n",
        "    \n",
        "    # ========================================\n",
        "    #               Training\n",
        "    # ========================================\n",
        "    \n",
        "    print(\"\")\n",
        "    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))\n",
        "    print('Training...')\n",
        "\n",
        "    # 시작 시간 설정\n",
        "    t0 = time.time()\n",
        "\n",
        "    # 로스 초기화\n",
        "    total_loss = 0\n",
        "\n",
        "    # 훈련모드로 변경\n",
        "    model.train()\n",
        "        \n",
        "    # 데이터로더에서 배치만큼 반복하여 가져옴\n",
        "    for step, batch in enumerate(train_dataloader):\n",
        "        # 경과 정보 표시\n",
        "        if step % 500 == 0 and not step == 0:\n",
        "            elapsed = format_time(time.time() - t0)\n",
        "            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))\n",
        "\n",
        "        # 배치를 GPU에 넣음\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # 배치에서 데이터 추출\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "\n",
        "        # Forward 수행                \n",
        "        outputs = model(b_input_ids, \n",
        "                        token_type_ids=None, \n",
        "                        attention_mask=b_input_mask, \n",
        "                        labels=b_labels)\n",
        "        \n",
        "        # 로스 구함\n",
        "        loss = outputs[0]\n",
        "\n",
        "        # 총 로스 계산\n",
        "        total_loss += loss.item()\n",
        "\n",
        "        # Backward 수행으로 그래디언트 계산\n",
        "        loss.backward()\n",
        "\n",
        "        # 그래디언트 클리핑\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "        # 그래디언트를 통해 가중치 파라미터 업데이트\n",
        "        optimizer.step()\n",
        "\n",
        "        # 스케줄러로 학습률 감소\n",
        "        scheduler.step()\n",
        "\n",
        "        # 그래디언트 초기화\n",
        "        model.zero_grad()\n",
        "\n",
        "    # 평균 로스 계산\n",
        "    avg_train_loss = total_loss / len(train_dataloader)            \n",
        "\n",
        "    print(\"\")\n",
        "    print(\"  Average training loss: {0:.2f}\".format(avg_train_loss))\n",
        "    print(\"  Training epcoh took: {:}\".format(format_time(time.time() - t0)))\n",
        "        \n",
        "    # ========================================\n",
        "    #               Validation\n",
        "    # ========================================\n",
        "\n",
        "    print(\"\")\n",
        "    print(\"Running Validation...\")\n",
        "\n",
        "    #시작 시간 설정\n",
        "    t0 = time.time()\n",
        "\n",
        "    # 평가모드로 변경\n",
        "    model.eval()\n",
        "\n",
        "    # 변수 초기화\n",
        "    eval_loss, eval_accuracy = 0, 0\n",
        "    nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "    # 데이터로더에서 배치만큼 반복하여 가져옴\n",
        "    for batch in validation_dataloader:\n",
        "        # 배치를 GPU에 넣음\n",
        "        batch = tuple(t.to(device) for t in batch)\n",
        "        \n",
        "        # 배치에서 데이터 추출\n",
        "        b_input_ids, b_input_mask, b_labels = batch\n",
        "        \n",
        "        # 그래디언트 계산 안함\n",
        "        with torch.no_grad():     \n",
        "            # Forward 수행\n",
        "            outputs = model(b_input_ids, \n",
        "                            token_type_ids=None, \n",
        "                            attention_mask=b_input_mask)\n",
        "        \n",
        "        # 로스 구함\n",
        "        logits = outputs[0]\n",
        "\n",
        "        # CPU로 데이터 이동\n",
        "        logits = logits.detach().cpu().numpy()\n",
        "        label_ids = b_labels.to('cpu').numpy()\n",
        "        \n",
        "        # 출력 로짓과 라벨을 비교하여 정확도 계산\n",
        "        tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "        eval_accuracy += tmp_eval_accuracy\n",
        "        nb_eval_steps += 1\n",
        "\n",
        "    print(\"  Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "    print(\"  Validation took: {:}\".format(format_time(time.time() - t0)))\n",
        "\n",
        "print(\"\")\n",
        "print(\"Training complete!\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "======== Epoch 1 / 4 ========\n",
            "Training...\n",
            "  Batch   500  of  4,219.    Elapsed: 0:03:17.\n",
            "  Batch 1,000  of  4,219.    Elapsed: 0:06:34.\n",
            "  Batch 1,500  of  4,219.    Elapsed: 0:09:50.\n",
            "  Batch 2,000  of  4,219.    Elapsed: 0:13:07.\n",
            "  Batch 2,500  of  4,219.    Elapsed: 0:16:24.\n",
            "  Batch 3,000  of  4,219.    Elapsed: 0:19:41.\n",
            "  Batch 3,500  of  4,219.    Elapsed: 0:22:57.\n",
            "  Batch 4,000  of  4,219.    Elapsed: 0:26:14.\n",
            "\n",
            "  Average training loss: 0.38\n",
            "  Training epcoh took: 0:27:40\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.86\n",
            "  Validation took: 0:00:57\n",
            "\n",
            "======== Epoch 2 / 4 ========\n",
            "Training...\n",
            "  Batch   500  of  4,219.    Elapsed: 0:03:17.\n",
            "  Batch 1,000  of  4,219.    Elapsed: 0:06:33.\n",
            "  Batch 1,500  of  4,219.    Elapsed: 0:09:50.\n",
            "  Batch 2,000  of  4,219.    Elapsed: 0:13:07.\n",
            "  Batch 2,500  of  4,219.    Elapsed: 0:16:23.\n",
            "  Batch 3,000  of  4,219.    Elapsed: 0:19:40.\n",
            "  Batch 3,500  of  4,219.    Elapsed: 0:22:57.\n",
            "  Batch 4,000  of  4,219.    Elapsed: 0:26:14.\n",
            "\n",
            "  Average training loss: 0.29\n",
            "  Training epcoh took: 0:27:40\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.87\n",
            "  Validation took: 0:00:57\n",
            "\n",
            "======== Epoch 3 / 4 ========\n",
            "Training...\n",
            "  Batch   500  of  4,219.    Elapsed: 0:03:17.\n",
            "  Batch 1,000  of  4,219.    Elapsed: 0:06:34.\n",
            "  Batch 1,500  of  4,219.    Elapsed: 0:09:51.\n",
            "  Batch 2,000  of  4,219.    Elapsed: 0:13:08.\n",
            "  Batch 2,500  of  4,219.    Elapsed: 0:16:25.\n",
            "  Batch 3,000  of  4,219.    Elapsed: 0:19:42.\n",
            "  Batch 3,500  of  4,219.    Elapsed: 0:22:59.\n",
            "  Batch 4,000  of  4,219.    Elapsed: 0:26:16.\n",
            "\n",
            "  Average training loss: 0.22\n",
            "  Training epcoh took: 0:27:42\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.87\n",
            "  Validation took: 0:00:57\n",
            "\n",
            "======== Epoch 4 / 4 ========\n",
            "Training...\n",
            "  Batch   500  of  4,219.    Elapsed: 0:03:17.\n",
            "  Batch 1,000  of  4,219.    Elapsed: 0:06:34.\n",
            "  Batch 1,500  of  4,219.    Elapsed: 0:09:51.\n",
            "  Batch 2,000  of  4,219.    Elapsed: 0:13:08.\n",
            "  Batch 2,500  of  4,219.    Elapsed: 0:16:25.\n",
            "  Batch 3,000  of  4,219.    Elapsed: 0:19:42.\n",
            "  Batch 3,500  of  4,219.    Elapsed: 0:22:59.\n",
            "  Batch 4,000  of  4,219.    Elapsed: 0:26:16.\n",
            "\n",
            "  Average training loss: 0.18\n",
            "  Training epcoh took: 0:27:42\n",
            "\n",
            "Running Validation...\n",
            "  Accuracy: 0.87\n",
            "  Validation took: 0:00:57\n",
            "\n",
            "Training complete!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hxlXEBA0WefL",
        "colab_type": "text"
      },
      "source": [
        "에폭마다 훈련셋과 검증셋을 반복하여 학습을 수행합니다. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6BVbl4Zjatzn",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **테스트셋 평가**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c5KHb6RkbHdj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        },
        "outputId": "b6c5da8a-c5ee-4190-b821-7fde62214c46"
      },
      "source": [
        "#시작 시간 설정\n",
        "t0 = time.time()\n",
        "\n",
        "# 평가모드로 변경\n",
        "model.eval()\n",
        "\n",
        "# 변수 초기화\n",
        "eval_loss, eval_accuracy = 0, 0\n",
        "nb_eval_steps, nb_eval_examples = 0, 0\n",
        "\n",
        "# 데이터로더에서 배치만큼 반복하여 가져옴\n",
        "for step, batch in enumerate(test_dataloader):\n",
        "    # 경과 정보 표시\n",
        "    if step % 100 == 0 and not step == 0:\n",
        "        elapsed = format_time(time.time() - t0)\n",
        "        print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(test_dataloader), elapsed))\n",
        "\n",
        "    # 배치를 GPU에 넣음\n",
        "    batch = tuple(t.to(device) for t in batch)\n",
        "    \n",
        "    # 배치에서 데이터 추출\n",
        "    b_input_ids, b_input_mask, b_labels = batch\n",
        "    \n",
        "    # 그래디언트 계산 안함\n",
        "    with torch.no_grad():     \n",
        "        # Forward 수행\n",
        "        outputs = model(b_input_ids, \n",
        "                        token_type_ids=None, \n",
        "                        attention_mask=b_input_mask)\n",
        "    \n",
        "    # 로스 구함\n",
        "    logits = outputs[0]\n",
        "\n",
        "    # CPU로 데이터 이동\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    label_ids = b_labels.to('cpu').numpy()\n",
        "    \n",
        "    # 출력 로짓과 라벨을 비교하여 정확도 계산\n",
        "    tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n",
        "    eval_accuracy += tmp_eval_accuracy\n",
        "    nb_eval_steps += 1\n",
        "\n",
        "print(\"\")\n",
        "print(\"Accuracy: {0:.2f}\".format(eval_accuracy/nb_eval_steps))\n",
        "print(\"Test took: {:}\".format(format_time(time.time() - t0)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  Batch   100  of  1,563.    Elapsed: 0:00:12.\n",
            "  Batch   200  of  1,563.    Elapsed: 0:00:25.\n",
            "  Batch   300  of  1,563.    Elapsed: 0:00:37.\n",
            "  Batch   400  of  1,563.    Elapsed: 0:00:49.\n",
            "  Batch   500  of  1,563.    Elapsed: 0:01:01.\n",
            "  Batch   600  of  1,563.    Elapsed: 0:01:14.\n",
            "  Batch   700  of  1,563.    Elapsed: 0:01:26.\n",
            "  Batch   800  of  1,563.    Elapsed: 0:01:38.\n",
            "  Batch   900  of  1,563.    Elapsed: 0:01:50.\n",
            "  Batch 1,000  of  1,563.    Elapsed: 0:02:03.\n",
            "  Batch 1,100  of  1,563.    Elapsed: 0:02:15.\n",
            "  Batch 1,200  of  1,563.    Elapsed: 0:02:27.\n",
            "  Batch 1,300  of  1,563.    Elapsed: 0:02:39.\n",
            "  Batch 1,400  of  1,563.    Elapsed: 0:02:52.\n",
            "  Batch 1,500  of  1,563.    Elapsed: 0:03:04.\n",
            "\n",
            "Accuracy: 0.87\n",
            "Test took: 0:03:11\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DbsNMA8Idc3K",
        "colab_type": "text"
      },
      "source": [
        "테스트셋의 정확도가 87%입니다. <BERT 톺아보기> 블로그에서는 같은 데이터로 88.7%를 달성하였습니다. 거기서는 한글 코퍼스로 사전훈련을 하여 새로운 모델을 만들었습니다. 반면에 우리는 BERT의 기본 모델인 bert-base-multilingual-cased를 사용했기 때문에 더 성능이 낮은 것 같습니다."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U7SzL1IBe1Dm",
        "colab_type": "text"
      },
      "source": [
        "<br>\n",
        "<br>\n",
        "\n",
        "# **새로운 문장 테스트**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Tb4v_VfEfGQB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 입력 데이터 변환\n",
        "def convert_input_data(sentences):\n",
        "\n",
        "    # BERT의 토크나이저로 문장을 토큰으로 분리\n",
        "    tokenized_texts = [tokenizer.tokenize(sent) for sent in sentences]\n",
        "\n",
        "    # 입력 토큰의 최대 시퀀스 길이\n",
        "    MAX_LEN = 128\n",
        "\n",
        "    # 토큰을 숫자 인덱스로 변환\n",
        "    input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "    \n",
        "    # 문장을 MAX_LEN 길이에 맞게 자르고, 모자란 부분을 패딩 0으로 채움\n",
        "    input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n",
        "\n",
        "    # 어텐션 마스크 초기화\n",
        "    attention_masks = []\n",
        "\n",
        "    # 어텐션 마스크를 패딩이 아니면 1, 패딩이면 0으로 설정\n",
        "    # 패딩 부분은 BERT 모델에서 어텐션을 수행하지 않아 속도 향상\n",
        "    for seq in input_ids:\n",
        "        seq_mask = [float(i>0) for i in seq]\n",
        "        attention_masks.append(seq_mask)\n",
        "\n",
        "    # 데이터를 파이토치의 텐서로 변환\n",
        "    inputs = torch.tensor(input_ids)\n",
        "    masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return inputs, masks"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C12NL1Fvgv4E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# 문장 테스트\n",
        "def test_sentences(sentences):\n",
        "\n",
        "    # 평가모드로 변경\n",
        "    model.eval()\n",
        "\n",
        "    # 문장을 입력 데이터로 변환\n",
        "    inputs, masks = convert_input_data(sentences)\n",
        "\n",
        "    # 데이터를 GPU에 넣음\n",
        "    b_input_ids = inputs.to(device)\n",
        "    b_input_mask = masks.to(device)\n",
        "            \n",
        "    # 그래디언트 계산 안함\n",
        "    with torch.no_grad():     \n",
        "        # Forward 수행\n",
        "        outputs = model(b_input_ids, \n",
        "                        token_type_ids=None, \n",
        "                        attention_mask=b_input_mask)\n",
        "\n",
        "    # 로스 구함\n",
        "    logits = outputs[0]\n",
        "\n",
        "    # CPU로 데이터 이동\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "\n",
        "    return logits"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZQezr0tljJlM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "83ee7417-05f1-4553-8188-75b31b277a50"
      },
      "source": [
        "logits = test_sentences(['연기는 별로지만 재미 하나는 끝내줌!'])\n",
        "\n",
        "print(logits)\n",
        "print(np.argmax(logits))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[-1.6218033  1.7296069]]\n",
            "1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-9MQ0SK0jofN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "c64d4970-3c48-4304-e56a-dd9ea1642bad"
      },
      "source": [
        "logits = test_sentences(['이딴게 영화냐 ㅉㅉ'])\n",
        "\n",
        "print(logits)\n",
        "print(np.argmax(logits))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[ 3.2080936 -3.184472 ]]\n",
            "0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iYYF_Bl9ewWh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# make function that determines positive or negative comment\n",
        "# 0 is negative, 1 is positive\n",
        "\n",
        "def good_or_bad(sentence):\n",
        "  logits = test_sentences([sentence])\n",
        "  determiner = np.argmax(logits)\n",
        "  return determiner"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nYL4kDpfexGi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "819e44b8-53c9-4271-b70b-e7c7d40fef6a"
      },
      "source": [
        "# 극찬 영화평 BERT로 확인해보기\n",
        "good_or_bad(\"와 개쩐다 정말 세계관 최강자들의 영화다\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hMXybtZwgJJx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f7f75b41-1320-4246-fa0a-94c7121204e3"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nsmc  sample_data  yonsei-exchange-life\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f_45wWVMhB6Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# define path for text dataset\n",
        "# abstract: 짧은 후기 제목만 있는 텍스트 데이터셋\n",
        "# specific: 문단으로 구성된 텍스트 데이터셋\n",
        "\n",
        "source = \"./data/univ_text_data\"\n",
        "source_abstract = \"./data/abstract\"\n",
        "source_specific = \"./data/specific\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lw7iU2fFgnkT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# define path for sentiment labeled dataset\n",
        "# abstract: 짧은 후기 제목만 있는 텍스트 데이터셋\n",
        "# specific: 문단으로 구성된 텍스트 데이터셋\n",
        "\n",
        "dest_abstract_sentiment = \"./data_sentiment/abstract\"\n",
        "dest_abstract_sentiment_bert = \"/data_sentiment/abstract_bert/\"\n",
        "dest_specific_sentiment = \"./data_sentiment/specific\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hsh0_E0UhJxQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# import file related library\n",
        "import os\n",
        "from os import fdopen, remove\n",
        "import glob\n",
        "from tempfile import mkstemp\n",
        "import shutil\n",
        "from shutil import move, copymode"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C8i0CLKlXy6U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "77e465f0-0910-4cbf-c4c5-9166abccfac0"
      },
      "source": [
        "%cd yonsei-exchange-program"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/yonsei-exchange-life\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DwjMhZvgX84R",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "ccb2d2b9-9da4-4b98-933f-54c26381d1bc"
      },
      "source": [
        "!ls"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "analyze_BERT_nsmc_corpus.ipynb\tgenerate_wordcloud.ipynb\n",
            "analyze_RNN.ipynb\t\tgenerate_wordcloud_konlpy.ipynb\n",
            "_archived\t\t\tgenerate_wordcloud_tfidf.ipynb\n",
            "best_model.h5\t\t\timg\n",
            "cluster_departments.ipynb\tLICENSE\n",
            "collect_reviews.ipynb\t\tREADME.md\n",
            "data\t\t\t\tREADME.pdf\n",
            "data_sentiment\t\t\trequirements.txt\n",
            "extract_keyword_tfidf.ipynb\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zXCREhqVhEqW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        },
        "outputId": "28da3a26-3dfe-4530-9017-0209e848f16a"
      },
      "source": [
        "# fetch all yonsei exchange review text datasets for each foreign universities\n",
        "abstract_yonsei_reviews = glob.glob(f\"{source_abstract}/*.csv\")\n",
        "abstract_yonsei_reviews[:5]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['./data/abstract/US000102_review_abstract.csv',\n",
              " './data/abstract/CZ000001_review_abstract.csv',\n",
              " './data/abstract/US000030_review_abstract.csv',\n",
              " './data/abstract/US000068_review_abstract.csv',\n",
              " './data/abstract/FR000002_review_abstract.csv']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 49
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uOr2mpkMhH9a",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "a4817163-c2b1-40fa-b606-353aa6a272a7"
      },
      "source": [
        "# look at sample dataset\n",
        "sample_file = abstract_yonsei_reviews[100]\n",
        "file_name = sample_file.split(\"/\")[-1]\n",
        "file_name_without_ext = file_name.split(\".\")[0]\n",
        "df_abstract = pd.read_csv(sample_file, encoding=\"utf-8\")\n",
        "df_abstract.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>제목</th>\n",
              "      <th>학과</th>\n",
              "      <th>과정</th>\n",
              "      <th>년도</th>\n",
              "      <th>href</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>86</td>\n",
              "      <td>ST CLOUD에서의 한학기</td>\n",
              "      <td>교육과학대학 체육교육학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2016</td>\n",
              "      <td>/partner/expReport.asp?id=13128&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>85</td>\n",
              "      <td>작은 마을 같았던 세인트 클라우드</td>\n",
              "      <td>실내건축학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>/partner/expReport.asp?id=12851&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>84</td>\n",
              "      <td>제가 느낀 것 위주로 써 보았습니다.(단점도 많습니다.)</td>\n",
              "      <td>스포츠레저학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>/partner/expReport.asp?id=12570&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>83</td>\n",
              "      <td>SCSU에서의 봄학기</td>\n",
              "      <td>화공생명공학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>/partner/expReport.asp?id=12302&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>82</td>\n",
              "      <td>Saint Cloud State University 에서의 한 학기</td>\n",
              "      <td>스포츠레저학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2014</td>\n",
              "      <td>/partner/expReport.asp?id=11813&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   No  ...                                           href\n",
              "0  86  ...  /partner/expReport.asp?id=13128&page=1&bgbn=R\n",
              "1  85  ...  /partner/expReport.asp?id=12851&page=1&bgbn=R\n",
              "2  84  ...  /partner/expReport.asp?id=12570&page=1&bgbn=R\n",
              "3  83  ...  /partner/expReport.asp?id=12302&page=1&bgbn=R\n",
              "4  82  ...  /partner/expReport.asp?id=11813&page=1&bgbn=R\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2xMefdi8iCnq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# labeling reviews in dataframe with BERT model\n",
        "def label_review_bert(series_object):\n",
        "  list_scores = []\n",
        "  list_titles = series_object.to_list()\n",
        "  for item in list_titles:\n",
        "    score = good_or_bad(item)\n",
        "    list_scores.append(score)\n",
        "  return list_scores"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jitQWRkniH6C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_abstract[\"BERT_SCORE\"] = label_review_bert(df_abstract[\"제목\"])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ota6FZZMjAye",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        },
        "outputId": "d27a38e3-278e-48c7-9bf6-03a091fa4eb6"
      },
      "source": [
        "df_abstract[[\"제목\", \"학과\", \"과정\", \"년도\", \"BERT_SCORE\"]].head(5)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>제목</th>\n",
              "      <th>학과</th>\n",
              "      <th>과정</th>\n",
              "      <th>년도</th>\n",
              "      <th>BERT_SCORE</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>ST CLOUD에서의 한학기</td>\n",
              "      <td>교육과학대학 체육교육학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2016</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>작은 마을 같았던 세인트 클라우드</td>\n",
              "      <td>실내건축학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>제가 느낀 것 위주로 써 보았습니다.(단점도 많습니다.)</td>\n",
              "      <td>스포츠레저학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>SCSU에서의 봄학기</td>\n",
              "      <td>화공생명공학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2015</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Saint Cloud State University 에서의 한 학기</td>\n",
              "      <td>스포츠레저학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2014</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                      제목             학과  과정    년도  BERT_SCORE\n",
              "0                        ST CLOUD에서의 한학기  교육과학대학 체육교육학과  학부  2016           1\n",
              "1                     작은 마을 같았던 세인트 클라우드         실내건축학과  학부  2015           0\n",
              "2        제가 느낀 것 위주로 써 보았습니다.(단점도 많습니다.)        스포츠레저학과  학부  2015           0\n",
              "3                            SCSU에서의 봄학기        화공생명공학과  학부  2015           1\n",
              "4  Saint Cloud State University 에서의 한 학기        스포츠레저학과  학부  2014           1"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IB9zQVh2aK0e",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_abstract.to_csv(f\".{dest_abstract_sentiment_bert}/sample_BERT.csv\",encoding=\"utf-8\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0ZiMZz6xaafO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!ls ./data_sentiment/abstract_bert/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5Zf_clgWFFko",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        },
        "outputId": "89c15802-1b7a-4382-d033-bbd8bace2916"
      },
      "source": [
        "# check for dataframes with nan values\n",
        "for csv_file in abstract_yonsei_reviews:\n",
        "  df_open = pd.read_csv(csv_file, encoding=\"utf-8\")\n",
        "  if df_open.isnull().values.any() == True:\n",
        "    print(csv_file)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "./data/abstract/US000085_review_abstract.csv\n",
            "./data/abstract/US000207_review_abstract.csv\n",
            "./data/abstract/CA000020_review_abstract.csv\n",
            "./data/abstract/JP000005_review_abstract.csv\n",
            "./data/abstract/DE000015_review_abstract.csv\n",
            "./data/abstract/US000254_review_abstract.csv\n",
            "./data/abstract/US000009_review_abstract.csv\n",
            "./data/abstract/JP000014_review_abstract.csv\n",
            "./data/abstract/US000250_review_abstract.csv\n",
            "./data/abstract/US000019_review_abstract.csv\n",
            "./data/abstract/JP000030_review_abstract.csv\n",
            "./data/abstract/NZ000005_review_abstract.csv\n",
            "./data/abstract/US000267_review_abstract.csv\n",
            "./data/abstract/JP000024_review_abstract.csv\n",
            "./data/abstract/JP000002_review_abstract.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3oWN_zYcF-d7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_with_nan = \"./data/abstract/US000254_review_abstract.csv\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uSEG2j2Fm0U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168
        },
        "outputId": "2ab04ad3-d24f-4af6-d3a6-6cbb505ffb83"
      },
      "source": [
        "# look into datframe\n",
        "df = pd.read_csv(df_with_nan, encoding=\"utf-8\")\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>제목</th>\n",
              "      <th>학과</th>\n",
              "      <th>과정</th>\n",
              "      <th>년도</th>\n",
              "      <th>href</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4</td>\n",
              "      <td>University of Wyoming 파견보고서</td>\n",
              "      <td>기계공학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2019</td>\n",
              "      <td>/partner/expReport.asp?id=16350&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>University of Wyoming</td>\n",
              "      <td>경영학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2005-2006</td>\n",
              "      <td>/partner/expReport.asp?id=2259&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>University of Wyoming</td>\n",
              "      <td>정치외교학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2003</td>\n",
              "      <td>/partner/expReport.asp?id=1051&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "      <td>사회학전공</td>\n",
              "      <td>학부</td>\n",
              "      <td>1998</td>\n",
              "      <td>/partner/expReport.asp?id=14&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   No  ...                                           href\n",
              "0   4  ...  /partner/expReport.asp?id=16350&page=1&bgbn=R\n",
              "1   3  ...   /partner/expReport.asp?id=2259&page=1&bgbn=R\n",
              "2   2  ...   /partner/expReport.asp?id=1051&page=1&bgbn=R\n",
              "3   1  ...     /partner/expReport.asp?id=14&page=1&bgbn=R\n",
              "\n",
              "[4 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BvqY1L_wFvf6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "0ef96ca6-06e9-4402-8422-d4c2f0b67649"
      },
      "source": [
        "# drop only when title is absent\n",
        "df = df.dropna(subset=[\"제목\"])\n",
        "df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>No</th>\n",
              "      <th>제목</th>\n",
              "      <th>학과</th>\n",
              "      <th>과정</th>\n",
              "      <th>년도</th>\n",
              "      <th>href</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4</td>\n",
              "      <td>University of Wyoming 파견보고서</td>\n",
              "      <td>기계공학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2019</td>\n",
              "      <td>/partner/expReport.asp?id=16350&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>University of Wyoming</td>\n",
              "      <td>경영학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2005-2006</td>\n",
              "      <td>/partner/expReport.asp?id=2259&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>University of Wyoming</td>\n",
              "      <td>정치외교학과</td>\n",
              "      <td>학부</td>\n",
              "      <td>2003</td>\n",
              "      <td>/partner/expReport.asp?id=1051&amp;page=1&amp;bgbn=R</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   No  ...                                           href\n",
              "0   4  ...  /partner/expReport.asp?id=16350&page=1&bgbn=R\n",
              "1   3  ...   /partner/expReport.asp?id=2259&page=1&bgbn=R\n",
              "2   2  ...   /partner/expReport.asp?id=1051&page=1&bgbn=R\n",
              "\n",
              "[3 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ne19S3KjB2-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# reflect bert score on dataframe, and save it as csv file on different directory\n",
        "def make_abstract_df_with_bert_score(csv_file):\n",
        "    file_name = csv_file.split(\"/\")[-1]\n",
        "    file_name_without_ext = file_name.split(\".\")[0]\n",
        "    df = pd.read_csv(csv_file, encoding=\"utf-8\")\n",
        "    \n",
        "    # DROPNA 안 하면 제목 기반으로 점수를 못 매김\n",
        "    df = df.dropna(subset=[\"제목\"])\n",
        "    cleaned_series = df['제목']\n",
        "    df[\"BERT_SCORE\"] = label_review_bert(cleaned_series)\n",
        "    df.to_csv(f\".{dest_abstract_sentiment_bert}/{file_name_without_ext}_BERT.csv\",encoding=\"utf-8\", index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "csJS6ArFsvzp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "1b3c1b93-e1a9-48b2-9fd8-9bb44e92f362"
      },
      "source": [
        "# number of universities datframe (consisted of abstract reviews)\n",
        "len(abstract_yonsei_reviews)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "470"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yEGtYknjs6OU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# label, save for all exchange university's reviews \n",
        "for csv_file in abstract_yonsei_reviews:\n",
        "  make_abstract_df_with_bert_score(csv_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFgi0niEK1zo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e8c5a349-9465-42fa-a4df-a690942f88ef"
      },
      "source": [
        "len(os.listdir(f\".{dest_abstract_sentiment_bert}\"))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "941"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 76
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j5HohBbDiHka",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "2cebb665-2d55-463d-f8fb-40dce11c4364"
      },
      "source": [
        "!git push --set-upstream origin master"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Counting objects: 329, done.\n",
            "Delta compression using up to 4 threads.\n",
            "Compressing objects: 100% (329/329), done.\n",
            "Writing objects: 100% (329/329), 356.75 KiB | 7.75 MiB/s, done.\n",
            "Total 329 (delta 2), reused 0 (delta 0)\n",
            "remote: Resolving deltas: 100% (2/2), completed with 2 local objects.\u001b[K\n",
            "To https://github.com/snoop2head/yonsei-exchange-life.git\n",
            "   d4cfb3a..421c598  master -> master\n",
            "Branch 'master' set up to track remote branch 'master' from 'origin'.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bPgy-QOQi81y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git pull"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8zROnu8CqD20",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        },
        "outputId": "53f58a14-ef93-42bd-dcc3-bf496f9037ad"
      },
      "source": [
        "!ls -a"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".\t\t\t   collect_reviews.ipynb\t    .git\n",
            "..\t\t\t   data\t\t\t\t    .gitignore\n",
            "analyze_BERT_nsmc.ipynb    data_sentiment\t\t    img\n",
            "analyze_RNN.ipynb\t   extract_keyword_tfidf.ipynb\t    LICENSE\n",
            "_archived\t\t   generate_wordcloud.ipynb\t    README.md\n",
            "best_model.h5\t\t   generate_wordcloud_konlpy.ipynb  README.pdf\n",
            "cluster_departments.ipynb  generate_wordcloud_tfidf.ipynb   requirements.txt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m2vZ0IlTqKno",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}