4/code/BiLSTM_CRF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from torchtext.legacy.data import Field, Dataset, Example,BucketIterator,Iterator\n",
    "from torchtext.vocab import Vectors\n",
    "\n",
    "from sklearn.metrics import f1_score,precision_score,recall_score\n",
    "\n",
    "import torch\n",
    "import torch.autograd as autograd\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import codecs\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "from torchcrf import CRF\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import time\n",
    "\n",
    "from torchtext.legacy.datasets import SequenceTaggingDataset\n",
    "# from BiLSTM_CRF import BiLSTM_CRF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x268f49449f0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "DEVICE=torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "SEED=2022\n",
    "torch.manual_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# label 如下所示\n",
    "# label_dict={\n",
    "#     'O':0,\n",
    "#     'B-ORG':1,\n",
    "#     'I-ORG':2,\n",
    "#     'B-MISC':3,\n",
    "#     'I-MISC':4,\n",
    "#     'B-PER':5,\n",
    "#     'I-PER':6,\n",
    "#     'B-LOC':7,\n",
    "#     'I-LOC':8,\n",
    "# }\n",
    "\n",
    "def zero_digits(s):\n",
    "    \"\"\"\n",
    "    Replace every digit in a string by a zero.\n",
    "    \"\"\"\n",
    "    return re.sub('\\d', '0', s)\n",
    "\n",
    "def load_sentences(path, lower, zeros):\n",
    "    \"\"\"\n",
    "    Load sentences. A line must contain at least a word and its tag.\n",
    "    Sentences are separated by empty lines.\n",
    "    \"\"\"\n",
    "    text=[]\n",
    "    label=[]\n",
    "    text_list=[]\n",
    "    label_list=[]\n",
    "    for line in codecs.open(path, 'r', 'utf-8'):\n",
    "        line = zero_digits(line.rstrip()) if zeros else line.rstrip()\n",
    "        if not line:\n",
    "            text_list.append(text)\n",
    "            label_list.append(label)\n",
    "            label=[]\n",
    "            text=[]\n",
    "        else:\n",
    "            sample=line.split()\n",
    "            word=sample[0]\n",
    "            nertag=sample[-1]\n",
    "            text.append(word)\n",
    "            label.append(nertag)\n",
    "    df_data=pd.DataFrame()\n",
    "    df_data['text']=text_list\n",
    "    df_data['label']=label_list\n",
    "\n",
    "    return df_data\n",
    "\n",
    "def laod_dataset(df,textfield,labelfield):\n",
    "\n",
    "    examples=[]\n",
    "    fields = [('text', textfield), ('label', labelfield)]\n",
    "    for index,row in df.iterrows():\n",
    "        examples.append(Example.fromlist([row['text'], row['label']], fields))\n",
    "    \n",
    "    dataset = Dataset(examples, fields)\n",
    "\n",
    "    return dataset\n",
    "\n",
    "root_path='../data/'\n",
    "train_path=root_path+'eng.train'\n",
    "valid_path=root_path+'eng.testa'\n",
    "test_path=root_path+'eng.testb'\n",
    "\n",
    "train_df=load_sentences(train_path,lower=True,zeros=True)\n",
    "valid_df=load_sentences(valid_path,lower=True,zeros=True)\n",
    "test_df=load_sentences(test_path,lower=True,zeros=True)\n",
    "\n",
    "# inlcude_length:return (length,text),use for the mask of CRF and LSTM\n",
    "TEXT=Field(sequential=True,use_vocab=True,include_lengths=True)\n",
    "LABEL = Field(sequential=True,use_vocab=True,include_lengths=True)\n",
    "\n",
    "\n",
    "\n",
    "train_data=laod_dataset(train_df,textfield=TEXT,labelfield=LABEL)\n",
    "valid_data=laod_dataset(valid_df,textfield=TEXT,labelfield=LABEL)\n",
    "test_data=laod_dataset(test_df,textfield=TEXT,labelfield=LABEL)\n",
    "\n",
    "EMBEDDING_DIM=100\n",
    "BATCH_SIZE=128\n",
    "MIN_FREQ=2\n",
    "\n",
    "TEXT.build_vocab(\n",
    "    train_data,\n",
    "    min_freq=MIN_FREQ,\n",
    "    vectors=\"glove.6B.{}d\".format(EMBEDDING_DIM),\n",
    "    unk_init=torch.Tensor.normal_\n",
    ")\n",
    "LABEL.build_vocab(train_data)\n",
    "\n",
    "train_iterator=BucketIterator(\n",
    "    train_data,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device=DEVICE,\n",
    "    sort_key=lambda x: len(x.TEXT)\n",
    ")\n",
    "\n",
    "test_iterator=BucketIterator(\n",
    "    test_data,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device=DEVICE,\n",
    "    sort_key=lambda x: len(x.TEXT)\n",
    ")\n",
    "\n",
    "valid_iterator=BucketIterator(\n",
    "    valid_data,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    device=DEVICE,\n",
    "    sort_key=lambda x: len(x.TEXT)\n",
    ")\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLSTM_CRF(nn.Module):\n",
    "    #定义模型中使用的所有层\n",
    "    def __init__(self, vocab_size, embedding_dim=100, hidden_dim=120, num_tags=10):\n",
    "        #构造函数\n",
    "        super().__init__()\n",
    "        #embeddding层\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        #lstm层\n",
    "        self.lstm = nn.LSTM(embedding_dim, \n",
    "                           hidden_dim,\n",
    "                           num_layers=1,\n",
    "                           bidirectional=True, \n",
    "\n",
    "                           batch_first=True)\n",
    "        self.crf=CRF(num_tags,batch_first=True)\n",
    "        self.fc = nn.Linear(hidden_dim*2, num_tags)\n",
    "\n",
    "    def _get_lstm_features(self, text,text_lengths):\n",
    "        embedded=self.embedding(text) #添加了padding的embedding结果\n",
    "\n",
    "        packed_embedded=pack_padded_sequence(embedded,text_lengths,batch_first=True,enforce_sorted=False)\n",
    "        packed_output,_=self.lstm(packed_embedded) #没有padding的输出\n",
    "        output,output_lengths=pad_packed_sequence(packed_output, batch_first=True) #返回输出，和没有padding的\n",
    "\n",
    "        output=self.fc(output)\n",
    "        return output,output_lengths\n",
    "\n",
    "    def neg_log_likelihood(self,text,tags,text_lengths,mask):\n",
    "              \n",
    "        out,_=self._get_lstm_features(text,text_lengths)\n",
    "        loss=self.crf(out,tags,mask=mask)\n",
    "        return -loss\n",
    "\n",
    "    def forward(self, text,text_lengths):\n",
    "        out,_=self._get_lstm_features(text,text_lengths)\n",
    "        predict=self.crf.decode(out)\n",
    "        return predict\n",
    "\n",
    "INPUT_DIM = len(TEXT.vocab)\n",
    "model = BiLSTM_CRF(INPUT_DIM).to(DEVICE)\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.01, weight_decay=1e-4)\n",
    "\n",
    "pretrained_embeddings = TEXT.vocab.vectors\n",
    "model.embedding.weight.data.copy_(pretrained_embeddings)\n",
    "model.embedding.weight.requires_grad = True\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def _metrics(predict,tags,text_lengths):\n",
    "    f1_list=[]\n",
    "    precision_list=[]\n",
    "    recall_list=[]\n",
    "\n",
    "    for i in range(len(text_lengths)):\n",
    "        y_true=tags[i,0:text_lengths[i]].cpu()\n",
    "        y_pre=predict[i][0:text_lengths[i]]\n",
    "        recall_list.append(recall_score(y_true,y_pre,average='macro',zero_division=0))\n",
    "        precision_list.append(precision_score(y_true,y_pre,average='macro',zero_division=0))\n",
    "        f1_list.append(f1_score(y_true,y_pre,average='macro',zero_division=0))\n",
    "    f1=np.mean(f1_list)\n",
    "    precision=np.mean(precision_list)\n",
    "    recall= np.mean(recall_list)\n",
    "\n",
    "    return f1,precision,recall\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, iterator, optimizer):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "\n",
    "    model.train() \n",
    "    epoch_f1=0\n",
    "    epoch_recall=0\n",
    "    epoch_precision=0\n",
    "    for batch in tqdm(iterator):\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        text=batch.text[0]\n",
    "        text=text.permute(1,0)\n",
    "        text_lengths=batch.text[1].cpu()\n",
    "\n",
    "        tags=batch.label[0]\n",
    "        tags=tags.permute(1,0)\n",
    "        mask=torch.zeros(tags.shape,dtype=torch.uint8)\n",
    "        for i in range(len(text_lengths)):\n",
    "            try:\n",
    "                pad_len=text_lengths[i]\n",
    "                mask[i,0:pad_len]=1\n",
    "            except:\n",
    "                print(i,len(text_lengths))\n",
    "        mask=mask.to(DEVICE)\n",
    "\n",
    "        loss=model.neg_log_likelihood(text,tags,text_lengths,mask)\n",
    "\n",
    "        predict=model(text,text_lengths)\n",
    "\n",
    "        loss.backward()\n",
    "        \n",
    "        f1,precision,recall=_metrics(predict,tags,text_lengths)\n",
    "        \n",
    "        optimizer.step()\n",
    "    \n",
    "        epoch_f1+=f1\n",
    "        epoch_precision+=precision\n",
    "        epoch_recall+=recall     \n",
    "        epoch_loss += loss.item()\n",
    "\n",
    "    epoch_f1/=len(iterator)\n",
    "    epoch_recall/=len(iterator)\n",
    "    epoch_precision/=len(iterator)\n",
    "    epoch_loss/=len(iterator)\n",
    "        \n",
    "    return epoch_loss ,epoch_f1,epoch_precision,epoch_recall\n",
    "\n",
    "def evaluate(model, iterator):\n",
    "    \n",
    "    epoch_loss = 0\n",
    "    epoch_f1=0\n",
    "    epoch_recall=0\n",
    "    epoch_precision=0\n",
    "\n",
    "    model.eval() \n",
    "    \n",
    "    with torch.no_grad():\n",
    "    \n",
    "        for batch in tqdm(iterator):\n",
    "\n",
    "            optimizer.zero_grad()\n",
    "            text=batch.text[0]\n",
    "            text=text.permute(1,0)\n",
    "            text_lengths=batch.text[1].cpu()\n",
    "            tags=batch.label[0]\n",
    "            tags=tags.permute(1,0)\n",
    "            mask=torch.zeros(tags.shape,dtype=torch.uint8)\n",
    "            for i in range(len(text_lengths)):\n",
    "                try:\n",
    "                    pad_len=text_lengths[i]\n",
    "                    mask[i,0:pad_len]=1\n",
    "                except:\n",
    "                    print(i,len(text))\n",
    "            mask=mask.to(DEVICE)\n",
    "            loss=model.neg_log_likelihood(text,tags,text_lengths,mask)\n",
    "            epoch_loss += loss.item()\n",
    "            predict=model(text,text_lengths)\n",
    "            \n",
    "            f1,precision,recall=_metrics(predict,tags,text_lengths)\n",
    "            \n",
    "            optimizer.step()\n",
    "        \n",
    "            epoch_f1+=f1\n",
    "            epoch_precision+=precision\n",
    "            epoch_recall+=recall     \n",
    "\n",
    "    epoch_f1/=len(iterator)\n",
    "    epoch_recall/=len(iterator)\n",
    "    epoch_precision/=len(iterator)\n",
    "    epoch_loss/=len(iterator)\n",
    "        \n",
    "    return epoch_loss,epoch_f1,epoch_precision,epoch_recall\n",
    "\n",
    "def epoch_time(start_time, end_time):\n",
    "    elapsed_time = end_time - start_time\n",
    "    elapsed_mins = int(elapsed_time / 60)\n",
    "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
    "    return elapsed_mins, elapsed_secs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/118 [00:00<?, ?it/s]C:\\ProgramData\\Anaconda3\\envs\\torch\\lib\\site-packages\\torchcrf\\__init__.py:249: UserWarning: where received a uint8 condition tensor. This behavior is deprecated and will be removed in a future version of PyTorch. Use a boolean condition instead. (Triggered internally at  ..\\aten\\src\\ATen\\native\\TensorCompare.cpp:255.)\n",
      "  score = torch.where(mask[i].unsqueeze(1), next_score, score)\n",
      "100%|██████████| 118/118 [01:03<00:00,  1.85it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 01 | Epoch Time: 1m 18s\n",
      "\tTrain Loss: 5799.635 | Train F1: 65.94% | Train Precision: 67.26% | Train Recall: 66.74% |\n",
      "\t Val. Loss: 880.835 |  Val. F1: 75.93% | Val Precision: 77.29% | Val Recall: 76.64% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:03<00:00,  1.86it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 02 | Epoch Time: 1m 18s\n",
      "\tTrain Loss: 754.719 | Train F1: 82.33% | Train Precision: 83.56% | Train Recall: 82.76% |\n",
      "\t Val. Loss: 528.503 |  Val. F1: 80.87% | Val Precision: 81.87% | Val Recall: 81.48% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:03<00:00,  1.87it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.88it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 03 | Epoch Time: 1m 18s\n",
      "\tTrain Loss: 230.303 | Train F1: 90.20% | Train Precision: 90.84% | Train Recall: 90.54% |\n",
      "\t Val. Loss: 480.436 |  Val. F1: 83.92% | Val Precision: 84.80% | Val Recall: 84.45% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:03<00:00,  1.87it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 04 | Epoch Time: 1m 17s\n",
      "\tTrain Loss: 147.948 | Train F1: 92.86% | Train Precision: 93.34% | Train Recall: 93.15% |\n",
      "\t Val. Loss: 401.246 |  Val. F1: 85.14% | Val Precision: 85.82% | Val Recall: 85.71% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.88it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 05 | Epoch Time: 1m 17s\n",
      "\tTrain Loss: 118.900 | Train F1: 94.23% | Train Precision: 94.60% | Train Recall: 94.51% |\n",
      "\t Val. Loss: 409.979 |  Val. F1: 84.86% | Val Precision: 85.48% | Val Recall: 85.63% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.88it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.89it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 06 | Epoch Time: 1m 17s\n",
      "\tTrain Loss: 101.016 | Train F1: 94.83% | Train Precision: 95.17% | Train Recall: 95.09% |\n",
      "\t Val. Loss: 392.830 |  Val. F1: 85.28% | Val Precision: 85.97% | Val Recall: 85.91% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:03<00:00,  1.86it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 07 | Epoch Time: 1m 18s\n",
      "\tTrain Loss: 93.250 | Train F1: 95.29% | Train Precision: 95.62% | Train Recall: 95.52% |\n",
      "\t Val. Loss: 403.610 |  Val. F1: 85.66% | Val Precision: 86.34% | Val Recall: 86.27% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.89it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 08 | Epoch Time: 1m 17s\n",
      "\tTrain Loss: 85.647 | Train F1: 95.41% | Train Precision: 95.72% | Train Recall: 95.65% |\n",
      "\t Val. Loss: 414.739 |  Val. F1: 85.98% | Val Precision: 86.73% | Val Recall: 86.48% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 09 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 80.422 | Train F1: 95.64% | Train Precision: 95.94% | Train Recall: 95.87% |\n",
      "\t Val. Loss: 389.016 |  Val. F1: 86.16% | Val Precision: 86.92% | Val Recall: 86.62% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 78.046 | Train F1: 95.78% | Train Precision: 96.05% | Train Recall: 96.02% |\n",
      "\t Val. Loss: 391.719 |  Val. F1: 86.06% | Val Precision: 86.78% | Val Recall: 86.63% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.89it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.92it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 74.868 | Train F1: 95.94% | Train Precision: 96.19% | Train Recall: 96.19% |\n",
      "\t Val. Loss: 420.893 |  Val. F1: 86.05% | Val Precision: 86.77% | Val Recall: 86.49% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 72.597 | Train F1: 96.00% | Train Precision: 96.26% | Train Recall: 96.24% |\n",
      "\t Val. Loss: 392.110 |  Val. F1: 86.06% | Val Precision: 86.68% | Val Recall: 86.70% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 71.026 | Train F1: 96.13% | Train Precision: 96.38% | Train Recall: 96.36% |\n",
      "\t Val. Loss: 414.120 |  Val. F1: 86.11% | Val Precision: 86.92% | Val Recall: 86.53% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:01<00:00,  1.91it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 70.904 | Train F1: 96.08% | Train Precision: 96.35% | Train Recall: 96.30% |\n",
      "\t Val. Loss: 391.371 |  Val. F1: 86.17% | Val Precision: 86.94% | Val Recall: 86.63% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:01<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 68.528 | Train F1: 96.16% | Train Precision: 96.39% | Train Recall: 96.40% |\n",
      "\t Val. Loss: 383.517 |  Val. F1: 85.95% | Val Precision: 86.55% | Val Recall: 86.59% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.89it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 67.656 | Train F1: 96.17% | Train Precision: 96.41% | Train Recall: 96.40% |\n",
      "\t Val. Loss: 391.921 |  Val. F1: 85.68% | Val Precision: 86.37% | Val Recall: 86.25% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:01<00:00,  1.91it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.95it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 66.517 | Train F1: 96.33% | Train Precision: 96.57% | Train Recall: 96.54% |\n",
      "\t Val. Loss: 382.063 |  Val. F1: 85.98% | Val Precision: 86.58% | Val Recall: 86.58% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:01<00:00,  1.91it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.91it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 65.194 | Train F1: 96.43% | Train Precision: 96.66% | Train Recall: 96.65% |\n",
      "\t Val. Loss: 388.119 |  Val. F1: 86.48% | Val Precision: 87.16% | Val Recall: 86.98% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.90it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.94it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 69.250 | Train F1: 96.28% | Train Precision: 96.51% | Train Recall: 96.49% |\n",
      "\t Val. Loss: 386.934 |  Val. F1: 86.51% | Val Precision: 87.06% | Val Recall: 87.12% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 118/118 [01:02<00:00,  1.89it/s]\n",
      "100%|██████████| 28/28 [00:14<00:00,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 | Epoch Time: 1m 16s\n",
      "\tTrain Loss: 65.838 | Train F1: 96.33% | Train Precision: 96.58% | Train Recall: 96.53% |\n",
      "\t Val. Loss: 392.938 |  Val. F1: 86.66% | Val Precision: 87.29% | Val Recall: 87.20% |\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "N_EPOCHS = 20\n",
    "\n",
    "best_valid_f1 = float('inf')\n",
    "\n",
    "\n",
    "train_loss_list=[]\n",
    "train_f1_list=[]\n",
    "train_precision_list=[]\n",
    "train_recall_list=[]\n",
    "\n",
    "valid_loss_list=[]\n",
    "valid_f1_list=[]\n",
    "valid_precision_list=[]\n",
    "valid_recall_list=[]\n",
    "\n",
    "\n",
    "\n",
    "for epoch in range(N_EPOCHS):\n",
    "\n",
    "    start_time = time.time()\n",
    "    \n",
    "    train_loss,train_f1,train_precision,train_recall = train(model, train_iterator, optimizer)\n",
    "    valid_loss,valid_f1,valid_precision,valid_recall = evaluate(model, valid_iterator)\n",
    "    \n",
    "    end_time = time.time()\n",
    "\n",
    "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
    "    \n",
    "    if best_valid_f1 < valid_f1:\n",
    "        best_valid_f1 = valid_f1\n",
    "        torch.save(model.state_dict(), 'tut1-model.pt')\n",
    "    \n",
    "    train_loss_list.append(train_loss)\n",
    "    valid_loss_list.append(valid_loss)\n",
    "\n",
    "    train_f1_list.append(train_f1)\n",
    "    train_precision_list.append(train_precision)\n",
    "    train_recall_list.append(train_recall)\n",
    "\n",
    "    valid_f1_list.append(valid_f1)\n",
    "    valid_precision_list.append(valid_precision)\n",
    "    valid_recall_list.append(valid_recall)\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')\n",
    "    print(f'\\tTrain Loss: {train_loss:.3f} | Train F1: {train_f1*100:.2f}% | Train Precision: {train_precision*100:.2f}% | Train Recall: {train_recall*100:.2f}% |')\n",
    "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. F1: {valid_f1*100:.2f}% | Val Precision: {valid_precision*100:.2f}% | Val Recall: {valid_recall*100:.2f}% |')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "![](../pic/BiLSTM_CRF-loss-20.jpg)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmNElEQVR4nO3deZwlZX3v8c/3nO6enqVnY5oBBhSMBASMSEYkxh0ckBiHJGogvmRQEiTizXbvNWgWomiumhuNZsEgoIMhghuIBi8ZQaNJZJmRfXMGBJlxmGlm33p6Ob/7Rz2nu/pM9/Tpme7TPae+79frvKrqqaeqnlNd/avnPPVUlSICMzMrhtJkF8DMzBrHQd/MrEAc9M3MCsRB38ysQBz0zcwKxEHfzKxAHPTNzArEQd8aRtLnJP3FZJdjKpH0fUm/O9nlsOJw0LdxJelpSXsk7ZS0RdK/SToGICIujYgrU77XS1o7wjqOlvR1Sc9L2ibpYUkXSXpNWu9OSbskRW56p6QXpCAakl5Ws86bU/rr6/gOvyjpq7ntPyjpTySVJR1bs92nJV2+n31Q/Rx1oPt0vEg6XNKXJf08fa//kvTKmjy/I+mZtH9vkTR/ssprE8NB3ybCr0fELOBIYAPw92Nc/kvAs8ALgcOAdwEbIuKHETErrfvklHduNS0ifpbSfgJcWF2ZpMOAXwG6RtuwpF8A7k7bf2lEzAHeDiwGOnJZ56ZyvA34C0lvqlnVr+fKNSsifj6mPTAxZgH3Ar8MzAeWA/8maRaApJOBfybb3wuB3cA/TU5RbaI46NuEiYhu4GvASQCSvijpo3Us+grgixGxKyL6IuK+iPjOGDZ9A/Dbkspp+gLgZqCnjmU/DPx3RPxJRKxP3+OJiPidiNhamzkiVgKPAKeOoXzDklSS9Oeppr1R0vWS5qR57ZL+RdImSVsl3StpYZp3kaSnJO2Q9FNJ7xxu/RHxVER8KiLWR0R/RFwNtAEnpCzvBL4VET+IiJ3AXwC/KaljuPXZoclB3yaMpBnAbwN3jXHRu4B/lHS+pBccwKZ/DjwKLEnTFwLX17nsWWQnqrpIOgM4BVgzlgKO4KL0eQPwIrKa+T+kecuAOcAxZL9+LgX2SJoJfBZ4c0R0AK8C7q+z7KeSBf1q2U8GHqjOj4gnyU6Uv3jA38imHAd9mwi3SNoKbAPeBPzNGJd/O/BDsprmTyXdL+kVY1zH9cCFkk4ka4r5UZ3LHQasryPf85L2AD8iawK5pWb+LalGvlVS7byRvBP4VKqR7wQ+CJwvqQXoTWV7caqlr4qI7Wm5CnCKpOmpFv/IaBuSNJusGe3DEbEtJc8i+5vlbWNos5Yd4hz0bSKcFxFzgXbg/cB/SDqi3oUjYktEXB4RJ5O1Ld9PFkQ1hjJ8A3hj2v6XxrDcJrJrEaNZQBYk/yfweqC1Zv55ETE3fc6Dgd5L1Qu7HxpmnUcBz+SmnwFayPbBl4DbgRvThdhPSmqNiF1kv6YuBdanC+cnpu09ktvea6orlTQd+BZwV0T8n9z2dgKza8o0G9hRx/6wQ4SDvk2YVCP9BtAPvPoA1/E88H/JAmLdPUkiYjfwHeD3GVvQ/y7wW3Vuoz8iPgV0A++rI/+luQu7fz1Mlp+TXbyuegHQR3YRuzciPhwRJ5E14byFdLE6Im6PiDeRnaweBz6f0k/Obe+HAJKmkf0qWQu8t2b7jwADvZ4kvQiYRnZh3JqEg75NGGWWAvOAx0bI017zkaRPSDpFUku6iPj7wJqI2DTGInwIeF1EPD2GZa4AXiXpb6q/TiS9OF1EnTvCMh8HPiCpfYzlq/Vl4I8lHZd61Pw1cFNE9El6g6SXpovT28maeyqSFkpamtr295LV1ivDrVxSK9n1ij3AsoiozXcD8OvKusbOBD4CfCMiXNNvIg76NhG+JWknWXD6GFmAGa6deRFZAMp/fgGYQdbbZivwFFnt961jLURE/Dwi/nOMyzxJ1r3zWOARSduArwMrGbmZ49+ALcDvjbWMNa4j+1XyA+CnZL8g/keadwRZwN5OdgL9j5S3BPwJ2a+EzcDryE6Sw6n+QlgCbK1t+kl/o0vJgv9Gsrb8UX/B2KFFfnOWmVlxuKZvZlYgDvpWOJK+U/OIhP31qDFrKm7eMTMrkJZ6MqVeC9eQ3XkYwHuAJ4CbyC54PQ28IyK2pL7UnwHOJXt2x0UR8eO0nmXAn6fVfjQilu9vuwsWLIhjjz12TF/IzKzoVq1a9XxEdA43r66avqTlwA8j4hpJbWS9Kz4EbI6Ijyt7yuC8iPhTSeeS9Tg4F3gl8JmIeGV6Wt9KsgdXBbAK+OWI2DLSdhcvXhwrV64c05c1Mys6SasiYvFw80Zt008PfHotcC1ARPSkB08tJXtKH2l4XhpfClwfmbuAuZKOBM4GVkTE5hToVwDnHPC3MjOzMavnQu5xZI+k/YKk+yRdk27cWFh9CiHwHNmt4pD1vX42t/zalDZS+hCSLpG0UtLKrq5Rn4RrZmZjUE/QbwFOA66KiJcDu4AhL42IrI1oXK4IR8TVEbE4IhZ3dg7bJGVmZgeonqC/FlgbEXen6a+RnQQ2pGYb0nBjmr+O7PGvVUentJHSzcysQUYN+hHxHPCspOqLFs4ke1b5rWTP+CYNv5nGbyV7pK3Ss8a3pWag24ElkuZJmkd2K/jt4/dVzMxsNHV12STrjXND6rnzFPBushPGVyRdTPYI2HekvLeR9dxZQ9Zl890AEbFZ0pVkr2sD+EhEbB6Xb2FmZnWZ0jdnucummdnYHVSXzUPRLfet4x2f+xH/evfPRs9sZlYgTRn0N+3q4Z6nN/OTDX4MuJlZXlMG/QWz2gDo2rl3kktiZja1NGXQ7+yYBkDXDgd9M7O85gz6s7Kg/7xr+mZmQzRn0HdN38xsWE0Z9OdMb6W1LHZ099Hd2z/ZxTEzmzKaMuhLYoGbeMzM9tGUQR8Gm3ie39kzySUxM5s6mjboV2v6btc3MxvUtEG/00HfzGwfTRv0F3RkN2i5Td/MbFDTBn3X9M3M9tW8Qb+jHXBN38wsr2mD/sDzd1zTNzMb0LRBf+CuXNf0zcwGNG3QX1Dtp++avpnZgKYN+h3TWpjWUmJXTz+7e/omuzhmZlNC0wb9IY9i2OG7cs3MoImDPuTb9bsnuSRmZlNDMYK+a/pmZkCTB/2B5++4B4+ZGdDkQd8vUzEzG6q5g/4sP3/HzCyvuYO+a/pmZkMUIui7pm9mlqkr6Et6WtJDku6XtDKlzZe0QtLqNJyX0iXps5LWSHpQ0mm59SxL+VdLWjYxX2mQX6RiZjbUWGr6b4iIUyNicZq+HLgjIo4H7kjTAG8Gjk+fS4CrIDtJAFcArwROB66onigmSj7oR8REbsrM7JBwMM07S4HlaXw5cF4u/frI3AXMlXQkcDawIiI2R8QWYAVwzkFsf1Qzp7Uwo63M3r4KO/f6UQxmZvUG/QD+XdIqSZektIURsT6NPwcsTOOLgGdzy65NaSOlDyHpEkkrJa3s6uqqs3gj88VcM7NB9Qb9V0fEaWRNN5dJem1+ZmRtJ+PSfhIRV0fE4ohY3NnZedDrG3j+zk7flWtmVlfQj4h1abgRuJmsTX5DarYhDTem7OuAY3KLH53SRkqfUH5topnZoFGDvqSZkjqq48AS4GHgVqDaA2cZ8M00fitwYerFcwawLTUD3Q4skTQvXcBdktImlLttmpkNaqkjz0LgZknV/P8aEf9P0r3AVyRdDDwDvCPlvw04F1gD7AbeDRARmyVdCdyb8n0kIjaP2zcZgbttmpkNGjXoR8RTwMuGSd8EnDlMegCXjbCu64Drxl7MA+cLuWZmg5r6jlwYfEG6m3fMzAoQ9P2CdDOzQYUJ+n5BuplZAYJ+/kUqfhSDmRVd0wf99tYyHe0t9PYH2/b0TnZxzMwmVdMHfRi8QcsXc82s6AoR9Bekdv2Nbtc3s4IrRNDv9PN3zMyAogR936BlZgYULOi7Td/Miq4QQb96V65r+mZWdIUI+m7eMTPLFCLoL3CXTTMzoCBB3zV9M7NMIYL+YTOzoL9pVw+Vih/FYGbFVYig39ZSYu6MVvorwZbd7qtvZsVViKAPuXflul3fzAqsMEF/4GLuDtf0zay4ChP0B1+m0j3JJTEzmzyFCfqu6ZuZFSjo+7WJZmYFDPp+baKZFVlhgv7A83dc0zezAitM0PdduWZmRQr6fv6OmVn9QV9SWdJ9kr6dpo+TdLekNZJuktSW0qel6TVp/rG5dXwwpT8h6exx/zb7MX9mG1L2KIa+/kojN21mNmWMpab/h8BjuelPAJ+OiBcDW4CLU/rFwJaU/umUD0knAecDJwPnAP8kqXxwxa9fS7nE/BltRMBmP4rBzAqqrqAv6Wjg14Br0rSANwJfS1mWA+el8aVpmjT/zJR/KXBjROyNiJ8Ca4DTx+E71M3t+mZWdPXW9P8O+ABQbRc5DNgaEX1pei2wKI0vAp4FSPO3pfwD6cMsM0DSJZJWSlrZ1dVV/zepg4O+mRXdqEFf0luAjRGxqgHlISKujojFEbG4s7NzXNc9+DIVN++YWTG11JHnV4G3SjoXaAdmA58B5kpqSbX5o4F1Kf864BhgraQWYA6wKZdelV+mIVzTN7OiG7WmHxEfjIijI+JYsguxd0bEO4HvAW9L2ZYB30zjt6Zp0vw7IyJS+vmpd89xwPHAPeP2TepQvUHL3TbNrKjqqemP5E+BGyV9FLgPuDalXwt8SdIaYDPZiYKIeETSV4BHgT7gsojoP4jtj5lr+mZWdGMK+hHxfeD7afwphul9ExHdwNtHWP5jwMfGWsjx4hekm1nRFeaOXHBN38ysWEHfr0w0s4IrVNCfN6ONckls3d1LT58fxWBmxVOooF8qicNmZj14Nu1ybd/MiqdQQR/82kQzK7bCBX2/IN3Miqy4Qd89eMysgAoX9P38HTMrssIFfdf0zazIChf0/YJ0MyuywgV91/TNrMiKF/T9/B0zK7DiBX3X9M2swAoX9OdMb6W1LHZ099Hd29AnO5uZTbrCBX1JfsSymRVW4YI+uInHzIqrkEHfN2iZWVEVMugPPFffNX0zK5hCBv0FHekGLQd9MyuYQgZ999U3s6IqZtDvaAdc0zez4ilk0K8+f8c1fTMrmkIG/cEXqTjom1mxFDLoL+iovjLRQd/MiqWQQb9jWgvTWkrs6uln196+yS6OmVnDFDLoSxpo4nG7vpkVyahBX1K7pHskPSDpEUkfTunHSbpb0hpJN0lqS+nT0vSaNP/Y3Lo+mNKfkHT2hH2rOvj5O2ZWRPXU9PcCb4yIlwGnAudIOgP4BPDpiHgxsAW4OOW/GNiS0j+d8iHpJOB84GTgHOCfJJXH8buMiZ+/Y2ZFNGrQj8zONNmaPgG8EfhaSl8OnJfGl6Zp0vwzJSml3xgReyPip8Aa4PTx+BIHolrT7/Lzd8ysQOpq05dUlnQ/sBFYATwJbI2I6lXQtcCiNL4IeBYgzd8GHJZPH2aZ/LYukbRS0squrq4xf6F6uaZvZkVUV9CPiP6IOBU4mqx2fuJEFSgiro6IxRGxuLOzc6I2Q+csP3/HzIpnTL13ImIr8D3gV4C5klrSrKOBdWl8HXAMQJo/B9iUTx9mmYZz7x0zK6J6eu90SpqbxqcDbwIeIwv+b0vZlgHfTOO3pmnS/DsjIlL6+al3z3HA8cA94/Q9xszNO2ZWRC2jZ+FIYHnqaVMCvhIR35b0KHCjpI8C9wHXpvzXAl+StAbYTNZjh4h4RNJXgEeBPuCyiJi0l9S6y6aZFdGoQT8iHgRePkz6UwzT+yYiuoG3j7CujwEfG3sxx9+C3ItUIoKsg5GZWXMr5B25ADOntTCjrczevgo7/SgGMyuIwgZ9cLu+mRVPoYP+Ar8r18wKptBBf/C1ib4r18yKodhBf6B5p3uSS2Jm1hiFDvoLXNM3s4IpdND3hVwzK5pCB/3qC9L9rlwzK4pCB30/f8fMisZBHzfvmFlxFDro55+/kz0TzsysuRU66Le3lulob6G3P9i2p3eyi2NmNuEKHfQhf4OWm3jMrPkVPugvSO36G92ub2YFUPig3+nn75hZgTjod/iuXDMrDgd9d9s0swIpfNCv3pXrC7lmVgSFD/qu6ZtZkRQ+6PtFKmZWJIUP+n7+jpkVSeGD/mEzs6C/aVcPlYofxWBmza3wQb+tpcTcGa30V4Itu91t08yaW+GDPuRu0HITj5k1OQd9ck/b3OGavpk1t1GDvqRjJH1P0qOSHpH0hyl9vqQVklan4byULkmflbRG0oOSTsuta1nKv1rSson7WmMz0G1zp1+QbmbNrZ6afh/wPyPiJOAM4DJJJwGXA3dExPHAHWka4M3A8elzCXAVZCcJ4ArglcDpwBXVE8Vkc7dNMyuKUYN+RKyPiB+n8R3AY8AiYCmwPGVbDpyXxpcC10fmLmCupCOBs4EVEbE5IrYAK4BzxvPLHCg/f8fMimJMbfqSjgVeDtwNLIyI9WnWc8DCNL4IeDa32NqUNlJ67TYukbRS0squrq6xFO+A+a5cMyuKuoO+pFnA14E/iojt+XmRvWtwXDq5R8TVEbE4IhZ3dnaOxypH5efvmFlR1BX0JbWSBfwbIuIbKXlDarYhDTem9HXAMbnFj05pI6VPOtf0zawo6um9I+Ba4LGI+FRu1q1AtQfOMuCbufQLUy+eM4BtqRnodmCJpHnpAu6SlDbp/CIVMyuKljry/CrwLuAhSfentA8BHwe+Iuli4BngHWnebcC5wBpgN/BugIjYLOlK4N6U7yMRsXk8vsTBmj+zDQk27+6hr79CS9m3L5hZcxo16EfEfwIaYfaZw+QP4LIR1nUdcN1YCtgILeUS82e0sWlXD5t39XD47PbJLpKZ2YRwlTYZvEHLTTxm1rwc9BNfzDWzInDQTwaev+MbtMysiTnoJ67pm1kROOgn1Ru0HPTNrJk56Cd+baKZFYGDfuInbZpZETjoJ67pm1kROOgnfmWimRWBg34yb0Yb5ZLYuruXnr7KZBfHzGxCOOgnpZI4bGbWg2fTLtf2zaw5Oejn+GKumTU7B/0cX8w1s2bnoJ/ju3LNrNk56Of4+Ttm1uwc9HNc0zezZuegn+Pn75hZs3PQz/GLVMys2Tno51Tvyn3eNX0za1IO+jmu6ZtZs3PQz5kzvZXWstjR3Ud3b/9kF8fMbNw56OdI8l25ZtbUHPRr+K5cM2tmDvo1XNM3s2bmoF+j03flmlkTc9Cv4btyzayZjRr0JV0naaOkh3Np8yWtkLQ6DeeldEn6rKQ1kh6UdFpumWUp/2pJyybm6xy86l25btM3s2ZUT03/i8A5NWmXA3dExPHAHWka4M3A8elzCXAVZCcJ4ArglcDpwBXVE8VU09nRDrimb2bNadSgHxE/ADbXJC8Flqfx5cB5ufTrI3MXMFfSkcDZwIqI2BwRW4AV7HsimRIGnr/jmr6ZNaEDbdNfGBHr0/hzwMI0vgh4NpdvbUobKX0fki6RtFLSyq6urgMs3oFzl00za2YHfSE3IgKIcShLdX1XR8TiiFjc2dk5Xqut2wJfyDWzJnagQX9DarYhDTem9HXAMbl8R6e0kdInRqUfHrgxG45Rx7QWprWU2N3Tz669fRNQODOzyXOgQf9WoNoDZxnwzVz6hakXzxnAttQMdDuwRNK8dAF3SUqbGD/8W7j5vXDD22DX82NaVJKbeMysadXTZfPLwI+AEyStlXQx8HHgTZJWA2elaYDbgKeANcDngfcBRMRm4Erg3vT5SEqbGItOgxmHwZN3wudeA8/8aEyL+65cM2tWLaNliIgLRph15jB5A7hshPVcB1w3ptIdqBefBe/9IXztPfDsXfDFX4OzroBX/QFIoy7umr6ZNavmvSN3ziK46NtZoI9+WPGX8OULYPfoPzBc0zezZtW8QR+g3ApLroTzvwztc+An34F/fh2sXbXfxao1/bVb9zSilGZmDdPcQb/qxHOz5p6jXg7bfgbXnQ13/zPE8D1NX7poDgBf+M+nufupTY0sqZnZhCpG0AeY90J4z+1w+nuh0gvf+QB89SLo3r5P1rNecjjvOuOF9PRX+L3rV7J6w47Gl9fMbAIUJ+gDtEyDcz8Jb/8itHXAo7fA1a+D5x4akk0Sf/XWk1ly0kK2d/dx0RfuZcP27kkpspnZeCpW0K86+Tfgku/DwlNg81Pw+TNh1fIhzT3lkvjsBS/ntBfMZd3WPSy77h52dPdOXpnNzMZBMYM+wIIXw+9+F067EPr3wrf+AG6+FHp2DWRpby1z7bJX8KIFM3n8uR1c+i+r6OmrTGKhzcwOTnGDPkDrdHjr38N5n4PWGfDgjfD5N8LGxweyzJvZxvL3nM6CWdP4rzWb+NOvP0iMcAHYzGyqK3bQrzr1Avi9O2HBCdD1OHz+DfDATQPNPcfMn8EXLnoFM9rK3HzfOj55+xOTXGAzswOjqVxrXbx4caxcubJxG9y7E779x/DQV7LplnboOAI6joKOI/hZ31z+5ZG9rK/M4y2/ehpnn3EqzD4q+8VgZjZFSFoVEYuHneegXyMCVn0R7vwo7K7zYW3tcwZODMxOw44jYcHxcPhJMLOzrsc/mJmNh/0F/VGfvVM4Eix+d/bZuwN2PAc71sP29dlwx3pWr1nN9q6fsVBbOKq8jVL3NujeBl2PDb/OGYdlwb/zRDj8Jdn44SfC9Cn5xkgza2IO+vszrSP7LDh+SPKLI/jQzQ/x5XueZf70Mt94z0s4tm37kBMD29ZC1xPZNYLdm+DpH2afvI4js5NA50sGTwadJ8C0WQ38klNApR/6e9Knd3DYOj3b/y3tU/uXUl8P7N0Oe7ZmJ//urdl0SztMm539Eqx+2mZBaQIvpVX6sx5ovbuh1AptM7P7U6by/rOGctA/AJK4cukpbNi+lzsf38i7bnySr//+qzj8iJfumzkCtq+DjY8Nfroey3oIVU8QT945dJm5L8guKpfbsruH+3uh0pd9+ntTWl82rPQNjlfnVfqz9HJbFjhb2rPeSa1p2NKepe9vXkt79qC6/t6hgXif4DzSeD150ydG6QZbakkn4Nnp07Hvp71mXtssKJWz/R8VIA2Dmuna+dVhBXp2piA+yqd39xgOnlLuRDAb2ucOPSm0z0nzZ6cAvjML4gPD2vHc9N6d0DfM86JKLVnwb+vIhtNm1TE9Kzt+yq3Z8qVydhIptQxOl/PTuU+5dXDfDxy76bgcOEb7ao7h6nGbO95VytZVbkvrrZanNaUPNy83DoP/C9GfxvsHxyPNq1Ry47n5g3+03KA6rprxNC8/HtX1VmrK0Dd8em0Zy9Pgl95e/7FVJ7fpH4TdPX1ccPVdPLB2Gy9dNIcbLzmDmdPqPI9WKrD1mXQieDT7RbDxMXj+J1kgLBRltdHqP2u5LfvH7t2dNbH1T/Gnnao8GLCnz02BuyP7BdC9Lav1V08QPTsnvjxts7ITeKU3OxkU7nhqEjMPh/+9+oAWdZv+BJnR1sK1F72C37rqv3lo3Tbed8OPuWbZYlrLdfx8L5Vg/nHZ58RzB9P7+7K7hDetzmoKpVYotwzWbobUploHa1X5+eXWLBD190Dvnqz219udG899Bubthr407O3O0qu1plKuVpUPzMOmtQ2WdyCQD5cnN79U3v++6tub1WL3bs9OAgPDmvHumvSIrOYlZbVG0nDEaYZOt81KwXzuvrXxIU02M+tvPunvSyeBrelEsH34Xw97t6caerUWPnPoeLU2XjuvZfq+zUd9PenXQO4XwZDpHYO/GPLz+vO18L7BmvjAr86+XC2+L1eTT+nSCMdtzTFcahk8ZgbGW3K/FKq/DvuyYfWX7sB47tdotbzVE131V4nK6f8kP96S/Z0HxmvySLm79GPk8YFBdTy9NlyltP58GVqyv8+QMpSztHwZqhWJCeCa/jh4+vld/OZV/83mXT28/ZeP5pNv+yXkNlQzmyT7q+n75qxxcOyCmVy7bDHtrSW+umotf/fdA/tJZmY20Rz0x8nLXzCPf7jgNEqCz9yxmhvv+dlkF8nMbB8O+uPorJMWcuV5pwDwZ7c8zB/fdD9X/+BJ/uMnXWzc3u1n9pjZpPOF3HH2zle+kA3buvnsnWu4+b513Hzf4Lx5M1o54YgOTjxiNice0cGJR87mFxfOYkab/wxm1hi+kDtBHl63jQfWbuXx9Tt44rkdPP7cdrZ39+2TT4IXzJ/BiUd0cMIRs3nJER2ccEQHLzxsJuWSLwab2di5y+YkOGXRHE5ZNNjlKiJYv607nQCyk8ATz+3gya6dPLNpN89s2s3tj2wYyD+tpcRhM9uYPb2V2e2tzJ7ekoatzG5vGSE9m541rYWWerqNmlnhOOg3iCSOmjudo+ZO5w0nHj6Q3tNX4anndw6eDNZnJ4Ofb+se+ByImW1lZkxrob21xPTWMtNby7S3lpneVh6cbquml/aZ395aprVcoqUkyiXRUhYtpVI2fgDTJf9qMZsSHPQnWVtLKbXxz2ZpLn3n3j627u5h+54+tnf3sn1PL9u7+9KwtyZ9cHpHdzbc1dPPrp7+EbfbaBK05k4C5XI6OVTTyhqcVyoNnCjKyl5dWVI2vzpeqknP8g6dX5LSfUrK7rciS6uOa2A8y5/dx7VvWrZMmk7ppdLQfAPrVc10zXay9WXj1JYpl0dSWjbtP/LrG/qdYGh+5dZ/UH8zGPK9a8s49DvmyzC4TPVvX3ssVL/TkOmR0ocsNzgv/90Hyzu4jvx2q+XL51PtumqWjch+ofdHUAmoVCK7XyyCSgSR0vsr1WmoRKTpwfVmx+PgPiuVGDhGq/tsSJ7S4PF9eEf7WP5kdWl40Jd0DvAZoAxcExEfb3QZDgWzpmXNNMwb+7KVSrCzp4/unn729PbT3VthT28/e3r66e7tHxjP5uXTKgNpe3r66asEfZUK/ZWgrz8b76tkB3Vvf9Cfpvv6s7Rq3t7+oK+/Qn8M5o2Anv4KTJ3zkNmUtmDWNFb++Vnjvt6GBn1JZeAfgTcBa4F7Jd0aEY82shzNrlRS1r7f3jrZRQEYqBENnEAqQX9/0Js7oVTT+yqVdILJak+VdJLpj6BSSbWsNK9/YDiYPpg3smerpdpXdpd8NqydR02+fHol1dqCNMzV6Aamc/mqy/VXgiDLGwwuF0PWlY1Ts+0grWfgDv+hyw6uc2APD93OkHkH+DdL6xqyb3Jlp3afQEqPwfLnyjEwzB0TQ4+Rwe86Uv7c191nPVGzjn22W5Mvv52R9mW1Jl4SlKu/9lItvZyvuUsDtfN8DT5f7ko6fvO/CPLj+TzV/5d5M9tG+zMdkEbX9E8H1kTEUwCSbgSWAg76TUyqNtOM8owdM5twje7isQh4Nje9NqUNkHSJpJWSVnZ1dTW0cGZmzW7K9euLiKsjYnFELO7s7Jzs4piZNZVGB/11wDG56aNTmpmZNUCjg/69wPGSjpPUBpwP3NrgMpiZFVZDL+RGRJ+k9wO3k3XZvC4iHmlkGczMiqzh/fQj4jbgtkZv18zMpuCFXDMzmzgO+mZmBTKlH60sqQt45iBWsQB4fpyKMxFcvoPj8h0cl+/gTOXyvTAihu3zPqWD/sGStHKkZ0pPBS7fwXH5Do7Ld3CmevlG4uYdM7MCcdA3MyuQZg/6V092AUbh8h0cl+/guHwHZ6qXb1hN3aZvZmZDNXtN38zMchz0zcwK5JAP+pLOkfSEpDWSLh9m/jRJN6X5d0s6toFlO0bS9yQ9KukRSX84TJ7XS9om6f70+ctGlS9XhqclPZS2v3KY+ZL02bQPH5R0WoPKdUJuv9wvabukP6rJ0/D9J+k6SRslPZxLmy9phaTVaTjsiy4lLUt5Vkta1sDy/Y2kx9Pf72ZJc0dYdr/HwgSW768krcv9Hc8dYdn9/r9PYPluypXtaUn3j7DshO+/gxbpBb+H4ofsoW1PAi8C2oAHgJNq8rwP+FwaPx+4qYHlOxI4LY13AD8ZpnyvB749yfvxaWDBfuafC3yH7D3PZwB3T9Lf+jmym04mdf8BrwVOAx7OpX0SuDyNXw58Ypjl5gNPpeG8ND6vQeVbArSk8U8MV756joUJLN9fAf+rjmNgv//vE1W+mvl/C/zlZO2/g/0c6jX9gdcvRkQPUH39Yt5SYHka/xpwpqovsJxgEbE+In6cxncAj1HzprBDxFLg+sjcBcyVdGSDy3Am8GREHMwd2uMiIn4AbK5Jzh9ny4Hzhln0bGBFRGyOiC3ACuCcRpQvIv49IvrS5F1k77KYFCPsv3rU8/9+0PZXvhQ73gF8eby32yiHetAf9fWL+TzpoN8GHNaQ0uWkZqWXA3cPM/tXJD0g6TuSTm5syYDsPdD/LmmVpEuGmV/Pfp5o5zPyP9pk7z+AhRGxPo0/BywcJs9U2I8A7yH75Tac0Y6FifT+1Px03QjNY1Nh/70G2BARq0eYP5n7ry6HetA/JEiaBXwd+KOI2F4z+8dkTRYvA/4euKXBxQN4dUScBrwZuEzSayehDCNKL9x5K/DVYWZPhf03RGS/86dkX2hJfwb0ATeMkGWyjoWrgF8ATgXWkzWhTEUXsP9a/pT+X4JDP+jX8/rFgTySWoA5wKaGlC7bZitZwL8hIr5ROz8itkfEzjR+G9AqaUGjype2uy4NNwI3k/2Mzpvs11y+GfhxRGyonTEV9l+yodrklYYbh8kzqftR0kXAW4B3phPTPuo4FiZERGyIiP6IqACfH2G7k73/WoDfBG4aKc9k7b+xONSDfj2vX7wVqPaSeBtw50gH/HhL7X/XAo9FxKdGyHNE9RqDpNPJ/iaNPCnNlNRRHSe74PdwTbZbgQtTL54zgG25poxGGLF2Ndn7Lyd/nC0DvjlMntuBJZLmpeaLJSltwkk6B/gA8NaI2D1CnnqOhYkqX/4a0W+MsN3Jft3qWcDjEbF2uJmTuf/GZLKvJB/sh6xnyU/Irur/WUr7CNnBDdBO1iywBrgHeFEDy/Zqsp/5DwL3p8+5wKXApSnP+4FHyHoi3AW8qsH770Vp2w+kclT3Yb6MAv4x7eOHgMUNLN9MsiA+J5c2qfuP7AS0Hugla1e+mOw60R3AauC7wPyUdzFwTW7Z96RjcQ3w7gaWbw1Ze3j1OKz2aDsKuG1/x0KDyveldGw9SBbIj6wtX5re5/+9EeVL6V+sHne5vA3ffwf78WMYzMwK5FBv3jEzszFw0DczKxAHfTOzAnHQNzMrEAd9M7MCcdA3MysQB30zswL5/7p544Ntgl8eAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "![](../pic/BiLSTM_CRF-F1-20.jpg)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAArb0lEQVR4nO3deXxcV3n/8c8zM1osyfImx3a8KolD4gBxqDAkQEjJgkOBsJU6UAhLm4Y2pS3QNhQaaCgFCqG/LmlLoPkBaZqENT9DQpOwhISWECtktZ04tpzEW2zZsiVr18w8vz/uHelqLFkja6SR7nzfr9d9zb3nnjvz6Gr0zNG5Z841d0dEROIrUeoARERkcinRi4jEnBK9iEjMKdGLiMScEr2ISMwp0YuIxJwSvYhIzCnRy5Qys383s78udRwi5USJXorOzJ41sx4z6zSzw2Z2p5ktB3D3q9z9M2G9C8xs9yjPsczMvmtmB82s3cyeNLP3mdlrwuftNLMuM/PIdqeZrTCz+8Lys/Oe8/th+QUF/Aynm9m3I6//uJl9xMySZrYq73WfNbNrjnMOcsvJo7zWfWbWm1f33HDfZ8zsCTNLm9mnx4j5JDO71cz2hjH/j5m9Iq/Ou8zsufDc3WFm88c6FzLzKdHLZHmTu9cBS4D9wD+P8/ibgV3ASmAB8B5gv7s/4O514XOfFdadmytz9+fDsm3Ae3NPZmYLgHOB1rFe2MxOBX4Vvv5L3H0O8NtAEzA7UnVuGMc7gL82s4vznupNkbjq3H3vcV726ry6vwzLtwN/Adw5VtxAHbAJ+A1gPvAN4E4zqwt/rrOArxCcy0VAN/CvBTyvzHBK9DKp3L0X+A6wBsDMvm5mf1vAoS8Hvu7uXe6edvdH3P1H43jpW4DfMbNkuH058H2gv4Bj/wb4X3f/iLvvC3+Op939Xe5+JL+yuzcDm4G144ivIO7+jfDnPlpA3RZ3/7K773P3jLvfCFQCLwqrvBv4gbvf7+6dwF8DbzOz2aM9p8SDEr1MKjOrAX4HeHCchz4I3GBmG8xsxQm89F5gC3BJuP1e4JsFHnsRwYdTQczslcCLCVrf04aZrSVI9Lm4zgIey+139x0EH3ynT3lwMqWU6GWy3GFmR4B24GLgi+M8/reBBwhanTvN7FEze/k4n+ObwHvN7AyCbpZfjnVAaAGwr4B6B82sB/glQRfIHXn77zCzI+GSvy/fP0Xq/rrAOEdlZvUE3V9/4+7tYXEdwe8jqp3h3VESQ0r0Mlne4u5zgWrgauDnZra40IPd/bC7X+PuZxH0Jz9KkDhtHDF8D3hd+Po3j+O4QwTXFsbSQJA8PwpcAFTk7X+Lu88Nl7fA4Kij3AXXv4rU/XCk7ssKCdLMNkee6zWR8lnAD4AH3f1zkUM6gfq8p6mngG4hmdmU6GVShX3F3wMywKtP8DkOAl8CTia4yFjocd3Aj4APMb5E/2Pg7QW+Rsbdvwz0An9YQP2rIhdc/24cMY30XGdFnusBADOrIvjPYjfwB3mHbAYGRyKZ2SlAFcGFa4kxJXqZVBa4DJgHbB2lTnXeYmb2BTN7sZmlwouFHwK2u/uhcYbwV8Br3f3ZcRzzKeA8M/ti7r8QMzvNzP7TzOaOcszngb8ws+pxxndcZlYRPmcCSIXnJzlaXYJrCz3AFe6ezatyC/CmcIhqLXAd8D13V4s+5pToZbL8wMw6gQ7gswSJZ/MI9ZYSJKbocipQQzBK5gjQQjDM8s3jDcLd97r7L8Z5zA6CoZirgM1m1g58F2hm9G6OO4HDwO+PN8YxfJXgnFwOfCJcf88odc8D3khwAfpIfrdOeP6vIkj4Bwj65sf8L0RmPtMdpkRE4k0tehGRmFOil7JkZj/Km3JgpJEwIrGgrhsRkZhLlTqAfA0NDb5q1apShyEiMqM8/PDDB9194Uj7pl2iX7VqFc3NzaUOQ0RkRjGz50bbpz56EZGYU6IXEYk5JXoRkZhTohcRiTklehGRmFOiFxGJOSV6EZGYm3bj6EVETkTvQIbOvjQ9/Rm6+tN092fo6c/Q3Z+huz89uN4zEGx39YX7BzL0hPWTCaO+uoL6WSnqZ1VQX13BnFkV4XpqcH1OuK8yNfG2srszkHHS2SwDGae+OsX47q8zNiV6EZlU7o47eG4dwu2gnMh270CWjp4B2nsG6OgdoKMnHVnPlaePqdPRO0B/On/6/clXXZEYTPpzZlVQkUyQzmbpzzjpTJZ0xhnIBo/pTJaBbPiYcQYyWdJZJ5MdPg3Ntr+9lMqUEr1IbLk7PQMZOnvTHO1L09WXHr7el+Zo79B6Z7i/M9yfdUgljYpkgorwMZVIUJkyUonE8PKkURk+BuUJslmnN52hbyBLXzpLXzpDXzpL70DwGJSH68PKg8d01ocl86lUmUxQV51iVkWSmsokNVUpasL1WZVhWWUqWK/IlaXC8mA94z7sQ6S9Z+iDpKMnXHrDD5+eAXoHsvQO9LG/o29CsacSFvweEoljEn8xKNGLFKg/naU9TAL5rcz27mC7ZyDDQDporfVnsvSnswyELbj+TG49V+6R/UFZz0CGSfg7nxbMwAAzCx/BCApz29UVycGuk1xLub66gjk1QddJtNtkWJ1ZFVRXjHjjrUnj7nT3Z4b955HOZEnlPjwTuQ/R4EM2+oEa3Z9KWNG7avIp0UtZyGadrv6gNZxrFR/tHRjWIs51CeS36HJL78DUdA1UVySoq6qgripJXXWKuqrIUp0a2leVoq46t15BbVWSZMKGugXCx9wHTTobfJiksz5YFtQLuhoGMlmSZlSlElRVJKiuSAbrqeRQWSpJVUWkLNyuTiWpTAWJ65hkPslJrFTMjNqqFLVVKZbMKXU0x6dELzNOV1+a1qN9HOzso/VoH62dfRwMH3P/ZucSeC6xd/alJ/y6qYQxJ2xRzg4fg2WoZVlTmRxstVWkElQmg26TwbJkUFYRlgX7h1p5NRVJUkkNhpPiUqKXknJ3+tJZuvqCURCHuvrCJN4fJvFeDh7tp7WzbzC5d/dnTui1hreKU8yuDpagvIK66tTgyIrcUh9Zr6lMxrZ1KvFWUKI3s/XAPwJJ4Gvu/vm8/SuBm4CFQBvwu+6+O9yXAZ4Iqz7v7uO+wbNMb+7OCx29PLXvKIe6+gcvFHYNXkDMBOv90fLM4Hp6nJ3SVakEC2dXBUtd8NhQV0XD7Crm1VREknjFYGKvrUyRTChJS3kaM9GbWRK4AbgY2A1sMrON7r4lUu1LwDfd/Rtm9jrgcwzdqb7H3dcWN2wplYFMlu0HOtm6r4MtezvYsq+Drfs6ONw9cMLPWZEM+zorU8yvrRxM4A2zK8NEXh0m82BfXVXxxxmLxFkhLfp1wHZ3bwEws9uAy4Bool8DfCRc/xlwRxFjlBJp7xkYTOhb9wVJ/Zn9nfRnjr0oObemgjMX17NkTvXgBaq6qmRkPRjGVhfZDvYlqUpN7WgJkXJTSKJfCuyKbO8GXpFX5zHgbQTdO28FZpvZAnc/BFSbWTOQBj7v7ndMOGopusNd/TQ/d5gn97QPttJ3H+4Zse6qBTWcuaSeNUvqWXNysCyur1YrW2SaKtbF2I8B/2Jm7wPuB/YAuStmK919j5mdAvzUzJ5w9x3Rg83sSuBKgBUrVhQpJDmefe09PLSzbXB55kDnMXWqUgnOWDybNSfXDyb2M5bUU1ela/giM0khf7F7gOWR7WVh2SB330vQosfM6oC3u/uRcN+e8LHFzO4DzgF25B1/I3AjQFNTU0y/LlI67s7Og11seraNX+1sY9OzbexqG95ar0olWLt8LmtXzA1a6kvqaWyo1VA/kRgoJNFvAlabWSNBgt8AvCtawcwagDZ3zwIfJxiBg5nNA7rdvS+s8yrg74sYv4wgk3WeeqGDTTvbeOjZNh7aeZiDncO/oj27KsVvrJrHusb5rFs1n5csm6O+cpGYGjPRu3vazK4G7iYYXnmTu282s+uAZnffCFwAfM7MnKDr5o/Cw88EvmJmWYIpkT+fN1pHiqSrL82tDz3P/2w/SPNzhznaO/wLQg11laxrnM/LVwXLmUvqNdxQpEyYT/XMQ2Noamry5ubmUocxY2Syzread3H9PduGtdqXzZs12Fpf1zifxoZaXSwViTEze9jdm0bap6tqM5S78/NtrXzurqd4ev9RANYun8v7zlvFusb5nDx3VokjFJHpQol+Btqyt4PP/WgrDzxzEAha73+5/gze+NIlarWLyDGU6GeQ/R29XH/P03z74d24w+zqFB9+3Wree95KXUgVkVEp0c8AXX1pbry/hRvvb6FnIEMqYbznvJV8+HWrmVdbWerwRGSaU6KfxjJZ59vNu7j+3m20Hg0utK4/azF/eekZNDbUljg6EZkplOinqZ9va+Xv7tw6eKH17OVz+eRvncnLV80vcWQiMtMo0U8zT73QwWfvHLrQunTuLP7y0jN440uWkNC4dxE5AUr000RPf4brfriZ2zftIhteaL36N0/jivNWTfm9MEUkXpTop4H+dJYP3fIw9z3dSiphvPfclXz4wtXM14VWESkCJfoSy2Sdj377Me57upV5NRXc8nuvZM3J9aUOS0RiRFMTlpC786mNT/KDx/ZSW5nk6+9fpyQvIkWnRF9C19+zjf988HkqUwm+ekUTZy+fW+qQRCSGlOhL5Kv3t/AvP9tOMmH8y+XncN6pDaUOSURiSom+BL61aRefvWsrAF94+0u55KzFJY5IROJMiX6K/feT+7jme48DcO0b1/CO31hW4ohEJO6U6KfQL545yIdvfZSsw4cvXM0HXt1Y6pBEpAwo0U+RR54/zJU3N9OfyXLFuSv5s4tWlzokESkTSvRT4OkXjvK+/7uJ7v4Mb1l7Mp9601maN15EpowS/STb1dbNe/7jV7T3DHDhGSfxxd8+W3PWiMiUUqKfRAc6enn3137FgaN9rGuczw3vfhkVSZ1yEZlayjqTpL17gPfe9BDPt3Xz4qX1fO2KJk1OJiIloUQ/Cbr707z/6w/x1AtHOWVhLd94/zrqqytKHZaIlCkl+iLrS2f4g5sf5tfPH+HkOdXc/MFXsKCuqtRhiUgZU6IvokzW+cjtj/HAMweZX1vJzb/3CpbOnVXqsESkzBWU6M1svZk9bWbbzeyaEfavNLOfmNnjZnafmS2L7LvCzJ4JlyuKGfx04u588o4nuPOJfdRVpfjmB9Zx6sK6UoclIjJ2ojezJHADcCmwBrjczNbkVfsS8E13fylwHfC58Nj5wKeAVwDrgE+Z2bzihT99/HxbK7c+tIuqVIKvXdHEi5fOKXVIIiJAYS36dcB2d29x937gNuCyvDprgJ+G6z+L7H89cK+7t7n7YeBeYP3Ew55+7npiHwAfuuBUXnnKghJHIyIypJBEvxTYFdneHZZFPQa8LVx/KzDbzBYUeCxmdqWZNZtZc2tra6GxTxuZrPPjrQcAuPTFS0ocjYjIcMW6GPsx4LVm9gjwWmAPkCn0YHe/0d2b3L1p4cKFRQpp6jz83GHauvpZuaCG0xepX15EppdC7hm7B1ge2V4Wlg1y972ELXozqwPe7u5HzGwPcEHesfdNIN5p6e7NLwDw+rMWaw4bEZl2CmnRbwJWm1mjmVUCG4CN0Qpm1mBmuef6OHBTuH43cImZzQsvwl4SlsWGu3PPliDRX7JmUYmjERE51piJ3t3TwNUECXor8C1332xm15nZm8NqFwBPm9k2YBHw2fDYNuAzBB8Wm4DrwrLYeOqFo+xq66GhrpJzVsRyQJGIzHCFdN3g7ncBd+WVXRtZ/w7wnVGOvYmhFn7s3LN5PwAXr1lEUrNSisg0pG/GTlCuf/6SNbrvq4hMT0r0E7CrrZst+zqorUxy7qkaOy8i05MS/QTcuyXotrngjJM0BbGITFtK9BMw1G2j0TYiMn0p0Z+gtq5+Nj3bRkXS+M0zTip1OCIio1KiP0E/2bqfrMO5pzbopiIiMq0p0Z+gu8Nhleq2EZHpTon+BHT3p3ngmWDytYuV6EVkmlOiPwH3bztIXzrLOSvmsqi+utThiIgclxL9CbhHX5ISkRlEiX6cBjJZfvJUMPf8JWep20ZEpj8l+nHatLON9p4BTjupTveEFZEZQYl+nPQlKRGZaZToxyGYez4cVnmW+udFZGYoaJpiCTy5p4N97b0srq/mpUvnlDocEZlp3CHdC70d0NseLH3tkfVOeNWHi/6ySvTjkOu2uXjNIhKae14kPtzh8E7obAXPQDYdLplwSQ8tnh2+Ha0z0AO9R6AvkshzST1Xluk/fizrroSK4g7bVqIfh8FbBmq0jcjMlc0GSX3vI7DvUdj7KOx7PGhZT4VkJVTPCZaq+qH16nDds0V/SSX6Au082MW2/Z3Mrk7xylM09/y04g59R6HnMJhBahZUhEtC00cDwTnKZoLWZKYfMgPhY19kvT84d3Unwax58Th340nqdYtgznJIVoAlg58/kQqX/O3UyHVS1ccm7uq5wxN6kVvrhVCiL1DuS1IXnnESFUldw540mYEgYXe3QU/bKI8j7M+mR36+ZFXwh1VREyb/muCPMbdeEflQqKgJ/4ATkcVGWR9lcQ9aZJ4FIuvDFs97jCzRLoHMwPDH7ABkwsdsemh9cH9Yd1jyDh/TfUE8hbIE1CyA2pOgtgFqFwYfALn12pPCx4agvGJWMX77E5PNQltLmNAfgX2PBUtfx7F16xbBkrVw8trw8RyoXzK18U4hJfoCabRNkfR1wpHnI8tzwXL4uWC798iJPW9FbdAKNYOB7qCvdKAnTHp9Qd9oubNE8MGXrAxarbnHVFiWSAXnrOtA8GHa1RoshaisC5L+7CVQfzLUL4U5y4av1zRAYgKNpGw2iG3Yeyf6XtoV/K7z1S2OJPTwMcZJfSRK9AU4cLSXXz9/mMpUgteevrDU4UxvAz3BH9yR5+HIs8FjLokfeQ66Dx3/eEsE/+rWzIdZ84c/jlQ2a36Q4Ef6d9g9aMnmEn+6N/Ih0A0D+ds9Qxfb8lvbI7bO88qymcL+E2Ck/wwsWBIpSFQECTiRjKyH3QPJiqAskRxaH1Y3ksRTVZGEXjm+rpjMAHQdHEr2uaXzQKQ8XO88AP2dwXL42dGfM1kZfBDMWRYk//qTh9bnLA0es+nxJ/Ko2UvyWuprYbYaZ0r0BfjxlgO4w2tOa6C2SqeMbCb4Izy4HQ49AwefgUPbg8fOF45/bLIS5q4Il5VD6/NWBY8TbfVFmYXdNpp4btySFUGrt5CWr3vwH1NXKxzdB+17oCNc2vdAx17o2B38l5D7D+5E1SyIvH8i76E5y2HucqiafeLPHWPKWgUo29E23W1DCTya0NtaRh8ilkgFrbTcH+C8leF6uF23qHiJXKYHM5g1N1gaVo9er78LOvYFST/6ARBdT6SGNwCiy5zlUKVpR05EQYnezNYD/wgkga+5++fz9q8AvgHMDetc4+53mdkqYCvwdFj1QXe/qjihT42jvQP87/ZDJAwuOjOGid49aHkd2AoHtsDBbUMt9eN1s8w+GRpOgwWrgz/uBauD7TnL4zFaQ4qvsjZ4jzScVupIys6Yid7MksANwMXAbmCTmW109y2Rap8EvuXu/2Zma4C7gFXhvh3uvraoUU+h+55upT+TZd2q+Syoq5q8F2rfA4/eEqzn+iznLA/6MYs1oqHrYJDMc0n9wNZgGWlUAgSjUBacFknkq4PtBaepZSUygxTSol8HbHf3FgAzuw24DIgmegfqw/U5wN5iBllKQ6NtJqk1f2QX/OIf4JGbR+8OqVkQXrRaFn4A5C5gLQ+26xZDMvKr7O2A1qeOTeqjjaCYNR8WnQULz4CFLxpK7PUnB/+Wi8iMVkiiXwrsimzvBl6RV+fTwD1m9sdALXBRZF+jmT0CdACfdPcH8l/AzK4ErgRYsWJFwcFPtr50hp/l5p4v9k1GjjwPD3wZHvnPYCw0Bme9FeY1Qvvu8ELW7qDvsvtQsOx7bOTnsmQw2mD2YujcD+27Rq5XWQcnnRkua4YeaxcqoYvEWLEuxl4OfN3drzezc4GbzezFwD5ghbsfMrPfAO4ws7PcfVhfgbvfCNwI0NTUNI5vdUyuX+44RGdfmjMWz2bFgpriPOnhZ4ME/+h/DSX4F78Dzv9zOOmMY+vnxg637x7+ARDd7twfXMjq2B0ck6yChacPT+YnnRn8B6CELlJ2Ckn0e4Dlke1lYVnUB4H1AO7+SzOrBhrc/QDQF5Y/bGY7gNOB5okGPhWK+iWptp3wwJfgsduCscKWgJe8M0jwC08f/bhEImipz14My5pGrpPuC1r+R/cFrfN5jcO7ckSkrBWSDTYBq82skSDBbwDelVfneeBC4OtmdiZQDbSa2UKgzd0zZnYKsBpoKVr0kyibde4NE/3rJ9I/f2gHPHB9kOA9/ELNSzfA+R87/lC08UhVwfzGYBERyTNmonf3tJldDdxNMHTyJnffbGbXAc3uvhH4KPBVM/szgguz73N3N7PzgevMbADIAle5e9uk/TRF9MiuI7Qe7WPp3FmsWVI/9gH5Du2A+78Ij38rTPBJOPtdQYJfcGrxAxYRGUVB/9+7+10EQyajZddG1rcArxrhuO8C351gjCUR/ZKUjadf++AzQYJ/4tvB1+ItCWt/F87/KMw/ZZKiFREZnTpyR+Du3LM5121TYP9831G482PwxLeCBJ9Iwdp3w2s+qi4VESkpJfoRbD/Qyc6DXcyrqaBp5byxD8hm4ftXwVM/DCaWetl74dUfCb7+LyJSYkr0I8iNtrnwzEWkCpl7/udfCJJ81Rz44D0jD5MUESkRzS41gtxNRi5ZU8Bom60/gJ9/PhhN846blORFZNpRos+zr72Hx3a3M6siyfljzT2/fwt87w+C9Ys+DasvOm51EZFSUKLPkxs7f/7pDVRXHGcWxu42uO1yGOgKvtl63oenKEIRkfFRos+TG21z3LltMmn4zvuD6QyWnA1v/mdNLSAi05YSfUR79wAPthwimTAuPPOk0Sveey203BdMN/A7t0BlkebBERGZBEr0ET99ej/prPOKxvnMrakcudKjt8KDNwTj5N/5zeD2ZSIi05gSfcRQt80oo212Pww/+JNg/Q1fhJXnTVFkIiInTok+1DuQ4efbghtzjDhb5dEX4PZ3B3ehb/pAsIiIzABK9KFf7Wyjuz/Di5fWc/LcvFv3pfvg9vcE0wCvOBfWf6E0QYqInAAl+tAz+48C8LIVeVMeuMOdH4XdDwW38nvnNyE1Sv+9iMg0pEQf2tHaBcApDbXDd2z6WnA/11Q1bLgF6o4zGkdEZBpSog+1tHYCcMrCuqHCnQ/Aj/4yWL/sBjh57dQHJiIyQUr0oZ0HgxZ9Y65Ff/g5+PYVwU1DXvUn8JJ3lDA6EZETp0QPHO0d4MDRPipTCZbOnQX9XXDbu6H7EJx2EVz4qVKHKCJywpTogWcPdgPQuKCWhAF3/CHsfwLmnwpv/xokjjPnjYjINKdED7QczPXP18Ivvgxb7oDK2XD5rTCrgBuPiIhMY0r0DI24uSj1CPzkM4DB278KC19U2sBERIpAiZ7gQuwya+XN2z8FOLzuE/CiS0sdlohIUSjREwyt/JPkd6lId8IZb4TXfKzUIYmIFE3ZJ3p3J3NwB29N/gK3JFzyt5pbXkRipewT/f6OPj6Y/S4py2JrL4f5jaUOSUSkqApK9Ga23syeNrPtZnbNCPtXmNnPzOwRM3vczN4Q2ffx8Linzez1xQy+GPa2PMlbk78gQ0JdNiISS2MmejNLAjcAlwJrgMvNbE1etU8C33L3c4ANwL+Gx64Jt88C1gP/Gj7ftDGn+Z9IWZaH565Xa15EYqmQFv06YLu7t7h7P3AbcFleHQfqw/U5wN5w/TLgNnfvc/edwPbw+aaHQzto3PND0p5g24uuKnU0IiKTopBEvxTYFdneHZZFfRr4XTPbDdwF/PE4jsXMrjSzZjNrbm1tLTD0InjgehJk+G7mfBqWa8y8iMRTsS7GXg583d2XAW8Abjazgp/b3W909yZ3b1q4cGGRQhrDoR3w2G2kSfAvmcs4dWHt2MeIiMxAqQLq7AGid8BeFpZFfZCgDx53/6WZVQMNBR5bGg9cD57he5kL2MMiViyoKXVEIiKTopBW9yZgtZk1mlklwcXVjXl1ngcuBDCzM4FqoDWst8HMqsysEVgNPFSs4E9Y2Jp3S/LP6ctYNq+GqtS0ukYsIlI0YyZ6d08DVwN3A1sJRtdsNrPrzOzNYbWPAr9vZo8BtwLv88Bm4FvAFuC/gT9y98xk/CDjErbm96x8C7t8UTCZmYhITBXSdYO730VwkTVadm1kfQvwqlGO/Szw2QnEWFxhax5Lct+iK+CpTk5pqBv7OBGRGar8vhkbtuZZezmPd80FoFEtehGJsfJK9JHWPK/5GC3h9MSn5t8QXEQkRsor0Uda88xvHLpPrFr0IhJj5ZPo81rz7d0DHOrqp6YyyeL66lJHJyIyacon0ee15neEtw9sbKjFNC2xiMRYeST6XGs+kRqcoXJn2D/fqP55EYm58kj0udb82UPzzQ/dEFxDK0Uk3uKf6Ie15j86WJy7EHuKWvQiEnPxT/QjtOaBwaGV+lasiMRdvBP9KK35bNaHhlaqRS8iMRfvRH//l0Zsze9t76EvnWXh7CpmV1eUMEARkckX30R/aAc8fvsxrXmIdNuoNS8iZSC+iX6U1jxELsSqf15EykA8E/1xWvMALa3h0ErNWikiZSCeif44rXmAFrXoRaSMxC/Rj9Gah6E+eo24EZFyEL9EP0Zrvncgw972HlIJY/l83SdWROIvXom+gNb8zoNduMOK+TVUJOP144uIjCRemW6M1jxoxI2IlJ/4JPoCWvMQGXGjycxEpEzEJ9Fv//GYrXkYGnGjC7EiUi5SpQ6gaF7xB7DilVCz4LjV9K1YESk38Un0AEvOPu5udx/sutF9YkWkXBTUdWNm683saTPbbmbXjLD/H8zs0XDZZmZHIvsykX0bixj7uLV19dPRm2Z2VYqFdVWlDEVEZMqM2aI3syRwA3AxsBvYZGYb3X1Lro67/1mk/h8D50Seosfd1xYt4gmIfiNW94kVkXJRSIt+HbDd3VvcvR+4DbjsOPUvB24tRnDFpvvEikg5KiTRLwV2RbZ3h2XHMLOVQCPw00hxtZk1m9mDZvaWUY67MqzT3NraWljkJ2CH7hMrImWo2MMrNwDfcfdMpGyluzcB7wL+j5mdmn+Qu9/o7k3u3rRw4cIihzREtw8UkXJUSKLfAyyPbC8Ly0aygbxuG3ffEz62APcxvP9+Sun2gSJSjgpJ9JuA1WbWaGaVBMn8mNEzZnYGMA/4ZaRsnplVhesNwKuALfnHToV0Jstzh5ToRaT8jDnqxt3TZnY1cDeQBG5y981mdh3Q7O65pL8BuM3dPXL4mcBXzCxL8KHy+ehonam050gPAxlnyZxqairj9fUBEZHjKSjjuftdwF15ZdfmbX96hOP+F3jJBOIrGvXPi0i5is9cN2PYodsHikiZKptErwuxIlKuyibRq+tGRMpV2ST6wRuOqOtGRMpMWST6rr40L3T0UplKsHTerFKHIyIypcoi0eda86sW1JBMaDIzESkvZZHodVcpESln5ZHodZ9YESljZZLo1aIXkfJVFok+10d/qoZWikgZin2ij94nVkMrRaQcxT7Rtx7to6s/w9yaCubVVpY6HBGRKRf7RL8j941Y9c+LSJmKfaJvCW8f2KhuGxEpU7FP9Ds1x42IlLnYJ/oWjbgRkTIX+0Q/ND2xum5EpDzFOtH3p7M839aNGaxcUFPqcERESiLWif75tm4yWWfZvFlUVyRLHY6ISEnEOtGr20ZEJOaJfugbsboQKyLlK9aJfvCuUhpxIyJlLNaJfvA+seq6EZEyVlCiN7P1Zva0mW03s2tG2P8PZvZouGwzsyORfVeY2TPhckURYx9T7luxatGLSDlLjVXBzJLADcDFwG5gk5ltdPctuTru/meR+n8MnBOuzwc+BTQBDjwcHnu4qD/FCNp7BjjY2U91RYLF9dWT/XIiItNWIS36dcB2d29x937gNuCy49S/HLg1XH89cK+7t4XJ/V5g/UQCLlR0xE1C94kVkTJWSKJfCuyKbO8Oy45hZiuBRuCn4z222HYe1IgbEREo/sXYDcB33D0znoPM7Eozazaz5tbW1qIE0qLJzEREgMIS/R5geWR7WVg2kg0MddsUfKy73+juTe7etHDhwgJCGpsSvYhIoJBEvwlYbWaNZlZJkMw35lcyszOAecAvI8V3A5eY2TwzmwdcEpZNuhZ9K1ZEBChg1I27p83saoIEnQRucvfNZnYd0OzuuaS/AbjN3T1ybJuZfYbgwwLgOndvK+6PcKxs1gf76BvVRy8iZW7MRA/g7ncBd+WVXZu3/elRjr0JuOkE4zshL3T00juQpaGukjmzKqbypUVEpp1YfjNW34gVERkSz0Svb8SKiAyKZ6JvzV2IVaIXEYlnoh+ctVJdNyIi8Uz0rRpxIyKSE7tE3zuQYc+RHpIJY8V83SdWRCR2if65Q924w4r5NVSmYvfjiYiMW+wyob4oJSIyXOwS/Y7BMfRK9CIiEMNEPzi0UmPoRUSAGCb6oXnoNbRSRARimOhzY+hPVYteRASIWaI/3NXPke4BaiuTLJxdVepwRESmhVgl+qE5buow031iRUQgZol+h+4qJSJyjFgl+p0HNZmZiEi+WCX63Bw3msxMRGRIrBJ9rkWvL0uJiAyJTaLPZJ1nD3UD6roREYmKTaLfc7iH/nSWxfXV1FYVdCtcEZGyEJtE36LJzERERhSbRN/Vl2Hh7CoNrRQRyRObPo7feukSfuulS8hkvdShiIhMK7Fp0eckE/pGrIhIVEGJ3szWm9nTZrbdzK4Zpc47zWyLmW02s/+KlGfM7NFw2ViswEVEpDBjdt2YWRK4AbgY2A1sMrON7r4lUmc18HHgVe5+2MxOijxFj7uvLW7YIiJSqEJa9OuA7e7e4u79wG3AZXl1fh+4wd0PA7j7geKGKSIiJ6qQRL8U2BXZ3h2WRZ0OnG5m/2NmD5rZ+si+ajNrDsvfMtILmNmVYZ3m1tbW8cQvIiJjKNaomxSwGrgAWAbcb2YvcfcjwEp332NmpwA/NbMn3H1H9GB3vxG4EaCpqUnDZkREiqiQFv0eYHlke1lYFrUb2OjuA+6+E9hGkPhx9z3hYwtwH3DOBGMWEZFxKCTRbwJWm1mjmVUCG4D80TN3ELTmMbMGgq6cFjObZ2ZVkfJXAVsQEZEpM2bXjbunzexq4G4gCdzk7pvN7Dqg2d03hvsuMbMtQAb4c3c/ZGbnAV8xsyzBh8rno6N1RvLwww8fNLPnJvAzNQAHJ3D8ZFN8E6P4JkbxTcx0jm/laDvMPV5d4mbW7O5NpY5jNIpvYhTfxCi+iZnu8Y0mdt+MFRGR4ZToRURiLo6J/sZSBzAGxTcxim9iFN/ETPf4RhS7PnoRERkuji16ERGJUKIXEYm5GZnox5o22cyqzOz2cP+vzGzVFMa23Mx+Fpmy+U9GqHOBmbVHpm++dqrii8TwrJk9Eb5+8wj7zcz+KTyHj5vZy6YwthdFzs2jZtZhZn+aV2dKz6GZ3WRmB8zsyUjZfDO718yeCR/njXLsFWGdZ8zsiimM74tm9lT4+/u+mc0d5djjvhcmMb5Pm9meyO/wDaMcO+Y06ZMU3+2R2J41s0dHOXbSz9+EufuMWgi+tLUDOAWoBB4D1uTV+UPg38P1DcDtUxjfEuBl4fpsgukg8uO7APhhic/js0DDcfa/AfgRYMArgV+V8Pf9AsGcSSU7h8D5wMuAJyNlfw9cE65fA3xhhOPmAy3h47xwfd4UxXcJkArXvzBSfIW8FyYxvk8DHyvg93/cv/fJii9v//XAtaU6fxNdZmKLvpBpky8DvhGufwe40Mym5NZT7r7P3X8drh8FtnLsbJ8zwWXANz3wIDDXzJaUII4LgR3uPpFvS0+Yu98PtOUVR99n3wDeMsKhrwfudfc2D6bxvhdYP0K9osfn7ve4ezrcfJBgnqqSGOX8FaKQv/cJO158Ye54J3BrsV93qszERF/ItMmDdcI3ejuwYEqiiwi7jM4BfjXC7nPN7DEz+5GZnTW1kQHgwD1m9rCZXTnC/kLO81TYwOh/YKU+h4vcfV+4/gKwaIQ60+U8foDgP7SRjPVemExXh11LN43S9TUdzt9rgP3u/swo+0t5/goyExP9jGBmdcB3gT9194683b8m6Io4G/hngknhptqr3f1lwKXAH5nZ+SWI4bgsmETvzcC3R9g9Hc7hIA/+h5+WY5XN7BNAGrhllCqlei/8G3AqsBbYR9A9Mh1dzvFb89P+b2kmJvpCpk0erGNmKWAOcGhKogtes4Igyd/i7t/L3+/uHe7eGa7fBVRYMLvnlPGh6aMPAN8n+Bc5qpDzPNkuBX7t7vvzd0yHcwjsz3VnhY8j3VmtpOfRzN4HvBF4d/hhdIwC3guTwt33u3vG3bPAV0d53VKfvxTwNuD20eqU6vyNx0xM9IVMm7wRyI1ueAfw09He5MUW9uf9B7DV3b88Sp3FuWsGZraO4PcwlR9EtWY2O7dOcNHuybxqG4H3hqNvXgm0R7oppsqoLalSn8NQ9H12BfD/RqiTm9l1Xtg1cUlYNuksuNPbXwBvdvfuUeoU8l6YrPii13zeOsrrFvL3PpkuAp5y990j7Szl+RuXUl8NPpGFYETINoKr8Z8Iy64jeEMDVBP8u78deAg4ZQpjezXBv/CPA4+GyxuAq4CrwjpXA5sJRhA8CJw3xefvlPC1HwvjyJ3DaIxGcFP4HcATQNMUx1hLkLjnRMpKdg4JPnD2AQME/cQfJLju8xPgGeDHwPywbhPwtcixHwjfi9uB909hfNsJ+rdz78PcSLSTgbuO916YovhuDt9bjxMk7yX58YXbx/y9T0V8YfnXc++5SN0pP38TXTQFgohIzM3ErhsRERkHJXoRkZhTohcRiTklehGRmFOiFxGJOSV6EZGYU6IXEYm5/w9CKy1MmeTZ5wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "![](../pic/BiLSTM_CRF-precision-20.jpg)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtXElEQVR4nO3de3xcdZ3/8ddnJvdL07RJS+8XKNACyqUUBBUQhcLvpyir/oo3cFGW3cVdWffnD2/I4rpell133cV1QbsqKqio2HWLiAIqCrRFKFCgUJpSeqFNmra5X2bm8/vjnElOpkkzaZNMMvN+Ph7nMec688lJ8p7vfM9lzN0REZH8Fct1ASIiMrYU9CIieU5BLyKS5xT0IiJ5TkEvIpLnFPQiInlOQS/DMrOvm9lncl1HITKzT5rZN7JY714zu3I8apLJR0EvAJjZNjPrNLM2M9tvZv9jZvMA3P1ad/9cuN75ZrZjiOeYa2Y/NrMmMztoZs+Y2VVm9obwedvMrN3MPDLdZmbzzeyhcP5rM57zp+H887P4GY43sx9FXv8pM/sbM4ub2cKM191mZjccZh+kh9lHuk9Hg7v/g7t/KIv1LnH3b4/264f79Gdm1mhmzWZ2n5mdkLHO9Wb2qpm1mNlqMysd7Trk6CjoJeqt7l4FzAL2AP82wu3vAF4BFgDTgfcDe9z9d+5eFT73SeG6U9Pz3H17OO8F4APpJzOz6cDrgMbhXtjMjgUeC1//FHevAd4FLAeqI6tODet4J/AZM3tLxlO9NVJXlbvvGtEeGLy2oqN9jhyaCqwBTgBmAuuAn6UXmtnFwA3AhQS/98XA3417lXJYCno5hLt3AXcDywDM7Ftm9vdZbHom8C13b3f3hLs/4e73juClvwf8HzOLh9NXAD8FerLY9u+AP7j737j77vDn2Ozu73H3A5kru/sGYBNw6gjq6xN+AvmCma0LW7I/M7Np4bL0p4erzWw78EA4/0/N7LnwE9N9ZrYg8nwnmdn9Yat5j5l9Mpx/k5l9NxwvM7Pvmtk+MztgZuvNbGakng+F4zEz+7SZvWxme83sO2ZWk1HblWa2Pfz086mhfk53X+fu33T3ZnfvBb4CnBC+CQNcCXzT3Te5+37gc8BVR7JPZewo6OUQZlYB/B/g0RFu+ihwq5mtMrP5R/DSu4BngYvC6Q8A38ly2zcTvDllxczOBk4GtoykwAwfAP6U4BNQAvhqxvLzgKXAxWZ2GfBJ4HKgHvgdcGdYSzXwK+AXwGzgOODXg7zelUANMI/gE9O1QOcg610VDhcQtLCrgH/PWOf1BK30C4EbzWxpVj8xvBF41d33hdMnARsjyzcCMyNvBDIBKOgl6h4zOwAcBN4C/OMIt38XQYB9BmgwsyfN7MwRPsd3gA+Y2YkE3SyPZLnddGB3Fus1mVkn8AjwNeCejOX3hK3lA2aWuSzTHe7+jLu3E/zM7458GgG4Kfx000kQyl9w9+fcPQH8A3Bq2Kr/3wTh+U/u3uXure7+2CCv1xv+nMe5e9LdH3f3lkHWey/wz+6+1d3bgE8AqzK6kP7O3TvdfSNBOL92kOcZwMzmArcCfxOZXUXw95KWHo92l0mOKegl6u3uPhUoA64DfmNmx2S7sbvvd/cb3P0kgv7cJwmC00ZQw0+AN4Wvf8cItttH0LIeTh1BOH0MOB8ozlj+dnefGg5vh76zjtIHZz8ZWfeVyPjL4XPVDbF8AfCv6TcRoBkwYA5BC/2lLGq/A7gPuMvMdpnZl80ss34IPhW8nFFbEcHvJO3VyHgHwT4h40B036cyM6sHfgl8zd3vjGzbBkyJTKfHW7P4eWScKOjlEGFr8SdAkuAj/pE8RxNwC0HoTBvBdh3AvcCfM7Kg/xXwJ1m+RtLd/xnoAv4ii/WvjRyc/YfIonmR8fkELe6m6KaR8VeAP4u8iUx193J3/0O4bHEWdfS6+9+5+zLgHIJPAh8YZNVdBG8s0doSBAfYh3uN6IHo7QBmVksQ8mvc/fMZm2xi4KeB1xIcgN+HTBgKejmEBS4DaoHnhlinLGMwM/uSmZ1sZkVhv/OfA1uO4J/+k8B57r5tBNt8FjjHzP4x/SnEzI4LD15OHWKbLwIfN7OyEdaX9j4zWxYe07gZuNvdk0Os+3XgE2Z2UlhbjZm9K1z2c2CWmX3UzErNrNrMzsp8AjO7wMxOCbuHWgjeWFKDvNadwPVmtsjMqgi6iX4QdhmNiJlNIfgU8Xt3v2GQVb4DXB3uh6nAp4FvjfR1ZGwp6CXqv82sjSBEPg9c6e6bBllvDsFBwOhwLFBBcJbMAWArQavybSMtwt13ufvDI9zmJYJTMRcCm8zsIPBjYANDdyP8D7Af+PBIawzdQRBqrxJ0d/3VYer7KfAlgm6XFuAZ4JJwWSvBMZG3hs/1IsGB1EzHEBxwbiF4A/4Ng3/qWR3O/y3QQPDJ5SMj/eFC7yA4m+qDg3XruPsvgC8DDwLbCbqJPnuEryVjxPTFIyIjZ2YPAd9192GvWhXJNbXoRUTynIJeJg0L7ufSNsjwyeG3Filc6roREclzatGLiOS5CXezpbq6Ol+4cGGuyxARmVQef/zxJnevH2zZhAv6hQsXsmHDhlyXISIyqZjZy0MtU9eNiEieU9CLiOQ5Bb2ISJ5T0IuI5DkFvYhInlPQi4jkOQW9iEiem3Dn0YuIjFR3IsnBzl46upN09CTp7E3S1ZukMxzvjI73hMvC8Y7eJF3hsnjMmFJWzJTyovCxmCllReFj//ya8mBZaVGMkXyBmrvTm3R6kim6e5P0JFP0JFJ0J/ofT58/dUTPmQ0FvYiMC3fHPfjarZQ7qfS090939gSBfbihZZB5Xb2Dff/K2CuJx/rCv7q8mOKY9YV2NMyjQT6cF/7+EkqKFPQieSeVClp5HT1J2rsTtHUn6OhJ0NYdme5O0N6T7BtPL2vvSQSP3UmS7hTFjJgZ8VhkyJwO1ylKj8eC8ZQ7iaTTm0yRSIWPSSeRStEbPibCFmki6SSSKXpTwWMi6STD8E5FH+mfHitFMaOmvJjK0iIqSuKUl8QpLw6Gssh43/whlifdaenspaWrl5bORPgYvsF0JQYu6+ylJ5miqa2HpraeEdVaUhSjtChGSXqIxygtilNSFCOZGv0dpaAXGYa7051I0dLVS2tXIhx6Mx6DobM3GbbckhkfycOWXW8q4zGY35ssjLvImkHMDCN8DKdjBmZGWXGcmvIiasr7u0dqIkPmdHqoKImPendHNrp6k33Bf7Czl2TKBwR4erw0Hu+bF4+Nf50KeslLqZTT2Ztu8YaP3Qk60i3inqAFPGB5TxDWbV0JWrsHhvp4BHFJUYyKkjiVJUVUlsapLC2iKmyhpscrS4uoDKejy4PHIoriRjLl/YP3j6dSTiI9L2x9p+elW/KxGBTFYhTHjaJYjKK4URyPURQziuL984vT8zOWB58UBoZ4NNRzEcZjqaw4TllxnBnVua7k8BT0Mil09CTY19ZDY1s3Ta3d4cflbprauvvnt3XT2tXfxTGaSuIxqsuKwqF4wHhVaRFTwvHyknhfSy4Y4gNbdodM9390z7cQlIlDQS9jLt310dGTpKMnET4G453heGc43dzRGwR4a3cY5EGgdxxBcJcXp1u+/a3kipKBreR0y7myJJg3MMiDx6rSIsqK42OwZ0TGh4Jejlgq5bzc3MGzu1p4dvdBnt/dyv6OnkiQJ+nsCfqtj/b4Ukk8Rl1VCXXVpdRVlQbjVcH49KoS6qtKqasu7euvrSgpyklfqMhEpKCXrHT2JNm8p5XndreEwd7Cc7tbsm5pp/ufK8IzHipKiigviVMZGa8oiTO1omRAiKfDvbq0SF0bIkcoq6A3s5XAvwJx4Bvu/sWM5QuA1UA90Ay8z913hMuSwNPhqtvd/W2jVLuMkaa27r4wTz9ubWwbtFV+zJQyls2ewtJZ1SydNYVjppT1BXn6NLeK4jhFcV2ELZIrwwa9mcWBW4G3ADuA9Wa2xt2fjax2C/Add/+2mb0J+ALw/nBZp7ufOrply2jpSaR4eucBHmtoZsO2/Tyz8yB7W7sPWS8eM46fUcmyWVNYNnsKy2bVsHRWNdOrSnNQtYiMRDYt+hXAFnffCmBmdwGXAdGgXwb8TTj+IHDPKNYoo6i9O8ET2w+wrmEf67Y188T2A4dcrVdVWsTSWdV9ob501hSOn1mtA5Iik1Q2QT8HeCUyvQM4K2OdjcDlBN077wCqzWy6u+8DysxsA5AAvuju92S+gJldA1wDMH/+/JH+DHIYBzp6WL9tfxjsQYs988q7JTOqWLFoGisWTePUeVOZV1tBTAcyRfLGaB2M/Vvg383sKuC3wE4gfZRugbvvNLPFwANm9rS7vxTd2N1vA24DWL58eWFcIjhGXj3Yxbptzaxr2Mf6hv1s3tM6YHk8Zrxmbg0rFk7jzEXTOHPhNKZVluSoWhEZD9kE/U5gXmR6bjivj7vvImjRY2ZVwJ+4+4Fw2c7wcauZPQScBgwIejk6yZTz0yd28h8PbeGlxvYBy0qKYpw6bypnhaF++oJaqkp1spVIIcnmP349sMTMFhEE/CrgPdEVzKwOaHb3FPAJgjNwMLNaoMPdu8N1zgW+PIr1FzR3575Ne/inX27mxb1tQNC/fsaC2r6umNfMraG0SH3rIoVs2KB394SZXQfcR3B65Wp332RmNwMb3H0NcD7wBTNzgq6bvww3Xwr8p5mlCL7k5IsZZ+vIEfrDlia+dN9mNr5yAIC5teVc/+bjuezU2TqVUUQGMB/Le4cegeXLl/uGDRtyXcaEtfGVA/zjfZt5eEsTAHVVpXzkTcexasU8tdxFCpiZPe7uywdbps7aSWLL3lZuue8FfrHpVQCqy4q49rxj+eC5C6ko0a9RRIamhJjgduzv4F9+9SI/+eMOUg6lRTGuOnchf37esUyt0NkyIjI8Bf0E1dTWzb8/sIXvP7adnmSKophxxYp5/NWFS5g5pSzX5YnIJKKgn2Baunr5xm+38o2HG+joSWIGl506m+vffDwL6ypzXZ6ITEIK+gkilXJW/76Bf39wCwc6egG48MQZ/O3FJ7B01pQcVycik5mCfgJwdz51zzPcuW47ACsWTuPjK09g+cJpOa5MRPKBgj7H3J2//5/nuHPddkqLYvzrqtO4+KSZuve6iIwaBX2OfeVXL/LNhxsojhtff/8ZXHDCjFyXJCJ5RpdQ5tB//uYlvvrrF4kZfHXVaQp5ERkTCvocueORbXzh3ucBuOVdr+WSU2bluCIRyVcK+hy4+/EdfOZnmwD4+7efzOWnz81xRSKSzxT042zt07v5+N0bAfjUpUt539kLclyRiOQ7Bf04evD5vfzVnU+Qcvjom5fw4TcuznVJIlIAFPTj5A8vNXHtdx8nkXI+/IZF/PWFS3JdkogUCAX9OHj85f186Nsb6E6keO9Z8/nkpUt1nryIjBsF/Rh7ZudBrvqvdXT0JLn8tDl87rKTFfIiMq4U9GNoy95WPrB6Ha1dCVaedAxffudriMUU8iIyvhT0Y+Tlfe285/bHaG7v4bzj6/nqFafpK/5EJCeUPGNg98FO3nP7Y+xt7easRdP4+vvOoKRIu1pEckPpM8oaW7t57+2PsfNAJ6fOm8o3rzqT8hJ9l6uI5I6CfhQd6Ojh/d98jK1N7SydNYVvf3AFVaW6b5yI5JaCfpR09Sa58r/W8/yrrSyur+SOq1dQU1Gc67JERBT0o2XNk7vY+MoB5taW870PnUVdVWmuSxIRART0o2btM7sBuO6C45hVU57jakRE+inoR8HBzl5+v6WJmMFbls3MdTkiIgNkFfRmttLMNpvZFjO7YZDlC8zs12b2lJk9ZGZzI8uuNLMXw+HK0Sx+ovj1c3voTTpnL57OdHXZiMgEM2zQm1kcuBW4BFgGXGFmyzJWuwX4jru/BrgZ+EK47TTgs8BZwArgs2ZWO3rlTwxrn34VQF8eIiITUjYt+hXAFnff6u49wF3AZRnrLAMeCMcfjCy/GLjf3ZvdfT9wP7Dy6MueOFq7evnti42YwcUnqdtGRCaebIJ+DvBKZHpHOC9qI3B5OP4OoNrMpme5LWZ2jZltMLMNjY2N2dY+ITzw/F56EinOXDCNGdVluS5HROQQo3Uw9m+B88zsCeA8YCeQzHZjd7/N3Ze7+/L6+vpRKml8/OKZdLfNMTmuRERkcNlctrkTmBeZnhvO6+Puuwhb9GZWBfyJux8ws53A+RnbPnQU9U4oHT0JHty8F4CVJyvoRWRiyqZFvx5YYmaLzKwEWAWsia5gZnVmln6uTwCrw/H7gIvMrDY8CHtROC8vPLS5ka7eFKfNn6pz50Vkwho26N09AVxHENDPAT90901mdrOZvS1c7Xxgs5m9AMwEPh9u2wx8juDNYj1wczgvL9wbdttcerLOthGRiSurO265+1pgbca8GyPjdwN3D7Htavpb+HmjqzfJA8/tAdRtIyITm66MPUK/faGR9p4kp8ypYd60ilyXIyIyJAX9EdLZNiIyWSjoj0B3Isn9YbfNJeqfF5EJTkF/BP6wZR+tXQmWzprCorrKXJcjInJYCvojsPbp4JbEl+ggrIhMAgr6EepNpvq6bS5V/7yITAIK+hF6dOs+DnT0smRGFcfNqM51OSIiw1LQj1DfLYnVbSMik4SCfgSSKeeXm3TveRGZXBT0I7CuoZl97T0sqqvkxGPUbSMik4OCfgTufab/bBszy3E1IiLZyepeNwKplPdfDauLpETkSPR2Qdur0LYXWl+Ftj3BkB7vPABX/xJGuSGpoM/SH7fvZ29rN3Nryzl5zpRclyMio6GjGbY/Att+Dy//HjqbobgSSiqguAJKKsPHcPqQeRnrJrqGDvG2PdB1cPiaulugrGZUf0wFfZbSZ9tcesosddtMJKkUdB2A9ibo2AexIiibAqVTgsfiilFvHU04yQS0NwYtxd5O8BS4h4+pwafJXB6uUzEN6o6HKXPyc7+1NQaB/vLvg3Dfu2l8Xz9WDFUzoWoGVB8TPFYdA9Uzw/nHBH+zo0xBnwV35xdh/7xuSTzGEj3QEYZ2e8ZjR1M43tw/3tkcBtcQYkX9oV86JWgpldX0z4uOl06BotJwwzDk+sLOBo5HHgasGy+F4vKw9Rd9LIdYfGT7IpmA9rB12PpqEOStrx463d54+H1wJEqqoG4J1J0A9ceHjydA7SKIT6LYaNkdhvrDwWPTCwOXx0th7nJYcC4sPBemzg/eLHs6oLc9fOyAnvbIY2fGvMi68TDIq48Jg3tmGOLhdHktxMb/0Ogk+o3lzsYdB9l1sItZNWWcOndqrsuZ/NyhdTc0Pg+NmyOPm4PgHqmyGqioC1qjnoKuluAjcndL8FG6s/nInne0Rd8ESjLfCCqgqCwIj9bd4Uf9vYBn99wVdUG4lFSBxcLBIuOxIeZHxrHgjaNpc/DGuuuJYIiKFcP0Y4NWf/0J/W8E05cEP1OuHdgedsM8HDzubxi4vKgc5q2Aha8Pwn3OGVBclptax5GCPgv3Pt3fmo/F8vDj7Fhxh4M7ImEeCfTuIfoqY0VQMb1/qKwLx+si4+n5YbjHi4euIdETBH7Xwf7w7zoYvBl0twx8U+g6CKlEUHfwA/T/HAPGM5eF0+6Q7A1aeb2d/S2/9JDsDoauA1nuQIPKGf0twurIUHUMVM8KllXOgKKSLJ8zS+37gsBv3By0ghs3Q9OLcHB7/+/yuYxtauYHLeKKaeHvaVr/76s8Om9a8Okpm64h9+DNr31v8AmubW/wCaa9KZzXGHTHtDcG0537B25fUgXzzw5b7K+HWaeO/r6aBBT0w3B31vadVqmzbQaVTMCBl4MgaNoMe8MgaHoBetoG36a8FuqXBq3C+hP7H6uPGd2+4aISKArfJHLJPfh00Rf+gzz2dAQH9KpnBiFeWX/4N7GxVDkdKs+BBecMnN/THv6e0+G/GRpfgOaXgjeBg9uze/70G3rmG0AqGYZ2Y3+IJzqzr7usBuafE3TDLDgXjnnN5OpqGiPaA8PYtKuFV5o7qa8u5YwFtbkuJ3fcg4/zTS/CvhfDxy3B0NwAqd7Bt6usHxjk6aGyLj8P9g3FrL+vnmm5rubIlVTC7FODISrZG/wdtO4O/k46m8NjKfsij+F4Z3PQAEifiTKcorLgU0tVffD3lB6qZhw6XT4tJ33gE52Cfhjpi6RWnnQM8ULotuntCvo1+wJ9S3+wH67LYcpcqDuu/6BdX6BPH7fSJYfixUFfff3x2a3f25XxZhAOsXgQ6pX1/cFeUlVYjYIxoKA/DHfvv4lZPt6SuLsNdm+EXX+EnX+E3U/C/m1Dn8FRUh2E+fQlwRkZ048Lh2ODlp5ItorLoHg2TJmd60oKgoL+MDbvaaWhqZ1plSWsWDiGH7d72uGpH8LTPwr6LqctHjjULjz6MxoS3fDqM/2hvuuPQR9r5lkdFgtec3oY5NFgr5qplpXIJKSgP4x7w9b8xSfNpCg+Bv1+zQ2w/hvwxB0Dr5hr+M2h61bPCoN/UcabwKLgHPCoZCI4GBoN9T3PHtqPHiuCmSfB7NNg9ukw5/Sg66UAz0oQyWcK+sO4dyzOtnGHrQ/CY7fBC7+gr0U990w488PBmQfNWwcO+18ODnK1hhd/ZKqoC0J/6nw4+ArsfmqQMxUs6DOffXoQ7HNOh5knF8Q5xCKFTkE/hC1723hhTxs15cW87thROKDY3Qob74J1t/VfnRcvgZP/BFZcEwTvUFLJ4Hz0AW8ADeGbQEN4JWkT7FjXv03twoGhPuu1UKpbK4sUoqyC3sxWAv8KxIFvuPsXM5bPB74NTA3XucHd15rZQoLLKjaHqz7q7teOTuljK33Lg7csm0nx0XTb7HspCPcnvx9clANBN8zyq+GMq4IzC4YTi0PtgmA49oKBy1Kp4FL4dMu/emYQ8BWT+BQ+ERlVwwa9mcWBW4G3ADuA9Wa2xt2fjaz2aeCH7v4fZrYMWAssDJe95O6njmrV46D/JmZHcLZNKgUv/Roe+0/Ycn///PmvC1rvS986ehfCxGLBmQtTZgdX/omIZMimRb8C2OLuWwHM7C7gMiAa9A6kjwjWALtGs8jxtq2pnWd3t1BdWsS5x43gisqug0HLfd3twZWCEFzscco7YcWfwazXjE3BIiKHkU3QzwFeiUzvAM7KWOcm4Jdm9hGgEnhzZNkiM3sCaAE+7e6/y3wBM7sGuAZg/vz5WRc/Vu4Nv2DkwqUzKC3K8o6DOzbAdy/vP3umZh6ceTWcfqW6UUQkp0brYOwVwLfc/Z/M7HXAHWZ2MrAbmO/u+8zsDOAeMzvJ3VuiG7v7bcBtAMuXL8/ydn1jJ90/n/UXgLfshrveG4T8vLPhnOvg+Et0jw0RmRCySaKdwLzI9NxwXtTVwEoAd3/EzMqAOnffC3SH8x83s5eA44ENR1v4WNmxv4ONOw5SURLnvOOzOFDa2wU/eF9wQHTBufCBn+XuRlQiIoPI5nSS9cASM1tkZiXAKmBNxjrbgQsBzGwpUAY0mll9eDAXM1sMLAG2jlbxYyH9vbAXnDiDsuJhum3c4efXw84NQVfNu7+jkBeRCWfYFr27J8zsOuA+glMnV7v7JjO7Gdjg7muAjwG3m9n1BAdmr3J3N7M3AjebWS+QAq519wnwDRBDS/fPX5rNRVKPfR02fj/4MoNV38/9rXBFRAaRVSeyu68lOGUyOu/GyPizwLmDbPdj4MdHWeO4efVgF4+/vJ+y4hjnnzBMt81LD8J9nwrG3/41nVEjIhOWbtwckT4Ie97x9VSWHuY9sHkr/Ogq8CS84WNw8uXjU6CIyBFQ0EesTXfbHO5sm+5WuPM9wb3Zj18JF3x6fIoTETlCCvpQY2s367c1UxKP8aYTZwy+UioFP70WGp8Lvhz58tv0bTYiMuEppUKPbt2HO7zu2OlUlw1x5sxvvwzP/xxKa2DVncH3U4qITHAK+lBDUzsAS2dNGXyF5/4bHvpC8MUc71wdfCGHiMgkoKAPpYN+Ud0g3+S0ZxP85M+C8TffBEvefOg6IiITlII+1B/0VQMXdDTDnVdAbzuc8i44569yUJ2IyJFT0IfSQb8w2qJPJoLTKA+8HHxxx9v+Td+ZKiKTjoIe2N/ew8HOXipL4tRXlfYvuP8zwfe3VtYHV74Wl+euSBGRI6SgBxr2hd029ZVYusX+xPfg0a9BrBjefQfUzM1hhSIiR05BDzQ0ht020yuDGTs2wM8/Goz/r1tgwetyU5iIyChQ0APbwhb94rrK/nvLJ3vgzA8F3+sqIjKJKeiBreGB2MW1RQPvLb/yi8NsKSIy8SnoCb4jFpw3bP687i0vInmn4IPe3WloaueS2Dqmb/mx7i0vInmn4IO+sbWb7p4ePl7yo2DGRZ/TveVFJK8UfNBvbWrnHfGHWcQuqF2og68ikncKPui379nPR4vCL8E6/xPqlxeRvFPwQV/z/J3MtSb2VSwO7mUjIpJnCjvoezo4e8dqALae/NcQi+e4IBGR0VfYQb/+dmqSzTyVWkT5a96e62pERMZE4QZ910H84a8AcEvi3SysrxpmAxGRyalwg/6Rr2Gd+3ksdSLPV5xJVWlRrisSERkThRn0Hc3wyK0A3NKr1ryI5Lesgt7MVprZZjPbYmY3DLJ8vpk9aGZPmNlTZnZpZNknwu02m9nFo1n8EXv4K9DTys7p57DeT2RR+q6VIiJ5aNigN7M4cCtwCbAMuMLMlmWs9mngh+5+GrAK+Fq47bJw+iRgJfC18Plyp2U3rLsNgLX1HwaC+9CLiOSrbFr0K4At7r7V3XuAu4DLMtZxYEo4XgPsCscvA+5y9253bwC2hM+XO7+7BRJdsPSt/KEz+DKRhWrRi0geyybo5wCvRKZ3hPOibgLeZ2Y7gLXAR0awLWZ2jZltMLMNjY2NWZZ+BPa/DI9/GzC44FNs29cBwGK16EUkj43WwdgrgG+5+1zgUuAOM8v6ud39Nndf7u7L6+vrR6mkQfzmS5Dqhde8m97pJ7C9uQMzmD+tYvhtRUQmqWzOKdwJzItMzw3nRV1N0AePuz9iZmVAXZbbjo/GzbDxTogVwfk3sGN/J8mUM2dqOWXFuiJWRPJXNq3u9cASM1tkZiUEB1fXZKyzHbgQwMyWAmVAY7jeKjMrNbNFwBJg3WgVPyIP/gN4Ck57P0xbTENTGwCL6tRtIyL5bdgWvbsnzOw64D4gDqx2901mdjOwwd3XAB8Dbjez6wkOzF7l7g5sMrMfAs8CCeAv3T05Vj/MkHZvhGfvgXgpvPH/AtDQFPTPK+hFJN9ldTmou68lOMganXdjZPxZ4Nwhtv088PmjqPHoPRC+/IoPQ01wLDjdol+ooBeRPJf/V8ZufwxevA9KquD11/fN3tbXoteBWBHJb/kd9O7wwOeC8bP/fMD3wDY0tQOwqE63PxCR/JbfQb/1Idj2Oyirgddd1ze7qzfJroOdxGPG3Nry3NUnIjIO8jfoo635cz8K5VP7Fm1v7sA9OH++OJ6/u0BEBPI56DevhZ2PQ+UMOOvPBiza2hh02yycrv55Ecl/+Rn0qVT/mTZv+BiUDDyzZts+9c+LSOHIz6Df9BPYuwmmzIXlHzxkcUNjOujVoheR/Jd/QZ/shQfD1vz5/w+KSg9ZpUEtehEpIPkX9E9+H5q3wrRj4bXvGXSVbeGplQvVoheRApBfQZ/oht98ORi/4JMQP/TC37buBHtbuykpijG7RqdWikj+y6+g3/Bf0LIDZpwEJ10+6Cp9rfnpFcRiNp7ViYjkRP4EfU978O1RAG/6NMQG/9Ea+oJe97gRkcKQP0G/7nZob4Q5Z8AJlwy5WrpFr++JFZFCkdXdKyeFM66E7hZYfD7Y0F0yffe4UYteRApE/gR9eS1ceOOwq/WfWqmgF5HCkD9dN1nqv2ulgl5ECkNBBf2Bjh4OdPRSWRKnvvrQC6lERPJRQQV93xk3dZXYYfrxRUTyScEGvYhIoSiooE+fWrlYQS8iBaSggn6rLpYSkQJUUEHfdx96XSwlIgWkYILe3fvvQ68WvYgUkIIJ+sa2btp7kkytKKa2siTX5YiIjJuCCfptTR2A+udFpPBkFfRmttLMNpvZFjO7YZDlXzGzJ8PhBTM7EFmWjCxbM4q1j0hDUxugK2JFpPAMe68bM4sDtwJvAXYA681sjbs/m17H3a+PrP8R4LTIU3S6+6mjVvERaghb9Ap6ESk02bToVwBb3H2ru/cAdwGXHWb9K4A7R6O40ZRu0etiKREpNNkE/Rzglcj0jnDeIcxsAbAIeCAyu8zMNpjZo2b29iG2uyZcZ0NjY2N2lY9Quo9eF0uJSKEZ7YOxq4C73T0ZmbfA3ZcD7wH+xcyOzdzI3W9z9+Xuvry+vn6US4JUyvvOoVeLXkQKTTZBvxOYF5meG84bzCoyum3cfWf4uBV4iIH99+Nid0sX3YkU9dWlVJXmzy34RUSykU3QrweWmNkiMyshCPNDzp4xsxOBWuCRyLxaMysNx+uAc4FnM7cda7pQSkQK2bDNW3dPmNl1wH1AHFjt7pvM7GZgg7unQ38VcJe7e2TzpcB/mlmK4E3li9GzdcaLvlVKRApZVv0Y7r4WWJsx78aM6ZsG2e4PwClHUd+o2KbbE4tIASuIK2P7vz6wIseViIiMv4II+m19QV+V40pERMZf3gd9Iplie3NwDv2C6WrRi0jhyfug37G/k0TKmTO1nLLieK7LEREZd3kf9P3fE6vWvIgUpoIJep1aKSKFKu+Dvu/WB7pYSkQKVN4HvVr0IlLoFPQiInkur4O+qzfJzgOdxGPGvGk6GCsihSmvg/6V5g7cYV5tOcXxvP5RRUSGlNfpt1X3uBERye+g36b+eRGR/A56HYgVEVHQi4jkvbwOel0sJSKSx0Hf3p1gT0s3JfEYs6eW57ocEZGcydugT7fmF0yvIB6zHFcjIpI7eRv0DTq1UkQEyOOgT59auVhBLyIFLm+DXhdLiYgE8jbodbGUiEggf4N+X/A9sQp6ESl0eRn0Bzt6aW7voaIkzozq0lyXIyKSU1kFvZmtNLPNZrbFzG4YZPlXzOzJcHjBzA5Ell1pZi+Gw5WjWPuQGiIXSpnp1EoRKWxFw61gZnHgVuAtwA5gvZmtcfdn0+u4+/WR9T8CnBaOTwM+CywHHHg83Hb/qP4UGRqa2gB124iIQHYt+hXAFnff6u49wF3AZYdZ/wrgznD8YuB+d28Ow/1+YOXRFJyNhib1z4uIpGUT9HOAVyLTO8J5hzCzBcAi4IGRbjuadLGUiEi/0T4Yuwq4292TI9nIzK4xsw1mtqGxsfGoi9CplSIi/bIJ+p3AvMj03HDeYFbR322T9bbufpu7L3f35fX19VmUNDR31+2JRUQisgn69cASM1tkZiUEYb4mcyUzOxGoBR6JzL4PuMjMas2sFrgonDdmmtp6aOtOUFNeTG1F8Vi+lIjIpDDsWTfunjCz6wgCOg6sdvdNZnYzsMHd06G/CrjL3T2ybbOZfY7gzQLgZndvHt0fYaC+e9DX6dRKERHIIugB3H0tsDZj3o0Z0zcNse1qYPUR1jdiDY1ht830ivF6SRGRCS3vroxNXyy1qK4qx5WIiEwM+Rf0jemuG7XoRUQgD4M+3Ue/WC16EREgz4I+lfLIxVJq0YuIQJ4F/astXXQnUtRVlVJdplMrRUQgz4K+/4pYteZFRNLyKui36opYEZFD5FXQb9PNzEREDpFXQd93j5vpCnoRkbT8Cvr0xVL1CnoRkbS8CfpEMsX28AvBF0xT0IuIpOVN0O880Eki5cyuKaO8JJ7rckREJoy8CfqtOhArIjKovAn6zp4k9dWlOrVSRCRDVrcpngwuPWUWl54yi2TKh19ZRKSA5E2LPi0e05eNiIhE5V3Qi4jIQAp6EZE8p6AXEclzCnoRkTynoBcRyXMKehGRPKegFxHJc+Y+sS4wMrNG4OWjeIo6oGmUyhkLqu/oqL6jo/qOzkSub4G71w+2YMIF/dEysw3uvjzXdQxF9R0d1Xd0VN/Rmej1DUVdNyIieU5BLyKS5/Ix6G/LdQHDUH1HR/UdHdV3dCZ6fYPKuz56EREZKB9b9CIiEqGgFxHJc5My6M1spZltNrMtZnbDIMtLzewH4fLHzGzhONY2z8weNLNnzWyTmf31IOucb2YHzezJcLhxvOqL1LDNzJ4OX3/DIMvNzL4a7sOnzOz0cazthMi+edLMWszsoxnrjOs+NLPVZrbXzJ6JzJtmZveb2YvhY+0Q214ZrvOimV05jvX9o5k9H/7+fmpmU4fY9rB/C2NY301mtjPyO7x0iG0P+/8+hvX9IFLbNjN7cohtx3z/HTV3n1QDEAdeAhYDJcBGYFnGOn8BfD0cXwX8YBzrmwWcHo5XAy8MUt/5wM9zvB+3AXWHWX4pcC9gwNnAYzn8fb9KcDFIzvYh8EbgdOCZyLwvAzeE4zcAXxpku2nA1vCxNhyvHaf6LgKKwvEvDVZfNn8LY1jfTcDfZvH7P+z/+1jVl7H8n4Abc7X/jnaYjC36FcAWd9/q7j3AXcBlGetcBnw7HL8buNDMxuWrp9x9t7v/MRxvBZ4D5ozHa4+yy4DveOBRYKqZzcpBHRcCL7n70VwtfdTc/bdAc8bs6N/Zt4G3D7LpxcD97t7s7vuB+4GV41Gfu//S3RPh5KPA3NF+3WwNsf+ykc3/+1E7XH1hdrwbuHO0X3e8TMagnwO8EpnewaFB2rdO+Id+EJg+LtVFhF1GpwGPDbL4dWa20czuNbOTxrcyABz4pZk9bmbXDLI8m/08HlYx9D9YrvfhTHffHY6/CswcZJ2Jsh//lOAT2mCG+1sYS9eFXUurh+j6mgj77w3AHnd/cYjludx/WZmMQT8pmFkV8GPgo+7ekrH4jwRdEa8F/g24Z5zLA3i9u58OXAL8pZm9MQc1HJaZlQBvA340yOKJsA/7ePAZfkKeq2xmnwISwPeGWCVXfwv/ARwLnArsJugemYiu4PCt+Qn/vzQZg34nMC8yPTecN+g6ZlYE1AD7xqW64DWLCUL+e+7+k8zl7t7i7m3h+Fqg2Mzqxqu+8HV3ho97gZ8SfESOymY/j7VLgD+6+57MBRNhHwJ70t1Z4ePeQdbJ6X40s6uA/w28N3wzOkQWfwtjwt33uHvS3VPA7UO8bq73XxFwOfCDodbJ1f4bickY9OuBJWa2KGzxrQLWZKyzBkif3fBO4IGh/shHW9if903gOXf/5yHWOSZ9zMDMVhD8HsbzjajSzKrT4wQH7Z7JWG0N8IHw7JuzgYORborxMmRLKtf7MBT9O7sS+Nkg69wHXGRmtWHXxEXhvDFnZiuBjwNvc/eOIdbJ5m9hrOqLHvN5xxCvm83/+1h6M/C8u+8YbGEu99+I5Ppo8JEMBGeEvEBwNP5T4bybCf6gAcoIPu5vAdYBi8exttcTfIR/CngyHC4FrgWuDde5DthEcAbBo8A547z/FoevvTGsI70PozUacGu4j58Glo9zjZUEwV0TmZezfUjwhrMb6CXoJ76a4LjPr4EXgV8B08J1lwPfiGz7p+Hf4hbgg+NY3xaC/u3032H6TLTZwNrD/S2MU313hH9bTxGE96zM+sLpQ/7fx6O+cP630n9zkXXHff8d7aBbIIiI5LnJ2HUjIiIjoKAXEclzCnoRkTynoBcRyXMKehGRPKegFxHJcwp6EZE89/8BUoRRsA4PimwAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "![](../pic/BiLSTM_CRF-recall-20.jpg)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAArr0lEQVR4nO3deZhcdZ3v8fe3q6v37qQ7aRLInhAIQWULIDoqikRgvOJydYIbbsNVB+/MqI8XZ9xuHGbU0dG596J3UCPKKLigDneEkU0cFzRpkMWEBLJBEkLSSaeX9FrL9/5xTnWfrlSnq9fqrvq8nuc8Z6/6dnX3p371O6fOMXdHRESKV1mhCxARkamloBcRKXIKehGRIqegFxEpcgp6EZEip6AXESlyCnoRkSKnoJdJY2b/18w+Weg6SpGZ3WJmfxdOX2pm+wtdk8wcCnoZEzPba2a9ZnbczI6Z2c/MbAmAu7/f3T8bbjdi2JjZYjO7w8yOmFmHmf3RzN5lZi8LH/e4mXWbmUfmj5vZUjN7MFx+TtZj/iRcfmkeP8MZZvbDyPM/bmYfNrOYmS3Pet69ZnbDSV6DzHDaeF/T6WZm15rZw2bWaWb7zewLZlYeWd8Uvp7dZvaMmb21kPXKxCnoZTz+i7vXAacCh4D/Pcb9bwX2AcuAecA7gEPu/it3rwsf++xw27mZZe7+bLjsKeCdmQczs3nAJUDraE9sZquA34fP/0J3nwO8GVgH1Ec2nRvW8V+BT5rZ5VkP9V8iddW5+3N5PLeZ2Uz4n6sB/gqYD1wMXAZ8NLL+JmAAWAC8DfiamZ2NzFoz4Y9OZil37wN+BKyF4d0Ho7gQuMXdu9096e5/cPe7x/DU3wX+zMxi4fw1wE8Iwmk0/xP4rbt/2N0Phj/HDnd/q7u3Z2/s7i3AVuDcMdQ3KPwEcqOZ/QboAVaa2Rozu9fM2sxsh5m9JbJ9tZl9KWxJd5jZr82sOlz3QzN7Plz+n+MNX3f/WvimOuDuBwhez5eGz1ELvAn4pLsfd/dfA3cSvBnLLKWgl3Ezsxrgz4DfjXHX3wE3mdkGM1s6jqd+DtgGrA/n3wl8J899X03w5pQXM3sx8AJg51gKzPIO4DqCTwytwL3A94BTgA3AV81sbbjtF4ELgJcATcDHgHS47m5gdbjfIwQBPRleTvBmBnAGkHT3pyLrH2PoE5bMQgp6GY+fmlk70AFcDvzjGPd/M/Ar4JPAHjN71MwuHONjfAd4p5mtIehmeSjP/eYBB/PY7oiZ9QIPAV8Ffpq1/qdm1h4O2euy3eLuW909CVwB7HX3b2U+zQB3AG8Ou3XeA/ylux9w95S7/9bd+wHcfZO7d4XznwHOMbM5+f3YuZnZewi6rb4YLqoDOrM262B4t5bMMgp6GY/Xu/tcoAq4HvilmS3Md2d3P+buN7j72QT9wI8SBKeNoYYfA68Kn//WMex3lODYwmjmE4TeR4BLgXjW+te7+9xweD0MnnWUOTj7N5Ft90WmlwEXR94k2gn6wReGz1kF7MouJjxQ/Dkz22VmncDeSJ0jMrO3RWq6O2vd64F/AK509yPh4uNAQ9bDNABdJ3semdkU9DJuYYvzx0AK+JNxPsYRgtbkaQRdFfnu10PQlfEBxhb09xH0QefzHCl3/yegD/hgHtu/P3Jw9u+jqyLT+4BfRt4kMgebPwAcCZ9rVY6HfytwNUHX0xxgebj8pG+O7v7dSE1XZpab2RXA1wkOKj8R2eUpoNzMVkeWncNQ147MQgp6GbfwLJKrgUbgyRG2qcoazMw+b2YvMLNyM6snCOud7n50jCX8DfAKd987hn0+DbzEzP4x8ynEzE43s381s7kj7PM54GNmVjXG+nL5d+AMM3uHmcXD4UIzO8vd08Am4J/M7LSwFX+JmVUSdJ30E3wiqQH+fuSnODkzexVB//6b3H1zdJ27dxN8WtpoZrVm9lKCN5ixvJnKDKOgl/H4f2Z2nKAv90bgWnfP1eJbBPRmDasIguonQDuwm6A743VjLcLdnwvPChnLPrsITsVcDmw1sw6CPvIWRu6e+BlwDPjzsdaY4/m7CA4ibyA4qPw88HmgMtzko8ATwBagLVxXRnBM4hngAMGB6LEeAI/6JMGngrtG6Nb5IFANHAZuAz4wwu9XZgnTHaZERIqbWvQiIkVOQS9Fx8zutuGXJ8h1JoxIyVDXjYhIkSsffZPpNX/+fF++fHmhyxARmVUefvjhI+7enGvdjAv65cuX09LSUugyRERmFTN7ZqR16qMXESlyCnoRkSKnoBcRKXIKehGRIqegFxEpcgp6EZEip6AXESlyM+48ehGR8Uik0vT0p+geSNIzkKInHPcORJb1J+lJpOjpTwXrEkm6w+megSSxMqOhKk5DdTkN1XEaquLMqY7TUB2Oq8oH5xuq4lSUz462soJeRMbN3elPpukdSNGbSJFKj3xJlej9w6I3E4veOaU/maarL0FXX5KuvgSdfcnB6cz4eH+wrDNreV8izXSrjsfC4C+noSpOZbyMZMpJpoMhlU6TTDmpdDAkB8fpofnU8OXbP3vlpL+BKOhFZoB02kmkg8A83h+0MoNxMBwPh2A6NcLyJA5UxMqoKC8jHisjHjPisTIqB+eDoaK8jIpwXby8bHCfZMrpSSTpGwhbuYnU4HRvImgd9ySS9A6k6R1I0ptIcZJsn1ZlBrWV5dRWlFNTEaOmMkZNvDwYV8SojpdTWxmjuiI2uE1muroi2Cbt0NGboLM3EYz7EnT2JukMp4evC37+3kSK57PvsjsB6Sm4/piCXmQUyVSazr4k7T0DtIf/5B09Cdp7BujoTdLeOzC4rC+ZIpFykqk0ybRnTafD1l56cHkiHYxnSliOR0WsjOqKGNXxGOWxofb5SHkVvZCiD1sOlfEy6qvKqa+MU1dVTn1V0FKuD6frB6fj4bpy6iqD6ZqK2LBPClPN3ekZSA2+IXT0JEimnViZUV5m4bgsGMcs9/IyIxYbvrxsCn4EBb0Uvcw/ZKZ1FnQJRKczLbZg3N47QHvPUKB39Senpc54zKiKx6irLA9appXl1FcGrdDayvLB5XXDpmOD29ZVllNmxkAyTSIVDAOpdDjvg8v6M+vD5ZltBlJpystsMLSDFm95ZDqyPB6LhPvs6KeebGY2+NqfRnWhyzkpBb3MKu7O8f4krV39wXC8nyPhuLWrP/xonaQz7LvNjE/WdzwaM5gTHoybGx6Im1tTwdzMspr44PrqihjlZUGXSXmsjPKyoHukPGbEy4JxdDoebhMrs2ltjUppUdBLQWUO5h3vT3K8L8nR7gFau/o5cnwoyIfNd/XTnxz7QbfqeIyG6uAjf0NVcEbF8OlgXTS851ZXMCdcVzYVn6dFpomCXibM3Tnc1c9Th7po6x6gq2/oIGLXsIOGKY73JQYPNGYOIibH2Nqujsdorq8MhrpK5tdX0FxXxfz6CppqKoIAD8+CyIR4vES7F0RAQS9j1N2fZMehLnY8HwxPHuxkx6Eu2nsS437MilgZdVVBX3RTTcVgiM+vGwrz6Hxtpf5sRcYir/8YM7sC+GcgBnzD3T+XtX4ZsAloBtqAt7v7/nBdCngi3PRZd3/dJNUuUyiZSrP3aDfbw0Df/nwX25/vZF9bb87tG6rKWbOwgVMaKqmvCk5xq6vKOnAYztdWBGdQ1IYHGivLY9P804mUllGD3sxiwE3A5cB+YIuZ3enu2yKbfRH4jrt/28xeBfwD8I5wXa+7nzu5ZctkSqTSbH2uk5a9bWw72Mn2g13sbD3OQI6+8HjMWNVcx5qF9aw5tYEzF9azZmE9CxuqdDBRZIbKp0V/EbDT3XcDmNntwNVANOjXAh8Op38B/HQSa5RJ1juQ4g/7jrFlzzE27z3KI8+005tInbDd4sZq1iys58yF9Zy5sIE1C+tZMb9W/d0is0w+Qb8I2BeZ3w9cnLXNY8AbCbp33gDUm9k8dz8KVJlZC5AEPufuP81+AjO7DrgOYOnSpWP9GWQUHT0JWp5pY/PeNrbsaeOJAx0kUsMPgK6cX8uFy5t40ZI5rFlYzxkL6qmviheoYhGZTJN1VOujwP8xs3cB/wkcADJNxGXufsDMVgIPmNkT7r4rurO73wzcDLBu3bpZ/B3BmeFQZx+b97SxZW8bm/e0seNQ17BvKZYZnH1aAxcub+LiFU2sW95Ec31l4QoWkSmVT9AfAJZE5heHywa5+3MELXrMrA54k7u3h+sOhOPdZvYgcB4wLOhl4va19fC1X+7iNzuP8MzRnmHrKmJlnLNkDhcub+LCFU1csKyRBrXWRUpGPkG/BVhtZisIAn4D8NboBmY2H2hz9zTwcYIzcDCzRqDH3fvDbV4KfGES6y95nX0JbnpgJ9/6zV4GUsHB09qKGBcsb+Ki5Y1cuLyJc5bMpSquM1tEStWoQe/uSTO7Hvg5wemVm9x9q5ltBFrc/U7gUuAfzMwJum7+Itz9LOBfzCxNcJOTz2WdrSPjlEyluW3zs3z5vqdp6x4A4A3nLeLdL13O2lMbSvb6IyJyIvMpuCTmRKxbt85bWloKXcaM5e78Ysdh/v6u7ew8fByAi5Y38YnXnsWLFs8tbHEiUjBm9rC7r8u1Tl8xnEWePNjJjT97kl/vPALAsnk1fPzKNbzm7IU6h11ERqSgnwUOd/bxpXue4gcP78M9+Bbqf79sNe+8ZPmsuZWZiBSOgn4G6x1I8Y1f7eZrv9xFz0CK8jLj7Zcs4y8vW01jbUWhyxORWUJBPwOl086/PXaAL/zHDg529AFw+doFfPzKNaxsritwdSIy2yjoZ5jNe9r4u59t4/H9HQCsPbWBT7z2LF6yan6BKxOR2UpBP0N09SX4H3c8zl1PPA/AgoZKPrr+TN54/mJiuumFiEyAgn4G6EukeN+3W/j9njaq4zH+2ytWct3LV1JToV+PiEyckqTAEqk013/vEX6/p40FDZX84L9dwrJ5tYUuS0SKiM7NK6B02vnYjx7nvicPM7cmzq3vvVghLyKTTkFfIO7Oxn/fxk/+cICaihi3vPsizlhQX+iyRKQIKegL5Cv3Pc0tv91LRayMr79zHecumVvokkSkSCnoC2DTr/fwz/c/TZnB/7rmXF56uk6dFJGpo6CfZnc8vJ+N/x5cwPNzb3wRV7zg1AJXJCLFTkE/je7ddoiP3fE4AJ/407N4y4VLRtlDRGTiFPTT5KFdR/mL7z1CKu1c/8rTed/LVha6JBEpEQr6afD4/nb+/DstDCTTvP3FS/nI+jMKXZKIlBAF/RTbebiLazdt5nh/ktedcxobX/cCXTteRKaVgn4K7T/Wwzu+uZljPQkuPbOZL73lHMp03RoRmWYK+ily5Hg/7/jmZg529LFuWSNfe9sFxHUfVxEpACXPFOjsS3Dtps3sOdLNWac28M13XUh1RazQZYlIiVLQT7K+RIr33dLC1uc6WT6vhu+85yLmVMcLXZaIlDAF/SRKpNJ88LuPsHlvGwsbqrj1vRfTXF9Z6LJEpMQp6CdJOu189IeP8cD2wzTWxLn1vRexpKmm0GWJiCjoJ8v92w/zb48+R214JcrVuhKliMwQeQW9mV1hZjvMbKeZ3ZBj/TIzu9/MHjezB81scWTdtWb2dDhcO5nFzyR3//EgAB985emcoytRisgMMmrQm1kMuAm4ElgLXGNma7M2+yLwHXd/EbAR+Idw3ybg08DFwEXAp82scfLKnxmSqTQPbD8MwGvOXljgakREhsunRX8RsNPdd7v7AHA7cHXWNmuBB8LpX0TWvwa4193b3P0YcC9wxcTLnlm27D1Ge0+ClfNrOf2UukKXIyIyTD5BvwjYF5nfHy6Legx4Yzj9BqDezObluS9mdp2ZtZhZS2tra761zxj3bHsegMvPXlDgSkRETjRZB2M/CrzCzP4AvAI4AKTy3dndb3b3de6+rrm5eZJKmh7uzj1bDwGwfq26bURk5inPY5sDQPTC6YvDZYPc/TnCFr2Z1QFvcvd2MzsAXJq174MTqHfG2XawkwPtvcyvq+Q8HYQVkRkonxb9FmC1ma0wswpgA3BndAMzm29mmcf6OLApnP45sN7MGsODsOvDZUXj3m1Ba/7ytafogmUiMiONGvTungSuJwjoJ4EfuPtWM9toZq8LN7sU2GFmTwELgBvDfduAzxK8WWwBNobLioa6bURkpsun6wZ3vwu4K2vZpyLTPwJ+NMK+mxhq4ReVfW09bDvYSW1FjEtWzSt0OSIiOembsRNw35NBa/7SM0+hKq6rU4rIzKSgn4BMt83la3VapYjMXAr6cTrWPcDmvW2UlxmvPPOUQpcjIjIiBf04PbD9MKm08+KV85hTo+vNi8jMpaAfp8xplev1bVgRmeEU9OPQl0jxy6eCSzW8+iwFvYjMbAr6cfj100foTaR44aI5nDa3utDliIiclIJ+HAa7bXS2jYjMAgr6MUqlffD8+fW69ryIzAIK+jF65NljHO0eYGlTDWcs0LXnRWTmU9CP0T1bg2vPr1+7ADNdxExEZr68rnUjAXfnnm3qthGRMXKH1AAkeiHZD8leSPTlGPfB2W+ASW5EKujH4OnDx3nmaA9NtRVcsKzobn0rIhmpJPR3Qn9XZOgcGvflWtcJA91DgZ3sC4M9HOP5PfeaP4Xyykn9cRT0Y5DptrlszSnEdO15kdkrnYb2Z6B1Oxx+Mhi3boeu54PgTvRM/nOWxSFeHYR4eTXEq6C86sRl6bxvzpc3Bf0YqNtGppT7pH9kL3npNHQ8C4e3Q+uT4Xg7HHnq5GFuZVBZD5UNWeN6qMpeFllXURcGeBje8eogzMurIFa4uFXQ5+lgRy+P7++gOh7jZavnF7qc0pROw8DxyEfn6LgjGKcSwbbugI8wJvc6CP4hKxvCf+bMeM7w+fKq/AM5lYDuI9B9GLpb4XhrMO4+HCw/Hi7PDGXlUN0I1U3BuKYxa74p93yuj/rpdBBmiZ6wS6EHBnog0R2Oo8u7g+4FTwc1lJVDWQxi8eHzg9PxrPnyIMiqm6BpJVTPncAvehxSSejYB0eejgT6k9D6VPDz5lK3EE5ZA81nDY3nLgl+zxW1RfWmq6DP031ha/7lZ8zXtecnKp2GvvZIyIWh190KvcdyhHgn9HcE43z7OadSWTzrjaABquYE40TP8GDvPTa2x04noetgMIxFvDYIfWwoyJO9Y3uMyVTdGAR+44pg3LRiaL7ulPGFaO8xOLY399C+D3yELo/aU04M9FPWBDWWCAV9nu4ZvDesum1ycofjh4KAyoR2dms105rtORIE2njEa3OHbObjdKwyEiIWTucYkxllLUv2nfgpIfuNJzUAPUeDYTRWBjXzgrCpnR+EXG1zMESnM4OnobctCLWecDxsvj33+kQ3dORoucZrgqGiJnjtKjLztVnrwmVWFvQRp5OQToTjzHwy+IQSnY8OqUTw+23bHdR14OFgyPU7bFoBjctPfBPwdI4g3xOM+zpO/lrXnwbzVkHzmkignxW+AZY2BX0eOnoTPLTrKGUWHIgtab3tcHQXHN05fGjbHXSr5KtqbiTwIkFX3TjUOs4O9MqGgvZzDkr0nfhJIzMfrw4CvTYM8ZqmoItjLCpqYM7i/Ld3D177njbAhwK9vBrKCvBVmcybftueIKTbdgfTbbuD+d5jcOiPwTAW8drgzSHXMHdp0DcuOc2A/5qZ78Edh0mmnRevbKKxtqLQ5Uy9RG/wjzkY5JFg7zky8n7VTTBnURja2S3YyHzNfCifxa9jvCoY6mbIm77Z0MHAmcAM6hcGw7JLTlzfe2x48LftGZoviwUt+1xhXju/qPrNp5OCPg9F123T1wEdB6DzAHTsh87nhqbb9gQHtUbqC4/XBB+P550+fGhaqY/Ikp/qRljUCIvOL3QlJUNBP4r+ZIoHtx8Gpvhqlf3HYeuPg3N4M6djlVeG59lWnbgsez4WD1o7A91hiO8fCvPOA5FgPwADXSevpaw8aEENBnkk2OtPVatKZJZR0I/ioV1H6R5IcdapDSxpqpn8J0j0wcPfgl99KTiQNV5WBrGK4GDiaOI10LAo6GZpWByOw/nGFUF/Z0y3RxQpFgr6UdwzVdeeTyXh0e/CL78QtL4BFl0Aiy8a+vp0si+8LkZf5GvV/bnXp5PBOFYJDacFB/MaFoXTWYFe3ahWuUgJySvozewK4J+BGPANd/9c1vqlwLeBueE2N7j7XWa2HHgS2BFu+jt3f//klD710mkfvMnI5ZMV9Ol00EXzi7+Htl3BslPOhld9As68cvwBnAqDvsi+6CEiEzdq0JtZDLgJuBzYD2wxszvdfVtks08AP3D3r5nZWuAuYHm4bpe7nzupVU+TR/e309rVz6K51Zx9WsPEHswddtwNv7hx6LSyppXwyr+Fs9848dPgYuUQ0/XxReRE+bToLwJ2uvtuADO7HbgaiAa9A5kknAM8N5lFFkq0NT+ha8/vfhDu/ywcaAnmGxbDKz4G575VfeEiMuXyCfpFwL7I/H7g4qxtPgPcY2YfAmqBV0fWrTCzPwCdwCfc/VfZT2Bm1wHXASxdujTv4qfa4E1Gzh5nt82+zXD/Rtgb/si1zfCyj8AF79aXO0Rk2kzWwdhrgFvc/Utmdglwq5m9ADgILHX3o2Z2AfBTMzvb3TujO7v7zcDNAOvWrZsBFzOBXa3H2dXazZzqOBctH+P54c8/AQ/8HTz1H8F81Rx4yX+Hi98PlepeEZHplU/QHwCWROYXh8ui3gtcAeDuD5lZFTDf3Q8D/eHyh81sF3AG0DLRwqdaptvmsjWnUB7Ls//8yM6gD37rj4P5eC28+P3wkg+V1AWURGRmySfotwCrzWwFQcBvAN6atc2zwGXALWZ2FlAFtJpZM9Dm7ikzWwmsBnZPWvVTaMzdNjvvg+9tCC4EFauAde+Fl3145nxNXkRK1qhB7+5JM7se+DnBqZOb3H2rmW0EWtz9TuAjwNfN7K8JDsy+y93dzF4ObDSzBJAG3u/ubVP200ySw519/GFfO5XlZbz8jObRdziyE374niDkX/hmuOzTwXWtRURmgLz66N39LoJTJqPLPhWZ3ga8NMd+dwB3TLDGaXffk4dxhz85fT41FaO8RH0dcNuG4CqGa14Lb7i5MFcMFBEZgRIph3u35dltk07BHe+Do08HX3p6w78o5EVkxlEqZTnen+Q3O49iBpedNUrQ378Rnr4nuDzvNd/TGTUiMiMp6LP8ckcrA6k065Y1Mr8ux304Mx7/IfzmK2AxeMu3g6s9iojMQAr6LPeE3TYnvbbNgUfgzuuD6Ss/DytePg2ViYiMj4I+IpFK80B47fkRbzLSdQhuf1twAbHzr4UL3zeNFYqIjJ2CPuL3u9vo6ktyxoI6VsyvPXGDZD98/+3Q9RwsvQSu+qKuFCkiM56CPiLTbbM+V2veHX72Ydi/Obgo2Vtund33PRWRkqGgD7mPcu353/8L/OFfobw6OMOmLo8vUomIzAAK+tDW5zo52NHHwoYqXrhozvCVux+En/9NMP36m+DUc6a9PhGR8VLQhx7f3wHAS06fR1lZpN+9bTf84FrwVHCJ4Re8qUAVioiMj4I+tLv1OACrmiNfeurrhNuugb52OONKeOUnClOciMgEKOhDu490A7CqOTzbJp2GH18HrduheQ28UdewEZHZSckVyrToV8wPW/S/uBGeuhuq5sKG70HVBO8ZKyJSIAp6YCCZZt+xXsxg2bwa+OOP4VdfBCuDN38L5q0qdIkiIuOmoAeebeshlXYWN1ZTdeSP8NMPBivW3wirXlXY4kREJkhBz1C3zYsaE3DbWyHZC+e+DV78gQJXJiIycZN1c/BZbc+RbuIk+Vj756F7Pyy+EF77ZV3eQESKglr0wO7Wbt4Wu49l3Y9B/WnwZ/8K5Se5RLGIyCyioAf2tx7lA+V3BjNX/SPUj3DlShGRWUhBD5x7+N9YYO0MNL8Q1vxpocsREZlUJR/0HR2dvDP9EwDKX/Vx9cuLSNEp+aDv+u3XWWDt7IytomzNVYUuR0Rk0pV20Cd6mffoVwG4f8G71ZoXkaJU2kH/8C1U9x/hifRyupddXuhqRESmRF5Bb2ZXmNkOM9tpZjfkWL/UzH5hZn8ws8fN7KrIuo+H++0ws9dMZvETkuiFX38ZgK8k38TKU+oLXJCIyNQYNejNLAbcBFwJrAWuMbO1WZt9AviBu58HbAC+Gu67Npw/G7gC+Gr4eIXX8i04foinY6u4P30+K5tz3CNWRKQI5NOivwjY6e673X0AuB24OmsbBzKXd5wDPBdOXw3c7u797r4H2Bk+XmEleuE3XwHgSwNvBCz3zcBFRIpAPkG/CNgXmd8fLov6DPB2M9sP3AV8aAz7YmbXmVmLmbW0trbmWfoEhK35gVNexH8kzqW5vpL6qvjUP6+ISAFM1sHYa4Bb3H0xcBVwq5nl/djufrO7r3P3dc3NU3zT7Uhr/qmzrgeMlWrNi0gRyyeMDwBLIvOLw2VR7wV+AODuDwFVwPw8951eYWueU8+lJX4hACujtw8UESky+QT9FmC1ma0wswqCg6t3Zm3zLHAZgJmdRRD0reF2G8ys0sxWAKuBzZNV/JgN9AyeacOlH2f30R4AtehFpKiNGvTungSuB34OPElwds1WM9toZq8LN/sI8Odm9hhwG/AuD2wlaOlvA/4D+At3T03FD5KXh78F3YfhtPPgjNewJ7xPrM64EZFiltf16N39LoKDrNFln4pMbwNeOsK+NwI3TqDGyTHQA7/+SjB9aXBNm92tmaBX142IFK/S+WZstDW/ej29AykOtPdSXmYsbqwudHUiIlOmNII+R2t+79GgNb90Xg3xWGm8DCJSmkoj4Vo2ha3582H1eoChbpv56rYRkeJW/EE/0DN43nymNQ9DNwRfpQOxIlLkij/oWzZBd2vYmh+6QuXu8IwbXfpARIpdcQf9QHfO1jwMBb3OuBGRYlfcQZ9pzS+6YFhr3t0Hu250Dr2IFLviDfqBbvjNPwfTWa35I8cH6OpL0lBVzrzaigIVKCIyPYo36KOt+dNfPWxVpjW/orkO0+0DRaTIFWfQn6Q1Dwxe+mCVDsSKSAkozqDf8s2wNb/uhNY8RA/EKuhFpPgVX9CP0pqHSNeNviwlIiWg+IJ+yzeh50jYmr8s5yZq0YtIKSmuoM+jNZ9IpXn2aA9m+rKUiJSG4gr6Ld8IWvOLLxyxNb+vrYdk2jltTjVV8dg0FygiMv2KJ+iHteZvyNmah8jFzNRtIyIloniCfss3oedo0Jpflbs1D0OnVur2gSJSKvK6w9SscN7boa8dlr9sxNY8wO4jmUsf6IwbESkNxRP0NU1w2adG3WxXq65aKSKlpXi6bvKkG4KLSKkpqaDv6kvQ2tVPVbyM0+boPrEiUhpKKugzZ9wsn1dLWZkuZiYipaG0gv6IrkEvIqWnpIJ+j24ILiIlKK+gN7MrzGyHme00sxtyrP+ymT0aDk+ZWXtkXSqy7s5JrH3MdulArIiUoFFPrzSzGHATcDmwH9hiZne6+7bMNu7+15HtPwScF3mIXnc/d9IqnoChb8WqRS8ipSOfFv1FwE533+3uA8DtwNUn2f4a4LbJKG4ypdPOniOZyxOrRS8ipSOfoF8E7IvM7w+XncDMlgErgAcii6vMrMXMfmdmrx9hv+vCbVpaW1vzq3yMnu/soy+RZn5dBXOq41PyHCIiM9FkH4zdAPzI3VORZcvcfR3wVuArZrYqeyd3v9nd17n7uubm5kkuKbBbB2JFpETlE/QHgCWR+cXhslw2kNVt4+4HwvFu4EGG999Pm93qthGREpVP0G8BVpvZCjOrIAjzE86eMbM1QCPwUGRZo5lVhtPzgZcC27L3nQ66PLGIlKpRz7px96SZXQ/8HIgBm9x9q5ltBFrcPRP6G4Db3d0ju58F/IuZpQneVD4XPVtnOg3dPlBdNyJSWvK6eqW73wXclbXsU1nzn8mx32+BF06gvkmTuSG4WvQiUmpK4puxfYkUB9p7iZUZSxprCl2OiMi0Komgf+ZoD+6wtKmGivKS+JFFRAaVROoNdtvojBsRKUGlEfRHdFcpESldpRH0usaNiJSw0gh6XYdeREpY0Qe9u+vLUiJS0oo+6Nu6B+joTVBXWU5zXWWhyxERmXZFH/R7IjcbMdN9YkWk9BR90A9dtVLdNiJSmoo+6HcNXrVSZ9yISGkq+qDXgVgRKXVFH/R7dENwESlxRR30yVSaZ47qW7EiUtqKOuj3H+slkXJOnVNFTUVeV2QWESk6RR306rYRESnyoN81eNVKnXEjIqWrqIN+t1r0IiJFHvStmXPoFfQiUrqKOugzffSrdHliESlhRRv0x/uTHOrsp6K8jNPmVhe6HBGRginaoN8TfiN2+bwaYmW6mJmIlK6iDfrBm43ojBsRKXHFG/S6xo2ICJBn0JvZFWa2w8x2mtkNOdZ/2cweDYenzKw9su5aM3s6HK6dxNpPaujUSrXoRaS0jXpdADOLATcBlwP7gS1mdqe7b8ts4+5/Hdn+Q8B54XQT8GlgHeDAw+G+xyb1p8hBp1aKiATyadFfBOx0993uPgDcDlx9ku2vAW4Lp18D3OvubWG43wtcMZGC8+HukVMrFfQiUtryCfpFwL7I/P5w2QnMbBmwAnhgLPua2XVm1mJmLa2trfnUfVKHOvvpGUjRVFvB3JqKCT+eiMhsNtkHYzcAP3L31Fh2cveb3X2du69rbm6ecBHqthERGZJP0B8AlkTmF4fLctnAULfNWPedNLuO6D6xIiIZ+QT9FmC1ma0wswqCML8zeyMzWwM0Ag9FFv8cWG9mjWbWCKwPl02pPa0640ZEJGPUs27cPWlm1xMEdAzY5O5bzWwj0OLumdDfANzu7h7Zt83MPkvwZgGw0d3bJvdHONHgl6V0IFZEZPSgB3D3u4C7spZ9Kmv+MyPsuwnYNM76xmXwy1LquhERKb5vxvYnU+w/1kOZwdJ5NYUuR0Sk4Iou6J892kPaYUlTDZXlsUKXIyJScEUX9LvUbSMiMkzRBX3mQOwKXbVSRAQowqDfo6tWiogMU3RBrxuCi4gMV3xB36objoiIRBVV0B/rHuBYT4LaihgLGioLXY6IyIxQVEGf6bZZ0VyLme4TKyICxRb06rYRETlBcQV9pkWvc+hFRAYVVdDr1EoRkRMVVdBnviy1SpcnFhEZVDRBn0o7e4/2ALBcXTciIoOKJugPHOtlIJlmQUMldZV5XX1ZRKQkFE3QD95sRGfciIgMUzRB3zOQorm+UgdiRUSyFE0fx1UvPJWrXngqqbSPvrGISAkpmhZ9RqxM34gVEYkquqAXEZHhFPQiIkVOQS8iUuQU9CIiRU5BLyJS5BT0IiJFTkEvIlLkzH1mfcHIzFqBZybwEPOBI5NUzlRQfROj+iZG9U3MTK5vmbs351ox44J+osysxd3XFbqOkai+iVF9E6P6Jmam1zcSdd2IiBQ5Bb2ISJErxqC/udAFjEL1TYzqmxjVNzEzvb6ciq6PXkREhivGFr2IiEQo6EVEitysDHozu8LMdpjZTjO7Icf6SjP7frj+92a2fBprW2JmvzCzbWa21cz+Msc2l5pZh5k9Gg6fmq76IjXsNbMnwudvybHezOx/ha/h42Z2/jTWdmbktXnUzDrN7K+ytpnW19DMNpnZYTP7Y2RZk5nda2ZPh+PGEfa9NtzmaTO7dhrr+0cz2x7+/n5iZnNH2PekfwtTWN9nzOxA5Hd41Qj7nvT/fQrr+36ktr1m9ugI+0756zdh7j6rBiAG7AJWAhXAY8DarG0+CPzfcHoD8P1prO9U4Pxwuh54Kkd9lwL/XuDXcS8w/yTrrwLuBgx4MfD7Av6+nyf4MkjBXkPg5cD5wB8jy74A3BBO3wB8Psd+TcDucNwYTjdOU33rgfJw+vO56svnb2EK6/sM8NE8fv8n/X+fqvqy1n8J+FShXr+JDrOxRX8RsNPdd7v7AHA7cHXWNlcD3w6nfwRcZmbTcuspdz/o7o+E013Ak8Ci6XjuSXY18B0P/A6Ya2anFqCOy4Bd7j6Rb0tPmLv/J9CWtTj6d/Zt4PU5dn0NcK+7t7n7MeBe4IrpqM/d73H3ZDj7O2DxZD9vvkZ4/fKRz//7hJ2svjA73gLcNtnPO11mY9AvAvZF5vdzYpAObhP+oXcA86aluoiwy+g84Pc5Vl9iZo+Z2d1mdvb0VgaAA/eY2cNmdl2O9fm8ztNhAyP/gxX6NVzg7gfD6eeBBTm2mSmv43sIPqHlMtrfwlS6Puxa2jRC19dMeP1eBhxy96dHWF/I1y8vszHoZwUzqwPuAP7K3TuzVj9C0BVxDvC/gZ9Oc3kAf+Lu5wNXAn9hZi8vQA0nZWYVwOuAH+ZYPRNew0EefIafkecqm9nfAknguyNsUqi/ha8Bq4BzgYME3SMz0TWcvDU/4/+XZmPQHwCWROYXh8tybmNm5cAc4Oi0VBc8Z5wg5L/r7j/OXu/une5+PJy+C4ib2fzpqi983gPh+DDwE4KPyFH5vM5T7UrgEXc/lL1iJryGwKFMd1Y4Ppxjm4K+jmb2LuC1wNvCN6MT5PG3MCXc/ZC7p9w9DXx9hOct9OtXDrwR+P5I2xTq9RuL2Rj0W4DVZrYibPFtAO7M2uZOIHN2w38FHhjpj3yyhf153wSedPd/GmGbhZljBmZ2EcHvYTrfiGrNrD4zTXDQ7o9Zm90JvDM8++bFQEekm2K6jNiSKvRrGIr+nV0L/FuObX4OrDezxrBrYn24bMqZ2RXAx4DXuXvPCNvk87cwVfVFj/m8YYTnzef/fSq9Gtju7vtzrSzk6zcmhT4aPJ6B4IyQpwiOxv9tuGwjwR80QBXBx/2dwGZg5TTW9icEH+EfBx4Nh6uA9wPvD7e5HthKcAbB74CXTPPrtzJ87sfCOjKvYbRGA24KX+MngHXTXGMtQXDPiSwr2GtI8IZzEEgQ9BO/l+C4z/3A08B9QFO47TrgG5F93xP+Le4E3j2N9e0k6N/O/B1mzkQ7DbjrZH8L01TfreHf1uME4X1qdn3h/An/79NRX7j8lszfXGTbaX/9JjroEggiIkVuNnbdiIjIGCjoRUSKnIJeRKTIKehFRIqcgl5EpMgp6EVEipyCXkSkyP1/YnPWakYC0e8AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def drawpic(train_loss_list=[],test_loss_list=[],epoch_number=10,title='1',root_path='./'):\n",
    "    # make data\n",
    "    x = [i for i in range(epoch_number)]\n",
    "    # plot\n",
    "    fig, ax = plt.subplots()\n",
    "    plt.title(title)\n",
    "    ax.plot(x, train_loss_list, linewidth=2.0)\n",
    "    ax.plot(x, test_loss_list, linewidth=2.0)\n",
    "    path=root_path+title+'.jpg'\n",
    "    print('![]({})'.format(path))\n",
    "    plt.savefig(path)\n",
    "    plt.show()\n",
    "root_path='../pic/'\n",
    "title='BiLSTM_CRF-loss-{}'.format(str(N_EPOCHS))\n",
    "\n",
    "drawpic(train_loss_list=train_loss_list,test_loss_list=valid_loss_list,epoch_number=N_EPOCHS,title=title,root_path=root_path)\n",
    "\n",
    "title='BiLSTM_CRF-F1-{}'.format(str(N_EPOCHS))\n",
    "\n",
    "drawpic(train_loss_list=train_f1_list,test_loss_list=valid_f1_list,epoch_number=N_EPOCHS,title=title,root_path=root_path)\n",
    "\n",
    "title='BiLSTM_CRF-precision-{}'.format(str(N_EPOCHS))\n",
    "\n",
    "drawpic(train_loss_list=train_precision_list,test_loss_list=valid_precision_list,epoch_number=N_EPOCHS,title=title,root_path=root_path)\n",
    "\n",
    "title='BiLSTM_CRF-recall-{}'.format(str(N_EPOCHS))\n",
    "\n",
    "drawpic(train_loss_list=train_recall_list,test_loss_list=valid_recall_list,epoch_number=N_EPOCHS,title=title,root_path=root_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'tut1-model.pt'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_36652/1920486072.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload_state_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'tut1-model.pt'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(f, map_location, pickle_module, **pickle_load_args)\u001b[0m\n\u001b[0;32m    592\u001b[0m         \u001b[0mpickle_load_args\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'encoding'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'utf-8'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    593\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 594\u001b[1;33m     \u001b[1;32mwith\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mopened_file\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    595\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0m_is_zipfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopened_file\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    596\u001b[0m             \u001b[1;31m# The zipfile reader is going to advance the current file position.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m_open_file_like\u001b[1;34m(name_or_buffer, mode)\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_open_file_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    229\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0m_is_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 230\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    231\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    232\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;34m'w'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\torch\\lib\\site-packages\\torch\\serialization.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, name, mode)\u001b[0m\n\u001b[0;32m    209\u001b[0m \u001b[1;32mclass\u001b[0m \u001b[0m_open_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_opener\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    210\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 211\u001b[1;33m         \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_open_file\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    212\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    213\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__exit__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'tut1-model.pt'"
     ]
    }
   ],
   "source": [
    "model.load_state_dict(torch.load('tut1-model.pt'))\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "b69a98d3df882577ba469635c4ab08c5ae67eaedfd3a57f311f98966a6edb2d0"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('torch': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}