exps/slavic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Without cls load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "data_dir = \"~/ae/work/data/training_data_v10/processed/\"\n",
    "train_path = data_dir + \"brexit_parsed.csv\"\n",
    "valid_path = data_dir + \"asia_bibi_parsed.csv\"\n",
    "model_dir = \"/home/eartemov/ae/work/models/multilingual_L-12_H-768_A-12\"\n",
    "# init_checkpoint_pt = os.path.join(model_dir, \"pytorch_model.bin\")\n",
    "init_checkpoint_pt = \"/home/eartemov/ae/work/models/bert-base-multilingual-cased.tar.gz\"\n",
    "bert_config_file = os.path.join(model_dir, \"bert_config.json\")\n",
    "# vocab_file = os.path.join(model_dir, \"vocab.txt\")\n",
    "token = \"96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\"\n",
    "vocab_file = \"/home/eartemov/models/bert-base-multilingual-cased/\" + token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import BertNerData as NerData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \r"
     ]
    }
   ],
   "source": [
    "data = NerData.create(train_path, valid_path, vocab_file, is_cls=False, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "119547"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.tokenizer.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.4.1'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.models.bert_models import BertBiLSTMAttnNMTJoint, BertBiLSTMAttnNCRF, BertBiLSTMAttnNCRFJoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = BertBiLSTMAttnNCRFJoint.create(\n",
    "#          len(data.label2idx), len(data.cls2idx), bert_config_file, init_checkpoint_pt,\n",
    "#          enc_hidden_dim=1024, rnn_layers=1, num_heads=6, input_dropout=0.5, nbest=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.modeling:loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased.tar.gz from cache at /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9\n",
      "INFO:pytorch_pretrained_bert.modeling:extracting archive file /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9 to temp dir /tmp/tmpmla39agv\n",
      "INFO:pytorch_pretrained_bert.modeling:Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 119547\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "model = BertBiLSTMAttnNCRF.create(\n",
    "        len(data.label2idx), bert_config_file, init_checkpoint_pt,\n",
    "       enc_hidden_dim=1024, rnn_layers=1, num_heads=6, input_dropout=0.5, nbest=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7370256"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_n_trainable_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import NerLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 150\n",
    "learner = NerLearner(model, data,\n",
    "                     best_model_path=\"/home/eartemov/ae/work/models/AGRR-2019/slavic_without_clf.cpt\",\n",
    "                     # best_model_path=\"/home/eartemov/ae/work/models/slavic-base.cpt\",\n",
    "                     lr=0.0001, clip=1.0, sup_labels=data.id2label[1:],\n",
    "                     t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learner.fit(num_epochs, target_metric=\"f1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(valid_path, learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "# preds_res, preds_cls = learner.predict(dl)\n",
    "preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite.metrics import flat_classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5457"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(true_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert pred_tokens == true_tokens\n",
    "tokens_report = flat_classification_report(true_labels, pred_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Without report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       I_EVT       0.45      0.39      0.42        54\n",
      "       I_LOC       0.67      0.90      0.77      1422\n",
      "         I_O       0.99      1.00      0.99     72152\n",
      "       I_ORG       0.85      0.60      0.70      1876\n",
      "       I_PER       0.90      0.77      0.83      4121\n",
      "       I_PRO       0.37      0.60      0.46       233\n",
      "\n",
      "    accuracy                           0.97     79858\n",
      "   macro avg       0.70      0.71      0.70     79858\n",
      "weighted avg       0.97      0.97      0.97     79858\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(tokens_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9715244559092389, 0.6951485358141007)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "pl = []\n",
    "for line in pred_labels:\n",
    "    for p in line:\n",
    "        pl.append(p)\n",
    "        \n",
    "tl = []\n",
    "for line in true_labels:\n",
    "    for p in line:\n",
    "        tl.append(p)\n",
    "accuracy_score(tl, pl), f1_score(tl, pl, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_cls = [x.cls for x in dl.dataset]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "          bg       0.95      1.00      0.97      1288\n",
      "          cs       1.00      0.98      0.99      1170\n",
      "          pl       0.96      1.00      0.98      1456\n",
      "          ru       1.00      0.94      0.97      1543\n",
      "\n",
      "    accuracy                           0.98      5457\n",
      "   macro avg       0.98      0.98      0.98      5457\n",
      "weighted avg       0.98      0.98      0.98      5457\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(flat_classification_report([true_cls], [preds_cls]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import tokens2spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_spans = tokens2spans(pred_tokens, pred_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_spans = tokens2spans(true_tokens, true_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.plot_metrics import get_bert_span_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import voting_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         PER      0.769     0.688     0.726      2846\n",
      "         LOC      0.642     0.872     0.739      1477\n",
      "         ORG      0.692     0.623     0.656       984\n",
      "         EVT      0.297     0.407     0.344        27\n",
      "         PRO      0.332     0.581     0.423       167\n",
      "\n",
      "   micro avg      0.688     0.721     0.704      5501\n",
      "   macro avg      0.546     0.634     0.578      5501\n",
      "weighted avg      0.706     0.721     0.706      5501\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(get_bert_span_report(dl, preds_res, fn=first_choicer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### With cls load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "data_dir = \"~/ae/work/data/training_data_v10/processed/\"\n",
    "train_path = data_dir + \"brexit_parsed.csv\"\n",
    "valid_path = data_dir + \"asia_bibi_parsed.csv\"\n",
    "model_dir = \"/home/eartemov/ae/work/models/multilingual_L-12_H-768_A-12\"\n",
    "# init_checkpoint_pt = os.path.join(model_dir, \"pytorch_model.bin\")\n",
    "init_checkpoint_pt = \"/home/eartemov/ae/work/models/bert-base-multilingual-cased.tar.gz\"\n",
    "bert_config_file = os.path.join(model_dir, \"bert_config.json\")\n",
    "# vocab_file = os.path.join(model_dir, \"vocab.txt\")\n",
    "token = \"96435fa287fbf7e469185f1062386e05a075cadbf6838b74da22bf64b080bc32.99bcd55fc66f4f3360bc49ba472b940b8dcf223ea6a345deb969d607ca900729\"\n",
    "vocab_file = \"/home/eartemov/models/bert-base-multilingual-cased/\" + token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import BertNerData as NerData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \r"
     ]
    }
   ],
   "source": [
    "data = NerData.create(train_path, valid_path, vocab_file, is_cls=True, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "119547"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.tokenizer.vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.4.1'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.models.bert_models import BertBiLSTMAttnNMTJoint, BertBiLSTMAttnNCRF, BertBiLSTMAttnNCRFJoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pytorch_pretrained_bert.modeling:loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased.tar.gz from cache at /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9\n",
      "INFO:pytorch_pretrained_bert.modeling:extracting archive file /home/eartemov/.pytorch_pretrained_bert/731c19ddf94e294e00ec1ba9a930c69cc2a0fd489b25d3d691373fae4c0986bd.4e367b0d0155d801930846bb6ed98f8a7c23e0ded37888b29caa37009a40c7b9 to temp dir /tmp/tmpk1rqwwzz\n",
      "INFO:pytorch_pretrained_bert.modeling:Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 119547\n",
      "}\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "model = BertBiLSTMAttnNCRFJoint.create(\n",
    "         len(data.label2idx), len(data.cls2idx), bert_config_file, init_checkpoint_pt,\n",
    "         enc_hidden_dim=1024, rnn_layers=1, num_heads=6, input_dropout=0.5, nbest=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = BertBiLSTMAttnNCRF.create(\n",
    "#         len(data.label2idx), bert_config_file, init_checkpoint_pt,\n",
    "#        enc_hidden_dim=1024, rnn_layers=1, num_heads=6, input_dropout=0.5, nbest=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8945684"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_n_trainable_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import NerLearner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 150\n",
    "learner = NerLearner(model, data,\n",
    "                     best_model_path=\"/home/eartemov/ae/work/models/AGRR-2019/slavic_clf.cpt\",\n",
    "                     # best_model_path=\"/home/eartemov/ae/work/models/slavic-base.cpt\",\n",
    "                     lr=0.0001, clip=1.0, sup_labels=data.id2label[1:],\n",
    "                     t_total=num_epochs * len(data.train_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# learner.fit(num_epochs, target_metric=\"f1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                     \r"
     ]
    }
   ],
   "source": [
    "dl = get_bert_data_loader_for_predict(valid_path, learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)\n",
    "# preds_res = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn_crfsuite.metrics import flat_classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res)\n",
    "true_tokens, true_labels = bert_labels2tokens(dl, [x.labels for x in dl.dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5457"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(true_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert pred_tokens == true_tokens\n",
    "tokens_report = flat_classification_report(true_labels, pred_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### With cls report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       I_EVT       0.51      0.39      0.44        54\n",
      "       I_LOC       0.73      0.88      0.80      1422\n",
      "         I_O       0.99      1.00      0.99     72152\n",
      "       I_ORG       0.81      0.64      0.72      1876\n",
      "       I_PER       0.91      0.80      0.85      4121\n",
      "       I_PRO       0.46      0.55      0.50       233\n",
      "\n",
      "    accuracy                           0.97     79858\n",
      "   macro avg       0.74      0.71      0.72     79858\n",
      "weighted avg       0.97      0.97      0.97     79858\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(tokens_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9743043902927697, 0.717417953930747)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, f1_score\n",
    "\n",
    "pl = []\n",
    "for line in pred_labels:\n",
    "    for p in line:\n",
    "        pl.append(p)\n",
    "        \n",
    "tl = []\n",
    "for line in true_labels:\n",
    "    for p in line:\n",
    "        tl.append(p)\n",
    "accuracy_score(tl, pl), f1_score(tl, pl, average=\"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import tokens2spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_spans = tokens2spans(pred_tokens, pred_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "true_spans = tokens2spans(true_tokens, true_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.plot_metrics import get_bert_span_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import voting_choicer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "         PER      0.811     0.754     0.782      2846\n",
      "         LOC      0.703     0.846     0.768      1477\n",
      "         ORG      0.661     0.641     0.651       984\n",
      "         EVT      0.375     0.444     0.407        27\n",
      "         PRO      0.385     0.533     0.447       167\n",
      "\n",
      "   micro avg      0.732     0.750     0.741      5501\n",
      "   macro avg      0.587     0.644     0.611      5501\n",
      "weighted avg      0.740     0.750     0.742      5501\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(get_bert_span_report(dl, preds_res, fn=first_choicer))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predict test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_path = data_dir + \"test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules import BertNerData as NerData\n",
    "from modules.models.bert_models import BertBiLSTMAttnNMTJoint, BertBiLSTMAttnNCRFJoint\n",
    "from modules import NerLearner\n",
    "\n",
    "data = NerData.create(train_path, test_path, vocab_file, is_cls=True, batch_size=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{False: 1, 'O': 2, True: 0}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.cls2idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "build CRF...\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "torch.cuda.set_device(4)\n",
    "\n",
    "model = BertBiLSTMAttnNCRFJoint.create(\n",
    "        len(data.label2idx), 2, bert_config_file, init_checkpoint_pt,\n",
    "        enc_hidden_dim=1024, rnn_layers=1, num_heads=6, input_dropout=0.5, nbest=11)\n",
    "# model = BertBiLSTMAttnNMTJoint.create(\n",
    "#    len(data.label2idx), len(data.cls2idx), bert_config_file, init_checkpoint_pt,\n",
    "#    enc_hidden_dim=128, rnn_layers=1, dec_embedding_dim=32, dec_hidden_dim=128, input_dropout=0.5, nbest=11)\n",
    "# model = torch.nn.DataParallel(model, [2, 3])\n",
    "num_epochs = 150\n",
    "learner = NerLearner(model, data,\n",
    "                     best_model_path=\"/home/aaemeljanov/models/AGRR-2019/slavic-all.cpt\",\n",
    "                     lr=0.0001, clip=1.0, sup_labels=data.id2label[1:],\n",
    "                     t_total=num_epochs * len(data.train_dl))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "learner.load_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                       \r"
     ]
    }
   ],
   "source": [
    "from modules.data.bert_data import get_bert_data_loader_for_predict\n",
    "\n",
    "dl = get_bert_data_loader_for_predict(test_path, learner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "preds_res, preds_cls = learner.predict(dl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import bert_labels2tokens, first_choicer\n",
    "\n",
    "pred_tokens, pred_labels = bert_labels2tokens(dl, preds_res, fn=first_choicer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.utils.utils import tokens2spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_spans = tokens2spans(pred_tokens, pred_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### To annotation df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf = pd.read_csv(test_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf[\"spans\"] = pred_spans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = []\n",
    "annotations = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "for path, group in tdf.groupby(\"path\"):\n",
    "    path = path.split(\"|\")[1].replace(\"/home/aaemeljanov/data/\", \"\")\n",
    "    l = [r for _, r in group.iterrows()]\n",
    "    spans = {}\n",
    "    for r in l:\n",
    "        for s in r.spans:\n",
    "            if s[1] != \"O\":\n",
    "                spans[s[0]] = s[1]\n",
    "    annotation = \"\\n\".join([\"{}\\tNormalForm\\t{}\".format(k, spans[k]) for k in spans])\n",
    "    paths.append(path)\n",
    "    annotations.append(annotation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'raw/nord_stream/bg/nord_stream_bg.txt_file_1003.txt'"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Германия\tNoralForm\tLOC\n",
      "Украйна\tNoralForm\tLOC\n",
      "Швеция\tNoralForm\tLOC\n",
      "Русия\tNoralForm\tLOC\n",
      "Витренко\tNoralForm\tPER\n",
      "Европа“\tNoralForm\tLOC\n",
      "Европа\tNoralForm\tLOC\n",
      "Дания\tNoralForm\tLOC\n",
      "Юрий Витренко\tNoralForm\tPER\n"
     ]
    }
   ],
   "source": [
    "print(annotations[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df = pd.DataFrame({\"paths\": paths, \"annotations\": annotations})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>annotations</th>\n",
       "      <th>paths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nШвеция\\tNoralForm\\tL...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Куртика\\tNoralForm\\tPER\\nпоток\\tNoralForm\\tLOC...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Варшава\\tNoralForm\\tLOC\\nЕС\\tNoralForm\\tORG\\nП...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         annotations  \\\n",
       "0  Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...   \n",
       "1  Германия\\tNoralForm\\tLOC\\nШвеция\\tNoralForm\\tL...   \n",
       "2  Куртика\\tNoralForm\\tPER\\nпоток\\tNoralForm\\tLOC...   \n",
       "3  Варшава\\tNoralForm\\tLOC\\nЕС\\tNoralForm\\tORG\\nП...   \n",
       "4  Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...   \n",
       "\n",
       "                                               paths  \n",
       "0  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "1  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "2  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "3  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "4  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df.to_csv(\"/home/aaemeljanov/data/test_pred_slavic.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "tdf = pd.read_csv(\"test_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>tokens</th>\n",
       "      <th>tags</th>\n",
       "      <th>lang</th>\n",
       "      <th>path</th>\n",
       "      <th>domain</th>\n",
       "      <th>cltags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>германия</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>украйна</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>швеция</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>русия</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>витренко</td>\n",
       "      <td>PER</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>PER-4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>европа</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>дания</td>\n",
       "      <td>LOC</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>LOC-6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>юрий_витренко</td>\n",
       "      <td>PER</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>PER-7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>11</td>\n",
       "      <td>бисер_петков</td>\n",
       "      <td>PER</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>PER-8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>12</td>\n",
       "      <td>георге_иванова</td>\n",
       "      <td>PER</td>\n",
       "      <td>bg</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "      <td>nord_stream</td>\n",
       "      <td>PER-9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0          tokens tags lang  \\\n",
       "0           0        германия  LOC   bg   \n",
       "1           1         украйна  LOC   bg   \n",
       "2           2          швеция  LOC   bg   \n",
       "3           3           русия  LOC   bg   \n",
       "4           4        витренко  PER   bg   \n",
       "5           5          европа  LOC   bg   \n",
       "6           7           дания  LOC   bg   \n",
       "7           8   юрий_витренко  PER   bg   \n",
       "8          11    бисер_петков  PER   bg   \n",
       "9          12  георге_иванова  PER   bg   \n",
       "\n",
       "                                                path       domain cltags  \n",
       "0  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-0  \n",
       "1  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-1  \n",
       "2  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-2  \n",
       "3  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-3  \n",
       "4  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  PER-4  \n",
       "5  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-5  \n",
       "6  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  LOC-6  \n",
       "7  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  PER-7  \n",
       "8  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  PER-8  \n",
       "9  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  nord_stream  PER-9  "
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdf.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_df = pd.read_csv(\"/home/aaemeljanov/data/test_pred_slavic.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>annotations</th>\n",
       "      <th>paths</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nШвеция\\tNoralForm\\tL...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Куртика\\tNoralForm\\tPER\\nпоток\\tNoralForm\\tLOC...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Варшава\\tNoralForm\\tLOC\\nЕС\\tNoralForm\\tORG\\nП...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...</td>\n",
       "      <td>raw/nord_stream/bg/nord_stream_bg.txt_file_100...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         annotations  \\\n",
       "0  Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...   \n",
       "1  Германия\\tNoralForm\\tLOC\\nШвеция\\tNoralForm\\tL...   \n",
       "2  Куртика\\tNoralForm\\tPER\\nпоток\\tNoralForm\\tLOC...   \n",
       "3  Варшава\\tNoralForm\\tLOC\\nЕС\\tNoralForm\\tORG\\nП...   \n",
       "4  Германия\\tNoralForm\\tLOC\\nУкрайна\\tNoralForm\\t...   \n",
       "\n",
       "                                               paths  \n",
       "0  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "1  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "2  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "3  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  \n",
       "4  raw/nord_stream/bg/nord_stream_bg.txt_file_100...  "
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "paths = []\n",
    "annotations = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "for path, group in tdf.groupby(\"path\"):\n",
    "    l = [r for _, r in group.iterrows()]\n",
    "    text = \"{}-{}\".format(path.split(\"/\")[2], path.split(\".\")[-2].split(\"_\")[-1])\n",
    "    for r in l:\n",
    "        if isinstance(r.tokens, str):\n",
    "            tokens = r.tokens.replace(\"_\", \" \")\n",
    "            tag = r.tags\n",
    "            cltags = r.cltags\n",
    "            normal_form = \"NormalForm\"\n",
    "            line = \"\\n{}\\t{}\\t{}\\t{}\".format(tokens, normal_form, tag, cltags)\n",
    "            text+=line\n",
    "    path = \"/home/aaemeljanov/data/annotated/\" + path[4:-3] + \"out\"\n",
    "    with open(path, \"w\", encoding=\"utf-8\") as f:\n",
    "        f.write(text)\n",
    "    paths.append(path)\n",
    "    annotations.append(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2394"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(annotations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2394"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "nord_stream  ryanair\r\n"
     ]
    }
   ],
   "source": [
    "!ls /home/aaemeljanov/data/TESTDATA_BSNLP_2019_shared_task_pred/annotated/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pl-raw/ryanair/pl/pl-ryanair-eiki\r\n",
      "waterford\tNormalForm\tLOC\tLOC-3008\r\n",
      "izraelu\tNormalForm\tLOC\tLOC-3009\r\n",
      "cyprze\tNormalForm\tLOC\tLOC-3010\r\n",
      "wrocławskiego\tNormalForm\tLOC\tLOC-3011\r\n",
      "strachowice\tNormalForm\tEVT\tEVT-3012\r\n",
      "tony\tNormalForm\tPER\tPER-3013\r\n",
      "ryana\tNormalForm\tPER\tPER-3014\r\n",
      "poznania\tNormalForm\tLOC\tLOC-3015\r\n",
      "boeing 737-8as\tNormalForm\tPRO\tPRO-3016\r\n",
      "londyn-gatwick\tNormalForm\tLOC\tLOC-3017"
     ]
    }
   ],
   "source": [
    "!cat /home/aaemeljanov/data/TESTDATA_BSNLP_2019_shared_task_pred/annotated/ryanair/pl/pl-ryanair-eiki.out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ryanair wiki\r\n",
      "pl\r\n",
      "1970-01-01\r\n",
      "\r\n",
      "Ryanair – irlandzkie linie lotnicze z siedzibą w Dublinie. Największy w Europie i jeden z największych na świecie niskobudżetowych przewoźników lotniczych. Flota Ryanair liczy 437 samolotów typu Boeing 737-800.\r",
      "\r\n",
      "\r",
      "\r\n",
      "\r",
      "\r\n",
      "Spółka została założona w 1985 r. przez Tony’ego Ryana i jego dzieci. Pierwsze połączenie z irlandzkiego miasta Waterford (WAT) do portu lotniczego Londyn-Gatwick (LGW) zostało uruchomione w lipcu 1985 r. Obecnie (dane za rok 2012) Ryanair przewozi około 80 mln pasażerów rocznie w 25 krajach Europy oraz Izraelu, Cyprze i Maroku. Linie od samego początku rozwijają się dynamicznie. W 1985 r. przewiozły 5 tys. pasażerów, w 1986 r. – 82 tys. pasażerów, w 2005 – 27,6 mln pasażerów, w 2012 – 79,6 mln pasażerów, a w 2014 - 90,6 mln pasażerów.\r",
      "\r\n",
      "\r",
      "\r\n",
      "W Polsce Ryanair obecny jest od marca 2005 r. Pierwszy lot odbył się 24 marca 2005 r. z wrocławskiego lotniska Strachowice (WRO) do Londynu-Stansted (STN). W marcu 2012 roku na wrocławskim lotnisku Ryanair zbazował jeden samolot typu Boeing 737-8AS. 12 grudnia 2012 Ryanair ogłosił również uruchomienie bazy w Krakowie (KRK).\r",
      "\r\n",
      "\r",
      "\r\n",
      "W 2018 roku Ryanair uruchomił w Polsce linie czarterowe Ryanair Sun. Pierwszy lot odbył się 26 kwietnia 2018, z Poznania do Zakintos.\r\n"
     ]
    }
   ],
   "source": [
    "!cat /home/aaemeljanov/data/TESTDATA_BSNLP_2019_shared_task_pred/raw/ryanair/pl/pl-ryanair-eiki.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}