lectures/08_Prepare_unimorph_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Python and Natural Language Technologies\n",
    "\n",
    "__Lecture 8-2, Data preparation__\n",
    "\n",
    "__March 30, 2021__\n",
    "\n",
    "__Judit Ács__\n",
    "\n",
    "This is a helper notebook that downloads and prepares the dataset used in Lecture 8."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import subprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "unimorph_path = \"data/unimorph_repos/hun\"\n",
    "os.makedirs(\"data\", exist_ok=True)\n",
    "os.makedirs(\"data/unimorph_repos\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipe = subprocess.Popen(f\"git clone git@github.com:unimorph/hun.git {unimorph_path}\",\n",
    "                        shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)\n",
    "stdout, stderr = pipe.communicate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b''"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stdout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b\"fatal: destination path 'data/unimorph_repos/hun' already exists and is not an empty directory.\\n\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stderr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>infl</th>\n",
       "      <th>tags</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitáción</td>\n",
       "      <td>N;ON+ESS;SG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációként</td>\n",
       "      <td>N;FRML;SG</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációkban</td>\n",
       "      <td>N;IN+ESS;PL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációkhoz</td>\n",
       "      <td>N;AT+ALL;PL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitáció</td>\n",
       "      <td>N;NOM;SG</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lemma            infl         tags\n",
       "0  gravitáció     gravitáción  N;ON+ESS;SG\n",
       "1  gravitáció  gravitációként    N;FRML;SG\n",
       "2  gravitáció  gravitációkban  N;IN+ESS;PL\n",
       "3  gravitáció  gravitációkhoz  N;AT+ALL;PL\n",
       "4  gravitáció      gravitáció     N;NOM;SG"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hun = pd.read_table(f\"{unimorph_path}/hun\", names=['lemma', 'infl', 'tags'], skip_blank_lines=True)\n",
    "hun.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "hun['pos'] = hun.tags.str[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>infl</th>\n",
       "      <th>tags</th>\n",
       "      <th>pos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>146230</th>\n",
       "      <td>megdönt</td>\n",
       "      <td>megdöntöttünk</td>\n",
       "      <td>V;IND;PST;INDF;1;PL</td>\n",
       "      <td>V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>485702</th>\n",
       "      <td>elvág</td>\n",
       "      <td>elvágnának</td>\n",
       "      <td>V;COND;PRS;INDF;3;PL</td>\n",
       "      <td>V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>311457</th>\n",
       "      <td>transzformál</td>\n",
       "      <td>transzformálnotok</td>\n",
       "      <td>V;NFIN;2;PL</td>\n",
       "      <td>V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>248036</th>\n",
       "      <td>sodródik</td>\n",
       "      <td>sodródjon</td>\n",
       "      <td>V;SBJV;PRS;INDF;3;SG</td>\n",
       "      <td>V</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338166</th>\n",
       "      <td>ventilál</td>\n",
       "      <td>ventilálták</td>\n",
       "      <td>V;IND;PST;DEF;3;PL</td>\n",
       "      <td>V</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               lemma               infl                  tags pos\n",
       "146230       megdönt      megdöntöttünk   V;IND;PST;INDF;1;PL   V\n",
       "485702         elvág         elvágnának  V;COND;PRS;INDF;3;PL   V\n",
       "311457  transzformál  transzformálnotok           V;NFIN;2;PL   V\n",
       "248036      sodródik          sodródjon  V;SBJV;PRS;INDF;3;SG   V\n",
       "338166      ventilál        ventilálták    V;IND;PST;DEF;3;PL   V"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hun[hun.tags.str[0]=='V'].sample(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_case(tags):\n",
    "    if tags[0] == 'N':\n",
    "        return tags.split(';')[1]\n",
    "    return None\n",
    "\n",
    "hun['case'] = hun['tags'].apply(get_case)\n",
    "hun = hun[hun.pos=='N']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PRP       25260\n",
       "NOM       25260\n",
       "ON+ABL    25260\n",
       "ON+ESS    25260\n",
       "IN+ALL    25260\n",
       "INST      25260\n",
       "TRANS     25260\n",
       "AT+ESS    25260\n",
       "IN+ABL    25260\n",
       "AT+ALL    25260\n",
       "DAT       25260\n",
       "ON+ALL    25260\n",
       "ACC       25260\n",
       "TERM      25260\n",
       "AT+ABL    25253\n",
       "IN+ESS    25252\n",
       "FRML      25248\n",
       "Name: case, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(hun.case.nunique())\n",
    "hun.case.value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12782, numpy.ndarray, 429393)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lemmas = hun.lemma.unique()\n",
    "len(lemmas), type(lemmas), len(hun)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(12)\n",
    "np.random.shuffle(lemmas)\n",
    "train_size = int(0.8 * len(lemmas))\n",
    "dev_size = int(0.1 * len(lemmas))\n",
    "train_lemmas = lemmas[:train_size]\n",
    "dev_lemmas = lemmas[train_size:train_size+dev_size]\n",
    "test_lemmas = lemmas[train_size+dev_size:]\n",
    "\n",
    "train_lemmas = set(train_lemmas)\n",
    "dev_lemmas = set(dev_lemmas)\n",
    "test_lemmas = set(test_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 0, 0)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_lemmas & dev_lemmas), len(train_lemmas & test_lemmas), len(test_lemmas & dev_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(343484, 42752, 43157)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hun_train = hun[hun.lemma.isin(train_lemmas)]\n",
    "hun_dev = hun[hun.lemma.isin(dev_lemmas)]\n",
    "hun_test = hun[hun.lemma.isin(test_lemmas)]\n",
    "len(hun_train), len(hun_dev), len(hun_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(\"data\", exist_ok=True)\n",
    "os.makedirs(\"data/unimorph\", exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hun_train.to_csv(\"data/unimorph/hun_train.tsv\", sep=\"\\t\", index=False)\n",
    "hun_dev.to_csv(\"data/unimorph/hun_dev.tsv\", sep=\"\\t\", index=False)\n",
    "hun_test.to_csv(\"data/unimorph/hun_test.tsv\", sep=\"\\t\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>infl</th>\n",
       "      <th>tags</th>\n",
       "      <th>pos</th>\n",
       "      <th>case</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitáción</td>\n",
       "      <td>N;ON+ESS;SG</td>\n",
       "      <td>N</td>\n",
       "      <td>ON+ESS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációként</td>\n",
       "      <td>N;FRML;SG</td>\n",
       "      <td>N</td>\n",
       "      <td>FRML</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációkban</td>\n",
       "      <td>N;IN+ESS;PL</td>\n",
       "      <td>N</td>\n",
       "      <td>IN+ESS</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitációkhoz</td>\n",
       "      <td>N;AT+ALL;PL</td>\n",
       "      <td>N</td>\n",
       "      <td>AT+ALL</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gravitáció</td>\n",
       "      <td>gravitáció</td>\n",
       "      <td>N;NOM;SG</td>\n",
       "      <td>N</td>\n",
       "      <td>NOM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        lemma            infl         tags pos    case\n",
       "0  gravitáció     gravitáción  N;ON+ESS;SG   N  ON+ESS\n",
       "1  gravitáció  gravitációként    N;FRML;SG   N    FRML\n",
       "2  gravitáció  gravitációkban  N;IN+ESS;PL   N  IN+ESS\n",
       "3  gravitáció  gravitációkhoz  N;AT+ALL;PL   N  AT+ALL\n",
       "4  gravitáció      gravitáció     N;NOM;SG   N     NOM"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hun_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}