word-tokenization/data-preparation-and-post-processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T08:19:35.741983Z",
     "start_time": "2019-09-12T08:19:35.323151Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T08:19:43.555275Z",
     "start_time": "2019-09-12T08:19:43.440900Z"
    }
   },
   "outputs": [],
   "source": [
    "data = []\n",
    "with open (\"../kaggle-competition/train.txt\", \"r\") as ftxt, \\\n",
    "    open (\"../kaggle-competition/train_label.txt\", \"r\") as flabel :\n",
    "    for t, l in zip(ftxt, flabel):\n",
    "        t = t.strip()\n",
    "        l = l.strip()\n",
    "\n",
    "        data.append({\n",
    "            \"txt\": t,\n",
    "            \"label\": l\n",
    "        })\n",
    "df_train = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T08:19:49.840676Z",
     "start_time": "2019-09-12T08:19:49.826456Z"
    }
   },
   "outputs": [],
   "source": [
    "def sampling_class(df, cls_name, n=20, seed=71):\n",
    "    \n",
    "    df_class = df_train[df_train.label==cls_name]\n",
    "    \n",
    "    total = len(df_class)\n",
    "    np.random.seed(seed)\n",
    "    indices = np.random.permutation(total)[:n]\n",
    "    return df_class.iloc[indices].txt.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T08:19:51.210467Z",
     "start_time": "2019-09-12T08:19:51.158301Z"
    }
   },
   "outputs": [],
   "source": [
    "seeds = dict(zip(['neg', 'neu', 'pos', 'q'], range(4)))\n",
    "\n",
    "n = 40\n",
    "sampling_data = []\n",
    "\n",
    "for k, v in seeds.items():\n",
    "    txts = sampling_class(df_train, k, n=n, seed=v)\n",
    "    for t in txts:\n",
    "        t = t.replace(\"|\", \"\\|\")\n",
    "        sampling_data.append(dict(\n",
    "            label=k,\n",
    "            raw=t,\n",
    "            tokenised=t\n",
    "        ))\n",
    "        \n",
    "\n",
    "pd.DataFrame(sampling_data).to_csv(\"./sampling-%d.csv\" % n, header=True, index=False)\n",
    "#             note = \"\"\"\n",
    "# ################\n",
    "# ### label: %s\n",
    "# ### raw: %s\n",
    "# ################\n",
    "#             \"\"\" % (k, t)\n",
    "#             f.write(\"\\n%s\\n\" % note.strip())\n",
    "#             f.write(\"::> %s\\n\" % t)\n",
    "\n",
    "# # ab = sampling_class(df_train, \"neu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-09-12T08:19:52.294514Z",
     "start_time": "2019-09-12T08:19:52.270380Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'neg', 'neu', 'pos', 'q'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(df_train.label.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Postprocessing\n",
    "\n",
    "Google Spreadsheet: https://docs.google.com/spreadsheets/d/1F_qT33T2iy0tKbflnVC8Ma-EoWEHimV3NmNRgLjN00o/edit#gid=1302375309"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"https://docs.google.com/spreadsheets/d/e/2PACX-1vRm-f8qstNhxICHzEfhbCacJNQSAZptP-6ockKwsxyck5vtl7e1-A2726Qj2hgp4Oht7WfcbdivQNPT/pub?gid=1302375309&single=true&output=csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we have 1000 samples\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(filepath)\n",
    "print(\"we have %d samples\" % len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "should_removed = ~df.label.apply(lambda x: len(x.split(\"-\")) > 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_filtered = df[should_removed]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "we have 993 after samples\n"
     ]
    }
   ],
   "source": [
    "print(\"we have %d after samples\" % len(df_filtered))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"wisesight-%d-samples-tokenised.txt\" % len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(filename, \"w\") as ft, open(filename.replace(\".txt\", \".label\"), \"w\") as fl:\n",
    "         for l in df_filtered.tokenised.values:\n",
    "            l = l.strip()\n",
    "            ft.write(\"%s\\n\" % l.replace(\"|\", \"\"))\n",
    "            fl.write(\"%s\\n\" % l)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}