bi-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bi-LSTM with synonym augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "with open('data_files/abortion/sentences2.pkl', 'rb') as f:\n",
    "    sentences = pickle.load(f)\n",
    "prol = sentences[0]\n",
    "proc = sentences[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data augmentation via synonym substitution"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### nltk synonyms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "\n",
    "concerns = {'NN': wn.NOUN,'JJ':wn.ADJ,'VB':wn.VERB,'RB':wn.ADV}\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "def synonimize(word, pos=None):\n",
    "    \"\"\" Get synonyms of the word / lemma \"\"\" \n",
    "    try:\n",
    "        # map part of speech tags to wordnet\n",
    "#        pos = {'NN': wn.NOUN,'JJ':wn.ADJ,'VB':wn.VERB,'RB':wn.ADV}[pos[:2]]\n",
    "        pos = {'NN': wn.NOUN,'JJ':wn.ADJ}[pos[:2]]\n",
    "    except:\n",
    "        # or just return the original word\n",
    "#        print(word)\n",
    "        return [word]\n",
    "    \n",
    "    \n",
    "    synsets = wn.synsets(word)#, pos)\n",
    "    synonyms = []\n",
    "    for synset in synsets:\n",
    "        names = synset.lemma_names()\n",
    "        for sim in  names:\n",
    "            synonyms.append(sim)\n",
    "    \n",
    "    # return list of synonyms or just the original word\n",
    "    return synonyms or [word]\n",
    "\n",
    "def vary_sentence(sentence):\n",
    "    \"\"\" Create variations of a sentence using synonyms \"\"\"\n",
    "    words = nltk.word_tokenize(sentence)\n",
    "    pos_tags = nltk.pos_tag(words)\n",
    "\n",
    "    words = []\n",
    "    for (word, pos) in pos_tags:\n",
    "        synonyms = synonimize(word, pos)\n",
    "        picked = np.random.choice(synonyms)\n",
    "     \n",
    "        words.append(picked)\n",
    "\n",
    "    return \" \".join(words)\n",
    "\n",
    "def vary_by_noun(sentence):\n",
    "    \"\"\" Create variations of a sentence using synonyms \"\"\"\n",
    "    words = nltk.word_tokenize(sentence)\n",
    "    pos_tags = nltk.pos_tag(words)\n",
    "\n",
    "    words = []\n",
    "    for (word, pos) in pos_tags:\n",
    "        if pos[:2] == 'NN':\n",
    "            synonyms = synonimize(word, pos)\n",
    "            picked = np.random.choice(synonyms)\n",
    "            words.append(picked)\n",
    "        else:\n",
    "            words.append(word)\n",
    "\n",
    "    return \" \".join(words)\n",
    "\n",
    "def vary_by_adjective(sentence):\n",
    "    words = nltk.word_tokenize(sentence)\n",
    "    pos_tags = nltk.pos_tag(words)\n",
    "\n",
    "    words = []\n",
    "    for (word, pos) in pos_tags:\n",
    "        if pos[:2] == 'JJ':\n",
    "            synonyms = synonimize(word, pos)\n",
    "            picked = np.random.choice(synonyms)\n",
    "            words.append(picked)\n",
    "        else:\n",
    "            words.append(word)\n",
    "\n",
    "    return \" \".join(words)\n",
    "\n",
    "def vary_by_noun_or_adj(sentence, noun_prob=0.5):\n",
    "    if np.random.random() < noun_prob:\n",
    "        return vary_by_noun(sentence)\n",
    "    else:\n",
    "        return vary_by_adjective(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# redo = False\n",
    "# if redo:\n",
    "#     n_syns=2\n",
    "#     print(len(prol),len(proc))\n",
    "#     for k in range(n_syns):\n",
    "#         prol = [[str(k)]+[str(vary_sentence(k))] for k in prol]\n",
    "#         proc = [[str(k)]+[str(vary_sentence(k))] for k in proc]\n",
    "#         import itertools\n",
    "#         prol = list(itertools.chain(*prol))\n",
    "#         proc = list(itertools.chain(*proc))\n",
    "#         print(len(prol),len(proc))\n",
    "#         with open('data_files/abortion/sentences_syn2.pkl', 'wb') as f:\n",
    "#             pickle.dump([prol,proc], f)\n",
    "\n",
    "# else:\n",
    "#     [prol,proc] = pickle.load('data_files/abortion/sentences_syn2.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(len(prol),len(proc))\n",
    "# with open('data_files/abortion/sentences_syn2.pkl', 'wb') as f:\n",
    "#     pickle.dump([prol,proc], f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "52968 48124\n",
      "41505 39306\n"
     ]
    }
   ],
   "source": [
    "print(len(prol),len(proc))\n",
    "prol = [p for p in prol if p]\n",
    "proc = [p for p in proc if p]\n",
    "prol = [p for p in prol if p[0]!=\">\"]\n",
    "proc = [p for p in proc if p[0]!=\">\"]\n",
    "\n",
    "# combine two sentences\n",
    "prol = [str(prol[k:k+1][0])+str(prol[k+1:k+2]) for k in range(len(prol))]\n",
    "proc = [str(proc[k:k+1][0])+str(proc[k+1:k+2]) for k in range(len(proc))]\n",
    "print(len(prol),len(proc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # num_words we use\n",
    "# num_words = 50000\n",
    "\n",
    "# # if we want two-sentence input# prolife is zero prochoice is 1\n",
    "\n",
    "# X = prol + proc\n",
    "# print(len(X))\n",
    "# print(\"num sentences total: \", len(X), \"using {} words\".format(num_words))\n",
    "\n",
    "\n",
    "# # prolife is zero prochoice is 1\n",
    "# Y = np.hstack((np.zeros(shape=len(prol)), np.ones(shape=len(proc))))\n",
    "\n",
    "# # split data into train and test set, and shuffle it\n",
    "# idx_train, idx_test, y_train, y_test = train_test_split(\n",
    "#                 np.array([range(len(Y))]).T, Y, test_size=0.2, random_state=0)\n",
    "# idx_train = np.array(idx_train.T[0],dtype='int')\n",
    "# idx_test = idx_test.T[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/yuki/anaconda2/envs/py36/lib/python3.6/site-packages/h5py/__init__.py:34: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "import keras\n",
    "num_words = 50000\n",
    "\n",
    "# no info leak here\n",
    "X = prol + proc\n",
    "Y = np.hstack((np.zeros(shape=len(prol)), np.ones(shape=len(proc))))\n",
    "\n",
    "# split data into train and test set, and shuffle it\n",
    "idx_train, idx_test, y_train, y_test = train_test_split(\n",
    "                np.array([range(len(Y))]).T, Y, test_size=0.2, random_state=0)\n",
    "idx_train = np.array(idx_train.T[0],dtype='int')\n",
    "idx_test = idx_test.T[0]\n",
    "\n",
    "X_train = [X[k] for k in idx_train]\n",
    "X_test = [X[k] for k in idx_test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64648\n",
      "193944\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train))\n",
    "X_train = [[str(k)]+[str(vary_by_noun_or_adj(str(k),0.2).replace(\"_\",\" \"))] +\n",
    "           [str(vary_by_noun_or_adj(str(k),0.2).replace(\"_\",\" \"))] for k in X_train]\n",
    "import itertools\n",
    "X_train = list(itertools.chain(*X_train))\n",
    "print(len(X_train))\n",
    "\n",
    "y_train = np.column_stack((y_train, y_train,y_train)).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('data_files/abortion/Xtrain3_ytrain3_02.pkl', 'wb') as f:\n",
    "     pickle.dump([X_train,y_train], f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prep for keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I get where this is going and what the idea is , but I definitely still have trauma from my decision to have an abortion [ ' I was married and pregnant ' ]\n",
      "[9, 74, 120, 27, 7, 143, 6, 36, 1, 375, 7, 22, 9, 602, 119, 18, 1669, 55, 45, 370, 3, 18, 31, 24, 2, 9, 40, 940, 6, 128, 2]\n"
     ]
    }
   ],
   "source": [
    "# changes words to numbers\n",
    "# tokenizing based on train data (i.e. will see new words in testset)\n",
    "tokenizer = keras.preprocessing.text.Tokenizer(num_words=num_words, lower=True,\n",
    "                                                filters='!\"\"#$%&()*+,-./:;<=>?@[\\]^_`{|}~ ”][',\n",
    "                                               split=\" \",oov_token=0)\n",
    "\n",
    "\n",
    "tokenizer.fit_on_texts(X_train)\n",
    "#X2 = np.array(tokenizer.texts_to_sequences([X[k] for k in range(len(X))]))\n",
    "print(X_train[95])\n",
    "x_train = np.array(tokenizer.texts_to_sequences(X_train))\n",
    "x_test = np.array(tokenizer.texts_to_sequences(X_test))\n",
    "print(x_train[95])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " I'm not aware of a ton of new research coming out in that window of time to support your statement[\"I didn't realize this--it's very disturbing if true\"]\n",
      "[184, 13, 756, 5, 4, 1220, 5, 292, 763, 931, 73, 12, 8, 3117, 5, 117, 3, 156, 38, 520, 9, 600, 751, 27, 112, 109, 3776, 20, 339]\n"
     ]
    }
   ],
   "source": [
    "print(X_train[912])\n",
    "print(x_train[912])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# embeddings_index = {}\n",
    "# maxlen = 100\n",
    "# EMBEDDING_DIM = 100\n",
    "\n",
    "# word_index = tokenizer.word_index\n",
    "# print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "# import os\n",
    "# f = open(os.path.join('','glove.6B.100d.txt'))\n",
    "# for line in f:\n",
    "#     values = line.split()\n",
    "#     word = values[0]\n",
    "#     coefs = np.asarray(values[1:], dtype='float32')\n",
    "#     embeddings_index[word] = coefs\n",
    "# f.close()\n",
    "\n",
    "\n",
    "\n",
    "# from keras.layers import Dense, Dropout, Embedding, LSTM, Bidirectional,CuDNNLSTM\n",
    "\n",
    "# embedding_matrix = np.zeros((len(word_index) + 1, EMBEDDING_DIM))\n",
    "# for word, i in word_index.items():\n",
    "#     embedding_vector = embeddings_index.get(word)\n",
    "#     if embedding_vector is not None:\n",
    "#         # words not found in embedding index will be all-zeros.\n",
    "#         embedding_matrix[i] = embedding_vector\n",
    "\n",
    "# embedding_layer = Embedding(len(word_index) + 1,\n",
    "#                             EMBEDDING_DIM,\n",
    "#                             weights=[embedding_matrix],\n",
    "#                             input_length=maxlen,\n",
    "#                             trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9626799488512148\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAADq1JREFUeJzt3X+o3fV9x/Hna3HazQ67mjgkMYuS\nIOaPzY6D2nV/uGJHrIuOIpuhsHaEBGEOB4MtsrGy/9Z/1iJ13QJKGBSd637l2gwrtiIDaY2tbZNm\nWW/F4iWlibPNYIw52/f+ON+428u9N+fcc07OPZ/7fMDlnu/nfM/3fN548vZz39/P+XxSVUiS2vUT\n0+6AJGmyTPSS1DgTvSQ1zkQvSY0z0UtS40z0ktQ4E70kNc5EL0mNM9FLUuMum3YHADZv3lw7duyY\ndjckaaa89NJLr1fVloudN9VEn2QvsHfnzp0cP358ml2RpJmT5DuDnDfV0k1VzVXVwauuumqa3ZCk\npk010SfZm+Tw+fPnp9kNSWqaI3pJapyzbiSpcZZuJKlxlm4kqXGWbiSpcZZuJKlxU/3CVFXNAXO9\nXu/ApXrPHYc+t+Jzr/75XZeqG5J0yayLJRAmYbWELkkbiTV6SWrculnrZj1Y6a8ASzqSZpnTKyWp\ncZZuJKlxJnpJapyJXpIaZ6KXpMb5zVhJapyzbiSpcZZuJKlxJnpJapyJXpIaZ6KXpMY1u3rlOLkG\njqRZ5vRKSWqc0yslqXEzX7pxgxFJWp03YyWpcSZ6SWqciV6SGmeil6TGmeglqXEmeklq3ESmVya5\nEnge+FhVPTWJ91gP/MaspFkw0Ig+yWNJziY5saR9T5LTSeaTHFr01B8BT46zo5KktRm0dHME2LO4\nIckm4BHgTmA3sC/J7iR3AN8EvjfGfkqS1mig0k1VPZ9kx5LmW4D5qnoFIMkTwD3AO4Er6Sf//05y\nrKp+NLYeS5KGMkqNfivw2qLjBeDWqnoAIMlHgddXSvJJDgIHAbZv3z5CNyRJqxll1k2Waau3H1Qd\nWe1GbFUdrqpeVfW2bNkyQjckSasZJdEvANctOt4GnBnmAi5TLEmTN0qifxHYleT6JJcD9wFHh7mA\nyxRL0uQNOr3yceAF4MYkC0n2V9VbwAPA08Ap4MmqOjnMmzuil6TJG3TWzb4V2o8Bx9b65lU1B8z1\ner0Da72GJGl1biUoSY1zK0FJapyLmklS4yzdSFLjpro5eKs3Y13VUtJ6YulGkhpnopekxlmjl6TG\nOb1Skhpn6UaSGmeil6TGWaOXpMZZo5ekxlm6kaTGmeglqXEmeklqnDdjJalx3oyVpMZZupGkxpno\nJalxJnpJatxUNx7ZaNyQRNI0OKKXpMY5vVKSGuf0SklqnKUbSWqciV6SGmeil6TGmeglqXEmeklq\nnIlekhpnopekxo19CYQkNwEPApuBZ6vq0+N+j9a4NIKkSRpoRJ/ksSRnk5xY0r4nyekk80kOAVTV\nqaq6H/hNoDf+LkuShjFo6eYIsGdxQ5JNwCPAncBuYF+S3d1zdwP/Cjw7tp5KktZkoERfVc8Dbyxp\nvgWYr6pXqupN4Angnu78o1X1y8CHx9lZSdLwRqnRbwVeW3S8ANya5HbgQ8AVwLGVXpzkIHAQYPv2\n7SN0Q5K0mlESfZZpq6p6DnjuYi+uqsPAYYBer1cj9EOStIpRplcuANctOt4GnBnmAi5TLEmTN0qi\nfxHYleT6JJcD9wFHh7mAyxRL0uQNOr3yceAF4MYkC0n2V9VbwAPA08Ap4MmqOjnMmzuil6TJG6hG\nX1X7Vmg/xio3XAe47hww1+v1Dqz1GpKk1bkEgiQ1zj1jJalx7hkrSY1zRC9JjXNEL0mN82asJDVu\n7OvRa3xcp17SOFijl6TGWaOXpMZZo5ekxpnoJalx1uglqXHW6CWpcZZuJKlxJnpJapyJXpIa581Y\nSWrcVJdAcIeptXFpBEnDsHQjSY0z0UtS40z0ktQ4E70kNc5EL0mNM9FLUuOcRy9JjXNRM0lqnKUb\nSWqciV6SGmeil6TGmeglqXEmeklqnIlekhpnopekxk1kPfokvwHcBVwDPFJVn5/E++jHuU69pOUM\nPKJP8liSs0lOLGnfk+R0kvkkhwCq6p+q6gDwUeC3xtpjSdJQhindHAH2LG5Isgl4BLgT2A3sS7J7\n0Sl/0j0vSZqSgRN9VT0PvLGk+RZgvqpeqao3gSeAe9L3ceBfquory10vycEkx5McP3fu3Fr7L0m6\niFFvxm4FXlt0vNC1/R5wB3BvkvuXe2FVHa6qXlX1tmzZMmI3JEkrGfVmbJZpq6p6GHj4oi9O9gJ7\nd+7cOWI3JEkrGXVEvwBct+h4G3Bm0Be7eqUkTd6oif5FYFeS65NcDtwHHB30xa5HL0mTN8z0yseB\nF4Abkywk2V9VbwEPAE8Dp4Anq+rkoNd0RC9Jkzdwjb6q9q3Qfgw4NrYeSZLGyq0EJalxE1kCYVBV\nNQfM9Xq9A9PsR+tcGkHa2FzUTJIaZ+lGkho31UTvrBtJmjxLN5LUuKnejHUJhOnyJq20MVi6kaTG\nWbqRpMaZ6CWpcU6vlKTGWaOXpMZZupGkxpnoJalxJnpJapw3YyWpcd6MlaTGTXUJBK1PKy2NAC6P\nIM0ia/SS1DgTvSQ1zkQvSY2zRq+huLSxNHsc0UtS45xHL0mNcx69JDXO0o0kNc5EL0mNM9FLUuNM\n9JLUOBO9JDXORC9JjTPRS1Ljxp7ok9yQ5NEknx33tSVJwxso0Sd5LMnZJCeWtO9JcjrJfJJDAFX1\nSlXtn0RnJUnDG3RRsyPAp4C/udCQZBPwCPABYAF4McnRqvrmuDup9c/FzqT1a6ARfVU9D7yxpPkW\nYL4bwb8JPAHcM+b+SZJGNEqNfivw2qLjBWBrkquT/BXwniQPrfTiJAeTHE9y/Ny5cyN0Q5K0mlHW\no88ybVVV/wHcf7EXV9Vh4DBAr9erEfohSVrFKCP6BeC6RcfbgDPDXMBliiVp8kZJ9C8Cu5Jcn+Ry\n4D7g6DAXcJliSZq8gUo3SR4Hbgc2J1kAPlZVjyZ5AHga2AQ8VlUnh3nzJHuBvTt37hyu15p5ztKR\nLp2BEn1V7Vuh/RhwbK1vXlVzwFyv1zuw1mtIklY31c3BHdFrKUf60vi5laAkNc5FzSSpcZZuNFEr\nlWIkXTqWbiSpcZZuJKlxU030fjNWkibP0o0kNc7SjSQ1zkQvSY1zeqUkjclq04mn+e1ua/SS1DhL\nN5LUOBO9JDXORC9JjTPRS1LjnHUjSUOatcX6nHUjSY2zdCNJjTPRS1LjTPSS1DgTvSQ1zkQvSY1z\neqWaNOz0t2kuOKXpm7XpksNyeqUkNc7SjSQ1zkQvSY0z0UtS40z0ktQ4E70kNc5EL0mNM9FLUuPG\n/oWpJFcCfwm8CTxXVZ8Z93tIkgY30Ig+yWNJziY5saR9T5LTSeaTHOqaPwR8tqoOAHePub+SpCEN\nWro5AuxZ3JBkE/AIcCewG9iXZDewDXitO+2H4+mmJGmtBkr0VfU88MaS5luA+ap6pareBJ4A7gEW\n6Cf7ga8vSZqcUWr0W/n/kTv0E/ytwMPAp5LcBcyt9OIkB4GDANu3bx+hG9oIJr3o1ErXd7Ezjcs0\nP2OjJPos01ZV9V/A71zsxVV1GDgM0Ov1aoR+SJJWMUppZQG4btHxNuDMMBdIsjfJ4fPnz4/QDUnS\nakZJ9C8Cu5Jcn+Ry4D7g6DAXcJliSZq8QadXPg68ANyYZCHJ/qp6C3gAeBo4BTxZVSeHeXNH9JI0\neQPV6Ktq3wrtx4Bja33zqpoD5nq93oG1XkOStLqpTn90RC9Jk+dWgpLUOL/QJEmNs3QjSY1L1fS/\nq5TkHPCdNb58M/D6GLuzHm2EGGFjxLkRYoSNEed6iPHnq2rLxU5aF4l+FEmOV1Vv2v2YpI0QI2yM\nODdCjLAx4pylGK3RS1LjTPSS1LgWEv3haXfgEtgIMcLGiHMjxAgbI86ZiXHma/SSpNW1MKKXJK1i\nZhP9CvvVzqTl9uRN8u4kzyT5Vvf7Z7v2JHm4i/vrSX5pej0fXJLrknwxyakkJ5M82LW3Fuc7knw5\nyde6OP+sa78+yZe6OP+2W/GVJFd0x/Pd8zum2f9hJNmU5KtJnuqOW4zx1STfSPJykuNd28x9Zmcy\n0a+yX+2sOsKSPXmBQ8CzVbULeLY7hn7Mu7qfg8CnL1EfR/UW8AdVdRNwG/C73X+z1uL8H+D9VfWL\nwM3AniS3AR8HPtHF+X1gf3f+fuD7VbUT+ER33qx4kP7KtRe0GCPAr1bVzYumUs7eZ7aqZu4HeC/w\n9KLjh4CHpt2vEWPaAZxYdHwauLZ7fC1wunv818C+5c6bpR/gn4EPtBwn8NPAV+hvsfk6cFnX/vbn\nl/4y3+/tHl/WnZdp932A2LbRT3LvB56iv+NcUzF2/X0V2LykbeY+szM5omf5/Wq3Tqkvk/JzVfVd\ngO73NV37zMfe/en+HuBLNBhnV9J4GTgLPAN8G/hB9fdwgB+P5e04u+fPA1df2h6vySeBPwR+1B1f\nTXsxAhTw+SQvdftcwwx+ZkfZM3aalt2v9pL3YjpmOvYk7wT+Hvj9qvrPZLlw+qcu0zYTcVbVD4Gb\nk7wL+EfgpuVO637PXJxJfh04W1UvJbn9QvMyp85sjIu8r6rOJLkGeCbJv61y7rqNc1ZH9CPvVzsD\nvpfkWoDu99mufWZjT/KT9JP8Z6rqH7rm5uK8oKp+ADxH/57Eu5JcGFgtjuXtOLvnrwLeuLQ9Hdr7\ngLuTvAo8Qb9880naihGAqjrT/T5L/3/atzCDn9lZTfQj71c7A44CH+kef4R+TftC+293d/hvA85f\n+DNyPUt/6P4ocKqq/mLRU63FuaUbyZPkp4A76N+w/CJwb3fa0jgvxH8v8IXqCrzrVVU9VFXbqmoH\n/X97X6iqD9NQjABJrkzyMxceA78GnGAWP7PTvkkwwk2SDwL/Tr/++cfT7s+IsTwOfBf4X/qjgv30\na5jPAt/qfr+7Ozf0Zxx9G/gG0Jt2/weM8Vfo/xn7deDl7ueDDcb5C8BXuzhPAH/atd8AfBmYB/4O\nuKJrf0d3PN89f8O0Yxgy3tuBp1qMsYvna93PyQt5ZhY/s34zVpIaN6ulG0nSgEz0ktQ4E70kNc5E\nL0mNM9FLUuNM9JLUOBO9JDXORC9Jjfs/4WaGNnni5mUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x12746c6a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "L = [len(x) for x in x_train]\n",
    "_ = plt.hist(L,  bins=50,log=True)\n",
    "print(np.sum(np.array(L)<70) / len(L))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data...\n",
      "193944 train sequences\n",
      "16163 test sequences\n",
      "Pad sequences (samples x time)\n",
      "removing len zero and len1 sentences...\n",
      "193944\n",
      "16163\n",
      "x_train shape: (193944, 70)\n",
      "x_test shape: (16163, 70)\n",
      "Train...\n",
      "Train on 193944 samples, validate on 16163 samples\n",
      "Epoch 1/50\n",
      "193944/193944 [==============================] - 483s 2ms/step - loss: 0.5628 - acc: 0.6994 - val_loss: 0.5565 - val_acc: 0.7117\n",
      "Epoch 2/50\n",
      "193944/193944 [==============================] - 498s 3ms/step - loss: 0.3954 - acc: 0.8138 - val_loss: 0.6121 - val_acc: 0.7221\n",
      "Epoch 3/50\n",
      "193944/193944 [==============================] - 478s 2ms/step - loss: 0.2907 - acc: 0.8662 - val_loss: 0.6134 - val_acc: 0.7468\n",
      "Epoch 4/50\n",
      "193944/193944 [==============================] - 478s 2ms/step - loss: 0.2102 - acc: 0.9077 - val_loss: 0.6682 - val_acc: 0.7537\n",
      "Epoch 5/50\n",
      "193944/193944 [==============================] - 726s 4ms/step - loss: 0.1486 - acc: 0.9376 - val_loss: 0.7740 - val_acc: 0.7629\n",
      "Epoch 6/50\n",
      "193944/193944 [==============================] - 1400s 7ms/step - loss: 0.1088 - acc: 0.9563 - val_loss: 0.8798 - val_acc: 0.7641\n",
      "Epoch 7/50\n",
      "143744/193944 [=====================>........] - ETA: 2:30 - loss: 0.0805 - acc: 0.9680"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-1a6d41cdaf2f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     52\u001b[0m           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m           \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m           validation_data=[x_test, y_test])\n\u001b[0m",
      "\u001b[0;32m~/anaconda2/envs/py36/lib/python3.6/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1040\u001b[0m                                         \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1041\u001b[0m                                         \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1042\u001b[0;31m                                         validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1043\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1044\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m~/anaconda2/envs/py36/lib/python3.6/site-packages/keras/engine/training_arrays.py\u001b[0m in \u001b[0;36mfit_loop\u001b[0;34m(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m    197\u001b[0m                     \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 199\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    200\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda2/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2659\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_legacy_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2661\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2662\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2663\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mpy_any\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mis_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda2/envs/py36/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2629\u001b[0m                                 \u001b[0msymbol_vals\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2630\u001b[0m                                 session)\n\u001b[0;32m-> 2631\u001b[0;31m         \u001b[0mfetched\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_callable_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0marray_vals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2632\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mfetched\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda2/envs/py36/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1449\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_created_with_new_api\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1450\u001b[0m           return tf_session.TF_SessionRunCallable(\n\u001b[0;32m-> 1451\u001b[0;31m               self._session._session, self._handle, args, status, None)\n\u001b[0m\u001b[1;32m   1452\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1453\u001b[0m           return tf_session.TF_DeprecatedSessionRunCallable(\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "from keras.preprocessing import sequence\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Embedding, LSTM,GRU, Bidirectional,CuDNNLSTM\n",
    "from keras.datasets import imdb\n",
    "\n",
    "\n",
    "# cut texts after this number of words\n",
    "# (among top max_features most common words)\n",
    "batch_size = 32\n",
    "\n",
    "\n",
    "maxlen = 70\n",
    "print('Loading data...')\n",
    "print(len(x_train), 'train sequences')\n",
    "print(len(x_test), 'test sequences')\n",
    "\n",
    "print('Pad sequences (samples x time)')\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=maxlen)\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=maxlen)\n",
    "\n",
    "print('removing len zero and len1 sentences...')\n",
    "\n",
    "nonzero1 = x_train.sum(1)>1\n",
    "x_train = x_train[nonzero1]\n",
    "y_train = y_train[nonzero1]\n",
    "\n",
    "print(len(x_train))\n",
    "nonzero2 = x_test.sum(1)>1\n",
    "x_test = x_test[nonzero2]\n",
    "y_test = y_test[nonzero2]\n",
    "print(len(x_test))\n",
    "\n",
    "print('x_train shape:', x_train.shape)\n",
    "print('x_test shape:', x_test.shape)\n",
    "y_train = np.array(y_train)\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(num_words, 50, input_length=maxlen))\n",
    "model.add(Bidirectional(GRU(30)))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "# try using different optimizers and different optimizer configs\n",
    "model.compile('adam', 'binary_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "print('Train...')\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=50,shuffle=True,\n",
    "          validation_data=[x_test, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example= ['I believe women should be able to choose their own fate they should not be restrained']\n",
    "# prolife is zero prochoice is 1\n",
    "ex = tokenizer.texts_to_sequences(example)\n",
    "ex = np.array(sequence.pad_sequences(ex, maxlen=maxlen))\n",
    "print(ex.shape)\n",
    "print(model.predict(ex))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "example = ['abortion is wrong It should be made illegal']\n",
    "ex = tokenizer.texts_to_sequences(example)\n",
    "ex = np.array(sequence.pad_sequences(ex, maxlen=maxlen))\n",
    "print(model.predict(ex))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save model and tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save('data_files/abortion/bilstm_syn3_02.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "# saving\n",
    "with open('data_files/abortion/tokenizer_syn3_02.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "# # loading\n",
    "# with open('tokenizer.pickle', 'rb') as handle:\n",
    "#     tokenizer = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "221px",
    "left": "966px",
    "right": "20px",
    "top": "120px",
    "width": "350px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}