Presentation_Tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset->Grouping User texts.\n",
      "\n",
      "Loaded 436 users...\n",
      "\n",
      "\n",
      "--------------- Thy time of Running ---------------\n",
      "Learning to judge age..\n",
      "Learning to judge gender..\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from argparse import ArgumentParser\n",
    "from sklearn.externals import joblib\n",
    "from tictacs import from_recipe\n",
    "from pan import ProfilingDataset\n",
    "import dill\n",
    "import cPickle as pickle\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "\n",
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "print('Loaded {} users...\\n'.format(len(dataset.entries)))\n",
    "# get config\n",
    "config = dataset.config\n",
    "tasks = config.tasks\n",
    "print('\\n--------------- Thy time of Running ---------------')\n",
    "for task in tasks:\n",
    "    print('Learning to judge %s..' % task)\n",
    "    # load data\n",
    "    X, y = dataset.get_data(task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pan import ProfilingDataset, createDocProfiles, create_target_prof_trainset\n",
    "from pan import preprocess\n",
    "\n",
    "task = 'gender'\n",
    "docs = createDocProfiles(dataset)\n",
    "X, y = create_target_prof_trainset(docs, task)\n",
    "print len(X)\n",
    "#X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436\n"
     ]
    }
   ],
   "source": [
    "#reload(preprocess)\n",
    "#reload(features)\n",
    "from pan import features\n",
    "from pan import preprocess\n",
    "#X, y = dataset.get_data('age')\n",
    "X, y = dataset.get_data('age')\n",
    "print len(X)\n",
    "#print X[0]\n",
    "#X = preprocess.preprocess(X)\n",
    "#print \"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\n",
    "#print X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(\"../DATA/graph-twitter/graph_gender_age/X.txt\", 'r') as inp:\n",
    "    X = pickle.load(inp)\n",
    "with open(\"../DATA/graph-twitter/graph_gender_age/y.txt\", 'r') as yinp:\n",
    "    y = pickle.load(yinp)\n",
    "X = preprocess.preprocess(X)\n",
    "y = [s.lower() for s in y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open(\"../DATA/graph-twitter/graph_gender_age/labeled.json\", 'r') as inp:\n",
    "    labeled = json.load(inp)\n",
    "X = []\n",
    "y = []\n",
    "for user in labeled:\n",
    "    y.append(user['gender'].lower())\n",
    "    X.append(' '.join(user['texts']))\n",
    "# for user in labeled:\n",
    "#     y.extend([user['gender'].lower() for t in user['texts']])\n",
    "#     X.extend(user['texts'])\n",
    "print len(X), len(y)\n",
    "#X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import regex as re\n",
    "import nltk\n",
    "import numpy\n",
    "from textblob.tokenizers import WordTokenizer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from pan.misc import _twokenize\n",
    "\n",
    "class CharFeatureExtractor(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    \"\"\" Model that extracts Char Based Features. Counter: 32 features \"\"\"\n",
    "\n",
    "    def __init__(self, norm=True, norm_axis=0):\n",
    "        self.letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j',\n",
    "                  'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',\n",
    "                  'u', 'v', 'w', 'x', 'y', 'z']\n",
    "        self.numbers = ['0', '1', '2', '3','4', '5', '6', '7', '8', '9']\n",
    "        self.special_c = ['!', '.', ':', '?', ';', ',', ')', '(', \n",
    "                          '-', '%', '$', '#', '@', '^', '&', '*', \n",
    "                          '=', '+', '/', '\"', \"'\", '<', '>', '|',\n",
    "                          '~', '`']\n",
    "        self.names = ['#C', '#a-z', '#upper', '#0-9', '#w', '#\\t'] + ['No_'+str(i) for i in self.special_c]\n",
    "        self.norm = norm\n",
    "        self.norm_axis = norm_axis\n",
    "\n",
    "    def fit(self, X, y=None):\n",
    "        return self\n",
    "\n",
    "    def transform(self, texts):\n",
    "        \"\"\" transform data\n",
    "\n",
    "        :texts: The texts to count capital words in\n",
    "        :returns: array\n",
    "\n",
    "        \"\"\"\n",
    "        from sklearn.preprocessing import normalize\n",
    "        \n",
    "        final_feat = []\n",
    "        for text in texts:\n",
    "            #text = text.encode('utf-8', 'ignore')\n",
    "            tmp = []\n",
    "            tmp.append(len(text))\n",
    "            if tmp[0] == 0:\n",
    "                tmp = [0 for i in self.names]\n",
    "            else:\n",
    "                tmp.append(sum([text.lower().count(l) for l in self.letters])/float(tmp[0]))\n",
    "                tmp.append(sum([l.isupper() for l in text])/float(tmp[0]))\n",
    "                tmp.append(sum([text.lower().count(l) for l in self.numbers])/float(tmp[0]))\n",
    "                tmp.append(text.count(' ')/float(tmp[0]))\n",
    "                tmp.append(text.count('\\t')/float(tmp[0]))\n",
    "                tmp.extend([text.lower().count(l)/float(tmp[0]) for l in self.special_c])\n",
    "            final_feat.append(tmp)\n",
    "        if self.norm:\n",
    "            final_feat2 = normalize(numpy.array(final_feat), norm='l1', axis=0)\n",
    "            #final_feat2 = normalize(numpy.array(final_feat), norm='l1', axis=1)\n",
    "            return final_feat2\n",
    "            #return normalize(numpy.array(final_feat), norm='l1', axis=self.norm_axis)\n",
    "        else:\n",
    "            return numpy.array(final_feat) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import regex as re\n",
    "import nltk\n",
    "import numpy\n",
    "from textblob.tokenizers import WordTokenizer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from pan.misc import _twokenize\n",
    "\n",
    "eyes = [':', ';']\n",
    "noses = ['', '-', '=', 'o', 'O', '0', '>']\n",
    "mouths = [')', 'D', ']', '>',  '(',  '[', '<', 'p', 'P', 'd', \"|\", '/', '*', '@', '!', '$', '3', 'X']\n",
    "emoticons = [\"*O\", \"*-*\", \"*O*\", \"*o*\", \"* *\", \"=}\",\"(:;)\", \"{;:]\", \"^_^\", \"^-^\"]\n",
    "for eye in eyes:\n",
    "    for nose in noses:\n",
    "        for mouth in mouths:\n",
    "            emoticons.append(eye+nose+mouth)\n",
    "\n",
    "\n",
    "eyes = [':', ';']\n",
    "noses = ['', '-']\n",
    "mouths = [')', 'D', ']', '>',  '(',  '[', '<', 'p', 'P', 'd']            \n",
    "tongue_no_emo = []\n",
    "for nose in noses:\n",
    "    for eye in eyes:\n",
    "        for mouth in mouths:\n",
    "            tongue_no_emo.append(eye+nose+mouth)\n",
    "    \n",
    "class EmoticonExtractor(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    \"\"\" Model that extracts Char Based Features. Counter: 32 features \"\"\"\n",
    "\n",
    "    def __init__(self, norm=True, norm_axis=0):\n",
    "        self.emoticons = emoticons\n",
    "        self.names = emoticons\n",
    "        self.norm = norm\n",
    "        self.norm_axis = norm_axis\n",
    "\n",
    "    def fit(self, texts, y=None):\n",
    "        \"\"\" Keep correct emoticons.\n",
    "\n",
    "        - texts: the texts to count the emoticons \n",
    "        - returns: Nothing, just updates emoticons and names attributes\n",
    "\n",
    "        \"\"\"\n",
    "        from sklearn.preprocessing import normalize\n",
    "        \n",
    "        final_feat = []\n",
    "        for text in texts:\n",
    "            #text = text.encode('utf-8', 'ignore')\n",
    "            tmp = [text.count(emo) for emo in self.emoticons]\n",
    "            #if ';od' in text:\n",
    "            #    print 'Kai omws'\n",
    "            final_feat.append(tmp)\n",
    "        final_feat2 = numpy.array(final_feat)\n",
    "        final_feat3 = final_feat2[:, numpy.nonzero(final_feat2.sum(axis=0))[0]]\n",
    "        #print 'Indexes'\n",
    "        #print numpy.nonzero(final_feat2.sum(axis=0))[0]\n",
    "        #print type(numpy.nonzero(final_feat2.sum(axis=0))[0])\n",
    "        #print 'Names'\n",
    "        #print numpy.array(self.names).shape\n",
    "        self.emoticons = numpy.array(self.emoticons)[numpy.nonzero(final_feat2.sum(axis=0))[0]]\n",
    "        self.names = numpy.array(self.names)[numpy.nonzero(final_feat2.sum(axis=0))[0]]\n",
    "        #print final_feat3.shape\n",
    "        return self\n",
    "\n",
    "    def transform(self, texts):\n",
    "        \"\"\" transform data\n",
    "\n",
    "        :texts: The texts to count capital words in\n",
    "        :returns: array\n",
    "\n",
    "        \"\"\"\n",
    "        from sklearn.preprocessing import normalize\n",
    "        \n",
    "        final_feat = []\n",
    "        for text in texts:\n",
    "            #text = text.encode('utf-8', 'ignore')\n",
    "            tmp = [text.count(emo) for emo in self.emoticons]\n",
    "            #if ';od' in text:\n",
    "            #    print 'Kai omws'\n",
    "            final_feat.append(tmp)\n",
    "        final_feat2 = numpy.array(final_feat)\n",
    "        #self.names = numpy.array(self.names)[numpy.nonzero(final_feat2.sum(axis=0))[0]]\n",
    "        if self.norm:\n",
    "            final_feat3 = normalize(numpy.array(final_feat2), norm='l1', axis=0)\n",
    "            return final_feat3\n",
    "            #return normalize(numpy.array(final_feat), norm='l1', axis=self.norm_axis)\n",
    "        else:\n",
    "            return final_feat3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "wanted_indexes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "split = 0.2\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y, random_state=100)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "\n",
    "#aa = EmoticonExtractor(emoticons)\n",
    "#aa = CharFeatureExtractor()\n",
    "#tr = aa.fit_transform(X_train)\n",
    "\n",
    "wanted_cols = aa.names[:5] + ['No_#', 'No_@', 'No_!', 'No_.', 'No_)', 'No_(', 'No_$', \"No_`\", \"No_~\", 'No_+']\n",
    "wanted_indexes = [i for i in xrange(len(aa.names))]\n",
    "# wanted_indexes = []\n",
    "# for i in wanted_cols:\n",
    "#     wanted_indexes.append(aa.names.index(i))\n",
    "\n",
    "tr2 = tr[:, wanted_indexes]\n",
    "cols = numpy.array(aa.names)[wanted_indexes]\n",
    "\n",
    "import pandas as pd\n",
    "data = pd.DataFrame(tr2, columns=cols.tolist())\n",
    "data[\"class\"] = y_train\n",
    "print(data.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_cmap(colors, position=None, bit=False):\n",
    "    '''\n",
    "    make_cmap takes a list of tuples which contain RGB values. The RGB\n",
    "    values may either be in 8-bit [0 to 255] (in which bit must be set to\n",
    "    True when called) or arithmetic [0 to 1] (default). make_cmap returns\n",
    "    a cmap with equally spaced colors.\n",
    "    Arrange your tuples so that the first color is the lowest value for the\n",
    "    colorbar and the last is the highest.\n",
    "    position contains values from 0 to 1 to dictate the location of each color.\n",
    "    '''\n",
    "    import matplotlib as mpl\n",
    "    import numpy as np\n",
    "    bit_rgb = np.linspace(0,1,256)\n",
    "    if position == None:\n",
    "        position = np.linspace(0,1,len(colors))\n",
    "    else:\n",
    "        if len(position) != len(colors):\n",
    "            sys.exit(\"position length must be the same as colors\")\n",
    "        elif position[0] != 0 or position[-1] != 1:\n",
    "            sys.exit(\"position must start with 0 and end with 1\")\n",
    "    if bit:\n",
    "        for i in range(len(colors)):\n",
    "            colors[i] = (bit_rgb[colors[i][0]],\n",
    "                         bit_rgb[colors[i][1]],\n",
    "                         bit_rgb[colors[i][2]])\n",
    "    cdict = {'red':[], 'green':[], 'blue':[]}\n",
    "    for pos, color in zip(position, colors):\n",
    "        cdict['red'].append((pos, color[0], color[0]))\n",
    "        cdict['green'].append((pos, color[1], color[1]))\n",
    "        cdict['blue'].append((pos, color[2], color[2]))\n",
    "\n",
    "    cmap = mpl.colors.LinearSegmentedColormap('my_colormap',cdict,256)\n",
    "    return cmap\n",
    "\n",
    "colors = [(255,153,255), (0,0,255)] # This example uses the 8-bit RGB\n",
    "my_cmap = make_cmap(colors, bit=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grouped = data.groupby('class')\n",
    "means = grouped.mean().T\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "### BAR PLOTS OF MEAN VALUE OF FEATURES FOR EACH CLASS ######\n",
    "\n",
    "grouped = data.groupby('class')\n",
    "#plt.figure()\n",
    "g2 = grouped.mean().T\n",
    "g2 = g2.div(g2.sum(axis=1), axis=0)\n",
    "g2.plot(kind='bar', figsize=(60,10), cmap=my_cmap, grid=True, layout='tight')\n",
    "plt.savefig('test1.png', facecolor='white', dpi=100)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list(g2.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "offline.plot."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import plotly.offline as offline\n",
    "\n",
    "\n",
    "offline.plot({'data': data, \n",
    "               'layout': layout, \n",
    "              },image='png')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "colors = {'female':'rgb(255,153,255)', 'male':'rgb(0,0,255)'}\n",
    "\n",
    "          \n",
    "data = []\n",
    "for col in g2.columns:\n",
    "    trace1 = go.Bar(\n",
    "        x=g2.index,\n",
    "        y=g2[col],\n",
    "        name=col,\n",
    "        marker={'color':colors[col]}\n",
    "        )\n",
    "    data.append(trace1)\n",
    "layout = go.Layout(\n",
    "    barmode='group'\n",
    ")\n",
    "\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig, filename='gender_char.png')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Single Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from pan.features import SOA_Model2, SOAC_Model2\n",
    "from pan import features\n",
    "\n",
    "aa = EmoticonExtractor(emoticons)\n",
    "c = CharFeatureExtractor(norm_axis=0, norm=True)\n",
    "svm2 = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced', probability=False)\n",
    "#w = WordFeatureExtractor(function_words_path='./English_Function_Words_Set/', norm=True, norm_axis=0)\n",
    "#soacw = SOACWrapper()\n",
    "#svm = LinearSVC(C=0.001,  class_weight='balanced')\n",
    "#svm2 = SVC(kernel='linear', C=1, gamma=1, class_weight='balanced', probability=False)\n",
    "#pipe1 = Pipeline([('EmoCount', aa), ('svm', svm2)])\n",
    "#pipe2 = Pipeline([('CharCount', c), ('svm', svm2)])\n",
    "#pipe2 = Pipeline([('CharCount', c), ('soac', soacw),('svm', svm2)])\n",
    "\n",
    "num_folds = 4\n",
    "split = 0.2\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y, random_state=100)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe1), ('1', pipe2)], voting='soft')\n",
    "#pipe2.fit(X_train, y_train)\n",
    "# trained_models = []\n",
    "params = {}\n",
    "#params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "for model in [pipe_emo]:\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=1, n_jobs=2, cv=num_folds, \n",
    "                                   refit=True, scoring='f1_weighted')\n",
    "    grid_search.fit(X_train,y_train)\n",
    "    print(grid_search.best_score_)\n",
    "    print(grid_search.best_estimator_) \n",
    "best = grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "predict = best.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ensembles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Preprocess(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    def __init__(self, preprocess=None, preprocess_flag=True):\n",
    "        self.flag = preprocess_flag\n",
    "        self.preprocess = preprocess\n",
    "    def fit(self, X, y=None):\n",
    "        \n",
    "        return self\n",
    "    \n",
    "    def transform(self, X):\n",
    "        \n",
    "        if self.flag:\n",
    "            if self.preprocess!= None:\n",
    "                return self.preprocess(X)\n",
    "            else:\n",
    "                print \"No preprocess func found\"\n",
    "        else:\n",
    "            return X\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3grams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Pre', Preprocess(preprocess=<function preprocess at 0x7fbf6ebf90c8>,\n",
       "        preprocess_flag=None)),\n",
       " ('3grams',\n",
       "  TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
       "          dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "          lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
       "          ngram_range=[2, 2], norm=u'l2', preprocessor=None, smooth_idf=True,\n",
       "          stop_words=None, strip_accents=None, sublinear_tf=False,\n",
       "          token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "          vocabulary=None)),\n",
       " ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from pan.preprocess import preprocess\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[2,2], max_features=5000) # age\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced',  probability=True) # age\n",
    "#grams3 = TfidfVectorizer(analyzer='word', ngram_range=[1,1], max_features=7500) # gender\n",
    "#svm = LinearSVC(kernel='linear', C=1, class_weight='balanced') # gender\n",
    "pipe_grams = Pipeline([('Pre', Preprocess(preprocess, True)), ('3grams',grams3), ('svm', svm)])\n",
    "pipe_grams.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Pre', Preprocess(preprocess=<function preprocess at 0x7fbf6ebf90c8>,\n",
       "        preprocess_flag=None)),\n",
       " ('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
       "        tokenizer_var='sklearn')),\n",
       " ('svm', SVC(C=0.1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None) # age\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced',  probability=True) # age\n",
    "#soac = features.SOAC_Model2(max_df=0.9, min_df=3, tokenizer_var='sklearn', max_features=None) # gender\n",
    "#svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced',  probability=True) # gender\n",
    "\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])+\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe_soac = Pipeline([('Pre', Preprocess(preprocess, True)), ('soac',soac), ('svm', svm)])\n",
    "pipe_soac.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pan.features import LSI_Model\n",
    "LSImodel = LSI_Model(num_topics=50, max_df=1.0, min_df=2, max_features=None) # age\n",
    "svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced',  probability=True) # age\n",
    "#LSImodel = LSI_Model(num_topics=50, max_df=1.0, min_df=2, max_features=None) # gender\n",
    "#svm = LinearSVC(C=10, class_weight='balanced') # gender\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe_lsi = Pipeline([('Pre', Preprocess(preprocess, True)), ('LSI',LSImodel), ('svm', svm)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mallet\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('Pre', Preprocess(preprocess=<function preprocess at 0x7fbf6ebf90c8>,\n",
       "      preprocess_flag=None)), ('LDA', LDA(lib='mallet', num_topics=90)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pan.features import LDA\n",
    "LDAmodel = LDA(lib='mallet', num_topics=90)\n",
    "svm = SVC(kernel='rbf', C=100, gamma=1, class_weight='balanced',  probability=True)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe_lda = Pipeline([('Pre', Preprocess(preprocess, True)), ('LDA', LDAmodel), ('svm', svm)])\n",
    "pipe_lda"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "c = CharFeatureExtractor(norm_axis=0, norm=True)\n",
    "svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced', probability=True) # age\n",
    "# svm = LinearSVC(C=1, class_weight='balanced') # gender\n",
    "pipe_char = Pipeline([('Pre', Preprocess(preprocess, False)), ('Char', c), ('svm', svm)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Emo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emo = EmoticonExtractor()\n",
    "svm = SVC(kernel='rbf', C=100, gamma=1, class_weight='balanced',  probability=True) # age\n",
    "# svm = LinearSVC(C=1, class_weight='balanced') # gender\n",
    "pipe_emo = Pipeline([('Pre', Preprocess(preprocess, False)), ('Emo', emo), ('svm', svm)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "\n",
    "from EnsembleDiversityTests import DiversityTests, BaseClassifiers\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "def print_overlaps(predictions, names, verbose=True):\n",
    "    N = len(names)\n",
    "    res = numpy.zeros([N,N])\n",
    "    temp = numpy.zeros([N,N])\n",
    "    for i in range(0, N):\n",
    "        for j in range(i+1, N):\n",
    "            temp[i,j] = len([m for l, m in enumerate(predictions[i]) if (m==predictions[j][l] and m==predictions[N-1][l])])/float(len(predictions[0]))\n",
    "            res[i,j] = len([(k,v) for k,v in zip(predictions[i], predictions[j]) if k==v])/float(len(predictions[0]))\n",
    "            if verbose:\n",
    "                print \"%s - %s : %0.3f  overlap | ground-truth coverage: %0.3f\" % (names[i],  names[j], 100*res[i,j], 100*temp[i,j])\n",
    "    return  [res, temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "best.steps[1][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble3(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Utilizing the neighborhood in all representations and also ground truth model.\n",
    "        Implementing a weighted voting scheme.\"\"\"\n",
    "\n",
    "    def __init__(self, models, k=3, weights= [2,1,3,0.7]):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\ ')\n",
    "        else:\n",
    "            self.models = models\n",
    "            # self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.weights = weights\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.representations = []\n",
    "            self.meta = None\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.doc_terms = None\n",
    "            self.tree = None\n",
    "            self.experts = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X_cv).toarray()\n",
    "            self.tree = BallTree(self.doc_terms, leaf_size=20)\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[1][1].transform(model.steps[0][1].transform(X_cv))\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            count = 0\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), self.k)  \n",
    "#             print 'Sample ' + y_real[i]\n",
    "            #print dist\n",
    "            #print type(dist)\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            #pass\n",
    "            y_pred.append(self.expert_decision(neigbors_indexes[0],  dist, X[i]))\n",
    "            \n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, neigbors_indexes, dist, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "#         print 'True'\n",
    "#         print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = self.weights[2]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[1][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_neig_pred, normalize=True))\n",
    "            if acc[-1] >self.weights[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(self.weights[1]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(self.weights[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "#                 print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_neig_pred, normalize=True))\n",
    "#                 print 'Predictions'\n",
    "#                 print model_pred\n",
    "#                 print 'Representations'\n",
    "#                 print model_neig_pred\n",
    "#                 print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "        total_pred.extend([n for i, n in enumerate(neigbors_true) for j in xrange(int(weights['true']*(self.k-i)))])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "#         print data\n",
    "#         best_model_ind = acc.index(max(acc))\n",
    "#         print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "#         print '='*50\n",
    "#         #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "326 55 55 436 436\n",
      "3grams\n",
      "541079\n",
      "soac\n",
      "541079\n",
      "emo\n",
      "541079\n",
      "space3\n",
      "Fit took: 54.221 seconds\n",
      "Predict took: 71.275 seconds\n",
      "OLA\n",
      "Fitting time 11.10\n",
      "LCA\n",
      "Fitting time 8.33\n",
      "KNE\n",
      "Fitting time 8.28\n",
      "KNU\n",
      "Fitting time 8.28\n",
      "majority\n",
      "weights\n",
      "accuracy\n",
      "optimal\n",
      "Round 0 took: 237.445 seconds\n",
      "326 55 55 436 436\n",
      "3grams\n",
      "579006\n",
      "soac\n",
      "579006\n",
      "emo\n",
      "579006\n",
      "space3\n",
      "Fit took: 46.000 seconds\n",
      "Predict took: 728.956 seconds\n",
      "OLA\n",
      "Fitting time 8.55\n",
      "LCA\n",
      "Fitting time 7.74\n",
      "KNE\n",
      "Fitting time 7.73\n",
      "KNU\n",
      "Fitting time 7.75\n",
      "majority\n",
      "weights\n",
      "accuracy\n",
      "optimal\n",
      "Round 1 took: 882.217 seconds\n",
      "326 55 55 436 436\n",
      "3grams\n",
      "453340\n",
      "soac\n",
      "453340\n",
      "emo\n",
      "453340\n",
      "space3\n",
      "Fit took: 47.249 seconds\n",
      "Predict took: 1300.671 seconds\n",
      "OLA\n",
      "Fitting time 7.36\n",
      "LCA\n",
      "Fitting time 7.37\n",
      "KNE\n",
      "Fitting time 7.39\n",
      "KNU\n",
      "Fitting time 7.38\n",
      "majority\n",
      "weights\n",
      "accuracy\n",
      "optimal\n",
      "Round 2 took: 1449.938 seconds\n",
      "326 55 55 436 436\n",
      "3grams\n",
      "265449\n",
      "soac\n",
      "265449\n",
      "emo\n",
      "265449\n",
      "space3\n",
      "Fit took: 47.490 seconds\n",
      "Predict took: 556.372 seconds\n",
      "OLA\n",
      "Fitting time 6.70\n",
      "LCA\n",
      "Fitting time 6.69\n",
      "KNE\n",
      "Fitting time 6.71\n",
      "KNU\n",
      "Fitting time 6.71\n",
      "majority\n",
      "weights\n",
      "accuracy\n",
      "optimal\n",
      "Round 3 took: 706.302 seconds\n",
      "326 55 55 436 436\n",
      "3grams\n",
      "187739\n",
      "soac\n",
      "187739\n",
      "emo\n",
      "187739\n",
      "space3\n",
      "Fit took: 46.364 seconds\n",
      "Predict took: 388.642 seconds\n",
      "OLA\n",
      "Fitting time 8.05\n",
      "LCA\n",
      "Fitting time 8.12\n",
      "KNE\n",
      "Fitting time 8.11\n",
      "KNU\n",
      "Fitting time 8.11\n",
      "majority\n",
      "weights\n",
      "accuracy\n",
      "optimal\n",
      "Round 4 took: 540.023 seconds\n",
      "Total time: 3816.017 seconds\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "import combinations\n",
    "import copy\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix,precision_recall_fscore_support, classification_report\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from pan.features import Metaclassifier\n",
    "import time\n",
    "import numpy\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "from pan import features\n",
    "from pan import preprocess\n",
    "#X, y = dataset.get_data('age')\n",
    "X, y = dataset.get_data('age')\n",
    "\n",
    "\n",
    "#pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "#pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "#pipe2 = Pipeline([('soa',soa), ('svm', svm)])\n",
    "\n",
    "### AGE ###\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='hard')\n",
    "#models = [pipe,pipe1,pipe2,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting', 'votingh']\n",
    "\n",
    "# Base Models\n",
    "base_models = [pipe_grams, pipe_soac, pipe_emo]\n",
    "base_model_names = ['3grams', 'soac', 'emo']\n",
    "\n",
    "# Meta Voting Models\n",
    "eclf = VotingClassifier(estimators=zip(base_model_names, base_models), voting='soft')\n",
    "eclfh = VotingClassifier(estimators=zip(base_model_names, base_models), voting='hard')\n",
    "voting_dic = {'votingf':eclf, 'votingh':eclfh}\n",
    "combinator_names = ['majority', 'weights', 'accuracy', 'optimal']\n",
    "#meta_models_names = ['votingf', 'votingh', 'space3', 'meta'] + combinator_names\n",
    "meta_models_names = ['space3', 'OLA', 'LCA', 'KNE', 'KNU'] + combinator_names\n",
    "#meta_models_names = []\n",
    "## all_models ##\n",
    "all_models_names = base_model_names + meta_models_names\n",
    "\n",
    "\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='hard')\n",
    "#models = [pipe,pipe1,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'voting', 'votingh']\n",
    "\n",
    "results = {'over':[]}\n",
    "for name in all_models_names:\n",
    "    results[name] = {'pred': [], 'conf': [], 'rep': [], 'acc': []}\n",
    "\n",
    "num_folds = 4\n",
    "train_split = 0.25\n",
    "meta_split = 0.5\n",
    "cv_rounds = 5\n",
    "seeds = list(xrange(cv_rounds))\n",
    "t0 = time.time()\n",
    "t1 = t0\n",
    "for j in xrange(cv_rounds):\n",
    "    X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=train_split, stratify=y, random_state=seeds[j])\n",
    "    for i, x in enumerate(X_train):\n",
    "        if len(x)==0:\n",
    "            X_train.remove(x)\n",
    "            y_train.remove(y_train[i])\n",
    "    for i, x in enumerate(X_cv):\n",
    "        if len(x)==0:\n",
    "            X_cv.remove(x)\n",
    "            y_cv.remove(y_cv[i])\n",
    "    if meta_split > 0:\n",
    "        X_meta, X_cv, y_meta, y_cv = train_test_split(X_cv, y_cv, test_size=meta_split, stratify=y_cv)\n",
    "        print len(X_train), len(X_cv), len(X_meta), len(X_cv) + len(X_train) + len(X_meta), len(X)\n",
    "    else:\n",
    "        print len(X_train), len(X_cv), len(X_cv) + len(X_train) , len(X)\n",
    "    trained_base_models = []\n",
    "    trained_all_models = []\n",
    "    predictions = []\n",
    "    base_predictions_cv = []\n",
    "    base_predictions_meta = []\n",
    "    for i, model in enumerate(base_models):\n",
    "        print base_model_names[i]\n",
    "        print len(X_train[0])\n",
    "        model.fit(X_train,y_train)\n",
    "        trained_base_models.append(model)\n",
    "        trained_all_models.append(model)\n",
    "        predict = model.predict(X_cv)\n",
    "        predictions.append(predict)\n",
    "        base_predictions_cv.append(predict)\n",
    "        base_predictions_meta.append(model.predict(X_meta))\n",
    "        results[base_model_names[i]]['pred'].append(predict)\n",
    "        results[base_model_names[i]]['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results[base_model_names[i]]['conf'].append(confusion_matrix(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "        results[base_model_names[i]]['rep'].append(classification_report(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "    #trained_all_models = copy.deepcopy(trained_base_models)\n",
    "    for name in meta_models_names:\n",
    "        print name\n",
    "        if name =='votingf' or name=='votingh':\n",
    "            model = voting_dic[name]\n",
    "            model.fit(X_train, y_train)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'space':\n",
    "            models_for_space = {}\n",
    "            cv_scores = []\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                models_for_space[base_model_names[i]] = base_trained_model\n",
    "                cv_scores.append(base_trained_model.score(X_meta, y_meta))\n",
    "            model = combinations.SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'space3':\n",
    "            models_for_space = {}\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                models_for_space[base_model_names[i]] = base_trained_model\n",
    "            model = SubSpaceEnsemble3(models_for_space, k=5, weights= [2,1,3,0.6])\n",
    "            model.fit(X_train, y_train)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'meta':\n",
    "            model_dic = {}\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                model_dic[base_model_names[i]] = base_trained_model\n",
    "            model = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'OLA' or name == 'LCA' or name== 'KNE' or name == 'KNU':\n",
    "            models11 = {}\n",
    "            models_tr11 = {}\n",
    "            if name == 'LCA':\n",
    "                for i, model1 in enumerate(trained_base_models[:]):\n",
    "                    models11[base_model_names[i]] = model1\n",
    "                    models_tr11[base_model_names[i]] = trained_base_models[i].steps[1][1]        \n",
    "            else:   \n",
    "                for i, model1 in enumerate(trained_base_models[:]):\n",
    "                    models11[base_model_names[i]] = model1\n",
    "                    models_tr11[base_model_names[i]] = trained_base_models[i].steps[1][1]        \n",
    "            model = combinations.Neigbors_DS(models11, models_tr11, k=3, scheme=name, common_neigh=False)\n",
    "            #print model\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name in combinator_names:\n",
    "            #print 'mpike'\n",
    "            model = combinations.Combinator(scheme=name, weights= [1/float(len(base_predictions_meta)) for i in xrange(len(base_predictions_meta))])\n",
    "            model.fit(base_predictions_meta, y_meta)\n",
    "            predict = model.predict(base_predictions_cv)\n",
    "        trained_all_models.append(model)\n",
    "        predictions.append(predict)\n",
    "        results[name]['pred'].append(predict)\n",
    "        results[name]['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results[name]['conf'].append(confusion_matrix(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "        results[name]['rep'].append(classification_report(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "    print('Round %d took: %0.3f seconds') % (j, time.time()-t1)\n",
    "    t1 = time.time()\n",
    "print('Total time: %0.3f seconds') % (time.time()-t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%%%%%%%%%%%%%%%%  3grams  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.425454545455\n",
      "Precision : 0.324\n",
      "Recall : 0.424\n",
      "F1 : 0.356\n",
      "Confusion matrix :\n",
      " [[  0.2   2.4   1.4   0.    0. ]\n",
      " [  0.    6.6   9.4   1.    0. ]\n",
      " [  0.    5.4  16.6   1.    0. ]\n",
      " [  0.    2.8   7.2   0.    0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  soac  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.447272727273\n",
      "Precision : 0.428\n",
      "Recall : 0.446\n",
      "F1 : 0.422\n",
      "Confusion matrix :\n",
      " [[  0.6   2.4   0.8   0.2   0. ]\n",
      " [  1.    6.    8.2   1.8   0. ]\n",
      " [  0.8   4.   15.4   2.8   0. ]\n",
      " [  0.6   0.6   6.2   2.6   0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  emo  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.294545454545\n",
      "Precision : 0.344\n",
      "Recall : 0.294\n",
      "F1 : 0.29\n",
      "Confusion matrix :\n",
      " [[ 0.6  2.4  0.8  0.   0.2]\n",
      " [ 1.8  8.4  3.   0.4  3.4]\n",
      " [ 1.8  9.4  6.   2.   3.8]\n",
      " [ 0.8  2.6  1.8  0.6  4.2]\n",
      " [ 0.   0.   0.4  0.   0.6]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  space  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.429090909091\n",
      "Precision : 0.428\n",
      "Recall : 0.428\n",
      "F1 : 0.386\n",
      "Confusion matrix :\n",
      " [[  0.2   2.4   1.2   0.2   0. ]\n",
      " [  0.2  10.    6.4   0.4   0. ]\n",
      " [  0.2   9.6  11.8   1.4   0. ]\n",
      " [  0.    2.4   6.    1.6   0. ]\n",
      " [  0.    0.2   0.8   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  OLA  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.418181818182\n",
      "Precision : 0.478\n",
      "Recall : 0.418\n",
      "F1 : 0.364\n",
      "Confusion matrix :\n",
      " [[  0.    2.8   1.2   0.    0. ]\n",
      " [  0.4   9.2   7.    0.4   0. ]\n",
      " [  0.2   9.8  12.4   0.6   0. ]\n",
      " [  0.    2.8   5.8   1.4   0. ]\n",
      " [  0.    0.4   0.6   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  LCA  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.421818181818\n",
      "Precision : 0.392\n",
      "Recall : 0.42\n",
      "F1 : 0.358\n",
      "Confusion matrix :\n",
      " [[  0.    3.    1.    0.    0. ]\n",
      " [  0.   11.8   4.4   0.4   0.4]\n",
      " [  0.   11.8  10.8   0.4   0. ]\n",
      " [  0.2   3.6   5.6   0.6   0. ]\n",
      " [  0.    0.4   0.6   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  KNE  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.425454545455\n",
      "Precision : 0.452\n",
      "Recall : 0.424\n",
      "F1 : 0.374\n",
      "Confusion matrix :\n",
      " [[  0.    2.8   1.2   0.    0. ]\n",
      " [  0.2   8.8   7.8   0.2   0. ]\n",
      " [  0.    8.8  13.2   1.    0. ]\n",
      " [  0.    2.6   6.    1.4   0. ]\n",
      " [  0.    0.2   0.8   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  KNU  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.345454545455\n",
      "Precision : 0.252\n",
      "Recall : 0.346\n",
      "F1 : 0.196\n",
      "Confusion matrix :\n",
      " [[  0.    2.4   1.6   0.    0. ]\n",
      " [  0.    9.8   6.6   0.    0.6]\n",
      " [  0.   13.8   9.2   0.    0. ]\n",
      " [  0.    5.6   4.4   0.    0. ]\n",
      " [  0.    0.6   0.4   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  majority  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.301818181818\n",
      "Precision : 0.298\n",
      "Recall : 0.304\n",
      "F1 : 0.282\n",
      "Confusion matrix :\n",
      " [[  0.2   1.8   1.8   0.    0.2]\n",
      " [  1.8   5.4   8.4   0.6   0.8]\n",
      " [  2.6   8.2  10.8   0.2   1.2]\n",
      " [  0.8   4.    4.8   0.2   0.2]\n",
      " [  0.2   0.6   0.2   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  weights  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.301818181818\n",
      "Precision : 0.298\n",
      "Recall : 0.304\n",
      "F1 : 0.282\n",
      "Confusion matrix :\n",
      " [[  0.2   1.8   1.8   0.    0.2]\n",
      " [  1.8   5.4   8.4   0.6   0.8]\n",
      " [  2.6   8.2  10.8   0.2   1.2]\n",
      " [  0.8   4.    4.8   0.2   0.2]\n",
      " [  0.2   0.6   0.2   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  accuracy  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.316363636364\n",
      "Precision : 0.308\n",
      "Recall : 0.318\n",
      "F1 : 0.296\n",
      "Confusion matrix :\n",
      " [[  0.2   1.6   1.8   0.    0.4]\n",
      " [  1.    4.4  10.    0.6   1. ]\n",
      " [  0.8   6.4  12.2   1.4   2.2]\n",
      " [  0.2   3.2   5.4   0.6   0.6]\n",
      " [  0.    0.6   0.4   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  optimal  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.290909090909\n",
      "Precision : 0.298\n",
      "Recall : 0.292\n",
      "F1 : 0.278\n",
      "Confusion matrix :\n",
      " [[  0.    1.8   1.4   0.4   0.4]\n",
      " [  0.8   4.2   9.8   0.8   1.4]\n",
      " [  1.2   6.4  11.2   1.8   2.4]\n",
      " [  0.4   3.4   5.6   0.6   0. ]\n",
      " [  0.2   0.6   0.2   0.    0. ]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
     ]
    }
   ],
   "source": [
    "for name in all_models_names:\n",
    "    print '%%%%%%%%%%%%%%%%  ' + name  + '  % %%%%%%%%%%%%%%%%%%%%%%%'\n",
    "    print '#################################'\n",
    "    mean_acc = 0\n",
    "    mean_prec = 0\n",
    "    mean_rec = 0\n",
    "    mean_f1 = 0\n",
    "    conf = numpy.zeros([5,5])\n",
    "    for i in xrange(cv_rounds):\n",
    "        mean_acc += results[name]['acc'][i]\n",
    "        #print results[key]['report'][i].split('     ')\n",
    "        mean_prec += float(results[name]['rep'][i].split('     ')[-4][2:])\n",
    "        mean_rec += float(results[name]['rep'][i].split('     ')[-3][2:])\n",
    "        mean_f1 += float(results[name]['rep'][i].split('     ')[-2][2:])\n",
    "        conf += results[name]['conf'][i]\n",
    "    mean_acc = mean_acc/float(cv_rounds)\n",
    "    mean_prec = mean_prec/float(cv_rounds)\n",
    "    mean_rec = mean_rec/float(cv_rounds)\n",
    "    mean_f1 = mean_f1/float(cv_rounds)\n",
    "    conf = conf/float(cv_rounds)\n",
    "    print('Accuracy : {}'.format(mean_acc))\n",
    "    print('Precision : {}'.format(mean_prec))\n",
    "    print('Recall : {}'.format(mean_rec))\n",
    "    print('F1 : {}'.format(mean_f1))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print '#################################'\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "copy.deepcopy(trained_base_models[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}