Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PAN 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset->Grouping User texts.\n",
      "\n",
      "Loaded 436 users...\n",
      "\n",
      "\n",
      "--------------- Thy time of Running ---------------\n",
      "Learning to judge age..\n",
      "Learning to judge gender..\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from argparse import ArgumentParser\n",
    "from sklearn.externals import joblib\n",
    "from tictacs import from_recipe\n",
    "from pan import ProfilingDataset\n",
    "import dill\n",
    "import cPickle as pickle\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "\n",
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "print('Loaded {} users...\\n'.format(len(dataset.entries)))\n",
    "# get config\n",
    "config = dataset.config\n",
    "tasks = config.tasks\n",
    "print('\\n--------------- Thy time of Running ---------------')\n",
    "all_models = {}\n",
    "for task in tasks:\n",
    "    print('Learning to judge %s..' % task)\n",
    "    # load data\n",
    "    X, y = dataset.get_data(task)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PAN 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset->Grouping User texts.\n",
      "\n",
      "Loaded 436 users...\n",
      "\n",
      "\n",
      "--------------- Thy time of Running ---------------\n",
      "Learning to judge age..\n",
      "Learning to judge gender..\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from argparse import ArgumentParser\n",
    "from sklearn.externals import joblib\n",
    "from tictacs import from_recipe\n",
    "from pan import ProfilingDataset\n",
    "import dill\n",
    "import cPickle as pickle\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "\n",
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "print('Loaded {} users...\\n'.format(len(dataset.entries)))\n",
    "# get config\n",
    "config = dataset.config\n",
    "tasks = config.tasks\n",
    "print('\\n--------------- Thy time of Running ---------------')\n",
    "all_models = {}\n",
    "for task in tasks:\n",
    "    print('Learning to judge %s..' % task)\n",
    "    # load data\n",
    "    X, y = dataset.get_data(task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from collections import Counter\n",
    "import pprint\n",
    "print \"Num of samples: \" + str(len(y))\n",
    "pprint.pprint(Counter(y))\n",
    "X, y = dataset.get_data('age')\n",
    "print len(X)\n",
    "\n",
    "X, X_cv, X, y_cv = train_test_split(X, y, test_size=0.4, random_state=42, stratify=y)\n",
    "X_cv, X_test, y_cv, y_test = train_test_split(X_cv, y_cv, test_size=0.5, random_state=42, stratify=y_cv)\n",
    "\n",
    "print len(X_cv), len(X_test), len(X) , len(X)+ len(X_cv) + len(X_test)\n",
    "pprint.pprint(Counter(y))\n",
    "pprint.pprint(Counter(y_cv))\n",
    "pprint.pprint(Counter(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "277792\n"
     ]
    }
   ],
   "source": [
    "from pan import ProfilingDataset, createDocProfiles, create_target_prof_trainset\n",
    "from pan import preprocess\n",
    "\n",
    "task = 'gender'\n",
    "docs = createDocProfiles(dataset)\n",
    "X, y = create_target_prof_trainset(docs, task)\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436\n"
     ]
    }
   ],
   "source": [
    "#reload(preprocess)\n",
    "#reload(features)\n",
    "from pan import features\n",
    "from pan import preprocess\n",
    "X, y = dataset.get_data('age')\n",
    "#X, y = dataset.get_data('gender')\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)\n",
    "#print \"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\n",
    "#print X[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3grams+soa+soac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset->Grouping User texts.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\n",
    "\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "task = 'gender'\n",
    "docs = createDocProfiles(dataset)\n",
    "X_inst_full, _ = create_target_prof_trainset(docs, task)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88\n",
      "Profile Based\n",
      "Train: 348 | Test: 88, 0.20 | Train + Test: 436 | Starting: 436\n",
      "Instance Based\n",
      "Train: 222762 | Test: 55030, 0.20 | Train + Test: 277792 | Starting: 277792\n"
     ]
    }
   ],
   "source": [
    "import random, copy\n",
    "from collections import Counter\n",
    "from pan import createDocProfiles, create_target_prof_trainset, ProfilingDataset\n",
    "from pan import preprocess\n",
    "\n",
    "\n",
    "test_split = 0.2\n",
    "task = 'age'\n",
    "N_test_samples = int(len(dataset.entries)*0.2)\n",
    "if N_test_samples % 2 !=0:\n",
    "    N_test_samples += 1\n",
    "print N_test_samples\n",
    "freq_labels = Counter([entry.__dict__[task] for entry in dataset.entries])\n",
    "flag_good_distr = False\n",
    "while not(flag_good_distr):\n",
    "    X_cv_prof = []\n",
    "    y_cv_prof = []\n",
    "    X_cv_inst = []\n",
    "    y_cv_inst = []\n",
    "    indexes = []\n",
    "    prof_indexes = []\n",
    "    indexes = random.sample(xrange(len(dataset.entries)), N_test_samples)\n",
    "    for ind in indexes:\n",
    "        X_cv_prof.append('\\n'.join(dataset.entries[ind].texts))\n",
    "        X_cv_inst.extend(dataset.entries[ind].texts)\n",
    "        y_cv_prof.append(dataset.entries[ind].__dict__[task])\n",
    "        y_cv_inst.extend([y_cv_prof[-1] for i in xrange(len(dataset.entries[ind].texts))])\n",
    "        prof_indexes.append(dataset.entries[ind].userid)\n",
    "    #break\n",
    "#print 'Number of texts: %d. Number of labels: %d. | True: %d' %(len(X_cv_inst), len(y_cv_inst), len(dataset.entries[ind].texts)) \n",
    "#print 'Numer of profiles:  %d. Number of labels: %d. | True: %d' %(len(X_cv_prof), len(y_cv_prof), len(indexes))\n",
    "    if len(list(set(y_cv_prof))) == len(list(set([entry.__dict__[task] for entry in dataset.entries]))):\n",
    "        flag_good_distr = True\n",
    "X_cv_prof = preprocess.preprocess(X_cv_prof)\n",
    "X_cv_inst = preprocess.preprocess(X_cv_inst)\n",
    "X_train_prof = []\n",
    "y_train_prof = []\n",
    "X_train_inst = []\n",
    "y_train_inst = []\n",
    "freq_test_labels = Counter(y_cv_prof)\n",
    "for j in xrange(len(dataset.entries)):\n",
    "    if j not in indexes:\n",
    "        X_train_prof.append('\\n'.join(dataset.entries[j].texts))\n",
    "        X_train_inst.extend(dataset.entries[j].texts)\n",
    "        y_train_prof.append(dataset.entries[j].__dict__[task])\n",
    "        y_train_inst.extend([y_train_prof[-1] for i in xrange(len(dataset.entries[j].texts))])\n",
    "#X_train_prof = preprocess.preprocess(X_train_prof)\n",
    "print 'Profile Based'\n",
    "print 'Train: %d | Test: %d, %0.2f | Train + Test: %d | Starting: %d' % (len(X_train_prof), len(X_cv_prof), len(X_cv_prof)/float(len(dataset.entries)), len(X_cv_prof) + len(X_train_prof), len(dataset.entries))\n",
    "print 'Instance Based'\n",
    "print 'Train: %d | Test: %d, %0.2f | Train + Test: %d | Starting: %d' % (len(X_train_inst), len(X_cv_inst), len(X_cv_inst)/float(len(X_inst_full)), len(X_cv_inst) + len(X_train_inst), len(X_inst_full))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~Bogas/943.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "data = [go.Bar(\n",
    "            x=sorted(list(set([entry.__dict__[task] for entry in dataset.entries]))),\n",
    "            y=[freq_labels[x_] for x_ in sorted(list(set([entry.__dict__[task] for entry in dataset.entries])))],\n",
    "            name='Ground'\n",
    "    ),\n",
    "       go.Bar(\n",
    "            x=sorted(list(set([entry.__dict__[task] for entry in dataset.entries]))),\n",
    "            y=[int(freq_test_labels[x_]/float(0.2)) for x_ in sorted(list(set([entry.__dict__[task] for entry in dataset.entries])))],\n",
    "            name='Sample'\n",
    "    )\n",
    "       ]\n",
    "\n",
    "py.iplot(data, filename='basic-bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220853"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X_train_inst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Profile\n",
      "To 0o mpainei edw sto profile\n",
      "Fitting 4 folds for each of 1 candidates, totalling 4 fits\n",
      "0.422413793103\n",
      "Pipeline(steps=[('3grams+soac', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=3000, min_df=1,\n",
      "        n...ax_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n",
      "Instance\n",
      "To 1o mpainei edw sto instance\n",
      "Fitting 4 folds for each of 1 candidates, totalling 4 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:  1.7min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.454203555747\n",
      "Pipeline(steps=[('3grams+soac', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=3000, min_df=1,\n",
      "        n...ax_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   4 out of   4 | elapsed:  3.3min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from pan.features import SOAC_Model2\n",
    "from pan import features\n",
    "\n",
    "# Profile Based\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[3,3], max_features=3000, stop_words='english')\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('3grams', grams3), ('soac', soac)])\n",
    "svm2 = LinearSVC(C=0.001, dual=False, class_weight='balanced')\n",
    "#svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=True)\n",
    "pipe1 = Pipeline([('3grams+soac',combined), ('svm', svm2)])\n",
    "#pipe = Pipeline([('3grams+soa',combined), ('svm', svm)])\n",
    "\n",
    "# Instance Based\n",
    "#svm2 = LinearSVC(C=0.001, dual=False, class_weight='balanced')\n",
    "pipe2 = Pipeline([('3grams+soac',combined), ('svm', svm2)])\n",
    "#pipe = Pipeline([('3grams+soa',combined), ('svm', svm)])\n",
    "trained_models = []\n",
    "names = ['Profile', 'Instance']\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "params = {}\n",
    "num_folds = 4\n",
    "for i, model in enumerate([pipe1, pipe2]):\n",
    "    print names[i]\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=1, n_jobs=2, cv=num_folds, refit=True)\n",
    "    if names[i] == 'Profile':\n",
    "        print 'To %do mpainei edw sto profile' % i\n",
    "        grid_search.fit(X_train_prof,y_train_prof)\n",
    "    elif names[i] == 'Instance':\n",
    "        print 'To %do mpainei edw sto instance' % i\n",
    "        grid_search.fit(X_train_inst,y_train_inst)\n",
    "    print(grid_search.best_score_)\n",
    "    print(grid_search.best_estimator_) \n",
    "    trained_models.append(grid_search.best_estimator_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#############Profile trained Model##############\n",
      "Test on Instance Test Data\n",
      "Accuracy : 0.424509945229\n",
      "Confusion matrix :\n",
      " [[    0   317  1844     0     0]\n",
      " [    0    36 19453     0     0]\n",
      " [    0    58 23526     0     0]\n",
      " [    0    17 10121     0     0]\n",
      " [    0     0   132     0     0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00      2161\n",
      "      25-34       0.08      0.00      0.00     19489\n",
      "      35-49       0.43      1.00      0.60     23584\n",
      "      50-64       0.00      0.00      0.00     10138\n",
      "      65-xx       0.00      0.00      0.00       132\n",
      "\n",
      "avg / total       0.21      0.42      0.26     55504\n",
      "\n",
      "#############################################\n",
      "Test on Profile Test Data\n",
      "Accuracy : 0.431818181818\n",
      "Confusion matrix :\n",
      " [[ 0  2  4  0  0]\n",
      " [ 0  4 27  0  0]\n",
      " [ 0  1 34  0  0]\n",
      " [ 0  0 15  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         6\n",
      "      25-34       0.57      0.13      0.21        31\n",
      "      35-49       0.42      0.97      0.59        35\n",
      "      50-64       0.00      0.00      0.00        15\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.37      0.43      0.31        88\n",
      "\n",
      "#############Instance trained Model##############\n",
      "Test on Instance Test Data\n",
      "Accuracy : 0.409375900836\n",
      "Confusion matrix :\n",
      " [[   63  1464   583    51     0]\n",
      " [  492  9017  9368   607     5]\n",
      " [  325  8863 12876  1517     3]\n",
      " [  151  3185  6035   766     1]\n",
      " [    1    29    94     8     0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.06      0.03      0.04      2161\n",
      "      25-34       0.40      0.46      0.43     19489\n",
      "      35-49       0.44      0.55      0.49     23584\n",
      "      50-64       0.26      0.08      0.12     10138\n",
      "      65-xx       0.00      0.00      0.00       132\n",
      "\n",
      "avg / total       0.38      0.41      0.38     55504\n",
      "\n",
      "#############################################\n",
      "Test on Profile Test Data\n",
      "Accuracy : 0.443181818182\n",
      "Confusion matrix :\n",
      " [[ 0  6  0  0  0]\n",
      " [ 0 24  6  1  0]\n",
      " [ 0 20 12  3  0]\n",
      " [ 0 10  2  3  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         6\n",
      "      25-34       0.40      0.77      0.53        31\n",
      "      35-49       0.57      0.34      0.43        35\n",
      "      50-64       0.43      0.20      0.27        15\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.44      0.44      0.40        88\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i, trained in enumerate(trained_models):\n",
    "    print '#############%s trained Model##############' %names[i]\n",
    "    print 'Test on Instance Test Data'\n",
    "    predict = trained.predict(X_cv_inst)\n",
    "    acc = accuracy_score(y_cv_inst, predict)\n",
    "    conf = confusion_matrix(y_cv_inst, predict, labels=sorted(list(set(y_cv_inst))))\n",
    "    rep = classification_report(y_cv_inst, predict, target_names=sorted(list(set(y_cv_inst))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))\n",
    "    print('#############################################')\n",
    "    print 'Test on Profile Test Data'\n",
    "    predict = trained.predict(X_cv_prof)\n",
    "    acc = accuracy_score(y_cv_prof, predict)\n",
    "    conf = confusion_matrix(y_cv_prof, predict, labels=sorted(list(set(y_cv_prof))))\n",
    "    rep = classification_report(y_cv_prof, predict, target_names=sorted(list(set(y_cv_prof))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 88 436 436\n",
      "Come to &quot;Algo Más Inesperado Que La Muerte&quot; Wednesday, March 24 at 8:00 pm until &lt;br /&gt;Saturday, April 24 at 11:00... http://fb.me/7kLR3SI 26823213\n",
      "277792 0 277792 277792\n",
      "277792 0 277792 277792\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "import random,  time\n",
    "from pan import createDocProfiles, create_target_prof_trainset, ProfilingDataset\n",
    "from pan import preprocess\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from pan.features import SOAC_Model2\n",
    "from pan import features\n",
    "\n",
    "# Read\n",
    "infolder = \"../pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "task = 'age'\n",
    "docs = createDocProfiles(dataset)\n",
    "X_inst_full, _ = create_target_prof_trainset(docs, task)\n",
    "\n",
    "# Create dataset\n",
    "\n",
    "test_split = 0.2\n",
    "N_test_samples = int(len(dataset.entries)*0.2)\n",
    "if N_test_samples % 2 !=0:\n",
    "    N_test_samples += 1\n",
    "print N_test_samples\n",
    "\n",
    "# Models\n",
    "\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[3,3], max_features=3000, stop_words='english')\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('3grams', grams3), ('soac', soac)])\n",
    "svm2 = LinearSVC(C=0.001, dual=False, class_weight='balanced')\n",
    "\n",
    "## Profile Based\n",
    "pipe1 = Pipeline([('3grams+soac',combined), ('svm', svm2)])\n",
    "\n",
    "## Instance Based\n",
    "pipe2 = Pipeline([('3grams+soac',combined), ('svm', svm2)])\n",
    "\n",
    "names = ['Profile', 'Instance']\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "num_folds = 4\n",
    "\n",
    "\n",
    "N = 5\n",
    "results = {'prof_prof':{'acc':[], 'conf':[], 'report':[]}, 'prof_inst':{'acc':[], 'conf':[], 'report':[]}, 'inst_prof':{'acc':[], 'conf':[], 'report':[]}, 'inst_inst':{'acc':[], 'conf':[], 'report':[]}}\n",
    "time_start = time.time()\n",
    "for ii in xrange(N):\n",
    "    t0 = time.time()\n",
    "    flag_good_distr = False\n",
    "    while not(flag_good_distr):\n",
    "        X_cv_prof = []\n",
    "        y_cv_prof = []\n",
    "        X_cv_inst = []\n",
    "        y_cv_inst = []\n",
    "        indexes = []\n",
    "        prof_indexes = []\n",
    "        indexes = random.sample(xrange(len(dataset.entries)), N_test_samples)\n",
    "        for ind in indexes:\n",
    "            X_cv_prof.append('\\n'.join(dataset.entries[ind].texts))\n",
    "            X_cv_inst.extend(dataset.entries[ind].texts)\n",
    "            y_cv_prof.append(dataset.entries[ind].__dict__[task])\n",
    "            y_cv_inst.extend([y_cv_prof[-1] for i in xrange(len(dataset.entries[ind].texts))])\n",
    "            prof_indexes.append(dataset.entries[ind].userid)\n",
    "        #break\n",
    "    #print 'Number of texts: %d. Number of labels: %d. | True: %d' %(len(X_cv_inst), len(y_cv_inst), len(dataset.entries[ind].texts))\n",
    "    #print 'Numer of profiles:  %d. Number of labels: %d. | True: %d' %(len(X_cv_prof), len(y_cv_prof), len(indexes))\n",
    "        if len(list(set(y_cv_prof))) == len(list(set([entry.__dict__[task] for entry in dataset.entries]))):\n",
    "            flag_good_distr = True\n",
    "    X_cv_prof = preprocess.preprocess(X_cv_prof)\n",
    "    X_cv_inst = preprocess.preprocess(X_cv_inst)\n",
    "    X_train_prof = []\n",
    "    y_train_prof = []\n",
    "    X_train_inst = []\n",
    "    y_train_inst = []\n",
    "    for j in xrange(len(dataset.entries)):\n",
    "        if j not in indexes:\n",
    "            X_train_prof.append('\\n'.join(dataset.entries[j].texts))\n",
    "            X_train_inst.extend(dataset.entries[j].texts)\n",
    "            y_train_prof.append(dataset.entries[j].__dict__[task])\n",
    "            y_train_inst.extend([y_train_prof[-1] for i in xrange(len(dataset.entries[j].texts))])\n",
    "    #X_train_prof = preprocess.preprocess(X_train_prof)\n",
    "    print 'Profile Based'\n",
    "    print 'Train: %d | Test: %d, %0.2f | Train + Test: %d | Starting: %d' % (len(X_train_prof), len(X_cv_prof), len(X_cv_prof)/float(len(dataset.entries)), len(X_cv_prof) + len(X_train_prof), len(dataset.entries))\n",
    "    print 'Instance Based'\n",
    "    print 'Train: %d | Test: %d, %0.2f | Train + Test: %d | Starting: %d' % (len(X_train_inst), len(X_cv_inst), len(X_cv_inst)/float(len(X_inst_full)), len(X_cv_inst) + len(X_train_inst), len(X_inst_full))\n",
    "    trained_models = []\n",
    "    for i, model in enumerate([pipe1, pipe2]):\n",
    "        print names[i]\n",
    "        grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=1, n_jobs=2, cv=num_folds, refit=True)\n",
    "        if names[i] == 'Profile':\n",
    "            #print 'To %do mpainei edw sto profile' % i\n",
    "            grid_search.fit(X_train_prof,y_train_prof)\n",
    "        elif names[i] == 'Instance':\n",
    "            #print 'To %do mpainei edw sto instance' % i\n",
    "            grid_search.fit(X_train_inst,y_train_inst)\n",
    "        print(grid_search.best_score_)\n",
    "        print(grid_search.best_params_)\n",
    "        trained_models.append(grid_search.best_estimator_)\n",
    "    for i, trained in enumerate(trained_models):\n",
    "        if names[i] == 'Profile':\n",
    "            key1 = 'prof_prof'\n",
    "            key2 = 'prof_inst'\n",
    "        else:\n",
    "            key1 = 'inst_prof'\n",
    "            key2 = 'inst_inst'\n",
    "        # Instance Test\n",
    "        predict = trained.predict(X_cv_inst)\n",
    "        acc = accuracy_score(y_cv_inst, predict)\n",
    "        conf = confusion_matrix(y_cv_inst, predict, labels=sorted(list(set(y_cv_inst))))\n",
    "        rep = classification_report(y_cv_inst, predict, target_names=sorted(list(set(y_cv_inst))))\n",
    "        results[key1]['acc'].append(acc)\n",
    "        results[key1]['conf'].append(conf)\n",
    "        results[key1]['report'].append(rep)\n",
    "        # Profile Test\n",
    "        predict = trained.predict(X_cv_prof)\n",
    "        acc = accuracy_score(y_cv_prof, predict)\n",
    "        conf = confusion_matrix(y_cv_prof, predict, labels=sorted(list(set(y_cv_prof))))\n",
    "        rep = classification_report(y_cv_prof, predict, target_names=sorted(list(set(y_cv_prof))))\n",
    "        results[key2]['acc'].append(acc)\n",
    "        results[key2]['conf'].append(conf)\n",
    "        results[key2]['report'].append(rep)\n",
    "    print 'Finished iteration %d in Start: %0.2f | Iteration: %0.2f  (seconds) ' % (ii, time.time()-time_start, time.time()-t0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  GENDER  N = 3  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "%%%%%%%%%%%%%%%%  Profile Model -- Instance Test  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'prof_prof'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-bc5a87eb2053>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mN\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m         \u001b[0mmean_acc\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mresults\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'acc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m         \u001b[1;31m#print results[key]['report'][i].split('     ')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m         \u001b[0mmean_prec\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'report'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'     '\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyError\u001b[0m: 'prof_prof'"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "gg = numpy.zeros([2,2])\n",
    "keys = ['prof_prof', 'prof_inst', 'inst_prof', 'inst_inst']\n",
    "names_2 = ['Profile Model -- Instance Test','Profile Model -- Profile Test', 'Instance Model -- Instance Test','Instance Model -- Profile Test',]\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~  ' + task.upper() + '  N = ' + str(N) + '  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\n",
    "for ii, key in enumerate(keys):\n",
    "    print '%%%%%%%%%%%%%%%%  ' + names_2[ii]  + '  % %%%%%%%%%%%%%%%%%%%%%%%'\n",
    "    print '#################################'\n",
    "    mean_acc = 0\n",
    "    mean_prec = 0\n",
    "    mean_rec = 0\n",
    "    mean_f1 = 0\n",
    "    conf = numpy.zeros([5,5])\n",
    "    for i in xrange(N):\n",
    "        mean_acc += results[key]['acc'][i]\n",
    "        #print results[key]['report'][i].split('     ')\n",
    "        mean_prec += float(results[key]['report'][i].split('     ')[-4][2:])\n",
    "        mean_rec += float(results[key]['report'][i].split('     ')[-3][2:])\n",
    "        mean_f1 += float(results[key]['report'][i].split('     ')[-2][2:])\n",
    "        conf += results[key]['conf'][i]\n",
    "    mean_acc = mean_acc/float(N)\n",
    "    mean_prec = mean_prec/float(N)\n",
    "    mean_rec = mean_rec/float(N)\n",
    "    mean_f1 = mean_f1/float(N)\n",
    "    conf = conf/float(N)\n",
    "    print('Accuracy : {}'.format(mean_acc))\n",
    "    print('Precision : {}'.format(mean_prec))\n",
    "    print('Recall : {}'.format(mean_rec))\n",
    "    print('F1 : {}'.format(mean_f1))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print '#################################'\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[3,3], max_features=5000, stop_words='english')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=None)\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=5000)\n",
    "countTokens = features.CountTokens()\n",
    "countHash = features.CountHash()\n",
    "countUrls = features.CountURLs()\n",
    "countReplies = features.CountReplies()\n",
    "svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced', probability=True)\n",
    "#svm = DecisionTreeClassifier()\n",
    "combined = FeatureUnion([('3grams', grams3), ('soa', soa)])\n",
    "#pipe = Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counts + SOA+SOAC. Ommit preprocess!!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(features)\n",
    "features.SOAC_Model2.__doc__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[3,3], max_features=5000, stop_words='english')\n",
    "countTokens = features.CountTokens()\n",
    "countHash = features.CountHash()\n",
    "countUrls = features.CountURLs()\n",
    "countReplies = features.CountReplies()\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=None)\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=5000)\n",
    "scaler = StandardScaler()#MinMaxScaler()#StandardScaler()\n",
    "#svm = DecisionTreeClassifier()\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "#combined = FeatureUnion([('soa', soa), ('soac', soac)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])\n",
    "combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe = Pipeline([('3grams', grams3), ('svm', svm)])\n",
    "#pipe = Pipeline([('soac',soac), ('svm', svm)])\n",
    "#pipe = Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grams3.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a= grams3.transform([X[1]])\n",
    "import pprint\n",
    "pprint.pprint(list(a.todense()[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from pan.features import LDA\n",
    "\n",
    "LDAmodel = LDA(num_topics=30, lib='sklearn')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=None)\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=5000)\n",
    "countTokens = features.CountTokens()\n",
    "countHash = features.CountHash()\n",
    "countUrls = features.CountURLs()\n",
    "countReplies = features.CountReplies()\n",
    "#svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced')\n",
    "svm = DecisionTreeClassifier()\n",
    "combined = FeatureUnion([('LDA', LDAmodel)])#, ('soa', soa), ('soac', soac)])\n",
    "pipe = Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Soft Voting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('3grams',\n",
       "  TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
       "          dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "          lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
       "          ngram_range=[2, 2], norm=u'l2', preprocessor=None, smooth_idf=True,\n",
       "          stop_words='english', strip_accents=None, sublinear_tf=False,\n",
       "          token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "          vocabulary=None)),\n",
       " ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from pan.features import SOA_Model2\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[2,2], max_features=5000, stop_words='english')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('3grams', grams3), ('soa', soa)])\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=True)\n",
    "pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "#pipe = Pipeline([('3grams+soa',combined), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
       "        tokenizer_var='sklearn')),\n",
       " ('svm', SVC(C=1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced', probability=True)\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])+\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soac.fit(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soac.counter.transform([X[1]]).todense().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('LSI', LSI_Model(num_topics=50)),\n",
       " ('svm', SVC(C=1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pan.features import LDA\n",
    "from pan.features import LSI_Model\n",
    "#LDAmodel = LDA(num_topics=30, lib='sklearn')\n",
    "LSImodel = LSI_Model(num_topics=50)\n",
    "svm = SVC(kernel='rbf', C=1, gamma=1, class_weight='balanced', probability=True)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe2 = Pipeline([('LSI',LSImodel), ('svm', svm)])\n",
    "\n",
    "countTokens = features.CountTokens()\n",
    "countHash = features.CountHash()\n",
    "countUrls = features.CountURLs()\n",
    "countReplies = features.CountReplies()\n",
    "combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "#svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=True)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "#pipe2 = Pipeline([('LDAmodel',LDAmodel), ('svm', svm)])\n",
    "pipe2.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "### Gender"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('combined', FeatureUnion(n_jobs=1,\n",
       "         transformer_list=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
       "          dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "          lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
       "          ngram_range=[3, 3], norm=u'l2', preprocessor=None, smooth_idf=True,\n",
       "          stop_words='english', strip_accents=None, sublinear_tf=False,\n",
       "          token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "          vocabulary=None))],\n",
       "         transformer_weights=None)),\n",
       " ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from pan.features import SOA_Model2\n",
    "from pan import features\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[3,3], max_features=5000, stop_words='english')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('3grams', grams3)])\n",
    "#svm = LinearSVC(C=0.001, dual=False, class_weight='balanced')\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=True)\n",
    "#pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "pipe = Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
       "        tokenizer_var='sklearn')),\n",
       " ('svm', SVC(C=0.1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from pan.features import SOAC_Model2\n",
    "from sklearn.svm import SVC\n",
    "soac = SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# LDA\n",
    "\n",
    "from pan.features import LDA\n",
    "LDAmodel = LDA(num_topics=120, lib='sklearn')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "countTokens = features.CountTokens()\n",
    "countHash = features.CountHash()\n",
    "countUrls = features.CountURLs()\n",
    "countReplies = features.CountReplies()\n",
    "combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe2 = Pipeline([('LDAmodel',LDAmodel), ('svm', svm)])\n",
    "#pipe2 = Pipeline([('soa',soa), ('svm', svm)])\n",
    "pipe2.steps                             "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# LSI\n",
    "\n",
    "from pan.features import LSI_Model\n",
    "LSImodel = LSI_Model(num_topics=100)\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe2 = Pipeline([('LSI',LSImodel), ('svm', svm)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2.58285953,  3.19662126,  3.19191807,  1.41978412,  2.51990408],\n",
       "       [ 2.39997484,  2.72140868,  2.73327336,  1.71105421,  2.35863521],\n",
       "       [ 3.57621963,  4.49868035,  4.85222854,  4.50983793,  4.3119122 ],\n",
       "       ..., \n",
       "       [ 3.05207618,  3.42534966,  3.55453814,  2.08951272,  2.96817928],\n",
       "       [ 3.74328351,  4.26783942,  2.50918718,  3.77818404,  3.64180076],\n",
       "       [ 1.21603409,  0.84187722,  1.38922593,  1.25588595,  1.18699204]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soac.fit_transform(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "304 132 436 436\n",
      "304 304\n",
      "304 132 436 436\n",
      "304 304\n",
      "Accuracy : 0.477272727273\n",
      "Confusion matrix :\n",
      " [[30 36]\n",
      " [33 33]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     female       0.48      0.45      0.47        66\n",
      "       male       0.48      0.50      0.49        66\n",
      "\n",
      "avg / total       0.48      0.48      0.48       132\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "\n",
    "num_folds = 4\n",
    "split = 0.3\n",
    "#X, y = dataset.get_data('age')\n",
    "X, y = dataset.get_data('gender')\n",
    "X = preprocess.preprocess(X)\n",
    "X = soac.transform(X)\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "print len(X_train), len(y_train)\n",
    "for i, x in enumerate(X_train):\n",
    "    if len(x)==0:\n",
    "        X_train.remove(x)\n",
    "        y_train.remove(y_train[i])\n",
    "for i, x in enumerate(X_cv):\n",
    "    if len(x)==0:\n",
    "        X_cv.remove(x)\n",
    "        y_cv.remove(y_cv[i])\n",
    "#X_meta, X_cv, y_meta, y_cv = train_test_split(X_cv, y_cv, test_size=0.5, stratify=y_cv)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "print len(X_train), len(y_train)\n",
    "bdt = AdaBoostClassifier(DecisionTreeClassifier(),\n",
    "                         algorithm=\"SAMME.R\",\n",
    "                         n_estimators=200)\n",
    "bdt.fit(X_train, y_train)\n",
    "predict = bdt.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy, copy\n",
    "\n",
    "def print_overlaps(predictions, names, verbose=True):\n",
    "    N = len(names)\n",
    "    res = numpy.zeros([N,N])\n",
    "    temp = numpy.zeros([N,N])\n",
    "    for i in range(0, N):\n",
    "        for j in range(i+1, N):\n",
    "            temp[i,j] = len([m for l, m in enumerate(predictions[i]) if (m==predictions[j][l] and m==predictions[N-1][l])])/float(len(predictions[0]))\n",
    "            res[i,j] = len([(k,v) for k,v in zip(predictions[i], predictions[j]) if k==v])/float(len(predictions[0]))\n",
    "            if verbose:\n",
    "                print \"%s - %s : %0.3f  overlap | ground-truth coverage: %0.3f\" % (names[i],  names[j], 100*res[i,j], 100*temp[i,j])\n",
    "    return  [res, temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split: 0.25\n",
      "323 55 378 436\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n",
      "First search took: 24.265 seconds\n",
      "Accuracy obtained in CV-data: 50.000\n",
      "[0.89, 0.11, 0.173, 6]\n",
      "Fit took: 38.383 seconds\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'print_overlaps' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-8-1ca3efea41f6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    101\u001b[0m         \u001b[1;31m# Meta model END ###\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    102\u001b[0m         \u001b[0mpredictions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 103\u001b[1;33m         \u001b[0mresults\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'3grams'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'over'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mprint_overlaps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel_names\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'space'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'meta'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'true'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    104\u001b[0m     \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Split %0.1f.: %0.3f seconds'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0msplit\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mt0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'print_overlaps' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from pan.features import Metaclassifier\n",
    "import time\n",
    "\n",
    "#pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "#pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "#pipe2 = Pipeline([('soa',soa), ('svm', svm)])\n",
    "\n",
    "### AGE ###\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='hard')\n",
    "#models = [pipe,pipe1,pipe2,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting', 'votingh']\n",
    "\n",
    "### GENDER ###\n",
    "eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), ('2', pipe2)], voting='soft')\n",
    "eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), ('2', pipe2)], voting='hard')\n",
    "models = [pipe,pipe1, pipe2, eclfh]\n",
    "model_names = ['3grams', 'soac', 'lsi', 'votingh']\n",
    "\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='hard')\n",
    "#models = [pipe,pipe1,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'voting', 'votingh']\n",
    "\n",
    "results = {}\n",
    "for name in model_names:\n",
    "    results[name] = {'pred': [], 'acc': [], 'conf': [], 'over': []}\n",
    "results['space'] = {'pred': [], 'acc': [], 'conf': [], 'over':[]}\n",
    "results['meta'] = {'pred': [], 'acc': [], 'conf': [], 'over':[]}\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "#params = {}\n",
    "num_folds = 4\n",
    "#splits = [0.25, 0.3, 0.4]\n",
    "splits = [0.25]\n",
    "N = 3\n",
    "t0 = time.time()\n",
    "for split in splits:\n",
    "    print \"Split: \" + str(split)  \n",
    "    for i in xrange(N):\n",
    "        #X, y = dataset.get_data('age')\n",
    "        #X, y = dataset.get_data('gender')\n",
    "        X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y)\n",
    "        for i, x in enumerate(X_train):\n",
    "            if len(x)==0:\n",
    "                X_train.remove(x)\n",
    "                y_train.remove(y_train[i])\n",
    "        for i, x in enumerate(X_cv):\n",
    "            if len(x)==0:\n",
    "                X_cv.remove(x)\n",
    "                y_cv.remove(y_cv[i])\n",
    "        if 'space' or 'meta' in results.keys():\n",
    "            X_meta, X_cv, y_meta, y_cv = train_test_split(X_cv, y_cv, test_size=0.5, stratify=y_cv)\n",
    "        print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "        trained_models = []\n",
    "        for i, model in enumerate(models):\n",
    "            if model_names[i] == 'voting' or model_names[i] == 'votingh':\n",
    "                params = {}\n",
    "            grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=0, n_jobs=3, cv=num_folds, refit=True)\n",
    "            grid_search.fit(X_train,y_train)\n",
    "            trained_models.append(grid_search.best_estimator_)\n",
    "        predictions = []\n",
    "        for i, model in enumerate(trained_models):\n",
    "            predict = model.predict(X_cv)\n",
    "            predictions.append(predict)\n",
    "            results[model_names[i]]['pred'].append(predict)\n",
    "            results[model_names[i]]['acc'].append(accuracy_score(y_cv, predict))\n",
    "            results[model_names[i]]['conf'].append(confusion_matrix(y_cv, predict, labels=list(set(y))))\n",
    "        # Space model ###\n",
    "        \n",
    "        models_for_space = {}\n",
    "        cv_scores = []\n",
    "        for name, model in zip(model_names, trained_models):\n",
    "            if name!='voting' and name!='votingh':\n",
    "                models_for_space[name] = model\n",
    "                cv_scores.append(model.score(X_meta, y_meta))\n",
    "        space = SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "        space.fit(X_meta, y_meta)\n",
    "        predict = space.predict(X_cv)\n",
    "        results['space']['pred'].append(predict)\n",
    "        results['space']['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results['space']['conf'].append(confusion_matrix(y_cv, predict, labels=list(set(y))))\n",
    "        predictions.append(predict)\n",
    "        \n",
    "        # Space model end ###\n",
    "        # Meta ###\n",
    "        model_dic = {}\n",
    "        for i, model in enumerate(trained_models):\n",
    "            if model_names[i] != 'voting' and model_names[i] !='votingh': \n",
    "                model_dic[model_names[i]] = model\n",
    "        Meta = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "        Meta.fit(X_meta, y_meta)\n",
    "        predict = Meta.predict(X_cv)\n",
    "        results['meta']['pred'].append(predict)\n",
    "        results['meta']['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results['meta']['conf'].append(confusion_matrix(y_cv, predict, labels=list(set(y))))\n",
    "        predictions.append(predict)\n",
    "        # Meta model END ###\n",
    "        predictions.append(y_cv)\n",
    "        results['3grams']['over'].append(print_overlaps(predictions, model_names+['space','meta', 'true'], False))\n",
    "    print('Split %0.1f.: %0.3f seconds') % (split, time.time()-t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trained_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Split: 0.25\n",
      "----------- Scores-----------\n",
      "\n",
      "Model: 3grams Accuracy: 0.366 Std: 0.047\n",
      "\n",
      "Model: soac Accuracy: 0.317 Std: 0.043\n",
      "\n",
      "Model: lsi Accuracy: 0.384 Std: 0.065\n",
      "\n",
      "Model: votingh Accuracy: 0.329 Std: 0.021\n",
      "\n",
      "Model: space Accuracy: 0.384 Std: 0.046\n",
      "\n",
      "Model: meta Accuracy: 0.232 Std: 0.055\n",
      "----------- Overlaps-----------\n",
      "3grams - soac : 53.075  overlap | ground-truth coverage: 18.900\n",
      "3grams - lsi : 54.355  overlap | ground-truth coverage: 19.562\n",
      "3grams - votingh : 83.591  overlap | ground-truth coverage: 29.293\n",
      "3grams - space : 73.805  overlap | ground-truth coverage: 28.047\n",
      "3grams - meta : 47.015  overlap | ground-truth coverage: 18.923\n",
      "3grams - true : 36.577  overlap | ground-truth coverage: 36.577\n",
      "soac - lsi : 40.292  overlap | ground-truth coverage: 14.646\n",
      "soac - votingh : 59.787  overlap | ground-truth coverage: 20.123\n",
      "soac - space : 68.249  overlap | ground-truth coverage: 24.355\n",
      "soac - meta : 45.095  overlap | ground-truth coverage: 15.241\n",
      "soac - true : 31.684  overlap | ground-truth coverage: 31.684\n",
      "lsi - votingh : 70.157  overlap | ground-truth coverage: 23.210\n",
      "lsi - space : 60.460  overlap | ground-truth coverage: 25.645\n",
      "lsi - meta : 31.818  overlap | ground-truth coverage: 12.245\n",
      "lsi - true : 38.440  overlap | ground-truth coverage: 38.440\n",
      "votingh - space : 71.380  overlap | ground-truth coverage: 26.835\n",
      "votingh - meta : 49.439  overlap | ground-truth coverage: 18.923\n",
      "votingh - true : 32.941  overlap | ground-truth coverage: 32.941\n",
      "space - meta : 51.235  overlap | ground-truth coverage: 20.123\n",
      "space - true : 38.395  overlap | ground-truth coverage: 38.395\n",
      "meta - true : 23.165  overlap | ground-truth coverage: 23.165\n",
      "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import statistics\n",
    "for i, split in enumerate(splits):\n",
    "    print 'Split: %0.2f' % split\n",
    "    print '----------- Scores-----------'\n",
    "    #for name in model_names:\n",
    "    for name in model_names + ['space'] + ['meta']:\n",
    "        tmp = results[name]['acc'][N*i:(N*i+N)]\n",
    "        print \n",
    "        print 'Model: %s Accuracy: %0.3f Std: %0.3f' % (name, statistics.mean(tmp), \n",
    "                                                          statistics.stdev(tmp))\n",
    "        #tmp_conf = copy.deepcopy(results[name]['conf'][N*i])\n",
    "        #for j in xrange(N*i+1, N*i+N):\n",
    "        #    tmp_conf += results[name]['conf'][j]\n",
    "        #tmp_conf /= N\n",
    "        #print('Confusion matrix :\\n {}'.format(tmp_conf))\n",
    "    print '----------- Overlaps-----------'\n",
    "    tmp_overlaps = copy.deepcopy(results['3grams']['over'][N*i][0])\n",
    "    tmp_gt_overlaps = copy.deepcopy(results['3grams']['over'][N*i][1])\n",
    "    for j in xrange(N*i+1, N*i+N):\n",
    "            tmp_overlaps += results['3grams']['over'][j][0]\n",
    "            tmp_gt_overlaps += results['3grams']['over'][j][1]\n",
    "    tmp_overlaps /= N\n",
    "    tmp_gt_overlaps /= N\n",
    "    print_names = model_names+['space', 'meta','true']\n",
    "    #print_names = model_names+['true']\n",
    "    for k in xrange(tmp_overlaps.shape[0]):\n",
    "        for v in xrange(k+1, tmp_overlaps.shape[0]):\n",
    "            print \"%s - %s : %0.3f  overlap | ground-truth coverage: %0.3f\" % (print_names[k],  print_names[v], 100*tmp_overlaps[k, v], 100*tmp_gt_overlaps[k,v])\n",
    "    print '%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\\n'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 4)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(predictions), len(model_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "Diversity Tests Report\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Details\n",
      "===============================================================\n",
      "Correlation: For +-1 perfect aggrement/disagreement\n",
      "Q-statistic: Q=0  => Independent. For q>0 predictors find the the same results\n",
      "Cohen's k: k->0  => High Disagreement => High Diversity\n",
      "Kohovi-Wolpert Variance -> Inf => High Diversity\n",
      "Conditional Accuracy Table: Conditional Probability that the row system predicts correctly, given\n",
      "                            that the column system also predicts correctly\n",
      "===============================================================\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Results\n",
      "---------------------------------------------------------------\n",
      "\n",
      "#####  Kohovi-Wolpert Variance:  0.125  #####\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "#### Pairwise Average Metrics: #####\n",
      "Avg. Cor: 0.000\n",
      "Avg. Q-statistic: 0.435\n",
      "Avg. Cohen's k: -0.064\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "###Conditional Accuracy Table###\n",
      "         3grams  soac  lsi  votingh  space  meta\n",
      "3grams     1.00  0.65 0.18     0.72   0.80  0.69\n",
      "soac       0.48  1.00 0.12     0.67   0.75  1.00\n",
      "lsi        0.13  0.12 1.00     0.44   0.20  0.12\n",
      "votingh    0.57  0.71 0.47     1.00   0.70  0.75\n",
      "space      0.70  0.88 0.24     0.78   1.00  0.94\n",
      "meta       0.48  0.94 0.12     0.67   0.75  1.00\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "#import EnsembleDiversityTests\n",
    "#reload(EnsembleDiversityTests)\n",
    "from EnsembleDiversityTests import DiversityTests\n",
    "#gg = DiversityTests(predictions[:-1], print_names[:-1], predictions[-1])\n",
    "gg = DiversityTests(predictions, model_names+['space', 'meta'], y_cv)\n",
    "gg.print_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NotFittedError",
     "evalue": "TfidfVectorizer - Vocabulary wasn't fitted.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNotFittedError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-29-72c384604ab6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mens\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mEnsemble\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclassifiers\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mpipe\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mdiv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDiversity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmetric\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'q'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mq\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdiv\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcalculate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mens\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      7\u001b[0m \u001b[0mq\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/brew/metrics/diversity/base.pyc\u001b[0m in \u001b[0;36mcalculate\u001b[1;34m(self, ensemble, X, y)\u001b[0m\n\u001b[0;32m     91\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     92\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mcalculate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mensemble\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 93\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mensemble\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'labels'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     94\u001b[0m         \u001b[0moracle\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mequal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnewaxis\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/brew/base.pyc\u001b[0m in \u001b[0;36moutput\u001b[1;34m(self, X, mode)\u001b[0m\n\u001b[0;32m    164\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassifiers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    165\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mclf\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclassifiers\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 166\u001b[1;33m                 \u001b[0mout\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    168\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/utils/metaestimators.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     35\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[1;31m# lambda, but not partial, allows help() to work with update_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m         \u001b[1;31m# update the docstring of the returned function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[0mupdate_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    201\u001b[0m         \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 203\u001b[1;33m             \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    204\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, raw_documents, copy)\u001b[0m\n\u001b[0;32m   1331\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'_tfidf'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'The tfidf vector is not fitted'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1332\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1333\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mTfidfVectorizer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1334\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_tfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, raw_documents)\u001b[0m\n\u001b[0;32m    861\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_validate_vocabulary\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    862\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 863\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_vocabulary\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    864\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    865\u001b[0m         \u001b[1;31m# use the same matrix-building strategy as fit_transform\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m_check_vocabulary\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    273\u001b[0m         \u001b[1;34m\"\"\"Check if vocabulary is empty or missing (not fit-ed)\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    274\u001b[0m         \u001b[0mmsg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\"%(name)s - Vocabulary wasn't fitted.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 275\u001b[1;33m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'vocabulary_'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmsg\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mmsg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    276\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    277\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvocabulary_\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_is_fitted\u001b[1;34m(estimator, attributes, msg, all_or_any)\u001b[0m\n\u001b[0;32m    676\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    677\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mall_or_any\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattr\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mattr\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mattributes\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 678\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0mNotFittedError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmsg\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m'name'\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mtype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    679\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    680\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNotFittedError\u001b[0m: TfidfVectorizer - Vocabulary wasn't fitted."
     ]
    }
   ],
   "source": [
    "from brew.metrics.diversity.base import Diversity\n",
    "from brew.base import Ensemble\n",
    "\n",
    "ens = Ensemble(classifiers=[pipe, pipe1, pipe2])\n",
    "div = Diversity(metric='q')\n",
    "q = div.calculate(ens, numpy.array(X_cv), numpy.array(y_cv))\n",
    "q\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "import EnsembleDiversityTests\n",
    "reload(EnsembleDiversityTests)\n",
    "from EnsembleDiversityTests import DiversityTests, BaseClassifiers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base Accuracies\n",
      "3grams : 41.82  ||  soac : 30.91  ||  lsi : 30.91  ||  votingh : 32.73  ||  space : 36.36  ||  meta : 29.09  \n",
      "Models Correct Aggrement Percentages\n",
      "         Only this Model   1-model aggree   2-model aggree   3-model aggree   4-model aggree   5-model aggree\n",
      "3grams             12.73             0.00             0.00             0.00           100.00             0.00\n",
      "soac                1.82             0.00             0.00             0.00           100.00             0.00\n",
      "lsi                16.36           100.00             0.00             0.00             0.00             0.00\n",
      "votingh             0.00             0.00             0.00             0.00             0.00           100.00\n",
      "space               0.00             0.00           100.00             0.00             0.00             0.00\n",
      "meta                0.00             0.00           100.00             0.00             0.00             0.00\n",
      "Predictions Distributions\n",
      "All correct : 1.82  || Some correct : 74.55 || All wrong: 23.64 \n",
      "Not all Correct Instances Distributions\n",
      "None Correct : 23.64  ||  1 correct : 30.91  ||  2 correct : 14.55  ||  3 correct : 5.45  ||  4 correct : 3.64  ||  5 correct : 20.00  \n"
     ]
    }
   ],
   "source": [
    "gg1 = BaseClassifiers(predictions, model_names+['space', 'meta'], y_cv, True)\n",
    "#gg1 = BaseClassifiers(predictions[:3], print_names[:3], predictions[-1], True)\n",
    "gg1.get_comparison_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 788,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base Accuracies\n",
      "3grams : 50.00  ||  soac : 53.03  ||  lsi : 50.00  \n",
      "Models Correct Aggrement Percentages\n",
      "        Only this Model   1-model aggree   2-model aggree\n",
      "3grams             0.00            87.50            12.50\n",
      "soac               7.58             0.00           100.00\n",
      "lsi               31.82           100.00             0.00\n",
      "Predictions Distributions\n",
      "All correct : 13.64  || Some correct : 75.76 || All wrong: 10.61 \n",
      "Not all Correct Instances Distributions\n",
      "None Correct : 10.61  ||  1 correct : 39.39  ||  2 correct : 36.36  \n"
     ]
    }
   ],
   "source": [
    "def autolabel(rects, ax):\n",
    "    # attach some text labels\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.text(rect.get_x() + rect.get_width()/2. ,1.0*height, '%0.2f' % float(height),ha='center', va='bottom')\n",
    "\n",
    "def base_level(predictions, names, true, print_flag=True):\n",
    "    import numpy, pandas\n",
    "    import matplotlib.pyplot as plt\n",
    "    \n",
    "    fig1 = None\n",
    "    N = len(true)\n",
    "    L = len(predictions)\n",
    "    num__pairs = L*(L-1)/2\n",
    "    correct = numpy.zeros([L,N])\n",
    "    correct_per = []\n",
    "    for i in xrange(L):\n",
    "        correct[i, :] =  numpy.core.defchararray.equal(predictions[i], true)\n",
    "        correct_per.append(numpy.sum(correct[i,:])/float(N))\n",
    "    #print correct.shape\n",
    "    #print \n",
    "    df = pandas.DataFrame(correct.T, columns = names)\n",
    "    #print df.head(20)\n",
    "    # Classifier Performance\n",
    "    print 'Base Accuracies'\n",
    "    acc = []\n",
    "    acc_s = ''\n",
    "    for name in names:\n",
    "        acc.append(df.sum(axis=0)[name]*100/float(N))\n",
    "        acc_s += '%s : %0.2f  ||  ' % (name, acc[-1])\n",
    "    acc_s = acc_s[:-4]\n",
    "    print acc_s\n",
    "    \n",
    "    if print_flag:\n",
    "        fig, ax = plt.subplots()\n",
    "        s = plt.bar([i for i in xrange(L)], acc, align='center', alpha=0.4)\n",
    "        plt.xticks([i for i in xrange(L)], names)\n",
    "        plt.ylabel('% Accuracy')\n",
    "        plt.title('Base Classifiers')\n",
    "        autolabel(s.patches, ax)\n",
    "        plt.show()\n",
    "\n",
    "    # Classifier versus the others\n",
    "    count_others = numpy.zeros([L,L])\n",
    "    for i, name1 in enumerate(names):\n",
    "        tmp = df[df.sum(axis=1)==1]\n",
    "        if not(tmp[tmp[name1]>0].empty):\n",
    "            if tmp[tmp[name1]>0].shape[0] == 0:\n",
    "                count_others[0, i] = 1*100/float(N)\n",
    "            else:\n",
    "                count_others[0, i] = tmp[tmp[name1]>0].shape[0]*100/float(N)\n",
    "    pairs_titles = [[] for i in xrange(0,L)]\n",
    "    pairs_titles[0] = names\n",
    "    #print pairs_titles\n",
    "    for i in xrange(0, L):\n",
    "        name1 = names[i]\n",
    "        #print name1\n",
    "        cyclic_names = [name1]+names[names.index(name1)+1:]+names[:names.index(name1)]\n",
    "        tmp = df[df.sum(axis=1)==2]\n",
    "        tmp2 = tmp[tmp[name1]>0]\n",
    "        #tmp2 = df[df[name1]>0]\n",
    "        #N_class = df[df[name1]>0].shape[0]\n",
    "        N_class = tmp2.shape[0]\n",
    "        #print tmp2\n",
    "        #print df[df[name1]>0]\n",
    "        for j in xrange(1, L):\n",
    "            name2 = cyclic_names[j]\n",
    "            #print name2            \n",
    "            if not(tmp2[tmp2[name2]>0].empty):\n",
    "                if tmp2[tmp2[name2]>0].shape[0] == 0:\n",
    "                     count_others[j, i] = 1*100/float(N_class)\n",
    "                else:\n",
    "                    count_others[j, i] = tmp2[tmp2[name2]>0].shape[0]*100/float(N_class)\n",
    "            #if name1 == 'soac':\n",
    "                #print name2\n",
    "                #print tmp2[tmp2[name2]>0]\n",
    "            pairs_titles[j].append(name1+ '-'+name2)\n",
    "        #print count_others[:,i]\n",
    "    #print count_others\n",
    "    count_pd = pandas.DataFrame(count_others.T, index=names, columns=['Only this Model']+ [' %d-model aggree' % i for i in xrange(1,L)])\n",
    "    print 'Models Correct Aggrement Percentages'\n",
    "    print count_pd.astype('float').to_string(float_format= lambda x: '%0.2f'%(x))\n",
    "    if print_flag:\n",
    "        top_labels = ['Only this Model']+ [' %d-model aggree' % i for i in xrange(1,L)]\n",
    "\n",
    "        colors = ['rgba(38, 24, 74, 0.8)', 'rgba(71, 58, 131, 0.8)',\n",
    "                  'rgba(122, 120, 168, 0.8)', 'rgba(164, 163, 204, 0.85)',\n",
    "                  'rgba(190, 192, 213, 1)']\n",
    "\n",
    "        x_data = count_others.T.tolist()[::-1]\n",
    "\n",
    "        #print x_data\n",
    "        y_data = ['The course was effectively<br>organized',\n",
    "                  'The course developed my<br>abilities and skills ' +\n",
    "                  'for<br>the subject', 'The course developed ' +\n",
    "                  'my<br>ability to think critically about<br>the subject',\n",
    "                  'I would recommend this<br>course to a friend']\n",
    "\n",
    "        y_data = names[::-1]\n",
    "        #print y_data\n",
    "        #print top_labels\n",
    "\n",
    "        traces = []\n",
    "\n",
    "        for i in range(0, len(x_data[0])):\n",
    "            for xd, yd in zip(x_data, y_data):\n",
    "                traces.append(go.Bar(\n",
    "                    x=xd[i],\n",
    "                    y=yd,\n",
    "                    orientation='h',\n",
    "                    marker=dict(\n",
    "                        color=colors[i],\n",
    "                        line=dict(\n",
    "                                color='rgb(248, 248, 249)',\n",
    "                                width=1)\n",
    "                    )\n",
    "                ))\n",
    "\n",
    "        layout = go.Layout(\n",
    "            xaxis=dict(\n",
    "                showgrid=False,\n",
    "                showline=False,\n",
    "                showticklabels=False,\n",
    "                zeroline=False,\n",
    "                domain=[0.15, 1]\n",
    "            ),\n",
    "            yaxis=dict(\n",
    "                showgrid=False,\n",
    "                showline=False,\n",
    "                showticklabels=False,\n",
    "                zeroline=False,\n",
    "            ),\n",
    "            barmode='stack',\n",
    "            paper_bgcolor='rgb(248, 248, 255)',\n",
    "            plot_bgcolor='rgb(248, 248, 255)',\n",
    "            margin=dict(\n",
    "                l=120,\n",
    "                r=10,\n",
    "                t=140,\n",
    "                b=80\n",
    "            ),\n",
    "            showlegend=False,\n",
    "        )\n",
    "\n",
    "        annotations = []\n",
    "\n",
    "        for yd, xd in zip(y_data, x_data):\n",
    "            # labeling the y-axis\n",
    "            annotations.append(dict(xref='paper', yref='y',\n",
    "                                    x=0.14, y=yd,\n",
    "                                    xanchor='right',\n",
    "                                    text=str(yd),\n",
    "                                    font=dict(family='Arial', size=14,\n",
    "                                              color='rgb(67, 67, 67)'),\n",
    "                                    showarrow=False, align='right'))\n",
    "            # labeling the first percentage of each bar (x_axis)\n",
    "            annotations.append(dict(xref='x', yref='y',\n",
    "                                    x=xd[0] / 2, y=yd,\n",
    "                                    text='%0.2f'%xd[0],\n",
    "                                    font=dict(family='Arial', size=14,\n",
    "                                              color='rgb(248, 248, 255)'),\n",
    "                                    showarrow=False))\n",
    "            # labeling the first Likert scale (on the top)\n",
    "            if yd == y_data[-1]:\n",
    "                annotations.append(dict(xref='x', yref='paper',\n",
    "                                        x=xd[0] / 2, y=1.1,\n",
    "                                        text=top_labels[0],\n",
    "                                        font=dict(family='Arial', size=14,\n",
    "                                                  color='rgb(67, 67, 67)'),\n",
    "                                        showarrow=False))\n",
    "            space = xd[0]\n",
    "            for i in range(1, len(xd)):\n",
    "                    # labeling the rest of percentages for each bar (x_axis)\n",
    "                    annotations.append(dict(xref='x', yref='y',\n",
    "                                            x=space + (xd[i]/2), y=yd, \n",
    "                                            text='%0.2f ' % xd[i],\n",
    "                                            font=dict(family='Arial', size=14,\n",
    "                                                      color='rgb(248, 248, 255)'),\n",
    "                                            showarrow=False))\n",
    "                    # labeling the Likert scale\n",
    "                    if yd == y_data[-1]:\n",
    "                        annotations.append(dict(xref='x', yref='paper',\n",
    "                                                x=space + (xd[i]/2), y=1.1,\n",
    "                                                text=top_labels[i],\n",
    "                                                font=dict(family='Arial', size=14,\n",
    "                                                          color='rgb(67, 67, 67)'),\n",
    "                                                showarrow=False))\n",
    "                    space += xd[i]\n",
    "\n",
    "        layout['annotations'] = annotations\n",
    "\n",
    "        fig1 = go.Figure(data=traces, layout=layout)\n",
    "        #py.iplot(fig,  filename='Results.png')\n",
    "        #py.image.save_as(fig, filename='Results.png')\n",
    "        #from IPython.display import Image\n",
    "        #Image('Results.png')\n",
    "        #py.image.ishow(fig)\n",
    "\n",
    "    # Error Distributions\n",
    "    all_cor = numpy.sum(df[df.apply(lambda x: min(x) == max(x), 1)]['3grams']==1)\n",
    "    all_wro = numpy.sum(df[df.apply(lambda x: min(x) == max(x), 1)]['3grams']==0)\n",
    "    disag = N - all_cor - all_wro\n",
    "    print 'Predictions Distributions'\n",
    "    print 'All correct : %0.2f  || Some correct : %0.2f || All wrong: %0.2f ' % \\\n",
    "                      (100*all_cor/float(N), 100*disag/float(N), 100*all_wro/float(N) )\n",
    "    \n",
    "    if print_flag:\n",
    "\n",
    "        fig, ax = plt.subplots()\n",
    "        s = plt.bar([1,2,3], [100*all_cor/float(N), 100*disag/float(N), 100*all_wro/float(N)], align='center', alpha=0.4)\n",
    "        plt.xticks([1,2,3], ['All correct', 'Some correct', 'All wrong'])\n",
    "        plt.ylabel('% Percentage of test dataset')\n",
    "        plt.title('Ensemble Decisions')\n",
    "        autolabel(s.patches, ax)\n",
    "        plt.show()\n",
    "    \n",
    "    # Wrong Instances\n",
    "    df_not_correct = df[df.sum(axis=1)!=L]\n",
    "    N_wrong = df_not_correct.shape[0]\n",
    "    counts = [all_wro*100/float(N)]\n",
    "    for i in xrange(1,L):\n",
    "        if not(df_not_correct[df_not_correct.sum(axis=1)==i].empty):\n",
    "            if df_not_correct[df_not_correct.sum(axis=1)==i].shape[0] == 0:\n",
    "                counts.append(1*100/flaot(N))\n",
    "            else:\n",
    "                counts.append(df_not_correct[df_not_correct.sum(axis=1)==i].shape[0]*100/float(N))\n",
    "    non_corr_s = '%s : %0.2f  ||  ' % ('None Correct', counts[0])\n",
    "    for i in xrange(1,L):\n",
    "        non_corr_s +='%d correct : %0.2f  ||  ' % (i, counts[i])\n",
    "    print 'Not all Correct Instances Distributions'\n",
    "    print non_corr_s[:-4]\n",
    "    if print_flag:\n",
    "        fig, ax = plt.subplots()\n",
    "        s = plt.bar([i for i in xrange(1,L+1)], counts, align='center', alpha=0.4)\n",
    "        plt.xticks([i for i in xrange(1,L+1)], ['None correct']+['%d correct'% i for i in xrange(1,L)])\n",
    "        plt.ylabel('% Percentage of not all correct instances')\n",
    "        plt.title('Distribution of not all correct instances')\n",
    "        autolabel(s.patches, ax)\n",
    "        plt.show()\n",
    "    \n",
    "    \n",
    "\n",
    "    return df, fig1\n",
    "\n",
    "    \n",
    "\n",
    "\n",
    "a, fig1 = base_level(predictions[:3], print_names[:3], predictions[-1])\n",
    "if fig1:\n",
    "    py.iplot(fig1)\n",
    "#a\n",
    "#df_not_correct = a[a.sum(axis=1)!=3]\n",
    "#df_not_correct[df_not_correct.sum(axis=1)==0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 789,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~Bogas/957.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 789,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "py.iplot(fig1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "object of type 'itertools.combinations' has no len()",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-39-5b0a84c4ca09>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0ml\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msample\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m500000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m51100\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mll\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mitertools\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcombinations\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m \u001b[1;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mll\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: object of type 'itertools.combinations' has no len()"
     ]
    }
   ],
   "source": [
    "import random, itertools, time\n",
    "\n",
    "time_start = time.time()\n",
    "\n",
    "l = random.sample(range(0,500000), 51100)\n",
    "print \"Created %d list of unique nodes in %0.2f secs.\" % (len(l), time.time()-time_start)\n",
    "ll = []\n",
    "for comb in itertools.combinations(l, 2):\n",
    "    ll.append(comb)\n",
    "print len(ll)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 740,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lsi', '3grams']"
      ]
     },
     "execution_count": 740,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(a.columns).index('soac')\n",
    "list(a.columns)[list(a.columns).index('soac')+1:] + list(a.columns)[:list(a.columns).index('soac')] \n",
    "#k = a[a.sum(axis=1)==1]\n",
    "#k[k[k['lsi']>0]['soac']>0].shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 711,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[31.818181818181817, 4.545454545454546, 0.0], [7.575757575757576, 0.0, 31.818181818181817], [0.0, 31.818181818181817, 4.545454545454546]]\n",
      "['lsi', 'soac', '3grams']\n",
      "['Model Alone', 'Agreement with 1 other model', 'Agreement with 2 other model']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~Bogas/945.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 711,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "top_labels = ['Model Alone']+ ['Agreement with %d other model' % i for i in xrange(1,3)]\n",
    "\n",
    "colors = ['rgba(38, 24, 74, 0.8)', 'rgba(71, 58, 131, 0.8)',\n",
    "          'rgba(122, 120, 168, 0.8)', 'rgba(164, 163, 204, 0.85)',\n",
    "          'rgba(190, 192, 213, 1)']\n",
    "\n",
    "x_data = [[21, 30, 21, 16, 12],\n",
    "          [24, 31, 19, 15, 11],\n",
    "          [27, 26, 23, 11, 13],\n",
    "          [29, 24, 15, 18, 14]]\n",
    "x_data = cc.T.tolist()[::-1]\n",
    "\n",
    "print x_data\n",
    "y_data = ['The course was effectively<br>organized',\n",
    "          'The course developed my<br>abilities and skills ' +\n",
    "          'for<br>the subject', 'The course developed ' +\n",
    "          'my<br>ability to think critically about<br>the subject',\n",
    "          'I would recommend this<br>course to a friend']\n",
    "\n",
    "y_data = print_names[:3][::-1]\n",
    "print y_data\n",
    "print top_labels\n",
    "traces = []\n",
    "\n",
    "for i in range(0, len(x_data[0])):\n",
    "    for xd, yd in zip(x_data, y_data):\n",
    "        traces.append(go.Bar(\n",
    "            x=xd[i],\n",
    "            y=yd,\n",
    "            orientation='h',\n",
    "            marker=dict(\n",
    "                color=colors[i],\n",
    "                line=dict(\n",
    "                        color='rgb(248, 248, 249)',\n",
    "                        width=1)\n",
    "            )\n",
    "        ))\n",
    "\n",
    "layout = go.Layout(\n",
    "    xaxis=dict(\n",
    "        showgrid=False,\n",
    "        showline=False,\n",
    "        showticklabels=False,\n",
    "        zeroline=False,\n",
    "        domain=[0.15, 1]\n",
    "    ),\n",
    "    yaxis=dict(\n",
    "        showgrid=False,\n",
    "        showline=False,\n",
    "        showticklabels=False,\n",
    "        zeroline=False,\n",
    "    ),\n",
    "    barmode='stack',\n",
    "    paper_bgcolor='rgb(248, 248, 255)',\n",
    "    plot_bgcolor='rgb(248, 248, 255)',\n",
    "    margin=dict(\n",
    "        l=120,\n",
    "        r=10,\n",
    "        t=140,\n",
    "        b=80\n",
    "    ),\n",
    "    showlegend=False,\n",
    ")\n",
    "\n",
    "annotations = []\n",
    "\n",
    "for yd, xd in zip(y_data, x_data):\n",
    "    # labeling the y-axis\n",
    "    annotations.append(dict(xref='paper', yref='y',\n",
    "                            x=0.14, y=yd,\n",
    "                            xanchor='right',\n",
    "                            text=str(yd),\n",
    "                            font=dict(family='Arial', size=14,\n",
    "                                      color='rgb(67, 67, 67)'),\n",
    "                            showarrow=False, align='right'))\n",
    "    # labeling the first percentage of each bar (x_axis)\n",
    "    annotations.append(dict(xref='x', yref='y',\n",
    "                            x=xd[0] / 2, y=yd,\n",
    "                            text='%0.2f'%xd[0],\n",
    "                            font=dict(family='Arial', size=14,\n",
    "                                      color='rgb(248, 248, 255)'),\n",
    "                            showarrow=False))\n",
    "    # labeling the first Likert scale (on the top)\n",
    "    if yd == y_data[-1]:\n",
    "        annotations.append(dict(xref='x', yref='paper',\n",
    "                                x=xd[0] / 2, y=1.1,\n",
    "                                text=top_labels[0],\n",
    "                                font=dict(family='Arial', size=14,\n",
    "                                          color='rgb(67, 67, 67)'),\n",
    "                                showarrow=False))\n",
    "    space = xd[0]\n",
    "    for i in range(1, len(xd)):\n",
    "            # labeling the rest of percentages for each bar (x_axis)\n",
    "            annotations.append(dict(xref='x', yref='y',\n",
    "                                    x=space + (xd[i]/2), y=yd, \n",
    "                                    text='%0.2f ' % xd[i],\n",
    "                                    font=dict(family='Arial', size=14,\n",
    "                                              color='rgb(248, 248, 255)'),\n",
    "                                    showarrow=False))\n",
    "            # labeling the Likert scale\n",
    "            if yd == y_data[-1]:\n",
    "                annotations.append(dict(xref='x', yref='paper',\n",
    "                                        x=space + (xd[i]/2), y=1.1,\n",
    "                                        text=top_labels[i],\n",
    "                                        font=dict(family='Arial', size=14,\n",
    "                                                  color='rgb(67, 67, 67)'),\n",
    "                                        showarrow=False))\n",
    "            space += xd[i]\n",
    "\n",
    "layout['annotations'] = annotations\n",
    "\n",
    "fig = go.Figure(data=traces, layout=layout)\n",
    "py.iplot(fig, filename='bar-colorscale')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------------------------------\n",
      "Diversity Tests Report\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Details\n",
      "===============================================================\n",
      "Correlation: For +-1 perfect aggrement/disagreement\n",
      "Q-statistic: Q=0  => Independent. For q>0 predictors find the the same results\n",
      "Cohen's k: k->0  => High Disagreement => High Diversity\n",
      "Kohovi-Wolpert Variance -> Inf => High Diversity\n",
      "Conditional Accuracy Table: Conditional Probability that the row system predicts correctly, given\n",
      "                            that the column system also predicts correctly\n",
      "===============================================================\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Results\n",
      "---------------------------------------------------------------\n",
      "\n",
      "#####  Kohovi-Wolpert Variance:  0.189  #####\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "#### Pairwise Average Metrics: #####\n",
      "Avg. Cor: 0.000\n",
      "Avg. Q-statistic: 0.106\n",
      "Avg. Cohen's k: 0.004\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "###Conditional Accuracy Table###\n",
      "         3grams  soac  lsi  voting  votingh  space  meta\n",
      "3grams     1.00  0.86 0.36    0.97     1.00   0.89  0.00\n",
      "soac       0.91  1.00 0.27    0.94     0.91   0.97  0.15\n",
      "lsi        0.36  0.26 1.00    0.33     0.36   0.29  0.64\n",
      "voting     0.97  0.89 0.33    1.00     0.97   0.89  0.03\n",
      "votingh    1.00  0.86 0.36    0.97     1.00   0.89  0.00\n",
      "space      0.94  0.97 0.30    0.94     0.94   1.00  0.12\n",
      "meta       0.00  0.14 0.64    0.03     0.00   0.11  1.00\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n"
     ]
    }
   ],
   "source": [
    "pred_a = ['male', 'female', 'male']\n",
    "pred_b = ['female', 'female', 'female']\n",
    "pred_c = ['male','male','male']\n",
    "names = ['a', 'b', 'c']\n",
    "truth = ['female', 'male', 'female']\n",
    "predictions_test= [pred_a,pred_b,pred_c]\n",
    "test_class = DiversityTests(predictions[:-1], print_names[:-1], predictions[-1])\n",
    "test_class.print_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "323 55 378 436\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'pipe' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-2e33785dfd92>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstratify\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m \u001b[0meclf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVotingClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"0\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\"2\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvoting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'soft'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m \u001b[0meclf2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mVotingClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mestimators\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"0\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m\"2\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpipe2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvoting\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'hard'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m \u001b[0mmodels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mpipe\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpipe1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpipe2\u001b[0m \u001b[1;33m,\u001b[0m\u001b[0meclf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0meclf2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'pipe' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from pan.features import Metaclassifier\n",
    "import time\n",
    "\n",
    "num_folds = 4\n",
    "split = 0.25\n",
    "#X, y = dataset.get_data('age')\n",
    "#X, y = dataset.get_data('gender')\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y)\n",
    "for i, x in enumerate(X_train):\n",
    "    if len(x)==0:\n",
    "        X_train.remove(x)\n",
    "        y_train.remove(y_train[i])\n",
    "for i, x in enumerate(X_cv):\n",
    "    if len(x)==0:\n",
    "        X_cv.remove(x)\n",
    "        y_cv.remove(y_cv[i])\n",
    "X_meta, X_cv, y_meta, y_cv = train_test_split(X_cv, y_cv, test_size=0.5, stratify=y_cv)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='soft')\n",
    "eclf2 = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='hard')\n",
    "models = [pipe,pipe1,pipe2 ,eclf, eclf2]\n",
    "model_names = ['3grams', 'soac', 'lsi', 'voting', 'votingh', 'space', 'Meta']\n",
    "meta_models = ['voting', 'votingh', 'space', 'Meta']\n",
    "#models = [pipe1, pipe]\n",
    "#model_names = ['soac', '3grams']\n",
    "trained_models = []\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "time_start = time.time()\n",
    "for i, _ in enumerate(model_names):\n",
    "    print \"Model: %s \" % model_names[i]\n",
    "    if model_names[i] == 'voting' or model_names[i]=='votingh':\n",
    "        params = {}\n",
    "    if model_names[i] != 'Meta' and model_names[i] != 'space':\n",
    "        grid_search = GridSearchCV(estimator=models[i], param_grid=params, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "        grid_search.fit(X_train,y_train)\n",
    "        print(grid_search.best_score_)\n",
    "        print(grid_search.best_params_)\n",
    "        trained_models.append(grid_search.best_estimator_)\n",
    "    if model_names[i] == 'Meta':\n",
    "            model_dic = {}\n",
    "            for i, model in enumerate(trained_models):\n",
    "                if not(model_names[i] in meta_models): \n",
    "                    model_dic[model_names[i]] = model\n",
    "            Meta = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "            Meta.fit(X_meta, y_meta)\n",
    "            trained_models.append(Meta)\n",
    "    if model_names[i] == 'space':\n",
    "            models_for_space = {}\n",
    "            cv_scores = []\n",
    "            for name, model in zip(model_names, trained_models):\n",
    "                if not(name in meta_models):\n",
    "                    models_for_space[name] = model\n",
    "                    cv_scores.append(model.score(X_meta, y_meta))\n",
    "            space = SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "            space.fit(X_meta, y_meta)\n",
    "            trained_models.append(space)\n",
    "    print \"Trained in %0.3f seconds\" % (time.time()- time_start)\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['3grams', 'soac', 'lsi', 'votingh']\n",
      "[Pipeline(steps=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
      "        ngram_range=[2, 2], norm=u'l2', preprocessor=None, smooth_idf=Tru...,\n",
      "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))]), Pipeline(steps=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn')), ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))]), Pipeline(steps=[('LSI', LSI_Model(num_topics=50)), ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))]), VotingClassifier(estimators=[('0', Pipeline(steps=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
      "        ngram_range=[2, 2], norm=u'l2', prepro...  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))]))],\n",
      "         voting='hard', weights=None), Metaclassifier(C=1.0,\n",
      "        models={'lsi': Pipeline(steps=[('LSI', LSI_Model(num_topics=50)), ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))]),...  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])},\n",
      "        weights='balanced'), Pipeline(steps=[('3grams', TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
      "        dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
      "        lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
      "        ngram_range=[2, 2], norm=u'l2', preprocessor=None, smooth_idf=Tru...',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])]\n"
     ]
    }
   ],
   "source": [
    "print model_names\n",
    "print trained_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First search took: 23.757 seconds\n",
      "Accuracy obtained in CV-data: 53.704\n",
      "[0.065, 0.935, 0.98, 6]\n",
      "Fit took: 39.553 seconds\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SubSpaceEnsemble4_2(N_rand=10,\n",
       "          cv_scores=[0.33333333333333331, 0.35185185185185186, 0.42592592592592593],\n",
       "          k=6,\n",
       "          models={'lsi': Pipeline(steps=[('LSI', LSI_Model(num_topics=100)), ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])...\n",
       "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])},\n",
       "          rand_split=0.6, weights=[0.065, 0.935, 0.98, 6])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta_models = ['voting', 'votingh', 'space', 'Meta']\n",
    "model_dic = {}\n",
    "for i, model in enumerate(trained_models):\n",
    "    if not(model_names[i] in meta_models): \n",
    "        model_dic[model_names[i]] = model\n",
    "Meta = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "Meta.fit(X_meta, y_meta)\n",
    "trained_models.append(Meta)\n",
    "\n",
    "models_for_space = {}\n",
    "cv_scores = []\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if not(name in meta_models):\n",
    "        models_for_space[name] = model\n",
    "        cv_scores.append(model.score(X_meta, y_meta))\n",
    "space = SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "space.fit(X_meta, y_meta)\n",
    "trained_models.append(space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['25-34', '35-49', '50-64', '18-24', '65-xx']\n",
      "['lsi', 'soac', '3grams']\n",
      "lsi\n",
      "soac\n",
      "3grams\n",
      "(65, 15) (65,)\n",
      "fit true\n",
      "First search took: 24.712 seconds\n",
      "Accuracy obtained in CV-data: 46.154\n",
      "[0.657, 0.343, 0.429, 6]\n",
      "Fit took: 39.954 seconds\n",
      "Trained in 2078.559 seconds\n"
     ]
    }
   ],
   "source": [
    "from pan.features import Metaclassifier\n",
    "model_dic = {}\n",
    "for i, model in enumerate(trained_models):\n",
    "    if model_names[i] != 'voting' and model_names[i] !='votingh': \n",
    "        model_dic[model_names[i]] = model\n",
    "Meta = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "Meta.fit(X_meta, y_meta)\n",
    "trained_models.append(Meta)\n",
    "models_for_space = {}\n",
    "cv_scores = []\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if name!='voting' and name!='votingh' and name!='Meta' and name!='space':\n",
    "        models_for_space[name] = model\n",
    "        cv_scores.append(model.score(X_meta, y_meta))\n",
    "space = SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "space.fit(X_meta, y_meta)\n",
    "trained_models.append(space)\n",
    "print \"Trained in %0.3f seconds\" % (time.time()- time_start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "trained_models.append(space)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 3grams\n",
      "Accuracy : 0.527272727273\n",
      "Confusion matrix :\n",
      " [[ 0  3  0  1  0]\n",
      " [ 0  8  9  0  0]\n",
      " [ 0  3 20  0  0]\n",
      " [ 0  2  7  1  0]\n",
      " [ 0  1  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.47      0.47      0.47        17\n",
      "      35-49       0.56      0.87      0.68        23\n",
      "      50-64       0.50      0.10      0.17        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.47      0.53      0.46        55\n",
      "\n",
      "Model: soac\n",
      "Accuracy : 0.327272727273\n",
      "Confusion matrix :\n",
      " [[ 0  2  1  1  0]\n",
      " [ 0 10  5  2  0]\n",
      " [ 0 11  6  6  0]\n",
      " [ 0  4  4  2  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.37      0.59      0.45        17\n",
      "      35-49       0.35      0.26      0.30        23\n",
      "      50-64       0.18      0.20      0.19        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.30      0.33      0.30        55\n",
      "\n",
      "Model: lsi\n",
      "Accuracy : 0.418181818182\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  0 17  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.42      1.00      0.59        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.25        55\n",
      "\n",
      "Model: votingh\n",
      "Accuracy : 0.254545454545\n",
      "Confusion matrix :\n",
      " [[ 2  2  0  0  0]\n",
      " [ 7  6  4  0  0]\n",
      " [14  3  6  0  0]\n",
      " [ 7  1  2  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.06      0.50      0.11         4\n",
      "      25-34       0.50      0.35      0.41        17\n",
      "      35-49       0.50      0.26      0.34        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.37      0.25      0.28        55\n",
      "\n",
      "Model: meta\n",
      "Accuracy : 0.163636363636\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 4  0 12  0  1]\n",
      " [ 8  0  9  0  6]\n",
      " [ 3  0  5  0  2]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.29      0.39      0.33        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.12      0.16      0.14        55\n",
      "\n",
      "Model: space\n",
      "Accuracy : 0.327272727273\n",
      "Confusion matrix :\n",
      " [[ 0  2  2  0  0]\n",
      " [ 0  5 12  0  0]\n",
      " [ 0 10 13  0  0]\n",
      " [ 0  1  9  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.28      0.29      0.29        17\n",
      "      35-49       0.35      0.57      0.43        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.23      0.33      0.27        55\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.py:1074: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'print_overlaps' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-40c89fa1688a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     23\u001b[0m \u001b[1;31m#model_names += ['True']\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;31m#print len([(i, j) for i,j in zip(predictions[0], predictions[1]) if i==j])/float(len(predictions[0]))\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprint_overlaps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodel_names2\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'true'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'print_overlaps' is not defined"
     ]
    }
   ],
   "source": [
    "import copy\n",
    "from sklearn.metrics import classification_report\n",
    "predictions = []\n",
    "model_names2 = copy.deepcopy(model_names)\n",
    "model_names2 +=  ['meta', 'space']\n",
    "for i, model in enumerate(trained_models):\n",
    "    print \"Model: \" + str(model_names2[i])\n",
    "    predict = model.predict(X_cv)\n",
    "    predictions.append(predict)\n",
    "    acc = accuracy_score(y_cv, predict)\n",
    "    conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "    rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))\n",
    "    \n",
    "#pred2 = copy.deepcopy(predictions)\n",
    "#pred2.append(y_space)\n",
    "#pred2.append(y_cv)\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting']\n",
    "#model_names += ['space']\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting', 'votingh']\n",
    "#model_names += ['True']\n",
    "#print len([(i, j) for i,j in zip(predictions[0], predictions[1]) if i==j])/float(len(predictions[0]))\n",
    "_ = print_overlaps(predictions+[y_cv], model_names2 + ['true'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class encoder(obj):\n",
    "    \n",
    "    def __init__(self):\n",
    "        print 'Ok'\n",
    "        \n",
    "    def fit(X, y=None):\n",
    "        \n",
    "        self.fit_transform(X)\n",
    "        return self\n",
    "    \n",
    "    def fit_transform(X, y=None):\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['male', 'female', 'male'], ['female', 'male', 'male'], ['male', 'female', 'female']]\n",
      "True\n",
      "[[1]\n",
      " [1]\n",
      " [0]]\n",
      "[[1 0 1]\n",
      " [0 1 1]\n",
      " [1 0 0]]\n",
      "True\n",
      "[[ 0.  1.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "[[ 0.  1.]\n",
      " [ 1.  0.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]\n",
      " [ 0.  1.]\n",
      " [ 0.  1.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]\n",
      " [ 1.  0.]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Weighter(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Simple Majoirty Voter(also supports weights)\"\"\"\n",
    "    \n",
    "    def __init__(self, models, weights=None):\n",
    "        \n",
    "        from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "        \n",
    "        if (not models):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.weights = None\n",
    "            self.lab = LabelEncoder()\n",
    "            self.ohe = OneHotEncoder()\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "                \n",
    "    def fit(X, y, weights=None):\n",
    "        \n",
    "        if y is None:\n",
    "            raise ValueError('We need y labels to supervise-fit!')\n",
    "        if self.weights:\n",
    "            print \"Not so much Vox Populi, Vox Dei, huh?\"\n",
    "        else:\n",
    "            X = self.lab.fit_transform(X).reshape(-1, 1)\n",
    "            X = self.ohe.fit_transform(X)\n",
    "            print X\n",
    "        return self\n",
    "\n",
    "pred1 = ['male', 'female', 'male']\n",
    "pred2 = ['female', 'male', 'female']\n",
    "pred3 = ['male', 'male', 'female']\n",
    "pred_t = []\n",
    "for i in range(len(pred1)):\n",
    "    pred_t.append([pred1[i], pred2[i], pred3[i]])\n",
    "print pred_t\n",
    "\n",
    "truth = pred3\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "\n",
    "truth_lab = lab.fit_transform(truth).reshape(-1, 1)\n",
    "print \"True\"\n",
    "print truth_lab\n",
    "print lab.transform(pred_t)\n",
    "X_new = ohe.fit_transform(numpy.array(truth_lab).reshape(-1,1)).todense()\n",
    "print 'True'\n",
    "print X_new\n",
    "print ohe.transform(lab.transform(pred_t).reshape(-1, 1)).todense()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class SimpleVoter(BaseEstimator, TransformerMixin)\n",
    "    \"\"\"Simple Majoirty Voter(also supports weights)\"\"\"\n",
    "    \n",
    "    def __init__(self, models, weights=None):\n",
    "        \n",
    "        if (not models):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.weights = None\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "                \n",
    "    def fit(X, y, weights=None):\n",
    "        \n",
    "        if y is None:\n",
    "            raise ValueError('We need y labels to supervise-fit!')\n",
    "        if self.weights:\n",
    "            print \"Not so much Vox Populi, Vox Dei, huh?\"\n",
    "            \n",
    "        else:\n",
    "            print \"Equality for all! No fitting needed\"\n",
    "            \n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Metaclassifier based on the document-term representation accuracy of the\n",
    "        base classifiers. Currently looking only at the nearest neighbor for\n",
    "        each instance and selecting the model that correctly classifies it.\"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.doc_terms = None\n",
    "            self.experts = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X_cv).toarray()\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "            count = 0\n",
    "            for i, y in enumerate(y_true):\n",
    "                possible_experts = []\n",
    "                for j, pred in enumerate(predictions):\n",
    "                    if pred[i] == y:\n",
    "                        possible_experts.append(j)\n",
    "                if possible_experts:\n",
    "                    possible_scores = [self.cv_scores[poss] for poss in possible_experts]\n",
    "                    self.experts.append(possible_experts[possible_scores.index(max(possible_scores))])\n",
    "                    count += 1\n",
    "                else:\n",
    "                    self.experts.append(self.cv_scores.index(max(self.cv_scores)))\n",
    "            print \"Chosen through expert: %0.2f\" % (100*count/float(len(y_true))) \n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            best_model_ind = self.find_sim_projection(X_transformed[i,:])\n",
    "            #print best_model_ind\n",
    "            #print self.models[self.ind2names[best_model_ind]].predict([X[i]])[0]\n",
    "            y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def find_sim_projection(self, x_sample):\n",
    "\n",
    "        from sklearn.metrics.pairwise import cosine_similarity\n",
    "        \n",
    "        cos = []\n",
    "        j = None\n",
    "        min_s = -10000\n",
    "        for i in range(0, self.doc_terms.shape[0]):\n",
    "            #print x_sample.reshape(1,-1).shape\n",
    "            #print self.doc_terms[i,:].reshape(1,-1).shape\n",
    "            temp = cosine_similarity(x_sample.reshape(1,-1), self.doc_terms[i,:].reshape(1,-1))[0][0]\n",
    "            if min_s < 0 or  temp > min_s:\n",
    "                min_s = temp\n",
    "                j = i\n",
    "        return self.experts[j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import time\n",
    "\n",
    "class SubSpaceEnsemble3(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Utilizing the neighborhood in all representations and also ground truth model.\n",
    "        Implementing a weighted voting scheme.\"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores, k=3, weights= [6,3,2,0.7]):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.weights = weights\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.representations = []\n",
    "            self.meta = None\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.doc_terms = None\n",
    "            self.tree = None\n",
    "            self.experts = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X_cv).toarray()\n",
    "            self.tree = BallTree(self.doc_terms, leaf_size=20)\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            count = 0\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        \n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), self.k)  \n",
    "            #print 'Sample ' + y_real[i]\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            y_pred.append(self.expert_decision(neigbors_indexes[0],  X[i]))\n",
    "            \n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, neigbors_indexes, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "        #print 'True'\n",
    "        #print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = self.weights[1]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "            if acc[-1] >self.weights[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(self.weights[2]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(self.weights[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "                #print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "                #print 'Predictions'\n",
    "                #print model_pred\n",
    "                #print 'Representations'\n",
    "                #print model_neig_pred\n",
    "                #print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "        total_pred.extend([n for j in xrange(int(weights['true'])) for n in neigbors_true])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "        #print data\n",
    "        best_model_ind = acc.index(max(acc))\n",
    "        #print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "        #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a = models['lda']\n",
    "a.steps[0][1].transform(X[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble2(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Best model base on the prediction of the nearest, according to each model, neighbor \"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores, k=10):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.trees = []\n",
    "            self.representations = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "                self.trees.append(BallTree(self.representations[-1], leaf_size=20))\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            return self\n",
    "\n",
    "    def predict(self, X, y_real):\n",
    "        \n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        y_pred = []\n",
    "        for i, x in enumerate(X):\n",
    "            print 'True: ' + y_real[i]\n",
    "            y_pred.append(self.expert_decision(x))  \n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        \n",
    "        possible_experts = []\n",
    "        sample_predictions = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = self.trees[model_i].query(temp_trans, 1)\n",
    "            #print \"Model neig\"\n",
    "            #print model_neig[0].tolist()[0]\n",
    "            if  self.predictions[model_i][model_neig[0].tolist()[0]] == self.true[model_neig[0].tolist()[0]]:\n",
    "                possible_experts.append(model_i)\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "        if possible_experts:\n",
    "            #print 'Possible experts:'\n",
    "            #print [self.ind2names[poss] for poss in possible_experts]\n",
    "            #print sample_predictions\n",
    "            possible_scores = [self.cv_scores[poss] for poss in possible_experts]\n",
    "            #print 'Selected: '\n",
    "            #print 'Place of best expert: %d ' % possible_scores.index(max(possible_scores))\n",
    "            #print 'Name:  ' + self.ind2names[possible_experts[possible_scores.index(max(possible_scores))]]\n",
    "            #print 'PRediction index: '\n",
    "            #print possible_scores.index(max(possible_scores))\n",
    "            #print 'PRediction : '\n",
    "            #print sample_predictions[possible_scores.index(max(possible_scores))]\n",
    "            return sample_predictions[possible_scores.index(max(possible_scores))]\n",
    "        else:\n",
    "            return self.models[self.ind2names[(self.cv_scores.index(max(self.cv_scores)))]].predict(x_sample)[0]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "models = {}\n",
    "cv_scores = []\n",
    "print len(y_cv), len(X_cv)\n",
    "for i, x in enumerate(X_cv):\n",
    "    if len(x)==0:\n",
    "        X_cv.remove(x)\n",
    "        y_cv.remove(y_cv[i])\n",
    "print len(y_cv), len(X_cv)\n",
    "\n",
    "print len(y_meta), len(X_meta)\n",
    "for i, x in enumerate(X_meta):\n",
    "    if len(x)==0:\n",
    "        X_meta.remove(x)\n",
    "        y_meta.remove(y_meta[i])\n",
    "print len(y_meta), len(X_meta)        \n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if name!='voting' and name!='votingh' and name!='space' and name!='meta':\n",
    "        models[name] = model\n",
    "        cv_scores.append(model.score(X_meta, y_meta))\n",
    "        \n",
    "w = [1,1,1,0.35]\n",
    "space = SubSpaceEnsemble4(models,cv_scores,k=3)\n",
    "space.fit(X_meta+X_train, y_meta+y_train)\n",
    "predict = space.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "print Counter(y_cv)\n",
    "print list(set(y))\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "\n",
    "import numpy, copy\n",
    "\n",
    "def print_overlaps(predictions, names, verbose=True):\n",
    "    N = len(names)\n",
    "    res = numpy.zeros([N,N])\n",
    "    temp = numpy.zeros([N,N])\n",
    "    for i in range(0, N):\n",
    "        for j in range(i+1, N):\n",
    "            #print i,j\n",
    "            #print N\n",
    "            #print l\n",
    "            #print len(predictions[j]), predictions[j]\n",
    "            #print len(predictions[N-1]), predictions[N-1]\n",
    "            temp[i,j] = len([m for l, m in enumerate(predictions[i]) if (m==predictions[j][l] and m==predictions[N-1][l])])/float(len(predictions[0]))\n",
    "            #print i,j\n",
    "            #predictions[i]\n",
    "            #predictions[j]\n",
    "            res[i,j] = len([(k,v) for k,v in zip(predictions[i], predictions[j]) if k==v])/float(len(predictions[0]))\n",
    "            #print res[i,j]\n",
    "            if verbose:\n",
    "                print \"%s - %s : %0.3f  overlap | ground-truth coverage: %0.3f\" % (names[i],  names[j], 100*res[i,j], 100*temp[i,j])\n",
    "    return  [res, temp]\n",
    "\n",
    "pred2 = copy.deepcopy(predictions)\n",
    "pred2.append(predict)\n",
    "\n",
    "pred2.append(y_cv)\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting']\n",
    "#model_names += ['space']\n",
    "#model_names += ['True']\n",
    "#print len([(i, j) for i,j in zip(predictions[0], predictions[1]) if i==j])/float(len(predictions[0]))\n",
    "print_overlaps(pred2, model_names+ ['space', 'true'])\n",
    "a = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#statsmodels.sandbox.stats.runs.mcnemar\n",
    "\n",
    "from statsmodels.sandbox.stats.runs import mcnemar, cochrans_q\n",
    "\n",
    "print mcnemar(predictions[2], pred2[5], exact=False, correction=True)\n",
    "print cochrans_q(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_names += ['space','meta']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24', '18-24',\n",
       "       '18-24', '18-24', '18-24', '18-24', '18-24', '18-24'], \n",
       "      dtype='|S5')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.0727272727273\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [17  0  0  0  0]\n",
      " [23  0  0  0  0]\n",
      " [10  0  0  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Accuracy : 0.290909090909\n",
      "Confusion matrix :\n",
      " [[ 2  1  0  0  1]\n",
      " [ 2  2  9  3  1]\n",
      " [ 3  5 12  1  2]\n",
      " [ 5  2  3  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Accuracy : 0.0727272727273\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [17  0  0  0  0]\n",
      " [23  0  0  0  0]\n",
      " [10  0  0  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Accuracy : 0.345454545455\n",
      "Confusion matrix :\n",
      " [[ 0  4  0  0  0]\n",
      " [ 0  4 13  0  0]\n",
      " [ 0  8 15  0  0]\n",
      " [ 0  6  4  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Accuracy : 0.0727272727273\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [17  0  0  0  0]\n",
      " [23  0  0  0  0]\n",
      " [10  0  0  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Accuracy : 0.254545454545\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [ 7  2  7  1  0]\n",
      " [10  4  8  1  0]\n",
      " [ 6  1  3  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Accuracy : 0.436363636364\n",
      "Confusion matrix :\n",
      " [[ 0  1  3  0  0]\n",
      " [ 0  4 13  0  0]\n",
      " [ 0  3 20  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Accuracy : 1.0\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [ 0 17  0  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0  0 10  0]\n",
      " [ 0  0  0  0  1]]\n"
     ]
    }
   ],
   "source": [
    "for predict1 in predictions:\n",
    "    acc = accuracy_score(y_cv, predict1)\n",
    "    conf = confusion_matrix(y_cv, predict1, labels=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = []\n",
    "for i, model in enumerate(trained_models+[space, Meta]):\n",
    "    print model_names[i]\n",
    "    predict1 = model.predict(X_cv)\n",
    "    predictions.append(predict1)\n",
    "    acc = accuracy_score(y_cv, predict1)\n",
    "    conf = confusion_matrix(y_cv, predict1, labels=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble4(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Best model base on the prediction of the k-nearest, according to each model, neighbor \"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores, k=6, weights=[0.6,0.2,0.3]):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.ind2names = {}\n",
    "            self.weights = weights\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.trees = []\n",
    "            self.representations = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random, time\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            t0 = time.time()\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "                self.trees.append(BallTree(self.representations[-1], leaf_size=20))\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            #print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        \n",
    "        import time\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i, x in enumerate(X):\n",
    "            #print 'True: ' + y_real[i]\n",
    "            y_pred.append(self.expert_decision(x)) \n",
    "        #print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        \n",
    "        possible_experts = []\n",
    "        sample_predictions = []\n",
    "        acc = []\n",
    "        possible_experts_sc = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            #print 'Model: ' + self.ind2names[model_i]\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = self.trees[model_i].query(temp_trans, self.k)\n",
    "            #print \"Model neig\"\n",
    "            #print model_neig[0].tolist()[0]\n",
    "            model_neig_pred = []\n",
    "            neigh_true = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "                neigh_true.append(self.true[model_n_i])\n",
    "            #print \"True_neighbors\"\n",
    "            #print neigh_true\n",
    "            #print \"Predicted neighbors\"\n",
    "            #print model_neig_pred\n",
    "            acc.append(accuracy_score(neigh_true, model_neig_pred, normalize=True))\n",
    "            #print 'Neig Accc: % 0.2f' % acc[-1]\n",
    "            predicted = self.models[self.ind2names[model_i]].predict([x_sample])[0]\n",
    "            proba = max(self.models[self.ind2names[model_i]].predict_proba([x_sample])[0])\n",
    "            #print 'Predicted Sample: %s with proba: %0.3f' % (predicted, 100*proba)\n",
    "            if  acc[-1] > self.weights[2]:\n",
    "                possible_experts.append(model_i)\n",
    "                possible_experts_sc.append(self.weights[1]*acc[-1]+self.weights[0]*proba)\n",
    "                sample_predictions.append(predicted)\n",
    "        if possible_experts:\n",
    "            #print 'Possible experts:'\n",
    "            #print [self.ind2names[poss] for poss in possible_experts]\n",
    "            #print sample_predictions\n",
    "            #print 'Selected: '\n",
    "            #print 'Place of best expert: %d ' % possible_scores.index(max(possible_scores))\n",
    "            #print 'Name:  ' + self.ind2names[possible_experts[possible_scores.index(max(possible_scores))]]\n",
    "            #print 'PRediction index: '\n",
    "            #print possible_scores.index(max(possible_scores))\n",
    "            #print 'PRediction : '\n",
    "            #print sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "            return sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "        else:\n",
    "            #print 'Selected2 from base model: ' + self.ind2names[(self.acc.index(max(acc)))]\n",
    "            #print self.models[self.ind2names[(self.acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "            return self.models[self.ind2names[(acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def Most_Common(lst):\n",
    "    data = Counter(lst)\n",
    "    return data.most_common(1)[0][0]\n",
    "\n",
    "class Neighbors(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Best model base on the k-nearest, according to each model, neighbor.\n",
    "        Each new sample is projected onto the representation space of each model.\n",
    "        Then the labels of the nearest neighbors, found in the train set, or the\n",
    "        prediction of the model for those neighbor or both of them, are used as\n",
    "        voted for the final label of the sample.\n",
    "        Args:\n",
    "            models: dictionary('name1':model1, 'name2':model2) containing the base models\n",
    "            k: number of neighboors to take into account\n",
    "            flag: viable choices are 'true', 'pred' and 'both. \n",
    "                  If 'true' the true labels of the neighbors are used\n",
    "                  If 'pred' the predicted labels of the neighbors are used\n",
    "                  If 'both' both are used\n",
    "            print_flag: enables printing, for dev\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, models, k=6, flag='both', print_flag='False'):\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.k = k\n",
    "            self.flag = flag\n",
    "            self.print_flag = print_flag\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.trees = []\n",
    "            self.representations = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random, time\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            t0 = time.time()\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "                self.trees.append(BallTree(self.representations[-1], leaf_size=20))\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            #print('Created Trees in : %0.3f seconds') % (time.time()-t0)\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        \n",
    "        import time\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i, x in enumerate(X):\n",
    "            if self.print_flag:\n",
    "                print 'True: ' + y_real[i]\n",
    "            y_pred.append(self.expert_decision(x)) \n",
    "        #print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        \n",
    "        possible_experts = []\n",
    "        sample_predictions = []\n",
    "        acc = []\n",
    "        possible_experts_sc = []\n",
    "        neigh_total = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            if self.print_flag:\n",
    "                print 'Model: ' + self.ind2names[model_i]\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = self.trees[model_i].query(temp_trans, self.k)\n",
    "            #print \"Model neig\"\n",
    "            #print model_neig[0].tolist()[0]\n",
    "            model_neig_pred = []\n",
    "            neigh_true = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                if self.flag == 'true':\n",
    "                    neigh_total.append(self.true[model_n_i])\n",
    "                    neigh_true.append(self.true[model_n_i])\n",
    "                elif self.flag == 'pred':\n",
    "                    neigh_total.append(self.predictions[model_i][model_n_i])\n",
    "                    model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "                elif self.flag == 'both':\n",
    "                    neigh_total.append(self.true[model_n_i])\n",
    "                    neigh_total.append(self.predictions[model_i][model_n_i])\n",
    "                    model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "                    neigh_true.append(self.true[model_n_i])\n",
    "                #neigh_true.append(self.true[model_n_i])\n",
    "            #print neigh_true\n",
    "            if self.print_flag and model_neig_pred:\n",
    "                print 'Predicted'\n",
    "                print model_neig_pred\n",
    "            if self.print_flag and neigh_true:\n",
    "                print 'True'\n",
    "                print neigh_true\n",
    "        if self.print_flag:\n",
    "            print neigh_total\n",
    "            print \"Final Prediction: %s\" % Most_Common(neigh_total)  \n",
    "        return Most_Common(neigh_total)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def param_grid_search(model, params, print_flag=True,)\n",
    "    \n",
    "    import time\n",
    "    from itertools import product\n",
    "    from sklearn.metrics import f1_score\n",
    "    \n",
    "    param_grid = list(product(*params.values()))\n",
    "    par_scores = []\n",
    "    time_start = time.time()\n",
    "    for par in param_grid:\n",
    "\n",
    "        mod = model(**par)\n",
    "        neigh.fit(X_train, y_train)\n",
    "        par_scores.append(f1_score(y_meta, neigh.predict(X_meta), average='micro'))\n",
    "        if print_flag:\n",
    "            print \"Params: {}\".format(par)\n",
    "            print \"Score: % 0.3f \" %par_scores[-1]\n",
    "            print \"Time: %0.3f\" %(time.time()- time_start)\n",
    "    ind = par_scores.index(max(par_scores))\n",
    "    print \"Best Params with F1 score: %0.3f \" % max(par_scores)\n",
    "    print param_grid[ind]\n",
    "    return param_grid[ind]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Params with F1 score: 0.491 \n",
      "(7, 'true')\n",
      "Accuracy : 0.436363636364\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  1 16  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       1.00      0.06      0.11        17\n",
      "      35-49       0.43      1.00      0.60        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.49      0.44      0.28        55\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report,f1_score\n",
    "import itertools, time\n",
    "\n",
    "neigh = Neighbors(models_for_space, k=3, flag='both', print_flag=False)\n",
    "params = {'k':[i for i in range(1,10)], 'flag':['true', 'pred', 'both']}\n",
    "param_grid = list(itertools.product(*params.values()))\n",
    "par_scores = []\n",
    "time_start = time.time()\n",
    "for par in param_grid:\n",
    "    #print \"Params: {}\".format(par)\n",
    "    neigh = Neighbors(models_for_space, k=par[0], flag=par[1], print_flag=False)\n",
    "    neigh.fit(X_train, y_train)\n",
    "    par_scores.append(f1_score(y_meta, neigh.predict(X_meta), average='micro'))\n",
    "    #print \"Score: % 0.3f \" %par_scores[-1]\n",
    "    #print \"Time: %0.3f\" %(time.time()- time_start)\n",
    "\n",
    "ind = par_scores.index(max(par_scores))\n",
    "print \"Best Params with F1 score: %0.3f \" % max(par_scores)\n",
    "print param_grid[ind]\n",
    "neigh = Neighbors(models_for_space, k=param_grid[ind][0], flag=param_grid[ind][1], print_flag=False)\n",
    "neigh.fit(X_train + X_meta, y_train + y_meta)\n",
    "predict = neigh.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 3grams\n",
      "Accuracy : 0.327272727273\n",
      "Micro F1-score : 0.327272727273\n",
      "Confusion matrix :\n",
      " [[ 3  1  0  0  0]\n",
      " [ 5  2  8  2  0]\n",
      " [ 3  6 13  1  0]\n",
      " [ 2  1  7  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.23      0.75      0.35         4\n",
      "      25-34       0.20      0.12      0.15        17\n",
      "      35-49       0.45      0.57      0.50        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.27      0.33      0.28        55\n",
      "\n",
      "Model: soac\n",
      "Accuracy : 0.363636363636\n",
      "Micro F1-score : 0.363636363636\n",
      "Confusion matrix :\n",
      " [[ 0  2  2  0  0]\n",
      " [ 0  5 12  0  0]\n",
      " [ 2  7 14  0  0]\n",
      " [ 1  1  7  1  0]\n",
      " [ 0  1  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.31      0.29      0.30        17\n",
      "      35-49       0.40      0.61      0.48        23\n",
      "      50-64       1.00      0.10      0.18        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.45      0.36      0.33        55\n",
      "\n",
      "Model: lsi\n",
      "Accuracy : 0.418181818182\n",
      "Micro F1-score : 0.418181818182\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  0 17  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.42      1.00      0.59        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.25        55\n",
      "\n",
      "Model: voting\n",
      "Accuracy : 0.381818181818\n",
      "Micro F1-score : 0.381818181818\n",
      "Confusion matrix :\n",
      " [[ 0  2  2  0  0]\n",
      " [ 0  8  9  0  0]\n",
      " [ 0 10 13  0  0]\n",
      " [ 0  1  9  0  0]\n",
      " [ 0  1  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.36      0.47      0.41        17\n",
      "      35-49       0.39      0.57      0.46        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.28      0.38      0.32        55\n",
      "\n",
      "Model: votingh\n",
      "Accuracy : 0.0727272727273\n",
      "Micro F1-score : 0.0727272727273\n",
      "Confusion matrix :\n",
      " [[ 4  0  0  0  0]\n",
      " [17  0  0  0  0]\n",
      " [23  0  0  0  0]\n",
      " [10  0  0  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.07      1.00      0.14         4\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.00      0.00      0.00        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.01      0.07      0.01        55\n",
      "\n",
      "Model: space\n",
      "Accuracy : 0.418181818182\n",
      "Micro F1-score : 0.418181818182\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  0 17  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.42      1.00      0.59        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.25        55\n",
      "\n",
      "Model: Meta\n",
      "Accuracy : 0.4\n",
      "Micro F1-score : 0.4\n",
      "Confusion matrix :\n",
      " [[ 0  3  1  0  0]\n",
      " [ 1  5  8  1  2]\n",
      " [ 1  3 16  0  3]\n",
      " [ 0  3  7  0  0]\n",
      " [ 0  0  0  0  1]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.36      0.29      0.32        17\n",
      "      35-49       0.50      0.70      0.58        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.17      1.00      0.29         1\n",
      "\n",
      "avg / total       0.32      0.40      0.35        55\n",
      "\n",
      "Model: neigh\n",
      "Accuracy : 0.436363636364\n",
      "Micro F1-score : 0.436363636364\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  1 16  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       1.00      0.06      0.11        17\n",
      "      35-49       0.43      1.00      0.60        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.49      0.44      0.28        55\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model_names2 = model_names + ['neigh']\n",
    "trained_models2 = trained_models+[neigh]\n",
    "predictions = []\n",
    "for i, model in enumerate(trained_models2):\n",
    "    print \"Model: \" + str(model_names2[i])\n",
    "    predict = model.predict(X_cv)\n",
    "    predictions.append(predict)\n",
    "    acc = accuracy_score(y_cv, predict)\n",
    "    conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "    rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Micro F1-score : {}'.format(f1_score(y_cv, predict, average='micro')))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150, 4)\n"
     ]
    }
   ],
   "source": [
    "from matplotlib.pyplot import scatter\n",
    "from matplotlib import pyplot as plt\n",
    "from tsne import bh_sne\n",
    "\n",
    "from sklearn.datasets import load_iris\n",
    "\n",
    "iris = load_iris()\n",
    "\n",
    "X11 = iris.data\n",
    "print X11.shape\n",
    "\n",
    "y11 = iris.target\n",
    "\n",
    "X_2d = bh_sne(X11)\n",
    "\n",
    "#plt.figure()\n",
    "#scatter(X_2d[:, 0], X_2d[:, 1], c=y11)\n",
    "#plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'trained_models' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-afee9bd83c46>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mLSI1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrained_models\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mX_tr2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLSI1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[0mX_tr2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'trained_models' is not defined"
     ]
    }
   ],
   "source": [
    "LSI1 = trained_models[2].steps[0][1]\n",
    "X_tr2 = LSI1.transform(X_cv)\n",
    "print X_tr2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tsne\n",
    "\n",
    "#LSI1 = trained_models2[2].steps[0][1]\n",
    "#X_tr = numpy.array([LSI1.transform(x) for x in X_cv])\n",
    "print \n",
    "X_2d = bh_sne(X_tr2)\n",
    "\n",
    "def mistakes_clustering(predictions, model_names2, y_cv, X_cv):\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report,f1_score\n",
    "neigh = Neighbors(models_for_space, k=3, flag='both', print_flag=False)\n",
    "g\n",
    "#neigh.fit(X_train + X_meta, y_train + y_meta)\n",
    "predict = neigh.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble4_2(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Best model base on the prediction of the k-nearest, according to each model, neighbor.\n",
    "        Implementing fitting with random weight searching for better results.\"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores, k=6, weights=[0.6,0.2,0.3, 6], N_rand=8, rand_split=0.6):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.ind2names = {}\n",
    "            self.weights = weights\n",
    "            self.N_rand = N_rand\n",
    "            self.rand_split = rand_split\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.trees = []\n",
    "            self.representations = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        import random, time\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            t0 = time.time()\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "                self.trees.append(BallTree(self.representations[-1], leaf_size=20))\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            N_rand1 = int(self.rand_split*self.N_rand)\n",
    "            poss_w = []\n",
    "            acc_ = []\n",
    "            pred = []\n",
    "            for i in xrange(N_rand1):  \n",
    "                tmp_w = [0.6,0.2,0.3, 6]\n",
    "                tmp_w[0] = round(random.random(),3)\n",
    "                tmp_w[1] = round(1 - tmp_w[0],3)\n",
    "                tmp_w[2] = round(random.uniform(0.2,0.8),3)\n",
    "                #tmp_w[3] = random.randint(1,10)\n",
    "                poss_w.append(tmp_w)\n",
    "                pred = self.find_weights(X_cv, tmp_w)\n",
    "                acc = accuracy_score(self.true, pred)\n",
    "                #print('Accuracy : {}'.format(acc))\n",
    "                acc_.append(acc)\n",
    "            print('First search took: %0.3f seconds') % (time.time()-t0)\n",
    "            tmp_w = poss_w[acc_.index(max(acc_))]\n",
    "            poss_w = []\n",
    "            acc_ = []\n",
    "            for i in xrange(self.N_rand-N_rand1):\n",
    "                tmp_w2 = tmp_w\n",
    "                tmp_w2[0] = round(random.uniform(tmp_w[0]-0.1, tmp_w[0]+0.1),3)\n",
    "                tmp_w2[1] = round(1 - tmp_w2[0],3)\n",
    "                tmp_w2[2] = round(random.uniform(tmp_w[2]-0.1, tmp_w[1]+0.1),3)\n",
    "                poss_w.append(tmp_w2)\n",
    "                pred = self.find_weights(X_cv, tmp_w2)\n",
    "                acc = accuracy_score(self.true, pred)\n",
    "                #print('Accuracy : {}'.format(acc))\n",
    "                acc_.append(acc)\n",
    "            self.weights = poss_w[acc_.index(max(acc_))]\n",
    "            self.k = self.weights[3]\n",
    "            print 'Accuracy obtained in CV-data: %0.3f' % (100*acc_[acc_.index(max(acc_))])\n",
    "            print self.weights\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            return self\n",
    "    \n",
    "    def find_weights(self, X_cv, w):\n",
    "        \n",
    "        y_pred = []\n",
    "        #t0 = time.time()\n",
    "        for x in X_cv:\n",
    "            #print 'True: ' + y_real[i]\n",
    "            y_pred.append(self.expert_fit_decision(x, w)) \n",
    "        #print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "    \n",
    "    \n",
    "    def expert_fit_decision(self, x_sample, w):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        \n",
    "        possible_experts = []\n",
    "        sample_predictions = []\n",
    "        acc = []\n",
    "        possible_experts_sc = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            #print 'Model: ' + self.ind2names[model_i]\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = self.trees[model_i].query(temp_trans, w[3])\n",
    "            #print \"Model neig\"\n",
    "            #print model_neig[0].tolist()[0]\n",
    "            model_neig_pred = []\n",
    "            neigh_true = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "                neigh_true.append(self.true[model_n_i])\n",
    "            #print \"True_neighbors\"\n",
    "            #print neigh_true\n",
    "            #print \"Predicted neighbors\"\n",
    "            #print model_neig_pred\n",
    "            acc.append(accuracy_score(neigh_true, model_neig_pred, normalize=True))\n",
    "            #print 'Neig Accc: % 0.2f' % acc[-1]\n",
    "            predicted = self.models[self.ind2names[model_i]].predict([x_sample])[0]\n",
    "            proba = max(self.models[self.ind2names[model_i]].predict_proba([x_sample])[0])\n",
    "            #print 'Predicted Sample: %s with proba: %0.3f' % (predicted, 100*proba)\n",
    "            if  acc[-1] > w[2]:\n",
    "                possible_experts.append(model_i)\n",
    "                possible_experts_sc.append(w[1]*acc[-1]+w[0]*proba)\n",
    "                sample_predictions.append(predicted)\n",
    "        if possible_experts:\n",
    "            #print 'Possible experts:'\n",
    "            #print [self.ind2names[poss] for poss in possible_experts]\n",
    "            #print sample_predictions\n",
    "            #print 'Selected: '\n",
    "            #print 'Place of best expert: %d ' % possible_scores.index(max(possible_scores))\n",
    "            #print 'Name:  ' + self.ind2names[possible_experts[possible_scores.index(max(possible_scores))]]\n",
    "            #print 'PRediction index: '\n",
    "            #print possible_scores.index(max(possible_scores))\n",
    "            #print 'PRediction : '\n",
    "            #print sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "            return sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "        else:\n",
    "            #print 'Selected2 from base model: ' + self.ind2names[(self.acc.index(max(acc)))]\n",
    "            #print self.models[self.ind2names[(self.acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "            return self.models[self.ind2names[(acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "        \n",
    "    \n",
    "    def predict(self, X):\n",
    "        \n",
    "        #import time\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        y_pred = []\n",
    "        #t0 = time.time()\n",
    "        for i, x in enumerate(X):\n",
    "            #print 'True: ' + y_real[i]\n",
    "            y_pred.append(self.expert_decision(x)) \n",
    "        #print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        \n",
    "        possible_experts = []\n",
    "        sample_predictions = []\n",
    "        acc = []\n",
    "        possible_experts_sc = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            #print 'Model: ' + self.ind2names[model_i]\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = self.trees[model_i].query(temp_trans, self.k)\n",
    "            #print \"Model neig\"\n",
    "            #print model_neig[0].tolist()[0]\n",
    "            model_neig_pred = []\n",
    "            neigh_true = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "                neigh_true.append(self.true[model_n_i])\n",
    "           # print \"True_neighbors\"\n",
    "           # print neigh_true\n",
    "           # print \"Predicted neighbors\"\n",
    "           # print model_neig_pred\n",
    "            acc.append(accuracy_score(neigh_true, model_neig_pred, normalize=True))\n",
    "            #print 'Neig Accc: % 0.2f' % acc[-1]\n",
    "            predicted = self.models[self.ind2names[model_i]].predict([x_sample])[0]\n",
    "            proba = max(self.models[self.ind2names[model_i]].predict_proba([x_sample])[0])\n",
    "            #print 'Predicted Sample: %s with proba: %0.3f' % (predicted, 100*proba)\n",
    "            if  acc[-1] > self.weights[2]:\n",
    "                possible_experts.append(model_i)\n",
    "                possible_experts_sc.append(self.weights[1]*acc[-1]+self.weights[0]*proba)\n",
    "                sample_predictions.append(predicted)\n",
    "        if possible_experts:\n",
    "           # print 'Possible experts:'\n",
    "            #print [self.ind2names[poss] for poss in possible_experts]\n",
    "            #print sample_predictions\n",
    "            #print possible_experts_sc\n",
    "            #print 'Selected: '\n",
    "            #print 'Place of best expert: %d ' % possible_scores.index(max(possible_scores))\n",
    "            #print 'Name:  ' + self.ind2names[possible_experts[possible_scores.index(max(possible_scores))]]\n",
    "            #print 'PRediction index: '\n",
    "            #print possible_scores.index(max(possible_scores))\n",
    "            #print 'PRediction : '\n",
    "            #print sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "            return sample_predictions[possible_experts_sc.index(max(possible_experts_sc))]\n",
    "        else:\n",
    "           # print 'Selected2 from base model: ' + self.ind2names[(acc.index(max(acc)))]\n",
    "           # print self.models[self.ind2names[(acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "            return self.models[self.ind2names[(acc.index(max(acc)))]].predict([x_sample])[0]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "cv_scores = []\n",
    "models = {}\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if name!='voting' and name!='votingh' and name!='space' and name!='meta':\n",
    "        models[name] = model\n",
    "        cv_scores.append(model.score(X_meta, y_meta))\n",
    "        \n",
    "\n",
    "w = [0.649, 0.351, 0.32, 6]\n",
    "print len(X_meta)\n",
    "print len(y_meta)\n",
    "space = SubSpaceEnsemble4_2(models,cv_scores,k=w[3], weights=w[:-1], N_rand=20, rand_split=0.6)\n",
    "space.fit(X_meta, y_meta)\n",
    "predict = space.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "import time\n",
    "\n",
    "class SubSpaceEnsemble3_2(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Utilizing the neighborhood in all representations and also ground truth model.\n",
    "        Implementing a weighted voting scheme.\"\"\"\n",
    "\n",
    "    def __init__(self, models, cv_scores, k=3, weights= [6,3,2,0.7, 3], N_rand=8, rand_split=0.6):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) or (not cv_scores):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.weights = weights\n",
    "            self.ind2names = {}\n",
    "            self.N_rand = N_rand\n",
    "            self.rand_split = rand_split\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.representations = []\n",
    "            self.meta = None\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.doc_terms = None\n",
    "            self.tree = None\n",
    "            self.experts = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X_cv).toarray()\n",
    "            self.tree = BallTree(self.doc_terms, leaf_size=20)\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            self.true = y_true\n",
    "            N_rand1 = int(self.rand_split*self.N_rand)\n",
    "            poss_w = []\n",
    "            acc_ = []\n",
    "            pred = []\n",
    "            for i in xrange(N_rand1):  \n",
    "                tmp_w = self.weights\n",
    "                tmp_w[0] = random.randint(1,10)\n",
    "                tmp_w[1] = random.randint(1,10)\n",
    "                tmp_w[2] = random.randint(1,10)\n",
    "                tmp_w[3] = round(random.uniform(0.2,0.8),3)\n",
    "                tmp_w[4] = random.randint(1,10)\n",
    "                poss_w.append(tmp_w)\n",
    "                pred = self.find_weights(X_cv, tmp_w)\n",
    "                acc = accuracy_score(self.true, pred)\n",
    "                #print('Accuracy : {}'.format(acc))\n",
    "                acc_.append(acc)\n",
    "            print('First search took: %0.3f seconds') % (time.time()-t0)\n",
    "            tmp_w = poss_w[acc_.index(max(acc_))]\n",
    "            poss_w = []\n",
    "            acc_ = []\n",
    "            for i in xrange(self.N_rand-N_rand1):\n",
    "                tmp_w2 = tmp_w\n",
    "                tmp_w2[0] = round(random.randint(tmp_w[0]-2, tmp_w[0]+2))\n",
    "                tmp_w2[1] = round(random.randint(tmp_w[1]-2, tmp_w[1]+2))\n",
    "                tmp_w2[2] = round(random.randint(tmp_w[2]-2, tmp_w[2]+2))\n",
    "                tmp_w2[3] = round(random.uniform(tmp_w[3]-0.1, tmp_w[3]+0.1),3)\n",
    "                poss_w.append(tmp_w2)\n",
    "                pred = self.find_weights(X_cv, tmp_w2)\n",
    "                acc = accuracy_score(self.true, pred)\n",
    "                #print('Accuracy : {}'.format(acc))\n",
    "                acc_.append(acc)\n",
    "            self.weights = poss_w[acc_.index(max(acc_))]\n",
    "            self.k = self.weights[4]\n",
    "            print 'Accuracy obtained in CV-data: %0.3f' % (100*acc_[acc_.index(max(acc_))])\n",
    "            print self.weights\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            return self\n",
    "    \n",
    "    def find_weights(self, X_cv, w):\n",
    "        \n",
    "        X_transformed = self.counter.transform(X_cv).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), w[4])  \n",
    "            #print 'Sample ' + y_real[i]\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            y_pred.append(self.expert_fit_decision(neigbors_indexes[0],  X[i], w))\n",
    "            \n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        return y_pred\n",
    "            \n",
    "    def expert_fit_decision(self, neigbors_indexes, x_sample, w):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "        #print 'True'\n",
    "        #print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = w[1]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "            #print 'acc-thres'\n",
    "            #print acc, w[3]\n",
    "            if acc[-1] >w[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(w[2]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(w[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "                #print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "                #print 'Predictions'\n",
    "                #print model_pred\n",
    "                #print 'Representations'\n",
    "                #print model_neig_pred\n",
    "                #print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "                #print \"weights\"\n",
    "                #print weights\n",
    "        total_pred.extend([n for j in xrange(int(weights['true'])) for n in neigbors_true])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        #print total_pred\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "        #print data\n",
    "        best_model_ind = acc.index(max(acc))\n",
    "        #print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "        #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]\n",
    "        \n",
    "\n",
    "    def predict(self, X, y_real):\n",
    "        \n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), self.k)  \n",
    "            print 'Sample ' + y_real[i]\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            y_pred.append(self.expert_decision(neigbors_indexes[0],  X[i]))\n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, neigbors_indexes, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "        #print 'True'\n",
    "        #print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = self.weights[1]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "            if acc[-1] >self.weights[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(self.weights[2]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(self.weights[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "                #print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "                #print 'Predictions'\n",
    "                #print model_pred\n",
    "                #print 'Representations'\n",
    "                #print model_neig_pred\n",
    "                #print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "        total_pred.extend([n for j in xrange(int(weights['true'])) for n in neigbors_true])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "        print data\n",
    "        best_model_ind = acc.index(max(acc))\n",
    "        print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "        #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "space.k = space.weights[4]\n",
    "predict = space.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "cv_scores = []\n",
    "models = {}\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if name!='voting' and name!='votingh' and name!='space' and name!='meta':\n",
    "        models[name] = model\n",
    "        cv_scores.append(model.score(X_meta, y_meta))\n",
    "        \n",
    "\n",
    "weights= [6,3,2,0.7, 3]\n",
    "print len(X_meta)\n",
    "print len(y_meta)\n",
    "space = SubSpaceEnsemble3_2(models,cv_scores,k=weights[4], weights=weights, N_rand=20, rand_split=0.6)\n",
    "space.fit(X_meta, y_meta)\n",
    "predict = space.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Metaclassifier2(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    \"\"\" A Linear Weights Metaclassifier based on the neighborhood of each sample.\n",
    "        The neighborhood is different per base model. For each sample we have\n",
    "        [N, N*k] votes, with N the number of base classifiers and k the number\n",
    "        of neighbors to look for. \"\"\"\n",
    "\n",
    "    def __init__(self, models, C=1.0, weights='balanced', k=3):\n",
    "\n",
    "        from sklearn.svm import LinearSVC\n",
    "        from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "        if not models:\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier')\n",
    "        self.models = models\n",
    "        self.weights = weights\n",
    "        self.C = C\n",
    "        self.k = 3\n",
    "        self.svc = LinearSVC(C=self.C, class_weight=self.weights)\n",
    "        self.lab_encoder = LabelEncoder()\n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            # import pprint\n",
    "            #print list(set(y_true))\n",
    "            # print len(y_true)\n",
    "            y_true = self.lab_encoder.fit_transform(y_true)\n",
    "            #print self.models.keys()\n",
    "            # print self.lab_encoder.classes_\n",
    "            # print self.models[self.models.keys()[1]].predict(X_cv)\n",
    "            #y_true = self.create_onehot(y_true)\n",
    "            # print \"Train X shape: \" + str(X_cv.shape) + \"train y_true \" + str(y_true.shape)\n",
    "            transformed_y = self.transform_to_y(X_cv)\n",
    "            #X = self.oh_encoder.transform(y_pred.T)\n",
    "            #print transformed_y.shape, y_true.T.shape\n",
    "            #print \"fit true\"\n",
    "            # print transformed_y\n",
    "            # print y_true\n",
    "            self.svc.fit(transformed_y, y_true.T)\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X = self.transform_to_y(X)\n",
    "        # print \"PRedict after\"\n",
    "        # print X.shape\n",
    "        # print X.T.shape\n",
    "        import pprint\n",
    "        # pprint.pprint(X)\n",
    "        # pprint.pprint(X.T)\n",
    "        # print \"Predict\"\n",
    "        y_pred = self.svc.predict(X)\n",
    "        # pprint.pprint(y_pred)\n",
    "        # pprint.pprint(self.lab_encoder.inverse_transform(y_pred)) \n",
    "        return self.lab_encoder.inverse_transform(y_pred)\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        # import numpy\n",
    "        # print \"Score\"\n",
    "        # print X.shape, numpy.array(y).shape\n",
    "        # transformed_y = self.transform_to_y(X)\n",
    "        # print 'edw ok'\n",
    "        # print self.svc.predict(transformed_y).shape\n",
    "        # print 'Transformed'\n",
    "        # print transformed_y.shape\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        # import pprint\n",
    "        # print \"Ture\"\n",
    "        # pprint.pprint(y)\n",
    "\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "    def create_onehot(self, l):\n",
    "\n",
    "        from numpy import zeros, vstack\n",
    "        #print \"L:\"\n",
    "        #from pprint import pprint as pprint\n",
    "        #print type(l)\n",
    "        # pprint(l)\n",
    "        l = list(l)\n",
    "        for i, el in enumerate(l):\n",
    "            temp = zeros([1, len(self.lab_encoder.classes_)], dtype=float)\n",
    "            #print(temp.shape)\n",
    "           # pprint(temp)\n",
    "            temp[0, el] = 1\n",
    "            if i == 0:\n",
    "                fin = temp\n",
    "            else:\n",
    "                fin = vstack((fin, temp))\n",
    "        #print \"onehot shape\" + str(fin.shape)\n",
    "        return fin\n",
    "\n",
    "    def transform_to_y(self, X):\n",
    "\n",
    "        from numpy import hstack\n",
    "\n",
    "        #print \"Train X shape: \" + str(X.shape)\n",
    "        for i, model in enumerate(self.models.values()):\n",
    "            #print self.models.keys()[i]\n",
    "            predict = model.predict(X)\n",
    "            #print type(predict)\n",
    "            #print predict.shape\n",
    "            #print predict\n",
    "            tmp_pred = self.create_onehot(self.lab_encoder.transform(predict))\n",
    "            #print type(tmp_pred)\n",
    "            if i == 0:\n",
    "                y_pred = tmp_pred\n",
    "            else:\n",
    "                y_pred = hstack((y_pred, tmp_pred))\n",
    "            predictions_n = self.neigh_model_pred(model, X, predict)\n",
    "            #print 'Num Pred'\n",
    "            #print len(predictions_n[0])\n",
    "            for neigh_dist in xrange(self.k):\n",
    "                tmp_pred_n = self.create_onehot(self.lab_encoder.transform(predictions_n[:, neigh_dist]))\n",
    "                y_pred = hstack((y_pred, tmp_pred_n))\n",
    "            #print \"y_pred: \" + str(y_pred.shape)\n",
    "        #print \"y_pred: \" + str(y_pred.shape)\n",
    "        #print len(self.lab_encoder.classes_)\n",
    "        #print y_pred\n",
    "        return y_pred\n",
    "\n",
    "\n",
    "    def neigh_model_pred(self, model, X, pred):\n",
    "\n",
    "        from sklearn.neighbors import BallTree\n",
    "        import numpy\n",
    "\n",
    "        # Expects a pipeline with two steps. Transform and Predict.\n",
    "        transf = model.steps[0][1].transform(X)\n",
    "        if hasattr(transf, \"toarray\"):\n",
    "            # print 'Exei'\n",
    "            representations = transf.toarray()\n",
    "        else:\n",
    "            representations = transf\n",
    "        ModelTree = BallTree(representations)\n",
    "        predictions = []\n",
    "        for i in xrange(representations.shape[0]):\n",
    "            _, neig_ind = ModelTree.query(representations[i,:].reshape(1,-1), self.k)\n",
    "            predictions.extend([pred[n_i] for n_i in neig_ind])\n",
    "        return numpy.array(predictions)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pan.features import Metaclassifier\n",
    "\n",
    "model_dic = {}\n",
    "for i, model in enumerate(trained_models):\n",
    "    if model_names[i] != 'voting' and model_names[i] !='votingh' and model_names[i] !='meta' and model_names[i] !='space': \n",
    "        model_dic[model_names[i]] = model\n",
    "model_dic['space'] = space\n",
    "#Meta = Metaclassifier2(models=model_dic, C=1, weights='balanced')\n",
    "Meta = Metaclassifier(models=model_dic, C=1, weights='balanced')\n",
    "params = {'C':[0.01, 0.1, 1, 10, 100]}\n",
    "params = {}\n",
    "grid = GridSearchCV(Meta, param_grid=params, verbose=1, n_jobs=-1, cv=3, refit=True)\n",
    "grid.fit(X_train + X_meta, y_train + y_meta)\n",
    "print 'Best params: {} score: {}'.format(grid.best_params_, grid.best_score_)\n",
    "#Meta.fit(X_meta, y_meta)\n",
    "#predict = Meta.predict(X_cv)\n",
    "predict = grid.predict(X_cv)\n",
    "#Meta.fit(X_meta, y_meta)\n",
    "#predict = Meta.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from cvxpy import *\n",
    "\n",
    "def f(w):\n",
    "    print \"Weights\"\n",
    "    print w\n",
    "    space = SubSpaceEnsemble3(models,cv_scores,k=10, weights=w)\n",
    "    space.fit(X_train + X_cv, y_train + y_cv)\n",
    "    score = 1- accuracy_score(y_meta, space.predict(X_meta))\n",
    "    print 'Score: ' + str(score)\n",
    "    return score\n",
    "\n",
    "n = 4\n",
    "w = Variable(n)\n",
    "objective = Minimize(f(w))\n",
    "constraints = [0 <= w]\n",
    "prob = Problem(objective, constraints)\n",
    "\n",
    "# The optimal objective is returned by prob.solve().\n",
    "result = prob.solve()\n",
    "# The optimal value for x is stored in x.value.\n",
    "print(w.value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "a = numpy.array([[  0, 207,  65, 161,  11,  61, 152,  37, 302,  25]])\n",
    "a[0].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "\n",
    "global X_train, X_meta, x_cv, y_train, y_meta, y_cv\n",
    "\n",
    "def f(w):\n",
    "    print \"Weights\"\n",
    "    print w\n",
    "    score = 1- accuracy_score(y_meta, space.predict(X_meta, w))\n",
    "    print 'Score: ' + str(score)\n",
    "    return score\n",
    "\n",
    "w = [3,2,1,0.35]\n",
    "#space = SubSpaceEnsemble3(models,cv_scores,k=3, weights=w)\n",
    "#space.fit(X_train + X_cv, y_train + y_cv)\n",
    "#print models.keys()\n",
    "#print cv_scores\n",
    "\n",
    "bnds = ((0, None), (0, None), (0, None), (0, 1))\n",
    "a = minimize(f, w,  method='SLSQP', bounds=bnds)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pan.features import Metaclassifier2\n",
    "\n",
    "model_dic = {}\n",
    "for i, model in enumerate(trained_models):\n",
    "    if model_names[i] != 'voting' and model_names[i] !='votingh' and model_names[i] !='meta' and model_names[i] !='space': \n",
    "        model_dic[model_names[i]] = model\n",
    "model_dic['space'] = space\n",
    "Meta = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "#grid = GridSearchCV(Meta, param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "#grid.fit(X_meta, y_meta)\n",
    "Meta.fit(X_train + X_meta, y_train + y_meta)\n",
    "predict = Meta.predict(X_cv)\n",
    "#Meta.fit(X_meta, y_meta)\n",
    "#predict = Meta.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "models['voting']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "models = {}\n",
    "cv_scores = []\n",
    "for name, model in zip(model_names, trained_models):\n",
    "    if name!='voting' and name!='space' and name!='meta':\n",
    "        models[name] = model\n",
    "        cv_scores.append(model.score(X_cv, y_cv))\n",
    "\n",
    "print models.keys()\n",
    "print cv_scores\n",
    "space = SubSpaceEnsemble(models, cv_scores)\n",
    "grid_search = GridSearchCV(SubSpaceEnsemble(models, cv_scores), param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "grid_search.fit(X_cv, y_cv)\n",
    "space.fit(X_cv, y_cv)\n",
    "y_space = grid_search.best_estimator_.predict(X_cv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(y_space[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = []\n",
    "for i, model in enumerate(trained_models):\n",
    "    predict = model.predict(X_cv)\n",
    "    predictions.append(predict)\n",
    "    acc = accuracy_score(y_cv, predict)\n",
    "    conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=list(set(y)))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "\n",
    "import numpy, copy\n",
    "\n",
    "def print_overlaps(predictions, names, verbose=True):\n",
    "    N = len(names)\n",
    "    res = numpy.zeros([N,N])\n",
    "    for i in range(0, N):\n",
    "        for j in range(i+1, N):\n",
    "            #print i,j\n",
    "            #predictions[i]\n",
    "            #predictions[j]\n",
    "            res[i,j] = len([(k,v) for k,v in zip(predictions[i], predictions[j]) if k==v])/float(len(predictions[0]))\n",
    "            #print res[i,j]\n",
    "            if verbose:\n",
    "                print \"%s - %s : %0.3f  overlap\" % (names[i],  names[j], 100*res[i,j])\n",
    "    return  res\n",
    "\n",
    "#pred2 = copy.deepcopy(predictions)\n",
    "#pred2.append(y_space)\n",
    "\n",
    "#pred2.append(y_cv)\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting']\n",
    "#model_names += ['space']\n",
    "#model_names += ['True']\n",
    "#print len([(i, j) for i,j in zip(predictions[0], predictions[1]) if i==j])/float(len(predictions[0]))\n",
    "#print_overlaps(pred2, model_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=True)\n",
    "\n",
    "#clf = AdaBoostClassifier(base_estimator=svm, n_estimators=100, learning_rate=1.0, algorithm='SAMME.R', random_state=42)\n",
    "\n",
    "clf = BaggingClassifier(base_estimator = svm, n_estimators=100, verbose=1, random_state=42)\n",
    "\n",
    "#X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, random_state=42, stratify=y)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "X_train_new = soac.transform(X_train)\n",
    "#for i, x in enumerate(X_train):\n",
    "#    if len(x)<=1 or y_train[i]<=1:\n",
    "#        print 'y'\n",
    "#        X_train.remove(x)\n",
    "#        y_train.remove(y_train[i])\n",
    "print len(X_train), len(y_train)\n",
    "clf.fit(X_train_new,y_train)\n",
    "predict= clf.predict(soac.transform(X_cv))\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=list(set(y)))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas, copy\n",
    "import matplotlib.pyplot as plt\n",
    "pred2 = copy.deepcopy(predictions)\n",
    "pred2.append(y_cv)\n",
    "pred2 = map(list, zip(*pred2))\n",
    "df = pandas.DataFrame(pred2, columns=model_names)\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print model_names\n",
    "print len(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.mlab as mlab\n",
    "import plotly.plotly as py\n",
    "import plotly.tools as tls\n",
    "from plotly.graph_objs import *\n",
    "\n",
    "tls.set_credentials_file(username=\"Bogas\",\n",
    "                             api_key=\"9s60rarm2w\")\n",
    "\n",
    "py.sign_in(username=\"Bogas\", api_key=\"9s60rarm2w\")\n",
    "\n",
    "pred2 = copy.deepcopy(predictions)\n",
    "pred2.append(y_cv)\n",
    "traces = []\n",
    "model_names += ['True']\n",
    "for i, pred in enumerate(pred2):\n",
    "    traces.append(Scatter(\n",
    "        x=range(0,len(y_cv)),\n",
    "        y=pred,\n",
    "        mode='markers+line',\n",
    "        type= 'scatter',\n",
    "        name= model_names[i]\n",
    "        )\n",
    "                )\n",
    "\n",
    "title1 = \"Results on test set for Ensemble Scheme\"\n",
    "layout = Layout(\n",
    "        width= 1200,\n",
    "        height= 800,\n",
    "        title= title1,\n",
    "        xaxis = {\"title\": 'Samples'},\n",
    "        yaxis = {\"title\": 'Classes', \"type\":'category'}\n",
    ")\n",
    "\n",
    "data = Data(traces)\n",
    "fig = Figure(data=data, layout=layout)\n",
    "#py.plot(fig, filename='Grey_70_cosine_vector_list bow')\n",
    "py.iplot(fig, filename=title1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "help(YAxis)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# predict class probabilities for all classifiers\n",
    "probas = [c.predict_proba(X_cv) for c in trained_models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from scipy import interp\n",
    "import numpy as np\n",
    "\n",
    "y_cv2 = label_binarize(y_cv, list(set(y)))\n",
    "pred2 = []\n",
    "for pred in predictions:\n",
    "    pred2.append(label_binarize(pred, list(set(y))))\n",
    "\n",
    "n_classes = len(list(set(y)))\n",
    "plt.figure()    \n",
    "for j, model in enumerate(trained_models):\n",
    "    # Compute ROC curve and ROC area for each class\n",
    "    fpr = dict()\n",
    "    tpr = dict()\n",
    "    roc_auc = dict()\n",
    "    for i in range(len(list(set(y)))):\n",
    "        fpr[i], tpr[i], _ = roc_curve(y_cv2[:, i], pred2[j][:, i])\n",
    "        roc_auc[i] = auc(fpr[i], tpr[i])\n",
    "    # Compute macro-average ROC curve and ROC area\n",
    "    # First aggregate all false positive rates\n",
    "    all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))\n",
    "    # Then interpolate all ROC curves at this points\n",
    "    mean_tpr = np.zeros_like(all_fpr)\n",
    "    for i in range(n_classes):\n",
    "        mean_tpr += interp(all_fpr, fpr[i], tpr[i])\n",
    "\n",
    "    # Finally average it and compute AUC\n",
    "    mean_tpr /= n_classes\n",
    "    fpr[\"macro\"] = all_fpr\n",
    "    tpr[\"macro\"] = mean_tpr\n",
    "    roc_auc[\"macro\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n",
    "\n",
    "    # Plot all ROC curves\n",
    "\n",
    "    plt.plot(fpr[\"macro\"], tpr[\"macro\"],\n",
    "         label=model_names[j]+' macro-area = {0:0.2f})'\n",
    "               ''.format(roc_auc[\"macro\"]),\n",
    "         linewidth=2)\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'k--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver operating characteristics to multi-class for ensemble methods')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "LDAA = grid_search.best_estimator_.steps[0][1].__dict__['transformer_list'][0][1]\n",
    "def print_top_words(model, feature_names, n_top_words):\n",
    "\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        topic_words = \" \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        print(\"#%d: \" % topic_idx + topic_words)\n",
    "        #print(\" \".join([feature_names[i]\n",
    "        #                for i in topic.argsort()[:-n_top_words - 1:-1]]))\n",
    "#print_top_words(LDAA.LDA, LDAA.counter.get_feature_names(), 10)\n",
    "\n",
    "def get_top_words(model, feature_names, n_top_words):\n",
    "     \n",
    "    feat = []\n",
    "    for topic_idx, topic in enumerate(model.components_):\n",
    "        topic_words = \" \".join([feature_names[i]\n",
    "                        for i in topic.argsort()[:-n_top_words - 1:-1]])\n",
    "        feat.append(\"#%d: \" % topic_idx + topic_words)\n",
    "        #print(\"#%d: \" % topic_idx + topic_words)\n",
    "    return feat\n",
    "get_top_words(LDAA.LDA, LDAA.counter.get_feature_names(), 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "feature_names = get_top_words(LDAA.LDA, LDAA.counter.get_feature_names(), 10)\n",
    "print len(feature_names)\n",
    "#soa_feat_names = [\"soa_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "#soac_feat_names = [\"soac_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "#feature_names += soa_feat_names\n",
    "#feature_names += soac_feat_names\n",
    "feature_names = [feat.encode('utf-8') for feat in feature_names]\n",
    "print len(feature_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Counts + soa + Soac Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'countTokens' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-72-437d3c6e55d3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mfeature_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcountTokens\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[0mfeature_names\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'numHash'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'numUrl'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'numRep'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;31m#soa_feat_names = [\"soa_prob_\"+str(i) for i in range(0, len(set(y)))]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'countTokens' is not defined"
     ]
    }
   ],
   "source": [
    "\n",
    "import copy\n",
    "feature_names = copy.deepcopy(countTokens.l)\n",
    "feature_names += ['numHash', 'numUrl', 'numRep']\n",
    "#soa_feat_names = [\"soa_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "#soac_feat_names = [\"soac_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "#feature_names += soa_feat_names\n",
    "#feature_names += soac_feat_names\n",
    "feature_names = [feat.encode('utf-8') for feat in feature_names]\n",
    "print len(countTokens.l), len(feature_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(features)\n",
    "#features.SOAC_Model2.__doc__\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=5000)\n",
    "#y[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "XX = [#\"I like playing video games very much :).\", \n",
    "     #\"Football games are the best!\",\n",
    "     #\"Being young forever is very funny and entertaining\",\n",
    "     # \"Football games are the best!\",\n",
    "      \"best games\",\n",
    "      \"best games\",\n",
    "     #\"World leaders should gather and decide for todays meeting!\",\n",
    "     #\"Problems nowadays seem to thrive everywhere\",\n",
    "     #\"Just got off from work today! Weekend is coming though, so it's alright...\",\n",
    "     #\"This weekend we are going of for 3 days..\",\n",
    "     \" Weekend alright...\",\n",
    "     \" Weekend alright...\",\n",
    "     \" Weekend alright...\",\n",
    "     \"Awful weather\",\n",
    "     \"Awful weather\",\n",
    "     \"Awful weather\",\n",
    "     \"Awful weather\",\n",
    "     \"Awful weather\"]\n",
    "yy = [\"18-24\",\n",
    "     \"18-24\",\n",
    "     \"25-34\",\n",
    "     \"25-34\",\n",
    "     \"25-34\",\n",
    "     \"35-49\",\n",
    "     \"35-49\",\n",
    "     \"35-49\",\n",
    "     \"35-49\",\n",
    "     \"35-49\",\n",
    "    ]\n",
    "#reload(preprocess)\n",
    "#reload(features)\n",
    "from pan import features\n",
    "from pan import preprocess\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "XX = preprocess.preprocess(XX)\n",
    "num_folds = 2\n",
    "grid_search = GridSearchCV(estimator=pipe, param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "grid_search.fit(XX,yy)\n",
    "print(grid_search.best_estimator_)\n",
    "print(grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from argparse import ArgumentParser\n",
    "from sklearn.externals import joblib\n",
    "from tictacs import from_recipe\n",
    "from pan import ProfilingDataset\n",
    "import dill\n",
    "import cPickle as pickle\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "\n",
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "print('Loaded {} users...\\n'.format(len(dataset.entries)))\n",
    "# get config\n",
    "config = dataset.config\n",
    "tasks = config.tasks\n",
    "print('\\n--------------- Thy time of Running ---------------')\n",
    "all_models = {}\n",
    "for task in tasks:\n",
    "    print('Learning to judge %s..' % task)\n",
    "    # load data\n",
    "    X, y = dataset.get_data(task)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def jointed_tasks(jointed=None, sep_list=None, delim='+', jointed_to_dist=False):\n",
    "    \n",
    "    if jointed_to_dist:\n",
    "        tmp = zip(*[j.split(delim) for j in jointed])\n",
    "        out = [list(tmp_) for tmp_ in tmp]\n",
    "        return out\n",
    "    else:\n",
    "        #out = [ for i in xrange(len(sep_list)) for j in xrange(len(sep_list[0]))\n",
    "        out = []\n",
    "        for j in xrange(len(sep_list[0])):\n",
    "            tmp = ''\n",
    "            for i in xrange(len(sep_list)):\n",
    "                tmp += sep_list[i][j]+'+'\n",
    "            tmp = tmp[:-1]\n",
    "            out.append(tmp)\n",
    "        return out\n",
    "\n",
    "y_dic = {}\n",
    "tasks = ['gender', 'age']\n",
    "for task in tasks:\n",
    "    _, y_dic[task] = dataset.get_data(task)\n",
    "y_both = jointed_tasks(sep_list=[y_dic[task] for task in tasks], jointed_to_dist=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "X, _, y, _ = train_test_split(X, y, train_size=10000, stratify=y, random_state=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pan import ProfilingDataset, createDocProfiles, create_target_prof_trainset\n",
    "from pan import preprocess\n",
    "\n",
    "task = 'gender'\n",
    "docs = createDocProfiles(dataset)\n",
    "X, y = create_target_prof_trainset(docs, task)\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('combined', FeatureUnion(n_jobs=1,\n",
       "         transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
       "        tokenizer_var='sklearn'))],\n",
       "         transformer_weights=None)),\n",
       " ('svm', SVC(C=0.1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#reload(features)\n",
    "from pan import features\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.svm import SVC\n",
    "#from pan.features import SOAC_Model2\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('soac', soac)])\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe1 = Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=5, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('soa', soa)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe2= Pipeline([('combined',combined), ('svm', svm)])\n",
    "pipe2.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "num_folds = 4\n",
    "grid_search1 = GridSearchCV(estimator=pipe1, param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "grid_search1.fit(X,y)\n",
    "print(grid_search1.best_estimator_)\n",
    "print(grid_search1.best_score_)\n",
    "grid_search2 = GridSearchCV(estimator=pipe2, param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "grid_search2.fit(X,y)\n",
    "print(grid_search2.best_estimator_)\n",
    "print(grid_search2.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 + 88 = 436\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "#reload(features)\n",
    "split = 0.2\n",
    "#X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y)\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y_both, test_size=split, stratify=y_both)\n",
    "print \"%d + %d = %d\" % (len(X_train), len(X_cv), len(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "pipe1.fit(X_train, y_train)\n",
    "predict = pipe1.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "num_folds = 4\n",
    "grid_search = GridSearchCV(estimator=pipe1, param_grid={}, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "grid_search.fit(X_train,y_train)\n",
    "print(grid_search.best_estimator_)\n",
    "print(grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(grid_search.best_estimator_)\n",
    "print(grid_search.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 88 436 436\n",
      "Fitting 2 folds for each of 6 candidates, totalling 12 fits\n",
      "0.241379310345\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  12 out of  12 | elapsed:   49.7s finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "\n",
    "num_folds = 2\n",
    "split = 0.2\n",
    "#X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=split, stratify=y, random_state=100)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe1), ('1', pipe2)], voting='soft')\n",
    "trained_models = []\n",
    "params = {}\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "for model in [pipe1]:\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=1, n_jobs=-1, cv=num_folds, refit=True)\n",
    "    grid_search.fit(X_train,y_train)\n",
    "    print(grid_search.best_score_)\n",
    "    print(grid_search.best_estimator_) \n",
    "    trained_models.append(grid_search.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 + 88 = 436\n",
      "0.204022988506\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n",
      "348 + 88 = 436\n",
      "0.258620689655\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n",
      "348 + 88 = 436\n",
      "0.218390804598\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n",
      "348 + 88 = 436\n",
      "0.204022988506\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n",
      "348 + 88 = 436\n",
      "0.172413793103\n",
      "Pipeline(steps=[('combined', FeatureUnion(n_jobs=1,\n",
      "       transformer_list=[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn'))],\n",
      "       transformer_weights=None)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
      "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
      "  max_iter=-1, probability=True, random_state=None, shrinking=True,\n",
      "  tol=0.001, verbose=False))])\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from pan import features\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "\n",
    "\n",
    "def jointed_tasks(jointed=None, sep_list=None, delim='+', jointed_to_dist=False):\n",
    "    \n",
    "    if jointed_to_dist:\n",
    "        tmp = zip(*[j.split(delim) for j in jointed])\n",
    "        out = [list(tmp_) for tmp_ in tmp]\n",
    "        return out\n",
    "    else:\n",
    "        #out = [ for i in xrange(len(sep_list)) for j in xrange(len(sep_list[0]))\n",
    "        out = []\n",
    "        for j in xrange(len(sep_list[0])):\n",
    "            tmp = ''\n",
    "            for i in xrange(len(sep_list)):\n",
    "                tmp += sep_list[i][j]+'+'\n",
    "            tmp = tmp[:-1]\n",
    "            out.append(tmp)\n",
    "        return out\n",
    "\n",
    "# DATASET\n",
    "y_dic = {}\n",
    "tasks = ['gender', 'age']\n",
    "for task in tasks:\n",
    "    _, y_dic[task] = dataset.get_data(task)\n",
    "y_both = jointed_tasks(sep_list=[y_dic[task] for task in tasks], jointed_to_dist=False)\n",
    "y_dic['both'] = y_both\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# MODEL\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('soac', soac)])\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "pipe1 = Pipeline([('combined',combined), ('svm', svm)])\n",
    "\n",
    "results = {}\n",
    "results2 = {}\n",
    "for task in tasks+['both']:\n",
    "    results[task] = {'acc':[], 'conf':[], 'report':[]}\n",
    "    results2[task] = {'acc':[], 'conf':[], 'report':[]}\n",
    "    \n",
    "y_train = {'age':[], 'gender':[], 'both':[]}\n",
    "y_cv = {'age':[], 'gender':[], 'both': []}\n",
    "    \n",
    "num_folds = 4\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "N = 5\n",
    "random_seeds = random.sample(xrange(1000), N)\n",
    "for i in xrange(N):\n",
    "\n",
    "    # SPLIT\n",
    "    split = 0.2\n",
    "    X_train, X_cv, y_train, y_cv = train_test_split(X, y_both, test_size=split, stratify=y_both, random_state=random_seeds[i])\n",
    "    \n",
    "    print \"%d + %d = %d\" % (len(X_train), len(X_cv), len(X))\n",
    "    for model in [pipe1]:\n",
    "        grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=0, n_jobs=-1, cv=num_folds, refit=True)\n",
    "        grid_search.fit(X_train,y_train)\n",
    "        print(grid_search.best_score_)\n",
    "        print(grid_search.best_estimator_) \n",
    "        trained_models.append(grid_search.best_estimator_)\n",
    "\n",
    "    # PREDICTIONS\n",
    "    predict = grid_search.predict(X_cv)\n",
    "    tasks = ['gender', 'age']\n",
    "    y_true = {'both': y_cv}\n",
    "    y_pred = {'both': predict}\n",
    "    for i, task in enumerate(tasks):\n",
    "        y_true[task] = jointed_tasks(jointed=y_cv, jointed_to_dist=True)[i]\n",
    "        y_pred[task] = jointed_tasks(jointed=predict, jointed_to_dist=True)[i]\n",
    "\n",
    "    # REPORT\n",
    "    for task in tasks+['both']:\n",
    "        #print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TASK : %s ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' % task\n",
    "        predict1 = y_pred[task]\n",
    "        y_cv1 = y_true[task]\n",
    "        acc = accuracy_score(y_cv1, predict1)\n",
    "        conf = confusion_matrix(y_cv1, predict1, labels=sorted(list(set(y_cv1))))\n",
    "        rep = classification_report(y_cv1, predict1, target_names=sorted(list(set(y_cv1))))\n",
    "        results[task]['acc'].append(acc)\n",
    "        results[task]['conf'].append(conf)\n",
    "        results[task]['report'].append(rep)\n",
    "        #print('Accuracy : {}'.format(acc))\n",
    "        #print('Confusion matrix :\\n {}'.format(conf))\n",
    "        #print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 + 88 = 436\n",
      "0.206896551724\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TASK : gender ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Found arrays with inconsistent numbers of samples: [ 0 88]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-424771dab8bd>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     82\u001b[0m         \u001b[0mpredict1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m         \u001b[0my_cv1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m         \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m         \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m         \u001b[0mrep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_names\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.pyc\u001b[0m in \u001b[0;36maccuracy_score\u001b[1;34m(y_true, y_pred, normalize, sample_weight)\u001b[0m\n\u001b[0;32m    170\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m     \u001b[1;31m# Compute accuracy for each possible representation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m     \u001b[0my_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_check_targets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    173\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0my_type\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'multilabel'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m         \u001b[0mdiffering_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcount_nonzero\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.pyc\u001b[0m in \u001b[0;36m_check_targets\u001b[1;34m(y_true, y_pred)\u001b[0m\n\u001b[0;32m     70\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0marray\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mindicator\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m     \"\"\"\n\u001b[1;32m---> 72\u001b[1;33m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[0mtype_true\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[0mtype_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[1;34m(*arrays)\u001b[0m\n\u001b[0;32m    174\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m         raise ValueError(\"Found arrays with inconsistent numbers of samples: \"\n\u001b[1;32m--> 176\u001b[1;33m                          \"%s\" % str(uniques))\n\u001b[0m\u001b[0;32m    177\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found arrays with inconsistent numbers of samples: [ 0 88]"
     ]
    }
   ],
   "source": [
    "import random\n",
    "\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from pan import features\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "\n",
    "\n",
    "def jointed_tasks(jointed=None, sep_list=None, delim='+', jointed_to_dist=False):\n",
    "    \n",
    "    if jointed_to_dist:\n",
    "        tmp = zip(*[j.split(delim) for j in jointed])\n",
    "        out = [list(tmp_) for tmp_ in tmp]\n",
    "        return out\n",
    "    else:\n",
    "        #out = [ for i in xrange(len(sep_list)) for j in xrange(len(sep_list[0]))\n",
    "        out = []\n",
    "        for j in xrange(len(sep_list[0])):\n",
    "            tmp = ''\n",
    "            for i in xrange(len(sep_list)):\n",
    "                tmp += sep_list[i][j]+'+'\n",
    "            tmp = tmp[:-1]\n",
    "            out.append(tmp)\n",
    "        return out\n",
    "\n",
    "# DATASET\n",
    "y_dic = {}\n",
    "tasks = ['gender', 'age']\n",
    "for task in tasks:\n",
    "    _, y_dic[task] = dataset.get_data(task)\n",
    "y_both = jointed_tasks(sep_list=[y_dic[task] for task in tasks], jointed_to_dist=False)\n",
    "y_dic['both'] = y_both\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# MODEL\n",
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('soac', soac)])\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "pipe1 = Pipeline([('combined',combined), ('svm', svm)])\n",
    "\n",
    "results = {}\n",
    "results2 = {}\n",
    "for task in tasks+['both']:\n",
    "    results[task] = {'acc':[], 'conf':[], 'report':[]}\n",
    "    results2[task] = {'acc':[], 'conf':[], 'report':[]}\n",
    "    \n",
    "y_train = {'age':[], 'gender':[], 'both':[]}\n",
    "y_cv = {'age':[], 'gender':[], 'both': []}\n",
    "    \n",
    "num_folds = 2\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "N = 5\n",
    "random_seeds = random.sample(xrange(1000), N)\n",
    "for i in xrange(N):\n",
    "\n",
    "    # SPLIT\n",
    "    split = 0.2\n",
    "    X_train, X_cv, y_train['both'], y_cv['both'] = train_test_split(X, y_both, test_size=split, stratify=y_both, random_state=random_seeds[i])\n",
    "    \n",
    "    print \"%d + %d = %d\" % (len(X_train), len(X_cv), len(X))\n",
    "    for model in [pipe1]:\n",
    "        grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=0, n_jobs=-1, cv=num_folds, refit=True)\n",
    "        grid_search.fit(X_train,y_train['both'])\n",
    "        print(grid_search.best_score_)\n",
    "    # PREDICTIONS\n",
    "    predict = grid_search.predict(X_cv)\n",
    "    tasks = ['gender', 'age']\n",
    "    y_true = {'both': y_cv}\n",
    "    y_pred = {'both': predict}\n",
    "    for i, task in enumerate(tasks):\n",
    "        #y_true[task] = jointed_tasks(jointed=y_cv['both'], jointed_to_dist=True)[i]\n",
    "        y_pred[task] = jointed_tasks(jointed=predict, jointed_to_dist=True)[i]\n",
    "\n",
    "    # REPORT\n",
    "    for task in tasks+['both']:\n",
    "        print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ TASK : %s ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~' % task\n",
    "        predict1 = y_pred[task]\n",
    "        y_cv1 = y_cv[task]\n",
    "        acc = accuracy_score(y_cv1, predict1)\n",
    "        conf = confusion_matrix(y_cv1, predict1, labels=sorted(list(set(y_cv1))))\n",
    "        rep = classification_report(y_cv1, predict1, target_names=sorted(list(set(y_cv1))))\n",
    "        results[task]['acc'].append(acc)\n",
    "        results[task]['conf'].append(conf)\n",
    "        results[task]['report'].append(rep)\n",
    "        \n",
    "        for model in [pipe1]:\n",
    "            grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=0, n_jobs=-1, cv=num_folds, refit=True)\n",
    "            grid_search.fit(X_train,y_train[task])\n",
    "            print(grid_search.best_score_)\n",
    "\n",
    "        # PREDICTIONS\n",
    "        predict = grid_search.predict(X_cv)\n",
    "        y_cv1 = y_cv['task']\n",
    "        acc = accuracy_score(y_cv1, predict)\n",
    "        conf = confusion_matrix(y_cv1, predict, labels=sorted(list(set(y_cv1))))\n",
    "        rep = classification_report(y_cv1, predict, target_names=sorted(list(set(y_cv1))))\n",
    "        print 'Task = %s 2' %task\n",
    "        results2[task]['acc'].append(acc)\n",
    "        results2[task]['conf'].append(conf)\n",
    "        results2[task]['report'].append(rep)\n",
    "        print 'Ypologisa to %s Report' % task\n",
    "        \n",
    "        \n",
    "    \n",
    "        #print('Accuracy : {}'.format(acc))\n",
    "        #print('Confusion matrix :\\n {}'.format(conf))\n",
    "        #print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'age': [],\n",
       " 'both': ['male+35-49',\n",
       "  'male+35-49',\n",
       "  'male+65-xx',\n",
       "  'female+35-49',\n",
       "  'male+35-49',\n",
       "  'female+35-49',\n",
       "  'male+25-34',\n",
       "  'female+50-64',\n",
       "  'male+50-64',\n",
       "  'male+25-34',\n",
       "  'male+35-49',\n",
       "  'female+25-34',\n",
       "  'male+50-64',\n",
       "  'female+35-49',\n",
       "  'female+50-64',\n",
       "  'female+35-49',\n",
       "  'female+35-49',\n",
       "  'female+18-24',\n",
       "  'male+25-34',\n",
       "  'female+65-xx',\n",
       "  'female+50-64',\n",
       "  'male+35-49',\n",
       "  'male+35-49',\n",
       "  'male+25-34',\n",
       "  'male+25-34',\n",
       "  'female+25-34',\n",
       "  'female+25-34',\n",
       "  'male+35-49',\n",
       "  'female+25-34',\n",
       "  'male+50-64',\n",
       "  'female+35-49',\n",
       "  'male+25-34',\n",
       "  'female+35-49',\n",
       "  'male+25-34',\n",
       "  'female+18-24',\n",
       "  'male+25-34',\n",
       "  'male+50-64',\n",
       "  'male+35-49',\n",
       "  'male+25-34',\n",
       "  'male+50-64',\n",
       "  'female+35-49',\n",
       "  'male+35-49',\n",
       "  'female+35-49',\n",
       "  'female+25-34',\n",
       "  'female+35-49',\n",
       "  'female+35-49',\n",
       "  'male+35-49',\n",
       "  'female+25-34',\n",
       "  'male+25-34',\n",
       "  'male+25-34',\n",
       "  'male+35-49',\n",
       "  'female+35-49',\n",
       "  'female+18-24',\n",
       "  'female+25-34',\n",
       "  'female+25-34',\n",
       "  'female+50-64',\n",
       "  'male+25-34',\n",
       "  'female+35-49',\n",
       "  'female+25-34',\n",
       "  'male+35-49',\n",
       "  'male+18-24',\n",
       "  'male+35-49',\n",
       "  'male+18-24',\n",
       "  'male+35-49',\n",
       "  'male+35-49',\n",
       "  'female+50-64',\n",
       "  'female+50-64',\n",
       "  'female+25-34',\n",
       "  'male+35-49',\n",
       "  'male+50-64',\n",
       "  'male+35-49',\n",
       "  'male+50-64',\n",
       "  'female+50-64',\n",
       "  'female+35-49',\n",
       "  'female+35-49',\n",
       "  'male+35-49',\n",
       "  'male+25-34',\n",
       "  'male+50-64',\n",
       "  'female+25-34',\n",
       "  'male+25-34',\n",
       "  'male+18-24',\n",
       "  'female+35-49',\n",
       "  'female+25-34',\n",
       "  'female+35-49',\n",
       "  'female+25-34',\n",
       "  'female+25-34',\n",
       "  'female+50-64',\n",
       "  'female+35-49'],\n",
       " 'gender': []}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%%%%%%%%%%%%%%%%  TRAINED ON JOINT  %%%%%%%%%%%%%%%%%%%%%%%%\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  GENDER  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.638636363636\n",
      "Precision : 0.642\n",
      "Recall : 0.64\n",
      "F1 : 0.638\n",
      "Confusion matrix :\n",
      " [[ 26.6  17.4]\n",
      " [ 14.4  29.6]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  AGE  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.381818181818\n",
      "Precision : 0.426\n",
      "Recall : 0.384\n",
      "F1 : 0.392\n",
      "Confusion matrix :\n",
      " [[  1.4   3.6   1.    0.    0. ]\n",
      " [  8.   13.    6.4   0.6   0. ]\n",
      " [  6.   11.   15.4   3.6   0. ]\n",
      " [  2.6   2.    7.6   3.8   0. ]\n",
      " [  0.    0.2   1.6   0.2   0. ]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  BOTH  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.240909090909\n",
      "Precision : 0.288\n",
      "Recall : 0.242\n",
      "F1 : 0.24\n",
      "Confusion matrix :\n",
      " [[ 0.8  0.4  0.8  0.   0.   0.   0.8  0.2  0.   0. ]\n",
      " [ 3.4  2.8  1.8  0.2  0.   0.2  4.6  1.   0.   0. ]\n",
      " [ 3.2  1.   3.4  2.2  0.   0.6  3.2  4.4  0.   0. ]\n",
      " [ 0.8  0.   2.4  2.6  0.   0.2  1.   0.8  0.2  0. ]\n",
      " [ 0.   0.   0.8  0.   0.   0.   0.   0.2  0.   0. ]\n",
      " [ 0.6  0.2  0.   0.   0.   0.   2.2  0.   0.   0. ]\n",
      " [ 3.6  0.8  0.2  0.4  0.   0.8  4.8  3.4  0.   0. ]\n",
      " [ 2.2  0.6  1.   1.   0.   0.   6.2  6.6  0.4  0. ]\n",
      " [ 1.4  0.4  1.   0.8  0.   0.2  0.6  3.4  0.2  0. ]\n",
      " [ 0.   0.   0.   0.2  0.   0.   0.2  0.6  0.   0. ]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "print '%%%%%%%%%%%%%%%%  TRAINED ON JOINT  %%%%%%%%%%%%%%%%%%%%%%%%'\n",
    "\n",
    "for task in tasks+['both']:\n",
    "    num_cl = len(list(set(y_dic[task])))\n",
    "    print '~~~~~~~~~~~~~~~~~~~~~~~~~~  ' + task.upper() + '  N = ' + str(N) + '  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\n",
    "    print '#################################'\n",
    "    mean_acc = 0\n",
    "    mean_prec = 0\n",
    "    mean_rec = 0\n",
    "    mean_f1 = 0\n",
    "    conf = numpy.zeros([num_cl,num_cl])\n",
    "    for i in xrange(N):\n",
    "        mean_acc += results[task]['acc'][i]\n",
    "        #print results[key]['report'][i].split('     ')\n",
    "        mean_prec += float(results[task]['report'][i].split('     ')[-4][2:])\n",
    "        mean_rec += float(results[task]['report'][i].split('     ')[-3][2:])\n",
    "        mean_f1 += float(results[task]['report'][i].split('     ')[-2][2:])\n",
    "        conf += results[task]['conf'][i]\n",
    "    mean_acc = mean_acc/float(N)\n",
    "    mean_prec = mean_prec/float(N)\n",
    "    mean_rec = mean_rec/float(N)\n",
    "    mean_f1 = mean_f1/float(N)\n",
    "    conf = conf/float(N)\n",
    "    print('Accuracy : {}'.format(mean_acc))\n",
    "    print('Precision : {}'.format(mean_prec))\n",
    "    print('Recall : {}'.format(mean_rec))\n",
    "    print('F1 : {}'.format(mean_f1))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print '#################################'\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration i = 0 !\n",
      "Task : gender !\n",
      "Mpika sto no both\n",
      "348 + 88 = 436\n",
      "0.655172413793\n",
      "Ypologisa to gender Report\n",
      "Task : age !\n",
      "Mpika sto no both\n",
      "348 + 88 = 436\n",
      "0.408045977011\n",
      "Ypologisa to age Report\n",
      "Task : both !\n",
      "Task = both 2\n",
      "Ypologisa to both Report\n",
      "Iteration i = 1 !\n",
      "Task : gender !\n",
      "Mpika sto no both\n",
      "348 + 88 = 436\n",
      "0.614942528736\n",
      "Ypologisa to gender Report\n",
      "Task : age !\n",
      "Mpika sto no both\n",
      "348 + 88 = 436\n",
      "0.465517241379\n",
      "Ypologisa to age Report\n",
      "Task : both !\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-77-196598797bf1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     85\u001b[0m             \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m             \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 87\u001b[1;33m             \u001b[0mrep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_names\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[1;34m'Task = %s 2'\u001b[0m \u001b[1;33m%\u001b[0m\u001b[0mtask\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m             \u001b[0mresults2\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtask\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'acc'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0macc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.pyc\u001b[0m in \u001b[0;36mclassification_report\u001b[1;34m(y_true, y_pred, labels, target_names, sample_weight, digits)\u001b[0m\n\u001b[0;32m   1385\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1386\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1387\u001b[1;33m         \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mtarget_names\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1388\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mr\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1389\u001b[0m             \u001b[0mvalues\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m\"{0:0.{1}f}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdigits\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from pan import features\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "\n",
    "\n",
    "def jointed_tasks(jointed=None, sep_list=None, delim='+', jointed_to_dist=False):\n",
    "    \n",
    "    if jointed_to_dist:\n",
    "        tmp = zip(*[j.split(delim) for j in jointed])\n",
    "        out = [list(tmp_) for tmp_ in tmp]\n",
    "        return out\n",
    "    else:\n",
    "        #out = [ for i in xrange(len(sep_list)) for j in xrange(len(sep_list[0]))\n",
    "        out = []\n",
    "        for j in xrange(len(sep_list[0])):\n",
    "            tmp = ''\n",
    "            for i in xrange(len(sep_list)):\n",
    "                tmp += sep_list[i][j]+'+'\n",
    "            tmp = tmp[:-1]\n",
    "            out.append(tmp)\n",
    "        return out\n",
    "\n",
    "# DATASET\n",
    "y_dic = {}\n",
    "y_true = {}\n",
    "y_pred = {}\n",
    "tasks = ['gender', 'age']\n",
    "for task in tasks:\n",
    "    _, y_dic[task] = dataset.get_data(task)\n",
    "y_both = jointed_tasks(sep_list=[y_dic[task] for task in tasks], jointed_to_dist=False)\n",
    "y_dic['both'] = y_both\n",
    "\n",
    "\n",
    "results2 = {}\n",
    "for task in tasks+['both']:\n",
    "    results2[task] = {'acc':[], 'conf':[], 'report':[]}\n",
    "\n",
    "num_folds = 2\n",
    "split = 0.2\n",
    "N = 5\n",
    "\n",
    "for jj in xrange(N):\n",
    "    print \"Iteration i = %d !\" % jj\n",
    "    for task in ['gender', 'age', 'both']:\n",
    "        print 'Task : %s !' % task\n",
    "        if task != 'both':\n",
    "            print 'Mpika sto no both'\n",
    "            # SPLIT\n",
    "            split = 0.2\n",
    "            X_train, X_cv, y_train, y_cv = train_test_split(X, y_dic[task], test_size=split, stratify=y_dic[task], random_state=random_seeds[jj])\n",
    "            print \"%d + %d = %d\" % (len(X_train), len(X_cv), len(X))\n",
    "\n",
    "            # MODEL\n",
    "            soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "            combined = FeatureUnion([('soac', soac)])\n",
    "            svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=True)\n",
    "            pipe1 = Pipeline([('combined',combined), ('svm', svm)])\n",
    "\n",
    "\n",
    "            params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "            for model in [pipe1]:\n",
    "                grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=0, n_jobs=-1, cv=num_folds, refit=True)\n",
    "                grid_search.fit(X_train,y_train)\n",
    "                print(grid_search.best_score_)\n",
    "                #print(grid_search.best_estimator_) \n",
    "                #trained_models.append(grid_search.best_estimator_)\n",
    "\n",
    "            # PREDICTIONS\n",
    "            predict = grid_search.predict(X_cv)\n",
    "            y_true[task] = y_cv\n",
    "            y_pred[task] = predict\n",
    "            acc = accuracy_score(y_cv, predict)\n",
    "            conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "            rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "            results2[task]['acc'].append(acc)\n",
    "            results2[task]['conf'].append(conf)\n",
    "            results2[task]['report'].append(rep)\n",
    "        elif task=='both':\n",
    "            predict = jointed_tasks(sep_list=[y_pred[task1] for task1 in tasks], jointed_to_dist=False)\n",
    "            y_cv = jointed_tasks(sep_list=[y_true[task2] for task2 in tasks], jointed_to_dist=False)\n",
    "            acc = accuracy_score(y_cv, predict)\n",
    "            conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "            rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "            print 'Task = %s 2' %task\n",
    "            results2[task]['acc'].append(acc)\n",
    "            results2[task]['conf'].append(conf)\n",
    "            results2[task]['report'].append(rep)\n",
    "        print 'Ypologisa to %s Report' % task\n",
    "        \n",
    "        #print('Accuracy : {}'.format(acc))\n",
    "        #print('Confusion matrix :\\n {}'.format(conf))\n",
    "        #print('Classification report :\\n {}'.format(rep))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'acc': [], 'conf': [], 'report': []}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%%%%%%%%%%%%%%%%  TRAINED ON DISJOINT  %%%%%%%%%%%%%%%%%%%%%%%%\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  GENDER  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.697727272727\n",
      "Precision : 0.71\n",
      "Recall : 0.696\n",
      "F1 : 0.694\n",
      "Confusion matrix :\n",
      " [[ 27.   17. ]\n",
      " [  9.6  34.4]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  AGE  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.388636363636\n",
      "Precision : 0.398\n",
      "Recall : 0.388\n",
      "F1 : 0.364\n",
      "Confusion matrix :\n",
      " [[  1.2   3.8   1.    0.    0. ]\n",
      " [  2.6  13.6  11.6   0.2   0. ]\n",
      " [  2.2  15.2  18.    1.6   0. ]\n",
      " [  0.6   6.    8.    1.4   0. ]\n",
      " [  0.    0.2   0.8   0.    0. ]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~  BOTH  N = 5  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "#################################\n",
      "Accuracy : 0.265909090909\n",
      "Precision : 0.34\n",
      "Recall : 0.266\n",
      "F1 : 0.28\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "print '%%%%%%%%%%%%%%%%  TRAINED ON DISJOINT  %%%%%%%%%%%%%%%%%%%%%%%%'\n",
    "\n",
    "for task in tasks+['both']:\n",
    "    num_cl = len(list(set(y_dic[task])))\n",
    "    print '~~~~~~~~~~~~~~~~~~~~~~~~~~  ' + task.upper() + '  N = ' + str(N) + '  ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\n",
    "    print '#################################'\n",
    "    mean_acc = 0\n",
    "    mean_prec = 0\n",
    "    mean_rec = 0\n",
    "    mean_f1 = 0\n",
    "    conf = numpy.zeros([num_cl,num_cl])\n",
    "    for i in xrange(N):\n",
    "        mean_acc += results2[task]['acc'][i]\n",
    "        #print results[key]['report'][i].split('     ')\n",
    "        mean_prec += float(results2[task]['report'][i].split('     ')[-4][2:])\n",
    "        mean_rec += float(results2[task]['report'][i].split('     ')[-3][2:])\n",
    "        mean_f1 += float(results2[task]['report'][i].split('     ')[-2][2:])\n",
    "        if task !='both':\n",
    "            conf += results2[task]['conf'][i]\n",
    "    mean_acc = mean_acc/float(N)\n",
    "    mean_prec = mean_prec/float(N)\n",
    "    mean_rec = mean_rec/float(N)\n",
    "    mean_f1 = mean_f1/float(N)\n",
    "    if task !='both':\n",
    "        conf = conf/float(N)\n",
    "    print('Accuracy : {}'.format(mean_acc))\n",
    "    print('Precision : {}'.format(mean_prec))\n",
    "    print('Recall : {}'.format(mean_rec))\n",
    "    print('F1 : {}'.format(mean_f1))\n",
    "    if task !='both':\n",
    "        print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print '#################################'\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['18-24', '25-34', '35-49', '50-64', '65-xx']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sorted(list(set(y_true['gender'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "predictions = []\n",
    "for model in trained_models:\n",
    "    predict = model.predict(X_cv)\n",
    "    predictions.append(predict)\n",
    "    acc = accuracy_score(y_cv, predict)\n",
    "    conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "predict = grid_search.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3grams + soa + Soac Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "#feature_names = grid_search.best_estimator_.steps[0][1].__dict__['transformer_list'][0][1].get_feature_names()\n",
    "#print len(set(y))\n",
    "feature_names = []\n",
    "soa_feat_names = [\"soa_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "soac_feat_names = [\"soac_prob_\"+str(i) for i in range(0, len(set(y)))]\n",
    "#feature_names += soa_feat_names\n",
    "feature_names += soac_feat_names\n",
    "print len(feature_names)\n",
    "feature_names = [feat.encode('utf-8') for feat in feature_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('combined', FeatureUnion(n_jobs=1, transformer_list=[('soac', Test_SOAC())],\n",
       "         transformer_weights=None)),\n",
       " ('svm',\n",
       "  LinearSVC(C=0.1, class_weight='balanced', dual=False, fit_intercept=True,\n",
       "       intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "       multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "       verbose=0))]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
      "      tokenizer_var='sklearn')\n"
     ]
    }
   ],
   "source": [
    "#a = pipe1.steps[0][1]\n",
    "a = trained_models[1].steps[0][1]\n",
    "#a = grid_search.best_estimator_.steps[0][1]\n",
    "print a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soacc.counter.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soacc = a.transformer_list[0][1]\n",
    "representatives = []\n",
    "voc2 = {}\n",
    "for word, ind in soacc.counter.vocabulary_.iteritems():\n",
    "    voc2[ind] = word\n",
    "for x in X_cv+X_train:\n",
    "    cc = soacc.counter.transform([x]).toarray()[0]\n",
    "    res = [voc2[word_ind] for word_ind in cc.nonzero()[0]]\n",
    "    representatives.append(res)\n",
    "    #break\n",
    "#print res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are transforming!\n",
      "LSI Transform:\n",
      "(436, 100)\n",
      "       soac_prob_0  soac_prob_1\n",
      "count   436.000000   436.000000\n",
      "mean      6.082526     6.027155\n",
      "std       1.759293     1.570929\n",
      "min       0.000000     0.000000\n",
      "25%       5.025053     5.356651\n",
      "50%       6.382039     6.257160\n",
      "75%       7.173403     6.919514\n",
      "max      12.577487    12.162403\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data = pd.DataFrame(a.transform(X), columns=feature_names)\n",
    "data[\"class\"] = y\n",
    "data['class_pred'] = grid_search.predict(X)\n",
    "data['text'] = X\n",
    "print(data.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are transforming!\n",
      "LSI Transform:\n",
      "(66, 100)\n",
      "       soac_prob_0  soac_prob_1\n",
      "count    66.000000    66.000000\n",
      "mean      5.904950     5.809555\n",
      "std       1.356484     1.215079\n",
      "min       2.760330     2.567185\n",
      "25%       5.156809     5.141473\n",
      "50%       6.274761     6.293834\n",
      "75%       6.920075     6.655433\n",
      "max       9.342234     7.765831\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data_cv = pd.DataFrame(a.transform(X_cv), columns=feature_names)\n",
    "data_cv[\"class\"] = y_cv\n",
    "data_cv['class_pred'] = grid_search.predict(X_cv)\n",
    "data_cv['text'] = X_cv\n",
    "print(data_cv.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We are transforming!\n",
      "LSI Transform:\n",
      "(300, 100)\n",
      "       soac_prob_0  soac_prob_1\n",
      "count   300.000000   300.000000\n",
      "mean      6.208870     6.173479\n",
      "std       1.756749     1.562239\n",
      "min       1.680606     1.123313\n",
      "25%       5.025053     5.407609\n",
      "50%       6.430265     6.264118\n",
      "75%       7.450018     7.114691\n",
      "max      12.577487    12.162403\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "data_train = pd.DataFrame(a.transform(X_train), columns=feature_names)\n",
    "data_train[\"class\"] = y_train\n",
    "data_train['class_pred'] = grid_search.predict(X_train)\n",
    "data_train['text'] = X_train\n",
    "print(data_train.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_full[data_full['class']!=data_full['class_pred']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(data.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soacc = a.transformer_list[0][1]\n",
    "voc = soacc.counter.vocabulary_\n",
    "print 'Voc: ' + str(len(voc))\n",
    "print soacc.term_table.shape\n",
    "#terms= ['marriage', 'pension']\n",
    "#graph_matrix = numpy.zeros([len(terms), soacc.term_table.shape[1]])\n",
    "j = 0 \n",
    "for term, index in voc.iteritems():\n",
    "    l = list(soacc.term_table[index,:])\n",
    "    if l.index(min(l))==3 and  min(l)<0.02 and min(l)!=0:\n",
    "        print term\n",
    "        print l\n",
    "        j += 1\n",
    "    if j==1000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "import numpy\n",
    "py.sign_in('Bogas', '9s60rarm2w')\n",
    "soacc = a.transformer_list[0][1]\n",
    "voc = soacc.counter.vocabulary_\n",
    "print 'Voc: ' + str(len(voc))\n",
    "print soacc.term_table.shape\n",
    "terms= ['dreamjob','lol', 'mortgage', 'booksellers', 'juvenile']\n",
    "graph_matrix = numpy.zeros([len(terms), soacc.term_table.shape[1]])\n",
    "j = 0\n",
    "for term in terms:\n",
    "    idx = voc[term]\n",
    "    print term\n",
    "    print soacc.term_table[idx,:]\n",
    "    graph_matrix[j, :] = soacc.term_table[idx,:]\n",
    "    j += 1\n",
    "    #plt.bar(numpy.arange(soacc.term_table.shape[1]), soacc.term_table[idx,:], color='r')\n",
    "    #plt.show()\n",
    "\n",
    "data = []\n",
    "names = sorted(list(set(y)))\n",
    "for i in range(0, soacc.term_table.shape[1]):\n",
    "    data.append(\n",
    "        go.Bar(\n",
    "        x=terms,\n",
    "        y=graph_matrix[:, i],\n",
    "        name=names[i]\n",
    "    )\n",
    "    )\n",
    "layout = go.Layout(\n",
    "    barmode='group'\n",
    ")\n",
    "fig = go.Figure(data=data, layout=layout)\n",
    "py.iplot(fig)\n",
    "#plot_url = py.plot(fig, filename='grouped-bar')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from numpy.random import randint\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "grouped = data.groupby('class')\n",
    "rowlength = grouped.ngroups/2                         # fix up if odd number of groups\n",
    "fig, axs = plt.subplots(figsize=(9,4), \n",
    "                        nrows=2, ncols=rowlength,     # fix as above\n",
    "                        gridspec_kw=dict(hspace=0.4)) # Much control of gridspec\n",
    "\n",
    "targets = zip(grouped.groups.keys(), axs.flatten())\n",
    "print targets\n",
    "grouped.get_group('18-24').hist(alpha=0.4)\n",
    "#for i, (key, ax) in enumerate(targets):\n",
    "#    ax.plot(grouped.get_group(key))\n",
    "#    ax.set_title('a=%s'%str(key))\n",
    "#ax.legend()\n",
    "#plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>class</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>soac_prob_0</th>\n",
       "      <td>5.509971</td>\n",
       "      <td>6.655081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>soac_prob_1</th>\n",
       "      <td>6.243043</td>\n",
       "      <td>5.811268</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "class          female      male\n",
       "soac_prob_0  5.509971  6.655081\n",
       "soac_prob_1  6.243043  5.811268"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grouped = data.groupby('class')\n",
    "grouped.mean().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "### BAR PLOTS OF MEAN VALUE OF FEATURES FOR EACH CLASS ######\n",
    "\n",
    "grouped = data.groupby('class')\n",
    "plt.figure()\n",
    "grouped.mean().T.plot(kind='bar', figsize=(60,10))\n",
    "plt.savefig('test1.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "##### Distribution over a feature for each class #####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "soac_prob_0\n",
      "soac_prob_1\n",
      "class\n",
      "class_pred\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "Empty 'DataFrame': no numeric data to plot",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-81-9a3e844df981>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     13\u001b[0m         \u001b[0mj\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     14\u001b[0m         \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 15\u001b[1;33m         \u001b[0mgrouped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkind\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'kde'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.8\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlegend\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgrouped\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgroups\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtitle\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     16\u001b[0m     \u001b[1;31m#g = grouped[key]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m     \u001b[1;31m#print grouped[key].mean()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mplot\u001b[1;34m(data, kind, ax, figsize, use_index, title, grid, legend, style, logx, logy, loglog, xticks, yticks, xlim, ylim, rot, fontsize, colormap, table, yerr, xerr, label, secondary_y, **kwds)\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    562\u001b[0m             \u001b[1;31m# exception below\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32min\u001b[0m \u001b[0m_plotting_methods\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 564\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcurried\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, *args, **kwargs)\u001b[0m\n\u001b[0;32m    669\u001b[0m         \u001b[1;31m# ignore SettingWithCopy here in case the user mutates\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    670\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0moption_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'mode.chained_assignment'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 671\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_python_apply_general\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    672\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    673\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_python_apply_general\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36m_python_apply_general\u001b[1;34m(self, f)\u001b[0m\n\u001b[0;32m    673\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_python_apply_general\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    674\u001b[0m         keys, values, mutated = self.grouper.apply(f, self._selected_obj,\n\u001b[1;32m--> 675\u001b[1;33m                                                    self.axis)\n\u001b[0m\u001b[0;32m    676\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    677\u001b[0m         return self._wrap_applied_output(keys, values,\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, f, data, axis)\u001b[0m\n\u001b[0;32m   1292\u001b[0m             \u001b[1;31m# group might be modified\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1293\u001b[0m             \u001b[0mgroup_axes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_get_axes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1294\u001b[1;33m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1295\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0m_is_indexed_like\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mres\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgroup_axes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1296\u001b[0m                 \u001b[0mmutated\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mf\u001b[1;34m(g)\u001b[0m\n\u001b[0;32m    665\u001b[0m         \u001b[1;33m@\u001b[0m\u001b[0mwraps\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    666\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 667\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    668\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    669\u001b[0m         \u001b[1;31m# ignore SettingWithCopy here in case the user mutates\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/core/groupby.pyc\u001b[0m in \u001b[0;36mcurried\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    553\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    554\u001b[0m             \u001b[1;32mdef\u001b[0m \u001b[0mcurried\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 555\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    556\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    557\u001b[0m             \u001b[1;31m# preserve the name so we can detect it when calling plot methods,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36mplot_series\u001b[1;34m(data, kind, ax, figsize, use_index, title, grid, legend, style, logx, logy, loglog, xticks, yticks, xlim, ylim, rot, fontsize, colormap, table, yerr, xerr, label, secondary_y, **kwds)\u001b[0m\n\u001b[0;32m   2517\u001b[0m                  \u001b[0myerr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0myerr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mxerr\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mxerr\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2518\u001b[0m                  \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlabel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msecondary_y\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msecondary_y\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2519\u001b[1;33m                  **kwds)\n\u001b[0m\u001b[0;32m   2520\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2521\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36m_plot\u001b[1;34m(data, x, y, subplots, ax, kind, **kwds)\u001b[0m\n\u001b[0;32m   2322\u001b[0m         \u001b[0mplot_obj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mklass\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msubplots\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msubplots\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0max\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkind\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkind\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2323\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2324\u001b[1;33m     \u001b[0mplot_obj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgenerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2325\u001b[0m     \u001b[0mplot_obj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdraw\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2326\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mplot_obj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36mgenerate\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    910\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mgenerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    911\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_args_adjust\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 912\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_compute_plot_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    913\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_setup_subplots\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    914\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_plot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/pandas/tools/plotting.pyc\u001b[0m in \u001b[0;36m_compute_plot_data\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1015\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_empty\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1016\u001b[0m             raise TypeError('Empty {0!r}: no numeric data to '\n\u001b[1;32m-> 1017\u001b[1;33m                             'plot'.format(numeric_data.__class__.__name__))\n\u001b[0m\u001b[0;32m   1018\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1019\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnumeric_data\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: Empty 'DataFrame': no numeric data to plot"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "grouped = data.groupby('class')\n",
    "import numpy\n",
    "ncol = 4 # pick one dimension\n",
    "nrow = (len(feature_names)+ ncol-1) / ncol # make sure enough subplots\n",
    "#fig, ax = plt.subplots(nrows=nrow, ncols=ncol, figsize=(6,6)) # create the axes\n",
    "j = 0\n",
    "for key in list(data.columns.values):\n",
    "#    ix = numpy.unravel_index(j, ax.shape)\n",
    "#    print ix\n",
    "    print key\n",
    "    if key!='class':\n",
    "        j += 1\n",
    "        plt.figure(j, figsize=(10,10))\n",
    "        grouped[key].plot(kind='kde', alpha=0.8, legend=grouped.groups.keys(), title=key)\n",
    "    #g = grouped[key]\n",
    "    #print grouped[key].mean()\n",
    "    #if j==1:\n",
    "    #    tmp = g.mean()\n",
    "    #else:\n",
    "    #    print g.mean()\n",
    "    #    tmp.append(g.mean())\n",
    "    #print tmp\n",
    "        plt.show()\n",
    "    #if j==2:\n",
    "    #    break\n",
    "#tmp\n",
    "    #break\n",
    "    #ax[ix] = grouped[key].plot(kind='kde', alpha=0.4, legend=grouped.groups.keys())\n",
    "    #break\n",
    "#for key in grouped.keys:\n",
    "#    grouped[key].plot(kind='kde', alpha=0.4, legend=grouped.groups.keys())\n",
    "#for key in grouped.groups.keys():\n",
    "#    b = grouped.get_group(key)\n",
    "#    b.plot('kin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for index, row in data_cv.iterrows():\n",
    "    print row['soac_prob_0']\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.590909090909\n",
      "Confusion matrix :\n",
      " [[21 12]\n",
      " [15 18]]\n"
     ]
    }
   ],
   "source": [
    "def linear_binary_pred(pandas_frame, a, b, featx, featy, upper_class, lower_class):\n",
    "    y = []\n",
    "    # Line is: y_pred = a*x+b\n",
    "    for index, row in pandas_frame.iterrows():\n",
    "        # If over the line -> male\n",
    "        if row[featy] > a*row[featx] + b:\n",
    "            y.append(upper_class)\n",
    "        else:\n",
    "            y.append(lower_class)\n",
    "    return y\n",
    "\n",
    "predict = linear_binary_pred(data_cv, 0.5, 3, 'soac_prob_0', 'soac_prob_1', 'female', 'male')\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.030000\n",
      "         Iterations: 8\n",
      "         Function evaluations: 27\n",
      "Best Line: y=0.5375*x + 2.7000 with linear optim score: 0.97000\n",
      "Accuracy : 0.575757575758\n",
      "Confusion matrix :\n",
      " [[22 11]\n",
      " [17 16]]\n",
      "Accuracy : 0.575757575758\n",
      "Confusion matrix :\n",
      " [[22 11]\n",
      " [17 16]]\n"
     ]
    }
   ],
   "source": [
    "from scipy.optimize import minimize, brute\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "\n",
    "global data_train, y_train\n",
    "\n",
    "def f(lin):\n",
    "    #print \"Weights\"\n",
    "    #print lin\n",
    "    score = 1 - accuracy_score(y_train, linear_binary_pred(data_train, lin[0], lin[1], 'soac_prob_0', 'soac_prob_1', 'female', 'male'))\n",
    "    #print 'Score: ' + str(score)\n",
    "    return score\n",
    "\n",
    "# Linear Optimization\n",
    "lin = [0.5,3]\n",
    "options={'disp': True, 'maxiter': 1000000, 'xtol': 0.01, 'ftol': 0.1}\n",
    "minim = minimize(f, lin, args=(), method='Nelder-Mead', options=options)\n",
    "print 'Best Line: y=%0.4f*x + %0.4f with linear optim score: %0.5f' % (minim.x[0], minim.x[1], 1-minim.fun)\n",
    "predict = linear_binary_pred(data_cv, minim.x[0], minim.x[1], 'soac_prob_0', 'soac_prob_1', 'female', 'male')\n",
    "\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_train))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "\n",
    "# Brute Grid Search\n",
    "\n",
    "#rranges = [(0,1),(-10,10)]\n",
    "#minim = brute(f, rranges,args=(), Ns=100)\n",
    "#print 'Best Line: y=%0.4f*x + %0.4f with grid search score: %0.5f' % (minim[0], minim[1], 1-f(minim))\n",
    "#predict = linear_binary_pred(data_cv, minim[0], minim[1], 'soac_prob_0', 'soac_prob_1', 'female', 'male')\n",
    "\n",
    "\n",
    "\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_train))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def linear_map(X, a ,b):\n",
    "    y = []\n",
    "    for x in X:\n",
    "        y.append(a*x+b)\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "xx = numpy.linspace(data['soac_prob_0'].min(), data['soac_prob_0'].max(), 1000)\n",
    "yy = linear_map(xx, minim.x[0],minim.x[1])\n",
    "#yy = linear_map(xx, minim[0],minim[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "rbf_svc = trained_models[0].steps[1][1]\n",
    "#rbf_svc = grid_search.best_estimator_.steps[1][1]\n",
    "gender2color = {'female':'r', 'male':'b'}\n",
    "gender2id = {'female':0, 'male':1}\n",
    "y_color = [gender2color[y]for y in list(data['class'])]\n",
    "y_id = [gender2id[y]for y in list(data['class'])]\n",
    "y_cv_id = [gender2id[y]for y in list(data_cv['class'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.636363636364\n",
      "Confusion matrix :\n",
      " [[20 13]\n",
      " [11 22]]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/matplotlib/collections.py:590: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  if self._edgecolors == str('face'):\n"
     ]
    }
   ],
   "source": [
    "# Linear SVM\n",
    "import matplotlib.pyplot as plt\n",
    "clf = SVC(kernel='linear', C=1)\n",
    "clf.fit(a.transform(X_train),y_train)\n",
    "predict = clf.predict(a.transform(X_cv))\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_train))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "\n",
    "w = clf.coef_[0]\n",
    "slope = -w[0] / w[1]\n",
    "yy_svm = slope * xx - (clf.intercept_[0]) / w[1]\n",
    "plt.scatter\n",
    "plt.plot(xx, yy_svm, 'k-')\n",
    "plt.scatter(data['soac_prob_0'], data['soac_prob_1'], c=y_id, cmap=plt.cm.Paired)\n",
    "plt.scatter(data_cv['soac_prob_0'], data_cv['soac_prob_1'], c=y_cv_id, s=80,cmap=plt.cm.Paired)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "rbf_svc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# RBF SCV\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "rbf_svc = trained_models[0].steps[1][1]\n",
    "#rbf_svc = grid_search.best_estimator_.steps[1][1]\n",
    "gender2color = {'female':'r', 'male':'b'}\n",
    "gender2id = {'female':0, 'male':1}\n",
    "y_color = [gender2color[y]for y in list(data['class'])]\n",
    "y_id = [gender2id[y]for y in list(data['class'])]\n",
    "y_cv_id = [gender2id[y]for y in list(data_cv['class'])]\n",
    "\n",
    "\n",
    "# create a mesh to plot in\n",
    "h = .02  # step size in the mesh\n",
    "\n",
    "\n",
    "# create a mesh to plot in\n",
    "x_min, x_max = data['soac_prob_0'].min() - 1, data['soac_prob_0'].max() + 1\n",
    "y_min, y_max = data['soac_prob_1'].min() - 1, data['soac_prob_1'].max() + 1\n",
    "xxx, yyy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "\n",
    "# Plot the decision boundary. For that, we will assign a color to each\n",
    "# point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "Z = clf.predict(np.c_[xxx.ravel(), yyy.ravel()])\n",
    "\n",
    "Z = Z.reshape(xxx.shape)\n",
    "Z = np.vectorize(gender2id.get)(Z)\n",
    "plt.contourf(xxx, yyy, Z, cmap=plt.cm.Paired, alpha=0.8)\n",
    "\n",
    "#plt.contour(xxx, yyy, Z, cmap=plt.cm.Paired)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Find the convex hull of RBF CONTOUR\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy\n",
    "\n",
    "rbf_svc = trained_models[0].steps[1][1]\n",
    "#rbf_svc = grid_search.best_estimator_.steps[1][1]\n",
    "gender2color = {'female':'r', 'male':'b'}\n",
    "gender2id = {'female':0, 'male':1}\n",
    "y_color = [gender2color[y]for y in list(data['class'])]\n",
    "y_id = [gender2id[y]for y in list(data['class'])]\n",
    "y_cv_id = [gender2id[y]for y in list(data_cv['class'])]\n",
    "\n",
    "\n",
    "# create a mesh to plot in\n",
    "h = .02  # step size in the mesh\n",
    "\n",
    "\n",
    "# create a mesh to plot in\n",
    "x_min, x_max = data['soac_prob_0'].min() - 1, data['soac_prob_0'].max() + 1\n",
    "y_min, y_max = data['soac_prob_1'].min() - 1, data['soac_prob_1'].max() + 1\n",
    "xxx, yyy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "\n",
    "# Plot the decision boundary. For that, we will assign a color to each\n",
    "# point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "Z = rbf_svc.predict(np.c_[xxx.ravel(), yyy.ravel()])\n",
    "\n",
    "Z = Z.reshape(xxx.shape)\n",
    "Z = np.vectorize(gender2id.get)(Z)\n",
    "points = np.vstack((xxx[Z.nonzero()], yyy[Z.nonzero()])).T\n",
    "hull = scipy.spatial.ConvexHull(points)\n",
    "hull_points = points[hull.vertices]\n",
    "plt.plot(hull_points[:,0],hull_points[:,1])\n",
    "    #plt.plot(points[simplex, 0], points[simplex, 1], 'k-')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "points[hull.vertices].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/plotly/plotly/plotly.py:236: UserWarning:\n",
      "\n",
      "Woah there! Look at all those points! Due to browser limitations, the Plotly SVG drawing functions have a hard time graphing more than 500k data points for line charts, or 40k points for other types of charts. Here are some suggestions:\n",
      "(1) Use the `plotly.graph_objs.Scattergl` trace object to generate a WebGl graph.\n",
      "(2) Trying using the image API to return an image instead of a graph URL\n",
      "(3) Use matplotlib\n",
      "(4) See if you can create your visualization with fewer data points\n",
      "\n",
      "If the visualization you're using aggregates points (e.g., box plot, histogram, etc.) you can disregard this warning.\n",
      "\n",
      "/usr/local/lib/python2.7/dist-packages/plotly/plotly/plotly.py:1441: UserWarning:\n",
      "\n",
      "Estimated Draw Time Too Long\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The draw time for this plot will be slow for all clients.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~Bogas/941.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#fig['data'].pop(4)\n",
    "import plotly.plotly as py\n",
    "py.iplot(fig, filename='Scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig['data'].append(svm_2feat_vis(rbf_svc, data, 'SOAC_RBF', 'rbf', gender2id=gender2id))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "dffemale = data[data['class']=='female']\n",
    "dfmale = data[data['class']=='male']\n",
    "data.head(2)\n",
    "fig = {\n",
    "    'data': [\n",
    "  \t\t{\n",
    "  \t\t\t'x': data_train[data_train['class']=='female'].soac_prob_0, \n",
    "        \t'y': data_train[data_train['class']=='female'].soac_prob_1, \n",
    "            'text': data_train[data_train['class']=='female'].text,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'female'},\n",
    "        {\n",
    "        \t'x': data_train[data_train['class']=='male'].soac_prob_0, \n",
    "        \t'y': data_train[data_train['class']=='male'].soac_prob_1, \n",
    "            'text': data_train[data_train['class']=='male'].text,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'male'},\n",
    "        {\n",
    "  \t\t\t'x': data_cv[data_cv['class']=='female'].soac_prob_0, \n",
    "        \t'y': data_cv[data_cv['class']=='female'].soac_prob_1, \n",
    "            'text': data_cv[data_cv['class']=='female'].text,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'female_test'},\n",
    "        {\n",
    "        \t'x': data_cv[data_train['class']=='male'].soac_prob_0, \n",
    "        \t'y': data_cv[data_train['class']=='male'].soac_prob_1, \n",
    "            'text': data_cv[data_cv['class']=='male'].text,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'male_test'},\n",
    "        {\n",
    "            'x': hull_points[:,0],\n",
    "            'y': hull_points[:,1],\n",
    "            'mode': 'markers',\n",
    "            'fillcolor': 'black',\n",
    "            'marker': {'color': 'rgba(0, 0, 0, 1)'},\n",
    "            'name': 'SVM_RBF'\n",
    "        },\n",
    "        {\n",
    "            'x': xx,\n",
    "            'y': yy,\n",
    "            'name':'Lin_Pred'\n",
    "        },\n",
    "        {\n",
    "            'x': xx,\n",
    "            'y': xx,\n",
    "            'name':'Max_Bayes'\n",
    "        },\n",
    "        {\n",
    "            'x': xx,\n",
    "            'y': yy_svm,\n",
    "            'name':'Linear SVM'\n",
    "        },\n",
    "        #go.Contour(\n",
    "        #    z= Z,\n",
    "        #    colorscale='jet',\n",
    "        #    contours=dict(\n",
    "        #    coloring='lines',\n",
    "        #),\n",
    "        #)\n",
    "    ],\n",
    "    'layout': {\n",
    "        'xaxis': {'title': 'Complimentary Probability of Female'},\n",
    "        'yaxis': {'title': \"Complimentary Probability of Male\"}\n",
    "    }\n",
    "}\n",
    "\n",
    "# IPython notebook\n",
    "# py.iplot(fig, filename='pandas/multiple-scatter')\n",
    "py.iplot(fig, filename='Scatter')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FeatureUnion(n_jobs=1, transformer_list=[('soac', Test_SOAC())],\n",
      "       transformer_weights=None)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#a = trained_models[1].steps[0][1]\n",
    "#a = grid_search.best_estimator_.steps[0][1]\n",
    "print a\n",
    "data = pd.DataFrame(a.transform(X), columns=feature_names)\n",
    "data[\"class\"] = y\n",
    "data['class_pred'] = grid_search.predict(X)\n",
    "data_cv = pd.DataFrame(a.transform(X_cv), columns=feature_names)\n",
    "data_cv[\"class\"] = y_cv\n",
    "data_cv['class_pred'] = grid_search.predict(X_cv)\n",
    "data_train = pd.DataFrame(a.transform(X_train), columns=feature_names)\n",
    "data_train[\"class\"] = y_train\n",
    "data_train['class_pred'] = grid_search.predict(X_train)\n",
    "gender2id = {'female':0, 'male':1}\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/pandas/core/frame.py:1825: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  \"DataFrame index.\", UserWarning)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'copy' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-bd8d5baaa336>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     31\u001b[0m         ]\n\u001b[0;32m     32\u001b[0m     }\n\u001b[1;32m---> 33\u001b[1;33m \u001b[0mtrained_models2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdeepcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrained_models\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     34\u001b[0m \u001b[0mtrained_models2\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mspace\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mMeta\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel_names\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'space'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'meta'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'copy' is not defined"
     ]
    }
   ],
   "source": [
    "fig = {\n",
    "    'data': [\n",
    "  \t\t{\n",
    "  \t\t\t'x': data_train[data_train['class']=='female'].soac_prob_0, \n",
    "        \t'y': data_train[data_train['class']=='female'].soac_prob_1, \n",
    "            'text': data_train[data_train['class']=='female'].class_pred,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'female'},\n",
    "        {\n",
    "        \t'x': data_train[data_train['class']=='male'].soac_prob_0, \n",
    "        \t'y': data_train[data_train['class']=='male'].soac_prob_1, \n",
    "            'text': data_train[data_train['class']=='male'].class_pred,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'male'},\n",
    "        {\n",
    "  \t\t\t'x': data_cv[data_cv['class']=='female'].soac_prob_0, \n",
    "        \t'y': data_cv[data_cv['class']=='female'].soac_prob_1, \n",
    "            'text': data_cv[data_cv['class']=='female'].class_pred,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'female_test'},\n",
    "        {\n",
    "        \t'x': data_cv[data_train['class']=='male'].soac_prob_0, \n",
    "        \t'y': data_cv[data_train['class']=='male'].soac_prob_1, \n",
    "            'text': data_cv[data_cv['class']=='male'].class_pred,\n",
    "            'marker': {'opacity': 0.6},\n",
    "        \t'mode': 'markers', \n",
    "        \t'name': 'male_test'}\n",
    "        ]\n",
    "    }\n",
    "trained_models2 = copy.deepcopy(trained_models)\n",
    "trained_models2 += [space, Meta]\n",
    "for i, name in enumerate(model_names + ['space'] + ['meta']):\n",
    "    print name\n",
    "    if 'steps' in trained_models2[i].__dict__.keys():\n",
    "        for step in trained_models2[i].steps:\n",
    "            if step[0]=='svm':\n",
    "                print step[1]\n",
    "        #print trained_models2[i].steps\n",
    "    else:\n",
    "        print trained_models2[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transform2d  = Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for model in trained_models:\n",
    "    if 'steps' in model.__dict__.keys():\n",
    "        for step in model.steps:\n",
    "            if step[0]=='soac':\n",
    "                transform2d = step[1]\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def svm_2feat_vis(clf, data_X, name, clf_type, gender2id=None, transformer=None, y_true=None, feat0='soac_prob_0', feat1='soac_prob_1'):\n",
    "    import numpy as np\n",
    "    import scipy\n",
    "    import pandas as pd\n",
    "    \n",
    "    vis_dic = {}\n",
    "    if clf_type == 'linear':\n",
    "        xx = numpy.linspace(data_X[feat0].min(), data_X[feat1].max(), 1000)\n",
    "        w = clf.coef_[0]\n",
    "        slope = -w[0] / w[1]\n",
    "        yy_lin = slope * xx - (clf.intercept_[0]) / w[1]\n",
    "        vis_dic['x'] = xx\n",
    "        vis_dic['y'] = yy_lin\n",
    "        vis_dic['name'] = name+ ' ('+clf_type+' SVM)'\n",
    "    elif clf_type == 'rbf':\n",
    "        xx = numpy.linspace(data_X[feat0].min(), data_X[feat1].max(), 1000)\n",
    "        h = .02\n",
    "        x_min, x_max = data_X[feat0].min() - 1, data_X[feat0].max() + 1\n",
    "        y_min, y_max = data_X[feat1].min() - 1, data_X[feat1].max() + 1\n",
    "        xxx, yyy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "        # Plot the decision boundary. For that, we will assign a color to each\n",
    "        # point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "        Z = clf.predict(np.c_[xxx.ravel(), yyy.ravel()])\n",
    "        Z = Z.reshape(xxx.shape)\n",
    "        Z = np.vectorize(gender2id.get)(Z)\n",
    "        points = np.vstack((xxx[Z.nonzero()], yyy[Z.nonzero()])).T\n",
    "        hull = scipy.spatial.ConvexHull(points)\n",
    "        hull_points = points[hull.vertices]\n",
    "        vis_dic['x'] = hull_points[:,0]\n",
    "        vis_dic['y'] = hull_points[:,1]\n",
    "        vis_dic['name'] = name + ' ('+clf_type+' SVM)'\n",
    "        vis_dic['mode'] = 'markers'\n",
    "    elif clf_type=='voting':\n",
    "        data = pd.DataFrame(transformer.transform(data_X), columns=[feat0, feat1])\n",
    "        data[\"class\"] = y_true\n",
    "        xx = np.array(data[feat0])\n",
    "        yy = np.array(data[feat1])\n",
    "        # Plot the decision boundary. For that, we will assign a color to each\n",
    "        # point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "        #print data.describe(10)\n",
    "        Z = clf.predict(data_X)\n",
    "        Z = Z.reshape(len(data_X))\n",
    "        Z = np.vectorize(gender2id.get)(Z)\n",
    "        vis_dic['x'] = xx[Z.nonzero()]\n",
    "        vis_dic['y'] = yy[Z.nonzero()]\n",
    "        points = np.vstack((xx[Z.nonzero()], yy[Z.nonzero()])).T\n",
    "        hull = scipy.spatial.ConvexHull(points)\n",
    "        hull_points = points[hull.vertices]\n",
    "        vis_dic['x'] = hull_points[:,0]\n",
    "        vis_dic['y'] = hull_points[:,1]\n",
    "        vis_dic['name'] = name + ' ('+clf_type+' SVM)'\n",
    "        vis_dic['mode'] = 'lines+markers'\n",
    "        vis_dic['line'] = line=dict(\n",
    "                            shape='spline'\n",
    "                            )\n",
    "    return vis_dic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig['data'].append(svm_2feat_vis(trained_models[0].steps[1][1], data, 'SOAC_Linear', 'linear', gender2id=gender2id, transformer=transform2d, y_true=y))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "fig['data'].append(svm_2feat_vis(trained_models2[2], X, 'SOAC_RBF', 'voting', gender2id=gender2id, transformer=transform2d, y_true=y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm, datasets\n",
    "\n",
    "# import some data to play with\n",
    "\n",
    "\n",
    "h = .02  # step size in the mesh\n",
    "\n",
    "\n",
    "# create a mesh to plot in\n",
    "x_min, x_max = data_full['soac_prob_0'].min() - 1, data['soac_prob_0'].max() + 1\n",
    "y_min, y_max = data_full['soac_prob_1'].min() - 1, data['soac_prob_1'].max() + 1\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "\n",
    "# title for the plots\n",
    "titles = ['SVC with RBF kernel']\n",
    "\n",
    "\n",
    "for i, clf in enumerate([rbf_svc]):\n",
    "    # Plot the decision boundary. For that, we will assign a color to each\n",
    "    # point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "\n",
    "    # Put the result into a color plot\n",
    "    Z = Z.reshape(xx.shape)\n",
    "    Z = np.vectorize(gender2id.get)(Z)\n",
    "    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)\n",
    "\n",
    "    # Plot also the training points\n",
    "    plt.scatter(data['soac_prob_0'], data['soac_prob_1'], c=y_id, cmap=plt.cm.Paired)\n",
    "    plt.scatter(data_cv['soac_prob_0'], data_cv['soac_prob_1'], c=y_cv_id, s=80,cmap=plt.cm.Paired)\n",
    "    plt.xlabel('soac_0')\n",
    "    plt.ylabel('soac1')\n",
    "    plt.xlim(xx.min(), xx.max())\n",
    "    plt.ylim(yy.min(), yy.max())\n",
    "    plt.xticks(())\n",
    "    plt.yticks(())\n",
    "    plt.title(titles[i])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(trained_models)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm, datasets\n",
    "\n",
    "# create a mesh to plot in\n",
    "x_min, x_max = data['soac_prob_0'].min() - 1, data['soac_prob_0'].max() + 1\n",
    "y_min, y_max = data['soac_prob_1'].min() - 1, data['soac_prob_1'].max() + 1\n",
    "xx, yy = np.meshgrid(np.arange(x_min, x_max, h),\n",
    "                     np.arange(y_min, y_max, h))\n",
    "\n",
    "\n",
    "\n",
    "for i, model in enumerate(trained_models+[space, Meta]):\n",
    "    # Plot the decision boundary. For that, we will assign a color to each\n",
    "    # point in the mesh [x_min, m_max]x[y_min, y_max].\n",
    "    plt.subplot(2, 2, i + 1)\n",
    "    plt.subplots_adjust(wspace=0.4, hspace=0.4)\n",
    "    if hasattr(model, 'steps'):\n",
    "        clf = model.steps[1][1]\n",
    "        print clf\n",
    "    #Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "    \n",
    "\n",
    "    # Put the result into a color plot\n",
    "    Z = Z.reshape(xx.shape)\n",
    "    Z = np.vectorize(gender2id.get)(Z)\n",
    "    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired, alpha=0.8)\n",
    "\n",
    "    # Plot also the training points\n",
    "    plt.scatter(data_train['soac_prob_0'], data_train['soac_prob_1'], c=y, cmap=plt.cm.Paired)\n",
    "    plt.xlabel('Sepal length')\n",
    "    plt.ylabel('Sepal width')\n",
    "    plt.xlim(xx.min(), xx.max())\n",
    "    plt.ylim(yy.min(), yy.max())\n",
    "    plt.xticks(())\n",
    "    plt.yticks(())\n",
    "    plt.title(model_names[i])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data['soac_prob_0']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import svm\n",
    "#from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "# we create 40 separable points\n",
    "rng = np.random.RandomState(0)\n",
    "n_samples_1 = 1000\n",
    "n_samples_2 = 100\n",
    "X = np.r_[1.5 * rng.randn(n_samples_1, 2),\n",
    "          0.5 * rng.randn(n_samples_2, 2) + [2, 2]]\n",
    "y = [0] * (n_samples_1) + [1] * (n_samples_2)\n",
    "\n",
    "# fit the model and get the separating hyperplane\n",
    "clf = svm.SVC(kernel='linear', C=1.0)\n",
    "clf.fit(X, y)\n",
    "\n",
    "w = clf.coef_[0]\n",
    "a = -w[0] / w[1]\n",
    "xx = np.linspace(-5, 5)\n",
    "yy = a * xx - clf.intercept_[0] / w[1]\n",
    "\n",
    "\n",
    "# get the separating hyperplane using weighted classes\n",
    "wclf = svm.SVC(kernel='linear', class_weight={1: 10})\n",
    "wclf.fit(X, y)\n",
    "\n",
    "ww = wclf.coef_[0]\n",
    "wa = -ww[0] / ww[1]\n",
    "wyy = wa * xx - wclf.intercept_[0] / ww[1]\n",
    "\n",
    "# plot separating hyperplanes and samples\n",
    "h0 = plt.plot(xx, yy, 'k-', label='no weights')\n",
    "h1 = plt.plot(xx, wyy, 'k--', label='with weights')\n",
    "plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Paired)\n",
    "plt.legend()\n",
    "\n",
    "plt.axis('tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print(__doc__)\n",
    "\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import datasets, svm\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "X = iris.data\n",
    "y = iris.target\n",
    "\n",
    "X = X[y != 0, :2]\n",
    "y = y[y != 0]\n",
    "\n",
    "n_sample = len(X)\n",
    "\n",
    "np.random.seed(0)\n",
    "order = np.random.permutation(n_sample)\n",
    "X = X[order]\n",
    "y = y[order].astype(np.float)\n",
    "\n",
    "X_train = X[:.9 * n_sample]\n",
    "y_train = y[:.9 * n_sample]\n",
    "X_test = X[.9 * n_sample:]\n",
    "y_test = y[.9 * n_sample:]\n",
    "\n",
    "# fit the model\n",
    "for fig_num, kernel in enumerate(('linear', 'rbf', 'poly')):\n",
    "    clf = svm.SVC(kernel=kernel, gamma=10)\n",
    "    clf.fit(X_train, y_train)\n",
    "\n",
    "    plt.figure(fig_num)\n",
    "    plt.clf()\n",
    "    plt.scatter(X[:, 0], X[:, 1], c=y, zorder=10, cmap=plt.cm.Paired)\n",
    "\n",
    "    # Circle out the test data\n",
    "    plt.scatter(X_test[:, 0], X_test[:, 1], s=80, facecolors='none', zorder=10)\n",
    "\n",
    "    plt.axis('tight')\n",
    "    x_min = X[:, 0].min()\n",
    "    x_max = X[:, 0].max()\n",
    "    y_min = X[:, 1].min()\n",
    "    y_max = X[:, 1].max()\n",
    "\n",
    "    XX, YY = np.mgrid[x_min:x_max:200j, y_min:y_max:200j]\n",
    "    Z = clf.decision_function(np.c_[XX.ravel(), YY.ravel()])\n",
    "\n",
    "    # Put the result into a color plot\n",
    "    Z = Z.reshape(XX.shape)\n",
    "    #plt.pcolormesh(XX, YY, Z > 0, cmap=plt.cm.Paired)\n",
    "    plt.contour(XX, YY, Z, colors=['k', 'k', 'k'], linestyles=['--', '-', '--'],\n",
    "                levels=[-.5, 0, .5])\n",
    "\n",
    "    plt.title(kernel)\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "ncol = 4 # pick one dimension\n",
    "nrow = (len(feature_names)+ ncol-1) / ncol # make sure enough subplots\n",
    "fig, ax = plt.subplots(nrows=nrow, ncols=ncol) # create the axes\n",
    "j = 0\n",
    "for i in feature_names: \n",
    "    ix = numpy.unravel_index(j, ax.shape)\n",
    "    #print ix\n",
    "    j += 1\n",
    "    ax[ix] = data.groupby('class').i.hist(alpha=0.4)   # go over a linear list of data # compute an appropriate index (1d or 2d)\n",
    "    #feat = feature_names[i]\n",
    "    #data.groupby('class').feat.hist(alpha=0.4, ax=ax[i])\n",
    "import matplotlib.pyplot as plt\n",
    "#%matplotlib outline\n",
    "plt.savefig('CameraEvolution.png', bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clf = grid_search.best_estimator_.steps[1][1]\n",
    "#import pydot\n",
    "import pyparsing\n",
    "\n",
    "#reload(pydot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pprint, numpy\n",
    "from operator import itemgetter\n",
    "\n",
    "feat_importance = zip(list(numpy.array(feature_names)[numpy.nonzero(clf.feature_importances_)]), list(clf.feature_importances_[numpy.nonzero(clf.feature_importances_)]))\n",
    "feat_importance = sorted(feat_importance, key=itemgetter(1))[::-1]\n",
    "feat_importance\n",
    "#for i in zip(list(numpy.array(feature_names)[numpy.nonzero(clf.feature_importances_)]), list(clf.feature_importances_([numpy.nonzero(clf.feature_importances_)]))):\n",
    "#    i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    ">>> with open(\"iris.dot\", 'w') as f:\n",
    "    f = tree.export_graphviz(clf, out_file=f, feature_names=feature_names,\n",
    "                         filled=True, rounded=True,  \n",
    "                         special_characters=True)\n",
    "#>>> import os\n",
    "#>>> os.unlink('iris.dot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    ">>> from sklearn.externals.six import StringIO\n",
    "from sklearn import tree\n",
    "import pydot\n",
    ">>> from IPython.display import Image  \n",
    ">>> dot_data = StringIO()  \n",
    ">>> tree.export_graphviz(clf,  out_file=dot_data,\n",
    "                         feature_names=feature_names,\n",
    "                         filled=True, rounded=True,  \n",
    "                         special_characters=True)  \n",
    ">>> graph = pydot.graph_from_dot_data(dot_data.getvalue())  \n",
    "#>>> Image(graph.create_png())   \n",
    ">>> graph.write_pdf(\"iris.pdf\") "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "from pan import ProfilingDataset, createDocProfiles, create_target_prof_trainset\n",
    "from pan import preprocess\n",
    "\n",
    "task = 'gender'\n",
    "\n",
    "#docs = createDocProfiles(dataset)\n",
    "#X, y = create_target_prof_trainset(docs, task)\n",
    "X, y = dataset.get_data(task)\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)\n",
    "#X, _, y, _ = train_test_split(X, y, train_size=100000, stratify=y, random_state=100)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 88 436 436\n",
      "Accuracy : 0.784090909091\n",
      "Confusion matrix :\n",
      " [[33 11]\n",
      " [ 8 36]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     female       0.80      0.75      0.78        44\n",
      "       male       0.77      0.82      0.79        44\n",
      "\n",
      "avg / total       0.79      0.78      0.78        88\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
    "import numpy\n",
    "\n",
    "\n",
    "\n",
    "class Test_SOAC(BaseEstimator, TransformerMixin):\n",
    "\n",
    "\n",
    "\n",
    "    def __init__(self, min_df=1, max_df=1.0, max_features=None):\n",
    "    \n",
    "        self.min_df = min_df\n",
    "        self.max_df = max_df\n",
    "        self.max_features = max_features\n",
    "        parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    # 'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': self.max_df,\n",
    "                    'min_df': self.min_df,\n",
    "                    'max_features': self.max_features,\n",
    "                    'use_idf': False\n",
    "                }\n",
    "        self.counter = TfidfVectorizer(**parameters)\n",
    "        self.term_table = None\n",
    "        self.labels = None\n",
    "        self.index2label = {}\n",
    "\n",
    "    def fit(self, X, y, sample_weight=None):\n",
    "#counter = TfidfVectorizer(**parameters)\n",
    "        try:\n",
    "            doc_term = self.counter.fit_transform(X)\n",
    "            #print'Doc Term'\n",
    "            #print type(doc_term)\n",
    "            #print doc_term.shape\n",
    "        except Exception, e:\n",
    "            print 'counter_transform'\n",
    "            print e\n",
    "        target_profiles = sorted(list(set(y)))\n",
    "        self.labels = target_profiles\n",
    "        for i, label in enumerate(self.labels):\n",
    "            self.index2label[i] = label\n",
    "        #print self.index2label\n",
    "        from collections import Counter\n",
    "        dd = Counter(y)\n",
    "        prior_row = numpy.zeros([1, len(target_profiles)])\n",
    "        for i, key in enumerate(sorted(dd.keys())):\n",
    "            dd[key] = dd[key]/float(len(y))\n",
    "            prior_row[0, i] = 1/dd[key]\n",
    "        try:\n",
    "            doc_prof = numpy.tile(prior_row, (doc_term.shape[0], 1))\n",
    "            #print'Doc prof'\n",
    "            #print type(doc_prof)\n",
    "            #print doc_prof.shape\n",
    "        except Exception, e:\n",
    "            print 'doc prof tiling'\n",
    "            print e\n",
    "        try:\n",
    "            for i in range(0, doc_term.shape[0]):\n",
    "                doc_prof[i, target_profiles.index(y[i])] = 0\n",
    "            #print'Doc prof Final'\n",
    "            #print type(doc_prof)\n",
    "            #print doc_prof.shape\n",
    "        except Exception, e:\n",
    "            print 'doc_prof 1'\n",
    "            print e\n",
    "        try:\n",
    "            term_prof = doc_term.transpose().dot(doc_prof)\n",
    "            #print'Term prof'\n",
    "            #print type(term_prof)\n",
    "            #print term_prof.shape\n",
    "        except Exception, e:\n",
    "            print \"Error in product\"\n",
    "            print e\n",
    "        try:\n",
    "            normalize(term_prof, norm='l1', axis=0, copy=False)\n",
    "            #print'Term prof norm1'\n",
    "            #print type(term_prof)\n",
    "            #print term_prof.shape\n",
    "        except Exception, e:\n",
    "            print \"Error in nrom per row\"\n",
    "            print e    \n",
    "        try:\n",
    "            normalize(term_prof, norm='l1', axis=1, copy=False)\n",
    "            #print'Term prof norm2'\n",
    "            #print type(term_prof)\n",
    "            #print term_prof.shape\n",
    "        except Exception, e:\n",
    "            print \"Error in nrom per col\"\n",
    "            print e\n",
    "        self.term_table = term_prof\n",
    "        return self\n",
    "    \n",
    "    def transform(self,X):\n",
    "        \n",
    "        doc_term2 = self.counter.transform(X)\n",
    "        doc_prof2 = doc_term2.dot(self.term_table)\n",
    "        for i in range(0, doc_prof2.shape[0]):\n",
    "            doc_prof2[i, :] = doc_prof2[i, :] - doc_prof2[i, :].min()\n",
    "        #print 'Final Doc Repr'\n",
    "        #print type(doc_prof2)\n",
    "        #print doc_prof2.shape\n",
    "        return doc_prof2\n",
    "    \n",
    "    def predict(self, X):\n",
    "        \n",
    "        transformed = self.transform(X)\n",
    "        return [self.index2label[t.argmin()] for t in transformed]\n",
    "    \n",
    "    def score(self, X, y_true):\n",
    "        \n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "\n",
    "Model = Test_SOAC()\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.2, stratify=y, random_state=100)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "Model.fit(X_train, y_train)\n",
    "#Model.score(X_cv, y_cv)\n",
    "predict = Model.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "348 348\n"
     ]
    }
   ],
   "source": [
    "print len(X_train), len(y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.772727272727\n",
      "Confusion matrix :\n",
      " [[32 12]\n",
      " [ 8 36]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "     female       0.80      0.73      0.76        44\n",
      "       male       0.75      0.82      0.78        44\n",
      "\n",
      "avg / total       0.78      0.77      0.77        88\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier, BaggingClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "#SVC(C=100, probability=True, kernel='linear')\n",
    "\n",
    "X_train2 = Model.transform(X_train)\n",
    "X_cv2 = Model.transform(X_cv)\n",
    "bdt = BaggingClassifier(SVC(C=10, probability=True, kernel='linear'), n_estimators=200)\n",
    "#bdt = AdaBoostClassifier(Model,\n",
    "#                         algorithm=\"SAMME\",\n",
    "#                         n_estimators=200)\n",
    "bdt.fit(X_train2, y_train)\n",
    "predict = bdt.predict(X_cv2)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y_cv))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y_cv))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "jj = 0\n",
    "for w,ind in Model.counter.vocabulary_.iteritems():\n",
    "    print w,ind\n",
    "    jj+=1\n",
    "    print Model.term_table[ind, :]\n",
    "    if jj == 1000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('soac', Test_SOAC(max_df=1.0, max_features=None, min_df=1)),\n",
       " ('svm',\n",
       "  LinearSVC(C=0.001, class_weight='balanced', dual=False, fit_intercept=True,\n",
       "       intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "       multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
       "       verbose=0))]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.naive_bayes import GaussianNB, MultinomialNB\n",
    "from pan.features import SOA_Predict\n",
    "#combined = FeatureUnion([('soac', Model)])\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=False)\n",
    "svmlin = LinearSVC(C=0.001, dual=False, class_weight='balanced')\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "gnb = GaussianNB()\n",
    "soa_pred = SOA_Predict()\n",
    "#pipe1 = Pipeline([('combined',combined), ('min_pred', soa_pred)])\n",
    "pipe1 = Pipeline([('soac',Model), ('svm', svmlin)])\n",
    "#pipe1 = Pipeline([('combined',combined), ('svm', svmlin)])\n",
    "#pipe1 = Pipeline([('combined',combined), ('NB', gnb)])\n",
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "222233 55559 277792 277792\n",
      "Accuracy : 0.770280242625\n",
      "Confusion matrix :\n",
      " [[17645  6408]\n",
      " [ 6355 25151]]\n"
     ]
    }
   ],
   "source": [
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.2, stratify=y, random_state=100)\n",
    "print len(X_train), len(X_cv), len(X_cv) + len(X_train), len(X)\n",
    "pipe1.fit(X_train, y_train)\n",
    "predict = pipe1.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['max_features', 'max_df', 'min_df']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Model.get_params().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 6 candidates, totalling 24 fits\n",
      "0.653161321676\n",
      "Pipeline(steps=[('soac', Test_SOAC(max_df=1.0, max_features=None, min_df=1)), ('svm', LinearSVC(C=0.001, class_weight='balanced', dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  24 out of  24 | elapsed:  1.5min finished\n"
     ]
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "\n",
    "params = {}\n",
    "params = {'svm__C': [0.001, 0.01, 0.1, 1, 10, 100]}\n",
    "#params = {'soac__max_df':[1.0, 0.9, 0.8], 'soac__min_df':[1, 3, 5], 'soac__max_features':[None, 5000, 10000]}\n",
    "trained_models = []\n",
    "for model in [pipe1]:\n",
    "    grid_search = GridSearchCV(estimator=model, param_grid=params, verbose=1, n_jobs=-1, cv=4, refit=True)\n",
    "    grid_search.fit(X_train,y_train)\n",
    "    print(grid_search.best_score_)\n",
    "    print(grid_search.best_estimator_) \n",
    "    trained_models.append(grid_search.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.653161321676\n",
      "{'svm__C': 0.001}\n"
     ]
    }
   ],
   "source": [
    "print(grid_search.best_score_)\n",
    "print(grid_search.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "277792 436\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-10-1b20d9ce5992>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0my_cv_inst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdocs\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mdoc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[1;33m==\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m             \u001b[0my_cv_inst\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "docs = createDocProfiles(dataset)\n",
    "X_inst, y_inst = create_target_prof_trainset(docs, task)\n",
    "X_prof, y_prof = dataset.get_data('gender')\n",
    "print len(X_inst), len(X_prof)\n",
    "_, X_cv, _, y_cv = train_test_split(X_inst, y_inst, test_size=0.2, stratify=y, random_state=100)\n",
    "y_cv_inst = []\n",
    "for x in X_cv:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.504587155963\n",
      "Confusion matrix :\n",
      " [[  2 216]\n",
      " [  0 218]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "X_test, y_test = dataset.get_data('gender')\n",
    "#predict = Model.predict(X_test)\n",
    "predict =pipe1.predict(X_test)\n",
    "acc = accuracy_score(y_test, predict)\n",
    "conf = confusion_matrix(y_test, predict, labels=sorted(list(set(y_test))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pan import features\n",
    "from pan import preprocess\n",
    "X, y = dataset.get_data('gender')\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "label_binarize() takes at least 2 arguments (1 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-51-4cb43abfbed8>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpreprocessing\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlabel_binarize\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mmodel\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtrained_models\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m     \u001b[1;32mprint\u001b[0m \u001b[0mlabel_binarize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: label_binarize() takes at least 2 arguments (1 given)"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import label_binarize\n",
    "for model in trained_models:\n",
    "    print model.predict([X_meta[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from scipy.optimize import minimize\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "\n",
    "global X_train, X_meta, x_cv, y_train, y_meta, y_cv\n",
    "\n",
    "def f(w):\n",
    "    print \"Weights\"\n",
    "    print w\n",
    "    score = 1- accuracy_score(y_meta, space.predict(X_meta, w))\n",
    "    print 'Score: ' + str(score)\n",
    "    return score\n",
    "\n",
    "w = [3,2,1,0.35]\n",
    "#space = SubSpaceEnsemble3(models,cv_scores,k=3, weights=w)\n",
    "#space.fit(X_train + X_cv, y_train + y_cv)\n",
    "#print models.keys()\n",
    "#print cv_scores\n",
    "\n",
    "bnds = ((0, None), (0, None), (0, None), (0, 1))\n",
    "a = minimize(f, w,  method='SLSQP', bounds=bnds)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['male', 'female', 'male'], ['female', 'male', 'male'], ['male', 'female', 'female']]\n",
      "True. LAbel\n",
      "[[1]\n",
      " [1]\n",
      " [0]]\n",
      "[[1 0 1]\n",
      " [0 1 1]\n",
      " [1 0 0]]\n",
      "True. oHE\n",
      "[[ 0.  1.]\n",
      " [ 0.  1.]\n",
      " [ 1.  0.]]\n",
      "[[ 0.  1.  1.  0.  0.  1.]\n",
      " [ 1.  0.  0.  1.  0.  1.]\n",
      " [ 0.  1.  1.  0.  1.  0.]]\n",
      "WEIGHTER\n",
      "True. LAbel\n",
      "[1 0 1 0 1 1 1 0 0]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/preprocessing/label.py:125: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "total size of new array must be unchanged",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-46-a9657e572262>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     67\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[1;34m'WEIGHTER'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     68\u001b[0m \u001b[0mW\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWeighter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;34m'first'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mpred1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'second'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mpred2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'third'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mpred3\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 69\u001b[1;33m \u001b[0mW\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpred_t\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtruth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-46-a9657e572262>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, weights)\u001b[0m\n\u001b[0;32m     39\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[1;34m\"True. LAbel\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 41\u001b[1;33m             \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mohe\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_transform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mN_samples\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_samples\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_labels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtodense\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     42\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     43\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: total size of new array must be unchanged"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "\n",
    "class Weighter(BaseEstimator, TransformerMixin):\n",
    "    \"\"\"Simple Majoirty Voter(also supports weights)\"\"\"\n",
    "    \n",
    "    import numpy\n",
    "    \n",
    "    def __init__(self, models, weights=None):\n",
    "        \n",
    "        from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "        \n",
    "        if (not models):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.weights = None\n",
    "            self.lab = LabelEncoder()\n",
    "            self.ohe = OneHotEncoder()\n",
    "            self.ind2names = {}\n",
    "            self.num_labels = 0\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "                \n",
    "    def fit(self, X, y, weights=None):\n",
    "        \n",
    "        if y is None:\n",
    "            raise ValueError('We need y labels to supervise-fit!')\n",
    "        \n",
    "        if self.weights:\n",
    "            print \"Not so much Vox Populi, Vox Dei, huh?\"\n",
    "        else:\n",
    "            self.num_labels = len(set(y))\n",
    "            N_samples = len(y)\n",
    "            if type(X) is numpy.array:\n",
    "                X = X.reshape(-1, 1)\n",
    "            else:\n",
    "                X = numpy.array(X).reshape(-1, 1)\n",
    "            X = self.lab.fit_transform(X)\n",
    "            print \"True. LAbel\"\n",
    "            print X\n",
    "            X = self.ohe.fit_transform(X.reshape(N_samples, N_samples*self.num_labels)).todense()\n",
    "            print X\n",
    "        return self\n",
    "\n",
    "pred1 = ['male', 'female', 'male']\n",
    "pred2 = ['female', 'male', 'female']\n",
    "pred3 = ['male', 'male', 'female']\n",
    "pred_t = []\n",
    "for i in range(len(pred1)):\n",
    "    pred_t.append([pred1[i], pred2[i], pred3[i]])\n",
    "print pred_t\n",
    "\n",
    "truth = pred3\n",
    "N_labels = len(set(truth))\n",
    "N_samples = len(pred_t)\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "lab = LabelEncoder()\n",
    "ohe = OneHotEncoder()\n",
    "truth_lab = lab.fit_transform(truth).reshape(-1, 1)\n",
    "print \"True. LAbel\"\n",
    "print truth_lab\n",
    "print lab.transform(pred_t)\n",
    "X_new = ohe.fit_transform(numpy.array(truth_lab).reshape(-1,1)).todense()\n",
    "print 'True. oHE'\n",
    "print X_new\n",
    "print ohe.transform(lab.transform(pred_t).reshape(-1, 1)).todense().reshape(N_samples, N_labels*N_samples)\n",
    "print 'WEIGHTER'\n",
    "W = Weighter({'first':pred1, 'second':pred2, 'third':pred3})\n",
    "W.fit(pred_t, truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n"
     ]
    }
   ],
   "source": [
    "predictions_meta = []\n",
    "print len(trained_models)\n",
    "for i, model in enumerate(trained_models):\n",
    "    predict = model.predict(X_meta)\n",
    "    predictions_meta.append(predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WEIGHTER\n",
      "Hurray! Equality for all!\n",
      "Model majority: 30.91%\n",
      "Using given weights: |  3.00 | 1.00 | 1.00 | 1.00 | 3.00 | 3.00 |\n",
      "Model weights: 32.73%\n",
      "Using found weights: |  18.00 | 19.00 | 23.00 | 11.00 | 19.00 | 29.00 |\n",
      "Model accuracy: 32.73%\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "f() takes exactly 4 arguments (3 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-296-61ee205541cb>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    180\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mscheme\u001b[0m \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'majority'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'weights'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'optimal'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    181\u001b[0m     \u001b[0mW\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWeighter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscheme\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mscheme\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 182\u001b[1;33m     \u001b[0mW\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    183\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Model {}: {}%\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mscheme\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m100\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mW\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    184\u001b[0m \u001b[1;31m#print W.predict(predictions_meta)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-267-09b1056e76ff>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, weights)\u001b[0m\n\u001b[0;32m     52\u001b[0m         \u001b[1;31m#print y\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscheme\u001b[0m  \u001b[1;32min\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'majority'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'weights'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 54\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_weights\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_tr\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     55\u001b[0m             \u001b[0mweights_string\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\" %.2f |\"\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[1;34m\"Using found weights: | %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mweights_string\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-267-09b1056e76ff>\u001b[0m in \u001b[0;36mfind_weights\u001b[1;34m(self, X, y, X_tr, y_tr)\u001b[0m\n\u001b[0;32m    148\u001b[0m             \u001b[0mw\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_models\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    149\u001b[0m             \u001b[0mbnds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnum_models\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 150\u001b[1;33m             \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mminimize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0margs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'SLSQP'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbnds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    151\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    152\u001b[0m         \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/_minimize.pyc\u001b[0m in \u001b[0;36mminimize\u001b[1;34m(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)\u001b[0m\n\u001b[0;32m    453\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'slsqp'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    454\u001b[0m         return _minimize_slsqp(fun, x0, args, jac, bounds,\n\u001b[1;32m--> 455\u001b[1;33m                                constraints, callback=callback, **options)\n\u001b[0m\u001b[0;32m    456\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'dogleg'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    457\u001b[0m         return _minimize_dogleg(fun, x0, args, jac, hess,\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/slsqp.pyc\u001b[0m in \u001b[0;36m_minimize_slsqp\u001b[1;34m(func, x0, args, jac, bounds, constraints, maxiter, ftol, iprint, disp, eps, callback, **unknown_options)\u001b[0m\n\u001b[0;32m    361\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    362\u001b[0m             \u001b[1;31m# Compute objective function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 363\u001b[1;33m             \u001b[0mfx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    364\u001b[0m             \u001b[1;31m# Compute the constraints\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    365\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mcons\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'eq'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[1;34m(*wrapper_args)\u001b[0m\n\u001b[0;32m    287\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    288\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 289\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    290\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: f() takes exactly 4 arguments (3 given)"
     ]
    }
   ],
   "source": [
    "import numpy\n",
    "\n",
    "class Combinator(BaseEstimator, TransformerMixin):\n",
    "    \"\"\" Static A posteriori Combinator of predictions.\n",
    "        \n",
    "        Args:\n",
    "            - scheme: String flag. Can be one of the following:\n",
    "                - 'majority': Simple Hard Majority Voting\n",
    "                - 'weights': Weighted Voting, with weights\n",
    "                             passed by user in the weights\n",
    "                             arg\n",
    "                - 'accuracy': Weights are calculated according\n",
    "                              to prediction accuracy over the\n",
    "                              meta train set\n",
    "                - 'optimal': The optimal weights are found, this\n",
    "                             is done by optimizing over the classification\n",
    "                             error\n",
    "            - weights: list or numpy.array(not sure?) containing as many\n",
    "                         weights as the models in the ensemble\n",
    "        Returns:\n",
    "            - The  ensemble Model. Needs to be fitted for the encoding part\n",
    "            \n",
    "    \n",
    "    \"\"\"\n",
    "    \n",
    "    import numpy\n",
    "    \n",
    "    def __init__(self, scheme='majority', weights=None):\n",
    "        \n",
    "        from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "        \n",
    "\n",
    "        self.scheme = scheme\n",
    "        self.weights = weights\n",
    "        self.transformation = self.voting\n",
    "        self.num_labels = 0\n",
    "        self.num_models = 0\n",
    "        self.lab = LabelEncoder()\n",
    "        self.ohe = OneHotEncoder()\n",
    "\n",
    "        if self.scheme == 'majority':\n",
    "            print \"Hurray! Equality for all!\"\n",
    "            self.weights = None\n",
    "        else:\n",
    "            print \"Not so much Vox Populi, Vox Dei, huh?\"\n",
    "            if self.scheme == 'weights':\n",
    "                if type(self.weights) in (numpy.array, numpy.ndarray):\n",
    "                    pass # It is from the optimization part\n",
    "                else: \n",
    "                    if not(self.weights):\n",
    "                        print \"Need weights for this scheme!\"\n",
    "                self.weights = weights\n",
    "                weights_string = \" %.2f |\"*len(self.weights) % tuple(self.weights)\n",
    "                print \"Using given weights: | %s\" % weights_string\n",
    "            else:\n",
    "                #print \"Will find the weights after fitting\"\n",
    "                pass\n",
    "                    \n",
    "                    \n",
    "                \n",
    "    def fit(self, X, y, weights=None):\n",
    "        \n",
    "        if y is None:\n",
    "            raise ValueError('We need y labels to supervise-fit!')\n",
    "        X_tr, y_tr  = self.fit_encoders(X, y)\n",
    "        if not(self.scheme  in ['majority', 'weights']):\n",
    "            self.find_weights(X, y, X_tr, y_tr)\n",
    "            weights_string = \" %.2f |\"*len(self.weights) % tuple(self.weights)\n",
    "            print \"Using found weights: | %s\" % weights_string\n",
    "        return self\n",
    "        \n",
    "    def transform(self, X):\n",
    "        \n",
    "        if type(X[0]) is numpy.array:\n",
    "            N_samples = X[0].shape[0]\n",
    "        else:\n",
    "            N_samples = len(X[0])\n",
    "        X = self.lab.transform(X)\n",
    "        X = self.ohe.transform(X.reshape(-1, 1)).todense().reshape(N_samples, -1)\n",
    "        prediction = self.transformation(X)\n",
    "        prediction = self.lab.inverse_transform(prediction.argmax(axis=1))\n",
    "        return prediction\n",
    "\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return self.transform(X)\n",
    "    \n",
    "    \n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "        \n",
    "    def fit_encoders(self, X, y):\n",
    "        self.num_labels = len(set(y))\n",
    "        N_samples = len(y)\n",
    "        #print \"N_smaples\"\n",
    "        #print N_samples\n",
    "        if type(X) is numpy.array:\n",
    "            y = y.reshape(-1, 1)\n",
    "        else:\n",
    "            y = numpy.array(y).reshape(-1, 1)\n",
    "        #print y\n",
    "        y = self.lab.fit_transform(y).reshape(-1,1)\n",
    "        #print 'label'\n",
    "        #print y\n",
    "        y = self.ohe.fit_transform(y).todense()\n",
    "        #print 'ohe'\n",
    "        #print y\n",
    "        X = self.lab.transform(X)\n",
    "        X = self.ohe.transform(X.T.reshape(-1, 1)).todense().reshape(N_samples, -1) # reshape(N_samples, N_samples*self.num_labels)\n",
    "        #print 'ohe'\n",
    "        #print X.shape\n",
    "        self.num_models = int(X.shape[1]/self.num_labels)\n",
    "        return X, y\n",
    "    \n",
    "    \n",
    "    def voting(self, X):\n",
    "        predictions = numpy.zeros([X.shape[0], self.num_labels])\n",
    "        if type(self.weights) in (numpy.array, numpy.ndarray):\n",
    "            pass\n",
    "        else: \n",
    "            if not(self.weights):\n",
    "                self.weights = [1 for i in xrange(self.num_models)]\n",
    "        for i in xrange(X.shape[0]):\n",
    "            #print X.shape\n",
    "            #print X\n",
    "            subarrays = numpy.split(X[i,:], self.num_models, axis=1)\n",
    "            #print \"subarrays\"\n",
    "            #print subarrays\n",
    "            votes = numpy.zeros([1, self.num_labels])\n",
    "            for model_index, subar in enumerate(subarrays):\n",
    "                #print subar\n",
    "                votes = numpy.vstack((votes, subar*self.weights[model_index]))\n",
    "            #print votes\n",
    "            pred_ = votes.sum(axis=0).argmax()\n",
    "            pred_ = self.ohe.transform(pred_).todense()\n",
    "            predictions[i,:] = pred_\n",
    "        return predictions\n",
    "    \n",
    "    def find_weights(self, X, y, X_tr, y_tr):\n",
    "        weights = [0 for i in xrange(self.num_models)]\n",
    "        if self.scheme == 'accuracy':\n",
    "            for i in xrange(X_tr.shape[0]):\n",
    "                subarrays = numpy.split(X_tr[i,:], self.num_models, axis=1)\n",
    "                for model_index, subar in enumerate(subarrays):\n",
    "                    if (subar==y_tr[i,:]).all():\n",
    "                        weights[model_index] += 1\n",
    "            self.weights = weights\n",
    "        if self.scheme == 'optimal':\n",
    "            from scipy.optimize import minimize\n",
    "            from sklearn.metrics import accuracy_score\n",
    "            #import time\n",
    "            \n",
    "            # global y\n",
    "            \n",
    "            w = [1 for i in xrange(self.num_models)]\n",
    "            bnds = tuple([(0, None) for i in xrange(self.num_models)])\n",
    "            a = minimize(f, w,  args=(Combinator, X, y), method='SLSQP', bounds=bnds)\n",
    "            self.weights = list(a.x)\n",
    "        return \n",
    "                   \n",
    "        \n",
    "def f(w, Combinator, x, y):\n",
    "    gg = Combinator(scheme='weights', weights=w)\n",
    "    gg.fit(x, y)\n",
    "    score = 1- gg.score(x, y)\n",
    "    #print 'Weights'\n",
    "    #print w\n",
    "    #print 'Score: ' + str(score)\n",
    "    return score         \n",
    "\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "#global y_meta, predictions_meta, y_cv, predictions\n",
    "print 'WEIGHTER'\n",
    "for scheme in ['majority', 'weights', 'accuracy', 'optimal']:\n",
    "    W = Weighter(scheme=scheme, weights= [3,1,1, 1, 3, 3])\n",
    "    W.fit(predictions_meta, y_meta)\n",
    "    print \"Model {}: {}%\".format(scheme, round(100*W.score(predictions, y_cv), 2))\n",
    "#print W.predict(predictions_meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  5127.44 | 2563.72 | 1242757.74 | 2563.72 | 1242757.74 | 1242757.74 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  2564.22 | 1282.36 | 621379.37 | 1282.36 | 621379.37 | 621379.37 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1282.61 | 641.68 | 310690.19 | 641.68 | 310690.19 | 310690.19 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  641.81 | 321.34 | 155345.59 | 321.34 | 155345.59 | 155345.59 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  321.40 | 161.17 | 77673.30 | 161.17 | 77673.30 | 77673.30 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  161.20 | 81.08 | 38837.15 | 81.08 | 38837.15 | 38837.15 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  81.10 | 41.04 | 19419.07 | 41.04 | 19419.07 | 19419.07 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  41.05 | 21.02 | 9710.04 | 21.02 | 9710.04 | 9710.04 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  21.03 | 11.01 | 4855.52 | 11.01 | 4855.52 | 4855.52 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  11.01 | 6.01 | 2428.26 | 6.01 | 2428.26 | 2428.26 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n"
     ]
    }
   ],
   "source": [
    "from scipy.optimize import minimize\n",
    "from sklearn.metrics import accuracy_score\n",
    "import time\n",
    "\n",
    "global y, predictions\n",
    "\n",
    "def f(w, Combinator, x, y):\n",
    "    gg = Combinator(scheme='weights', weights=w)\n",
    "    gg.fit(x, y)\n",
    "    score = 1- gg.score(x, y)\n",
    "    #print 'Weights'\n",
    "    #print w\n",
    "    #print 'Score: ' + str(score)\n",
    "    return score    \n",
    "\n",
    "w = [1 for i in xrange(6)]\n",
    "bnds = tuple([(0, None) for i in xrange(6)])\n",
    "a = minimize(f, w,  args=(Combinator, predictions_meta, y_meta), method='SLSQP', bounds=bnds)\n",
    "#print a        \n",
    "        \n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.70909090909090911"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.fun"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 3grams\n",
      "Accuracy : 0.333333333333\n",
      "Confusion matrix :\n",
      " [[ 0  1  2  0  0]\n",
      " [ 0  7 10  0  0]\n",
      " [ 0 12 11  0  0]\n",
      " [ 0  1  9  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.33      0.41      0.37        17\n",
      "      35-49       0.33      0.48      0.39        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.25      0.33      0.28        54\n",
      "\n",
      "Model: soac\n",
      "Accuracy : 0.351851851852\n",
      "Confusion matrix :\n",
      " [[0 3 0 0 0]\n",
      " [0 9 3 5 0]\n",
      " [1 9 6 7 0]\n",
      " [0 5 1 4 0]\n",
      " [0 0 0 1 0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.35      0.53      0.42        17\n",
      "      35-49       0.60      0.26      0.36        23\n",
      "      50-64       0.24      0.40      0.30        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.41      0.35      0.34        54\n",
      "\n",
      "Model: lsi\n",
      "Accuracy : 0.425925925926\n",
      "Confusion matrix :\n",
      " [[ 0  0  3  0  0]\n",
      " [ 0  0 17  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.43      1.00      0.60        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.43      0.25        54\n",
      "\n",
      "Model: votingh\n",
      "Accuracy : 0.203703703704\n",
      "Confusion matrix :\n",
      " [[ 2  1  0  0  0]\n",
      " [ 9  5  3  0  0]\n",
      " [13  6  4  0  0]\n",
      " [ 9  0  1  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.06      0.67      0.11         3\n",
      "      25-34       0.42      0.29      0.34        17\n",
      "      35-49       0.50      0.17      0.26        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.35      0.20      0.22        54\n",
      "\n",
      "Model: meta\n",
      "Accuracy : 0.351851851852\n",
      "Confusion matrix :\n",
      " [[ 2  0  1  0  0]\n",
      " [ 4  0 10  0  3]\n",
      " [ 3  0 16  0  4]\n",
      " [ 5  0  2  0  3]\n",
      " [ 0  0  0  0  1]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.14      0.67      0.24         3\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.55      0.70      0.62        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.09      1.00      0.17         1\n",
      "\n",
      "avg / total       0.24      0.35      0.28        54\n",
      "\n",
      "Model: space\n",
      "Accuracy : 0.537037037037\n",
      "Confusion matrix :\n",
      " [[ 0  1  2  0  0]\n",
      " [ 0  8  7  2  0]\n",
      " [ 0  2 21  0  0]\n",
      " [ 0  3  7  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.57      0.47      0.52        17\n",
      "      35-49       0.55      0.91      0.69        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.42      0.54      0.46        54\n",
      "\n",
      "Combinators\n",
      "Hurray! Equality for all!\n",
      "Model majority:\n",
      "Accuracy : 0.444444444444\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 2  7  6  1  1]\n",
      " [ 4  2 16  1  0]\n",
      " [ 2  2  6  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.58      0.41      0.48        17\n",
      "      35-49       0.55      0.70      0.62        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.42      0.44      0.42        54\n",
      "\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  3.00 | 1.00 | 1.00 | 1.00 | 3.00 | 3.00 |\n",
      "Model weights:\n",
      "Accuracy : 0.444444444444\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 3  5  7  1  1]\n",
      " [ 3  2 17  1  0]\n",
      " [ 2  1  6  1  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.56      0.29      0.38        17\n",
      "      35-49       0.55      0.74      0.63        23\n",
      "      50-64       0.33      0.10      0.15        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.48      0.44      0.43        54\n",
      "\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using found weights: |  18.00 | 19.00 | 23.00 | 11.00 | 19.00 | 29.00 |\n",
      "Model accuracy:\n",
      "Accuracy : 0.462962962963\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 2  7  6  1  1]\n",
      " [ 4  1 17  1  0]\n",
      " [ 2  2  6  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.64      0.41      0.50        17\n",
      "      35-49       0.57      0.74      0.64        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.45      0.46      0.44        54\n",
      "\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1.00 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  5127.44 | 2563.72 | 1242757.74 | 2563.72 | 1242757.74 | 1242757.74 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  2564.22 | 1282.36 | 621379.37 | 1282.36 | 621379.37 | 621379.37 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  1282.61 | 641.68 | 310690.19 | 641.68 | 310690.19 | 310690.19 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  641.81 | 321.34 | 155345.59 | 321.34 | 155345.59 | 155345.59 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  321.40 | 161.17 | 77673.30 | 161.17 | 77673.30 | 77673.30 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  161.20 | 81.08 | 38837.15 | 81.08 | 38837.15 | 38837.15 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  81.10 | 41.04 | 19419.07 | 41.04 | 19419.07 | 19419.07 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  41.05 | 21.02 | 9710.04 | 21.02 | 9710.04 | 9710.04 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  21.03 | 11.01 | 4855.52 | 11.01 | 4855.52 | 4855.52 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  11.01 | 6.01 | 2428.26 | 6.01 | 2428.26 | 2428.26 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Not so much Vox Populi, Vox Dei, huh?\n",
      "Using given weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Using found weights: |  6.01 | 3.50 | 1214.63 | 3.50 | 1214.63 | 1214.63 |\n",
      "Model optimal:\n",
      "Accuracy : 0.462962962963\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 1  8  6  1  1]\n",
      " [ 4  2 16  1  0]\n",
      " [ 1  5  3  0  1]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.12      0.33      0.18         3\n",
      "      25-34       0.50      0.47      0.48        17\n",
      "      35-49       0.62      0.70      0.65        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.43      0.46      0.44        54\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions_meta = []\n",
    "for i, model in enumerate(trained_models):\n",
    "    print \"Model: %s\" % (model_names2[i])\n",
    "    predict = model.predict(X_meta)\n",
    "    predictions_meta.append(predict)\n",
    "    acc = accuracy_score(y_meta, predict)\n",
    "    conf = confusion_matrix(y_meta, predict, labels=sorted(list(set(y))))\n",
    "    rep = classification_report(y_meta, predict, target_names=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))\n",
    "\n",
    "#print predictions_meta    \n",
    "\n",
    "print 'Combinators'\n",
    "trained_models3 = copy.deepcopy(trained_models)\n",
    "model_names3 = copy.deepcopy(model_names2)\n",
    "for scheme in ['majority', 'weights', 'accuracy', 'optimal']:\n",
    "    W = Combinator(scheme=scheme, weights= [3,1,1, 1, 3, 3])\n",
    "    W.fit(predictions_meta, y_meta)\n",
    "    trained_models.append(W)\n",
    "    model_names2.append(scheme)\n",
    "    print \"Model {}:\".format(scheme)\n",
    "    predict = W.predict(predictions_meta)\n",
    "    acc = accuracy_score(y_meta, predict)\n",
    "    conf = confusion_matrix(y_meta, predict, labels=sorted(list(set(y))))\n",
    "    rep = classification_report(y_meta, predict, target_names=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 319,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 3grams\n",
      "['35-49' '25-34' '25-34' '35-49' '35-49' '25-34' '35-49' '35-49' '25-34'\n",
      " '35-49' '35-49' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '35-49'\n",
      " '25-34' '35-49' '35-49' '25-34' '25-34' '35-49' '35-49' '35-49' '35-49'\n",
      " '25-34' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '25-34'\n",
      " '25-34' '25-34' '25-34' '35-49' '25-34' '25-34' '35-49' '25-34' '35-49'\n",
      " '35-49' '35-49' '25-34' '35-49' '35-49' '35-49' '25-34' '35-49' '25-34']\n",
      "['35-49' '25-34' '25-34' '35-49' '35-49' '25-34' '35-49' '35-49' '25-34'\n",
      " '35-49' '35-49' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '35-49'\n",
      " '25-34' '35-49' '35-49' '25-34' '25-34' '35-49' '35-49' '35-49' '35-49'\n",
      " '25-34' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '25-34'\n",
      " '25-34' '25-34' '25-34' '35-49' '25-34' '25-34' '35-49' '25-34' '35-49'\n",
      " '35-49' '35-49' '25-34' '35-49' '35-49' '35-49' '25-34' '35-49' '25-34']\n",
      "Model: soac\n",
      "['35-49' '25-34' '50-64' '50-64' '25-34' '25-34' '25-34' '25-34' '25-34'\n",
      " '50-64' '25-34' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '25-34'\n",
      " '18-24' '25-34' '25-34' '35-49' '25-34' '25-34' '35-49' '25-34' '25-34'\n",
      " '50-64' '50-64' '25-34' '50-64' '25-34' '50-64' '50-64' '50-64' '25-34'\n",
      " '50-64' '25-34' '25-34' '35-49' '25-34' '50-64' '35-49' '35-49' '50-64'\n",
      " '50-64' '25-34' '25-34' '35-49' '25-34' '50-64' '50-64' '50-64' '50-64']\n",
      "['35-49' '25-34' '50-64' '50-64' '25-34' '25-34' '25-34' '25-34' '25-34'\n",
      " '50-64' '25-34' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '25-34'\n",
      " '18-24' '25-34' '25-34' '35-49' '25-34' '25-34' '35-49' '25-34' '25-34'\n",
      " '50-64' '50-64' '25-34' '50-64' '25-34' '50-64' '50-64' '50-64' '25-34'\n",
      " '50-64' '25-34' '25-34' '35-49' '25-34' '50-64' '35-49' '35-49' '50-64'\n",
      " '50-64' '25-34' '25-34' '35-49' '25-34' '50-64' '50-64' '50-64' '50-64']\n",
      "Model: lsi\n",
      "['35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49']\n",
      "['35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49']\n",
      "Model: votingh\n",
      "['35-49' '25-34' '18-24' '18-24' '18-24' '25-34' '18-24' '18-24' '25-34'\n",
      " '18-24' '18-24' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '18-24'\n",
      " '18-24' '18-24' '18-24' '18-24' '25-34' '18-24' '35-49' '18-24' '18-24'\n",
      " '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '25-34'\n",
      " '18-24' '25-34' '25-34' '35-49' '25-34' '18-24' '35-49' '18-24' '18-24'\n",
      " '18-24' '18-24' '25-34' '35-49' '18-24' '18-24' '18-24' '18-24' '18-24']\n",
      "['35-49' '25-34' '18-24' '18-24' '18-24' '25-34' '18-24' '18-24' '25-34'\n",
      " '18-24' '18-24' '25-34' '35-49' '35-49' '25-34' '35-49' '25-34' '18-24'\n",
      " '18-24' '18-24' '18-24' '18-24' '25-34' '18-24' '35-49' '18-24' '18-24'\n",
      " '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '18-24' '25-34'\n",
      " '18-24' '25-34' '25-34' '35-49' '25-34' '18-24' '35-49' '18-24' '18-24'\n",
      " '18-24' '18-24' '25-34' '35-49' '18-24' '18-24' '18-24' '18-24' '18-24']\n",
      "Model: meta\n",
      "['35-49' '35-49' '35-49' '65-xx' '18-24' '35-49' '18-24' '18-24' '35-49'\n",
      " '65-xx' '18-24' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '18-24'\n",
      " '35-49' '18-24' '18-24' '35-49' '35-49' '18-24' '35-49' '18-24' '18-24'\n",
      " '35-49' '65-xx' '18-24' '65-xx' '18-24' '65-xx' '65-xx' '65-xx' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '65-xx'\n",
      " '65-xx' '18-24' '35-49' '35-49' '18-24' '65-xx' '35-49' '65-xx' '35-49']\n",
      "['35-49' '18-24' '65-xx' '65-xx' '18-24' '18-24' '18-24' '18-24' '18-24'\n",
      " '65-xx' '18-24' '18-24' '35-49' '35-49' '18-24' '35-49' '18-24' '18-24'\n",
      " '35-49' '18-24' '18-24' '35-49' '18-24' '18-24' '35-49' '18-24' '18-24'\n",
      " '65-xx' '65-xx' '18-24' '65-xx' '18-24' '65-xx' '65-xx' '65-xx' '18-24'\n",
      " '65-xx' '18-24' '18-24' '35-49' '18-24' '65-xx' '35-49' '35-49' '65-xx'\n",
      " '65-xx' '18-24' '18-24' '35-49' '18-24' '65-xx' '65-xx' '65-xx' '65-xx']\n",
      "AKYRO\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "Model: space\n",
      "['35-49', '25-34', '50-64', '35-49', '25-34', '35-49', '25-34', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '50-64', '25-34', '35-49', '35-49', '25-34', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49']\n",
      "['35-49', '25-34', '50-64', '35-49', '25-34', '35-49', '25-34', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '50-64', '25-34', '35-49', '35-49', '25-34', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49', '35-49', '25-34', '35-49', '35-49']\n"
     ]
    }
   ],
   "source": [
    "for i in xrange(len(predictions4)):\n",
    "    print \"Model: %s\" % (model_names3[i])\n",
    "    print predictions_meta[i]\n",
    "    print predictions4[i]\n",
    "    for j, lab in enumerate(predictions_meta[i]):\n",
    "        if lab != predictions4[i][j]:\n",
    "            print 'AKYRO'\n",
    "            print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\n",
    "            break"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 331,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['35-49' '35-49' '35-49' '65-xx' '18-24' '35-49' '18-24' '18-24' '35-49'\n",
      " '65-xx' '18-24' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '18-24'\n",
      " '35-49' '18-24' '18-24' '35-49' '35-49' '18-24' '35-49' '18-24' '18-24'\n",
      " '35-49' '65-xx' '18-24' '65-xx' '18-24' '65-xx' '65-xx' '65-xx' '35-49'\n",
      " '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '35-49' '65-xx'\n",
      " '65-xx' '18-24' '35-49' '35-49' '18-24' '65-xx' '35-49' '65-xx' '35-49']\n",
      "['35-49' '18-24' '65-xx' '65-xx' '18-24' '18-24' '18-24' '18-24' '18-24'\n",
      " '65-xx' '18-24' '18-24' '35-49' '35-49' '18-24' '35-49' '18-24' '18-24'\n",
      " '35-49' '18-24' '18-24' '35-49' '18-24' '18-24' '35-49' '18-24' '18-24'\n",
      " '65-xx' '65-xx' '18-24' '65-xx' '18-24' '65-xx' '65-xx' '65-xx' '18-24'\n",
      " '65-xx' '18-24' '18-24' '35-49' '18-24' '65-xx' '35-49' '35-49' '65-xx'\n",
      " '65-xx' '18-24' '18-24' '35-49' '18-24' '65-xx' '65-xx' '65-xx' '65-xx']\n"
     ]
    }
   ],
   "source": [
    "print trained_models[4].predict(X_meta)\n",
    "print trained_models3[4].predict(X_meta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: 3grams\n",
      "Accuracy : 0.333333333333\n",
      "Confusion matrix :\n",
      " [[ 0  1  2  0  0]\n",
      " [ 0  7 10  0  0]\n",
      " [ 0 12 11  0  0]\n",
      " [ 0  1  9  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.33      0.41      0.37        17\n",
      "      35-49       0.33      0.48      0.39        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.25      0.33      0.28        54\n",
      "\n",
      "Model: soac\n",
      "Accuracy : 0.351851851852\n",
      "Confusion matrix :\n",
      " [[0 3 0 0 0]\n",
      " [0 9 3 5 0]\n",
      " [1 9 6 7 0]\n",
      " [0 5 1 4 0]\n",
      " [0 0 0 1 0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.35      0.53      0.42        17\n",
      "      35-49       0.60      0.26      0.36        23\n",
      "      50-64       0.24      0.40      0.30        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.41      0.35      0.34        54\n",
      "\n",
      "Model: lsi\n",
      "Accuracy : 0.425925925926\n",
      "Confusion matrix :\n",
      " [[ 0  0  3  0  0]\n",
      " [ 0  0 17  0  0]\n",
      " [ 0  0 23  0  0]\n",
      " [ 0  0 10  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.43      1.00      0.60        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.43      0.25        54\n",
      "\n",
      "Model: votingh\n",
      "Accuracy : 0.203703703704\n",
      "Confusion matrix :\n",
      " [[ 2  1  0  0  0]\n",
      " [ 9  5  3  0  0]\n",
      " [13  6  4  0  0]\n",
      " [ 9  0  1  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.06      0.67      0.11         3\n",
      "      25-34       0.42      0.29      0.34        17\n",
      "      35-49       0.50      0.17      0.26        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.35      0.20      0.22        54\n",
      "\n",
      "Model: meta\n",
      "Accuracy : 0.351851851852\n",
      "Confusion matrix :\n",
      " [[ 2  0  1  0  0]\n",
      " [ 4  0 10  0  3]\n",
      " [ 3  0 16  0  4]\n",
      " [ 5  0  2  0  3]\n",
      " [ 0  0  0  0  1]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.14      0.67      0.24         3\n",
      "      25-34       0.00      0.00      0.00        17\n",
      "      35-49       0.55      0.70      0.62        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.09      1.00      0.17         1\n",
      "\n",
      "avg / total       0.24      0.35      0.28        54\n",
      "\n",
      "Model: space\n",
      "Accuracy : 0.537037037037\n",
      "Confusion matrix :\n",
      " [[ 0  1  2  0  0]\n",
      " [ 0  8  7  2  0]\n",
      " [ 0  2 21  0  0]\n",
      " [ 0  3  7  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         3\n",
      "      25-34       0.57      0.47      0.52        17\n",
      "      35-49       0.55      0.91      0.69        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.42      0.54      0.46        54\n",
      "\n",
      "Model: majority\n",
      "Accuracy : 0.444444444444\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 2  7  6  1  1]\n",
      " [ 4  2 16  1  0]\n",
      " [ 2  2  6  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.58      0.41      0.48        17\n",
      "      35-49       0.55      0.70      0.62        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.42      0.44      0.42        54\n",
      "\n",
      "Model: weights\n",
      "Accuracy : 0.444444444444\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 3  5  7  1  1]\n",
      " [ 3  2 17  1  0]\n",
      " [ 2  1  6  1  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.56      0.29      0.38        17\n",
      "      35-49       0.55      0.74      0.63        23\n",
      "      50-64       0.33      0.10      0.15        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.48      0.44      0.43        54\n",
      "\n",
      "Model: accuracy\n",
      "Accuracy : 0.462962962963\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 2  7  6  1  1]\n",
      " [ 4  1 17  1  0]\n",
      " [ 2  2  6  0  0]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.10      0.33      0.15         3\n",
      "      25-34       0.64      0.41      0.50        17\n",
      "      35-49       0.57      0.74      0.64        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.45      0.46      0.44        54\n",
      "\n",
      "Model: optimal\n",
      "Accuracy : 0.462962962963\n",
      "Confusion matrix :\n",
      " [[ 1  1  1  0  0]\n",
      " [ 1  8  6  1  1]\n",
      " [ 4  2 16  1  0]\n",
      " [ 1  5  3  0  1]\n",
      " [ 1  0  0  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.12      0.33      0.18         3\n",
      "      25-34       0.50      0.47      0.48        17\n",
      "      35-49       0.62      0.70      0.65        23\n",
      "      50-64       0.00      0.00      0.00        10\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.43      0.46      0.44        54\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions4 = []\n",
    "for i, model in enumerate(trained_models):\n",
    "    print \"Model: %s\" % (model_names2[i])\n",
    "    if not(model_names2[i] in ['majority', 'weights', 'accuracy', 'optimal']):\n",
    "        predict = model.predict(X_meta)\n",
    "        predictions4.append(predict)\n",
    "    else:\n",
    "        predict = model.predict(predictions4)\n",
    "        #print predictions4\n",
    "        #break\n",
    "    #print len(predictions4)\n",
    "    #print model_names3[i]\n",
    "    acc = accuracy_score(y_meta, predict)\n",
    "    conf = confusion_matrix(y_meta, predict, labels=sorted(list(set(y))))\n",
    "    rep = classification_report(y_meta, predict, target_names=sorted(list(set(y))))\n",
    "    print('Accuracy : {}'.format(acc))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 343,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.12, 0.33, 0.18]\n",
      "[0.5, 0.47, 0.48]\n",
      "[0.62, 0.7, 0.65]\n",
      "[0.0, 0.0, 0.0]\n",
      "[0.0, 0.0, 0.0]\n",
      "plotMat: [[0.12, 0.33, 0.18], [0.5, 0.47, 0.48], [0.62, 0.7, 0.65], [0.0, 0.0, 0.0], [0.0, 0.0, 0.0]]\n",
      "support: [3, 17, 23, 10, 1]\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "def show_values(pc, fmt=\"%.2f\", **kw):\n",
    "    '''\n",
    "    Heatmap with text in each cell with matplotlib's pyplot\n",
    "    Source: http://stackoverflow.com/a/25074150/395857 \n",
    "    By HYRY\n",
    "    '''\n",
    "    from itertools import izip\n",
    "    pc.update_scalarmappable()\n",
    "    ax = pc.get_axes()\n",
    "    for p, color, value in izip(pc.get_paths(), pc.get_facecolors(), pc.get_array()):\n",
    "        x, y = p.vertices[:-2, :].mean(0)\n",
    "        if np.all(color[:3] > 0.5):\n",
    "            color = (0.0, 0.0, 0.0)\n",
    "        else:\n",
    "            color = (1.0, 1.0, 1.0)\n",
    "        ax.text(x, y, fmt % value, ha=\"center\", va=\"center\", color=color, **kw)\n",
    "\n",
    "\n",
    "def cm2inch(*tupl):\n",
    "    '''\n",
    "    Specify figure size in centimeter in matplotlib\n",
    "    Source: http://stackoverflow.com/a/22787457/395857\n",
    "    By gns-ank\n",
    "    '''\n",
    "    inch = 2.54\n",
    "    if type(tupl[0]) == tuple:\n",
    "        return tuple(i/inch for i in tupl[0])\n",
    "    else:\n",
    "        return tuple(i/inch for i in tupl)\n",
    "\n",
    "\n",
    "def heatmap(AUC, title, xlabel, ylabel, xticklabels, yticklabels, figure_width=40, figure_height=20, correct_orientation=False, cmap='RdBu'):\n",
    "    '''\n",
    "    Inspired by:\n",
    "    - http://stackoverflow.com/a/16124677/395857 \n",
    "    - http://stackoverflow.com/a/25074150/395857\n",
    "    '''\n",
    "\n",
    "    # Plot it out\n",
    "    fig, ax = plt.subplots()    \n",
    "    #c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap='RdBu', vmin=0.0, vmax=1.0)\n",
    "    c = ax.pcolor(AUC, edgecolors='k', linestyle= 'dashed', linewidths=0.2, cmap=cmap)\n",
    "\n",
    "    # put the major ticks at the middle of each cell\n",
    "    ax.set_yticks(np.arange(AUC.shape[0]) + 0.5, minor=False)\n",
    "    ax.set_xticks(np.arange(AUC.shape[1]) + 0.5, minor=False)\n",
    "\n",
    "    # set tick labels\n",
    "    #ax.set_xticklabels(np.arange(1,AUC.shape[1]+1), minor=False)\n",
    "    ax.set_xticklabels(xticklabels, minor=False)\n",
    "    ax.set_yticklabels(yticklabels, minor=False)\n",
    "\n",
    "    # set title and x/y labels\n",
    "    plt.title(title)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel)      \n",
    "\n",
    "    # Remove last blank column\n",
    "    plt.xlim( (0, AUC.shape[1]) )\n",
    "\n",
    "    # Turn off all the ticks\n",
    "    ax = plt.gca()    \n",
    "    for t in ax.xaxis.get_major_ticks():\n",
    "        t.tick1On = False\n",
    "        t.tick2On = False\n",
    "    for t in ax.yaxis.get_major_ticks():\n",
    "        t.tick1On = False\n",
    "        t.tick2On = False\n",
    "\n",
    "    # Add color bar\n",
    "    plt.colorbar(c)\n",
    "\n",
    "    # Add text in each cell \n",
    "    show_values(c)\n",
    "\n",
    "    # Proper orientation (origin at the top left instead of bottom left)\n",
    "    if correct_orientation:\n",
    "        ax.invert_yaxis()\n",
    "        ax.xaxis.tick_top()       \n",
    "\n",
    "    # resize \n",
    "    fig = plt.gcf()\n",
    "    #fig.set_size_inches(cm2inch(40, 20))\n",
    "    #fig.set_size_inches(cm2inch(40*4, 20*4))\n",
    "    fig.set_size_inches(cm2inch(figure_width, figure_height))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "\n",
    "def plot_classification_report(classification_report, title='Classification report ', cmap='RdBu'):\n",
    "    '''\n",
    "    Plot scikit-learn classification report.\n",
    "    Extension based on http://stackoverflow.com/a/31689645/395857 \n",
    "    '''\n",
    "    lines = classification_report.split('\\n')\n",
    "\n",
    "    classes = []\n",
    "    plotMat = []\n",
    "    support = []\n",
    "    class_names = []\n",
    "    for line in lines[2 : (len(lines) - 2)]:\n",
    "        t = line.strip().split()\n",
    "        if len(t) < 2: continue\n",
    "        classes.append(t[0])\n",
    "        v = [float(x) for x in t[1: len(t) - 1]]\n",
    "        support.append(int(t[-1]))\n",
    "        class_names.append(t[0])\n",
    "        print(v)\n",
    "        plotMat.append(v)\n",
    "\n",
    "    print('plotMat: {0}'.format(plotMat))\n",
    "    print('support: {0}'.format(support))\n",
    "\n",
    "    xlabel = 'Metrics'\n",
    "    ylabel = 'Classes'\n",
    "    xticklabels = ['Precision', 'Recall', 'F1-score']\n",
    "    yticklabels = ['{0} ({1})'.format(class_names[idx], sup) for idx, sup  in enumerate(support)]\n",
    "    figure_width = 25\n",
    "    figure_height = len(class_names) + 7\n",
    "    correct_orientation = False\n",
    "    heatmap(np.array(plotMat), title, xlabel, ylabel, xticklabels, yticklabels, figure_width, figure_height, correct_orientation, cmap=cmap)\n",
    "    \n",
    "plot_classification_report(rep)\n",
    "plt.savefig('test_plot_classif_report.png', dpi=200, format='png', bbox_inches='tight')\n",
    "plt.close()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}