Ensemble_Tests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# PAN16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset->Grouping User texts.\n",
      "\n",
      "Loaded 436 users...\n",
      "\n",
      "\n",
      "--------------- Thy time of Running ---------------\n",
      "Learning to judge age..\n",
      "Learning to judge gender..\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from argparse import ArgumentParser\n",
    "from sklearn.externals import joblib\n",
    "from tictacs import from_recipe\n",
    "from pan import ProfilingDataset\n",
    "import dill\n",
    "import cPickle as pickle\n",
    "# from sklearn.neighbors import KNeighborsClassifier\n",
    "# from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "\n",
    "\n",
    "infolder = \"../DATA/pan16-author-profiling-training-dataset-2016-04-25/pan16-author-profiling-training-dataset-english-2016-02-29/\"\n",
    "outfolder = \"models/\"\n",
    "print('Loading dataset->Grouping User texts.\\n')\n",
    "dataset = ProfilingDataset(infolder)\n",
    "print('Loaded {} users...\\n'.format(len(dataset.entries)))\n",
    "# get config\n",
    "config = dataset.config\n",
    "tasks = config.tasks\n",
    "print('\\n--------------- Thy time of Running ---------------')\n",
    "for task in tasks:\n",
    "    print('Learning to judge %s..' % task)\n",
    "    # load data\n",
    "    X, y = dataset.get_data(task)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Instance Based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pan import ProfilingDataset, createDocProfiles, create_target_prof_trainset\n",
    "from pan import preprocess\n",
    "\n",
    "task = 'gender'\n",
    "docs = createDocProfiles(dataset)\n",
    "X, y = create_target_prof_trainset(docs, task)\n",
    "print len(X)\n",
    "X = preprocess.preprocess(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Profile Based"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "436\n"
     ]
    }
   ],
   "source": [
    "#reload(preprocess)\n",
    "#reload(features)\n",
    "from pan import features\n",
    "from pan import preprocess\n",
    "X, y = dataset.get_data('age')\n",
    "#X, y = dataset.get_data('gender')\n",
    "print len(X)\n",
    "#print X[0]\n",
    "X = preprocess.preprocess(X)\n",
    "#print \"$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\n",
    "#print X[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('3grams',\n",
       "  TfidfVectorizer(analyzer='word', binary=False, decode_error=u'strict',\n",
       "          dtype=<type 'numpy.int64'>, encoding=u'utf-8', input=u'content',\n",
       "          lowercase=True, max_df=1.0, max_features=5000, min_df=1,\n",
       "          ngram_range=[2, 2], norm=u'l2', preprocessor=None, smooth_idf=True,\n",
       "          stop_words='english', strip_accents=None, sublinear_tf=False,\n",
       "          token_pattern=u'(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "          vocabulary=None)),\n",
       " ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.svm import SVC\n",
    "from pan.features import SOA_Model2\n",
    "\n",
    "grams3 = TfidfVectorizer(analyzer='word', ngram_range=[2,2], max_features=5000, stop_words='english')\n",
    "soa = features.SOA_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "combined = FeatureUnion([('3grams', grams3), ('soa', soa)])\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=False)\n",
    "pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "pipe.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SOAC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('soac', SOAC_Model2(max_df=1.0, max_features=None, min_df=1, thres=0.1,\n",
       "        tokenizer_var='sklearn')),\n",
       " ('svm', SVC(C=0.1, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "    decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "    max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "    tol=0.001, verbose=False))]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soac = features.SOAC_Model2(max_df=1.0, min_df=1, tokenizer_var='sklearn', max_features=None)\n",
    "svm = SVC(kernel='rbf', C=0.1, gamma=1, class_weight='balanced', probability=False)\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies), \n",
    "#                          ('soa', soa), ('soac', soac)])+\n",
    "#combined = FeatureUnion([('count_tokens', countTokens), ('count_hash', countHash),\n",
    "#                         ('count_urls', countUrls), ('count_replies', countReplies)])\n",
    "pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "pipe1.steps"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LSI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pan.features import LSI_Model\n",
    "LSImodel = LSI_Model(num_topics=25, max_df=1.0, min_df=2, max_features=None)\n",
    "svm = SVC(kernel='rbf', C=10, gamma=1, class_weight='balanced', probability=False)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe2 = Pipeline([('LSI',LSImodel), ('svm', svm)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mallet\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('LDA', LDA(lib='mallet', num_topics=90)), ('svm', SVC(C=100, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pan.features import LDA\n",
    "LDAmodel = LDA(lib='mallet', num_topics=90)\n",
    "svm = SVC(kernel='rbf', C=100, gamma=1, class_weight='balanced', probability=False)\n",
    "#pipe2 = Pipeline([('counts',combined), ('svm', svm)])\n",
    "pipe2 = Pipeline([('LDA', LDAmodel), ('svm', svm)])\n",
    "pipe2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def print_overlaps(predictions, names, verbose=True):\n",
    "    N = len(names)\n",
    "    res = numpy.zeros([N,N])\n",
    "    temp = numpy.zeros([N,N])\n",
    "    for i in range(0, N):\n",
    "        for j in range(i+1, N):\n",
    "            temp[i,j] = len([m for l, m in enumerate(predictions[i]) if (m==predictions[j][l] and m==predictions[N-1][l])])/float(len(predictions[0]))\n",
    "            res[i,j] = len([(k,v) for k,v in zip(predictions[i], predictions[j]) if k==v])/float(len(predictions[0]))\n",
    "            if verbose:\n",
    "                print \"%s - %s : %0.3f  overlap | ground-truth coverage: %0.3f\" % (names[i],  names[j], 100*res[i,j], 100*temp[i,j])\n",
    "    return  [res, temp]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "reload(combinations)\n",
    "combinations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "303 65 64 432 436\n",
      "Couldn't import dot_parser, loading of dot files will not be possible.\n",
      "Fit took: 11.560 seconds\n",
      "Predict took: 108.211 seconds\n",
      "Fitting time 2.49\n",
      "Fitting time 2.41\n",
      "Fitting time 2.36\n",
      "Fitting time 2.49\n",
      "Round 0 took: 160.399 seconds\n",
      "301 66 65 432 436\n",
      "Fit took: 12.486 seconds\n",
      "Predict took: 22.893 seconds\n",
      "Fitting time 2.31\n",
      "Fitting time 2.18\n",
      "Fitting time 2.16\n",
      "Fitting time 2.11\n",
      "Round 1 took: 75.378 seconds\n",
      "302 65 65 432 436\n",
      "Fit took: 11.824 seconds\n",
      "Predict took: 68.530 seconds\n",
      "Fitting time 2.33\n",
      "Fitting time 2.37\n",
      "Fitting time 2.32\n",
      "Fitting time 2.31\n",
      "Round 2 took: 119.679 seconds\n",
      "304 64 64 432 436\n",
      "Fit took: 11.921 seconds\n",
      "Predict took: 72.626 seconds\n",
      "Fitting time 2.15\n",
      "Fitting time 2.10\n",
      "Fitting time 2.10\n",
      "Fitting time 2.11\n",
      "Round 3 took: 122.309 seconds\n",
      "302 65 65 432 436\n",
      "Fit took: 11.602 seconds\n",
      "Predict took: 25.270 seconds\n",
      "Fitting time 2.46\n",
      "Fitting time 2.47\n",
      "Fitting time 2.46\n",
      "Fitting time 2.47\n",
      "Round 4 took: 76.907 seconds\n",
      "Total time: 554.673 seconds\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "import combinations\n",
    "import copy\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix,precision_recall_fscore_support, classification_report\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from pan.features import Metaclassifier\n",
    "import time\n",
    "import numpy\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "#pipe = Pipeline([('3grams',grams3), ('svm', svm)])\n",
    "#pipe1 = Pipeline([('soac',soac), ('svm', svm)])\n",
    "#pipe2 = Pipeline([('soa',soa), ('svm', svm)])\n",
    "\n",
    "### AGE ###\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), (\"2\", pipe2)], voting='hard')\n",
    "#models = [pipe,pipe1,pipe2,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'lda', 'voting', 'votingh']\n",
    "\n",
    "# Base Models\n",
    "base_models = [pipe, pipe1, pipe2]\n",
    "base_model_names = ['3grams', 'soac', 'lsi']\n",
    "\n",
    "# Meta Voting Models\n",
    "eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), ('2', pipe2)], voting='soft')\n",
    "eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1), ('2', pipe2)], voting='hard')\n",
    "voting_dic = {'votingf':eclf, 'votingh':eclfh}\n",
    "combinator_names = ['majority', 'weights', 'accuracy', 'optimal']\n",
    "#meta_models_names = ['votingf', 'votingh', 'space3', 'meta'] + combinator_names\n",
    "meta_models_names = ['space3', 'OLA', 'LCA', 'KNE', 'KNU'] + combinator_names\n",
    "#meta_models_names = []\n",
    "## all_models ##\n",
    "all_models_names = base_model_names + meta_models_names\n",
    "\n",
    "\n",
    "#eclf = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='soft')\n",
    "#eclfh = VotingClassifier(estimators=[(\"0\", pipe), ('1', pipe1)], voting='hard')\n",
    "#models = [pipe,pipe1,eclf, eclfh]\n",
    "#model_names = ['3grams', 'soac', 'voting', 'votingh']\n",
    "\n",
    "results = {'over':[]}\n",
    "for name in all_models_names:\n",
    "    results[name] = {'pred': [], 'conf': [], 'rep': [], 'acc': []}\n",
    "\n",
    "num_folds = 4\n",
    "train_split = 0.3\n",
    "meta_split = 0.5\n",
    "cv_rounds = 5\n",
    "seeds = list(xrange(cv_rounds))\n",
    "t0 = time.time()\n",
    "t1 = t0\n",
    "for j in xrange(cv_rounds):\n",
    "    X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=train_split, stratify=y, random_state=seeds[j])\n",
    "    for i, x in enumerate(X_train):\n",
    "        if len(x)==0:\n",
    "            X_train.remove(x)\n",
    "            y_train.remove(y_train[i])\n",
    "    for i, x in enumerate(X_cv):\n",
    "        if len(x)==0:\n",
    "            X_cv.remove(x)\n",
    "            y_cv.remove(y_cv[i])\n",
    "    if meta_split > 0:\n",
    "        X_meta, X_cv, y_meta, y_cv = train_test_split(X_cv, y_cv, test_size=meta_split, stratify=y_cv)\n",
    "        print len(X_train), len(X_cv), len(X_meta), len(X_cv) + len(X_train) + len(X_meta), len(X)\n",
    "    else:\n",
    "        print len(X_train), len(X_cv), len(X_cv) + len(X_train) , len(X)\n",
    "    trained_base_models = []\n",
    "    predictions = []\n",
    "    base_predictions_cv = []\n",
    "    base_predictions_meta = []\n",
    "    for i, model in enumerate(base_models):\n",
    "        model.fit(X_train,y_train)\n",
    "        trained_base_models.append(model)\n",
    "        predict = model.predict(X_cv)\n",
    "        predictions.append(predict)\n",
    "        base_predictions_cv.append(predict)\n",
    "        base_predictions_meta.append(model.predict(X_meta))\n",
    "        results[base_model_names[i]]['pred'].append(predict)\n",
    "        results[base_model_names[i]]['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results[base_model_names[i]]['conf'].append(confusion_matrix(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "        results[base_model_names[i]]['rep'].append(classification_report(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "    trained_all_models = copy.deepcopy(trained_base_models)\n",
    "    for name in meta_models_names:\n",
    "        #print name\n",
    "        if name =='votingf' or name=='votingh':\n",
    "            model = voting_dic[name]\n",
    "            model.fit(X_train, y_train)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'space':\n",
    "            models_for_space = {}\n",
    "            cv_scores = []\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                models_for_space[base_model_names[i]] = base_trained_model\n",
    "                cv_scores.append(base_trained_model.score(X_meta, y_meta))\n",
    "            model = combinations.SubSpaceEnsemble4_2(models_for_space, cv_scores, k=6, weights=[0.65,0.35,0.32,6], N_rand=10, rand_split=0.6)\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'space3':\n",
    "            models_for_space = {}\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                models_for_space[base_model_names[i]] = base_trained_model\n",
    "            model = SubSpaceEnsemble3(models_for_space, k=5, weights= [2,1,3,0.6])\n",
    "            model.fit(X_train, y_train)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'meta':\n",
    "            model_dic = {}\n",
    "            for i, base_trained_model in enumerate(trained_base_models):\n",
    "                model_dic[base_model_names[i]] = base_trained_model\n",
    "            model = Metaclassifier(models=model_dic, C=1.0, weights='balanced')\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name == 'OLA' or name == 'LCA' or name== 'KNE' or name == 'KNU':\n",
    "            models11 = {}\n",
    "            models_tr11 = {}\n",
    "            if name == 'LCA':\n",
    "                for i, model1 in enumerate(trained_base_models[:]):\n",
    "                    models11[base_model_names[i]] = model1\n",
    "                    models_tr11[base_model_names[i]] = trained_base_models[i].steps[0][1]        \n",
    "            else:   \n",
    "                for i, model1 in enumerate(trained_base_models[:]):\n",
    "                    models11[base_model_names[i]] = model1\n",
    "                    models_tr11[base_model_names[i]] = trained_base_models[i].steps[0][1]        \n",
    "            model = Neigbors_DS(models11, models_tr11, k=3, scheme=name, common_neigh=False)\n",
    "            #print model\n",
    "            model.fit(X_meta, y_meta)\n",
    "            predict = model.predict(X_cv)\n",
    "        if name in combinator_names:\n",
    "            #print 'mpike'\n",
    "            model = combinations.Combinator(scheme=name, weights= [1/float(len(base_predictions_meta)) for i in xrange(len(base_predictions_meta))])\n",
    "            model.fit(base_predictions_meta, y_meta)\n",
    "            predict = model.predict(base_predictions_cv)\n",
    "        trained_all_models.append(model)\n",
    "        predictions.append(predict)\n",
    "        results[name]['pred'].append(predict)\n",
    "        results[name]['acc'].append(accuracy_score(y_cv, predict))\n",
    "        results[name]['conf'].append(confusion_matrix(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "        results[name]['rep'].append(classification_report(y_cv, predict, labels=sorted(list(set(y)))))\n",
    "    print('Round %d took: %0.3f seconds') % (j, time.time()-t1)\n",
    "    t1 = time.time()\n",
    "print('Total time: %0.3f seconds') % (time.time()-t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  0  4  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  0 27  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.42      1.00      0.59        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.24        65\n",
      "\n",
      "[[ 0  0  4  0  0]\n",
      " [ 0  0 20  0  0]\n",
      " [ 0  0 27  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        20\n",
      "      35-49       0.42      1.00      0.59        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.42      0.25        64\n",
      "\n",
      "[[ 0  0  4  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  0 28  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.42      1.00      0.60        28\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.42      0.25        66\n",
      "\n",
      "[[ 0  0  4  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  0 27  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.42      1.00      0.59        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.24        65\n",
      "\n",
      "[[ 0  0  4  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  0 27  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.42      1.00      0.59        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.42      0.24        65\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i, k in enumerate(results['lsi']['conf']):\n",
    "    print k\n",
    "    print results['lsi']['rep'][i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.425287356322\n",
      "Confusion matrix :\n",
      " [[ 0  0  5  0  0]\n",
      " [ 0  0 28  0  0]\n",
      " [ 0  0 37  0  0]\n",
      " [ 0  0 16  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         5\n",
      "      25-34       0.00      0.00      0.00        28\n",
      "      35-49       0.43      1.00      0.60        37\n",
      "      50-64       0.00      0.00      0.00        16\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.43      0.25        87\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predict = trained_base_models[2].predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "%%%%%%%%%%%%%%%%  3grams  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.378505244755\n",
      "Precision : 0.288\n",
      "Recall : 0.376\n",
      "F1 : 0.318\n",
      "Confusion matrix :\n",
      " [[  0.2   1.4   2.    0.4   0. ]\n",
      " [  0.    6.8  13.8   0.2   0. ]\n",
      " [  0.    9.2  17.6   0.4   0. ]\n",
      " [  0.    3.4   8.6   0.    0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  soac  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.292671911422\n",
      "Precision : 0.302\n",
      "Recall : 0.294\n",
      "F1 : 0.288\n",
      "Confusion matrix :\n",
      " [[  0.6   0.8   1.6   0.8   0.2]\n",
      " [  3.    4.2  11.    2.    0.6]\n",
      " [  2.4   6.   12.4   4.2   2.2]\n",
      " [  2.    2.6   5.4   1.6   0.4]\n",
      " [  0.    0.    0.4   0.4   0.2]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  lsi  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.418454254079\n",
      "Precision : 0.174\n",
      "Recall : 0.42\n",
      "F1 : 0.244\n",
      "Confusion matrix :\n",
      " [[  0.    0.    4.    0.    0. ]\n",
      " [  0.    0.   20.8   0.    0. ]\n",
      " [  0.    0.   27.2   0.    0. ]\n",
      " [  0.    0.   12.    0.    0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  space3  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.341668123543\n",
      "Precision : 0.288\n",
      "Recall : 0.342\n",
      "F1 : 0.308\n",
      "Confusion matrix :\n",
      " [[  0.4   1.2   2.2   0.2   0. ]\n",
      " [  0.8   6.4  11.8   1.8   0. ]\n",
      " [  0.6   9.4  14.8   2.4   0. ]\n",
      " [  0.2   3.8   7.4   0.6   0. ]\n",
      " [  0.    0.    0.8   0.2   0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  OLA  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.387778263403\n",
      "Precision : 0.334\n",
      "Recall : 0.388\n",
      "F1 : 0.304\n",
      "Confusion matrix :\n",
      " [[  0.2   0.6   3.    0.2   0. ]\n",
      " [  0.4   2.6  17.6   0.2   0. ]\n",
      " [  0.2   3.8  21.8   1.2   0.2]\n",
      " [  0.2   0.8  10.4   0.6   0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  LCA  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.418454254079\n",
      "Precision : 0.174\n",
      "Recall : 0.42\n",
      "F1 : 0.244\n",
      "Confusion matrix :\n",
      " [[  0.    0.    4.    0.    0. ]\n",
      " [  0.    0.   20.8   0.    0. ]\n",
      " [  0.    0.   27.2   0.    0. ]\n",
      " [  0.    0.   12.    0.    0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  KNE  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.381625874126\n",
      "Precision : 0.322\n",
      "Recall : 0.382\n",
      "F1 : 0.3\n",
      "Confusion matrix :\n",
      " [[  0.2   0.6   3.    0.2   0. ]\n",
      " [  0.4   2.8  17.4   0.2   0. ]\n",
      " [  0.2   4.2  21.4   1.2   0.2]\n",
      " [  0.4   1.2  10.    0.4   0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  KNU  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.418454254079\n",
      "Precision : 0.174\n",
      "Recall : 0.42\n",
      "F1 : 0.244\n",
      "Confusion matrix :\n",
      " [[  0.    0.    4.    0.    0. ]\n",
      " [  0.    0.   20.8   0.    0. ]\n",
      " [  0.    0.   27.2   0.    0. ]\n",
      " [  0.    0.   12.    0.    0. ]\n",
      " [  0.    0.    1.    0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  majority  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.33863490676\n",
      "Precision : 0.246\n",
      "Recall : 0.338\n",
      "F1 : 0.264\n",
      "Confusion matrix :\n",
      " [[  0.    1.    3.    0.    0. ]\n",
      " [  1.    1.6  17.6   0.6   0. ]\n",
      " [  2.4   4.   19.8   1.    0. ]\n",
      " [  0.6   2.    8.4   0.6   0.4]\n",
      " [  0.    0.2   0.8   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  weights  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.33863490676\n",
      "Precision : 0.246\n",
      "Recall : 0.338\n",
      "F1 : 0.264\n",
      "Confusion matrix :\n",
      " [[  0.    1.    3.    0.    0. ]\n",
      " [  1.    1.6  17.6   0.6   0. ]\n",
      " [  2.4   4.   19.8   1.    0. ]\n",
      " [  0.6   2.    8.4   0.6   0.4]\n",
      " [  0.    0.2   0.8   0.    0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  accuracy  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.347867132867\n",
      "Precision : 0.254\n",
      "Recall : 0.348\n",
      "F1 : 0.27\n",
      "Confusion matrix :\n",
      " [[  0.    0.8   3.2   0.    0. ]\n",
      " [  0.4   1.4  17.8   1.    0.2]\n",
      " [  1.2   4.2  20.4   1.4   0. ]\n",
      " [  0.4   1.8   8.4   0.8   0.6]\n",
      " [  0.    0.    0.8   0.2   0. ]]\n",
      "#################################\n",
      "%%%%%%%%%%%%%%%%  optimal  % %%%%%%%%%%%%%%%%%%%%%%%\n",
      "#################################\n",
      "Accuracy : 0.34488490676\n",
      "Precision : 0.264\n",
      "Recall : 0.344\n",
      "F1 : 0.276\n",
      "Confusion matrix :\n",
      " [[  0.    0.8   3.2   0.    0. ]\n",
      " [  0.8   2.4  17.2   0.4   0. ]\n",
      " [  2.4   4.2  19.4   1.2   0. ]\n",
      " [  0.6   2.    8.4   0.6   0.4]\n",
      " [  0.    0.    0.8   0.2   0. ]]\n",
      "#################################\n",
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
     ]
    }
   ],
   "source": [
    "for name in all_models_names:\n",
    "    print '%%%%%%%%%%%%%%%%  ' + name  + '  % %%%%%%%%%%%%%%%%%%%%%%%'\n",
    "    print '#################################'\n",
    "    mean_acc = 0\n",
    "    mean_prec = 0\n",
    "    mean_rec = 0\n",
    "    mean_f1 = 0\n",
    "    conf = numpy.zeros([5,5])\n",
    "    for i in xrange(cv_rounds):\n",
    "        mean_acc += results[name]['acc'][i]\n",
    "        #print results[key]['report'][i].split('     ')\n",
    "        mean_prec += float(results[name]['rep'][i].split('     ')[-4][2:])\n",
    "        mean_rec += float(results[name]['rep'][i].split('     ')[-3][2:])\n",
    "        mean_f1 += float(results[name]['rep'][i].split('     ')[-2][2:])\n",
    "        conf += results[name]['conf'][i]\n",
    "    mean_acc = mean_acc/float(cv_rounds)\n",
    "    mean_prec = mean_prec/float(cv_rounds)\n",
    "    mean_rec = mean_rec/float(cv_rounds)\n",
    "    mean_f1 = mean_f1/float(cv_rounds)\n",
    "    conf = conf/float(cv_rounds)\n",
    "    print('Accuracy : {}'.format(mean_acc))\n",
    "    print('Precision : {}'.format(mean_prec))\n",
    "    print('Recall : {}'.format(mean_rec))\n",
    "    print('F1 : {}'.format(mean_f1))\n",
    "    print('Confusion matrix :\\n {}'.format(conf))\n",
    "    print '#################################'\n",
    "print '~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ CLASSIFICATION RESULTS PER MODEL~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
      "============================ Model: 3grams ==================================+\n",
      "Accuracy : 0.369230769231\n",
      "Confusion matrix :\n",
      " [[ 0  2  2  0  0]\n",
      " [ 0  7 14  0  0]\n",
      " [ 1  9 17  0  0]\n",
      " [ 0  4  8  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.32      0.33      0.33        21\n",
      "      35-49       0.40      0.63      0.49        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.27      0.37      0.31        65\n",
      "\n",
      "============================================================================\n",
      "============================ Model: soac ==================================+\n",
      "Accuracy : 0.246153846154\n",
      "Confusion matrix :\n",
      " [[ 0  1  0  3  0]\n",
      " [ 0  3  5 13  0]\n",
      " [ 0  2  4 21  0]\n",
      " [ 0  1  2  9  0]\n",
      " [ 0  0  0  1  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.43      0.14      0.21        21\n",
      "      35-49       0.36      0.15      0.21        27\n",
      "      50-64       0.19      0.75      0.31        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.32      0.25      0.21        65\n",
      "\n",
      "============================================================================\n",
      "============================ Model: lsi ==================================+\n",
      "Accuracy : 0.353846153846\n",
      "Confusion matrix :\n",
      " [[ 2  1  0  1  0]\n",
      " [ 2 10  5  4  0]\n",
      " [ 3  7  8  9  0]\n",
      " [ 2  2  5  3  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.22      0.50      0.31         4\n",
      "      25-34       0.50      0.48      0.49        21\n",
      "      35-49       0.42      0.30      0.35        27\n",
      "      50-64       0.18      0.25      0.21        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.38      0.35      0.36        65\n",
      "\n",
      "============================================================================\n",
      "Base Accuracies\n",
      "3grams : 36.92  ||  soac : 24.62  ||  lsi : 35.38  \n",
      "Models Correct Aggrement Percentages\n",
      "        Only this Model   1-model aggree   2-model aggree\n",
      "3grams            21.54            28.57            71.43\n",
      "soac              12.31            60.00            40.00\n",
      "lsi               18.46            62.50            37.50\n",
      "Predictions Distributions\n",
      "All correct : 4.62  || Some correct : 67.69 || All wrong: 27.69 \n",
      "Not all Correct Instances Distributions\n",
      "None Correct : 27.69  ||  1 correct : 52.31  ||  2 correct : 15.38  \n",
      "---------------------------------------------------------------\n",
      "Diversity Tests Report\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Details\n",
      "===============================================================\n",
      "Correlation: For +-1 perfect aggrement/disagreement\n",
      "Q-statistic: Q=0  => Independent. For q>0 predictors find the the same results\n",
      "Cohen's k: k->0  => High Disagreement => High Diversity\n",
      "Kohovi-Wolpert Variance -> Inf => High Diversity\n",
      "Conditional Accuracy Table: Conditional Probability that the row system predicts correctly, given\n",
      "                            that the column system also predicts correctly\n",
      "===============================================================\n",
      "---------------------------------------------------------------\n",
      "\n",
      "Measures Results\n",
      "---------------------------------------------------------------\n",
      "\n",
      "#####  Kohovi-Wolpert Variance:  0.150  #####\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "#### Pairwise Average Metrics: #####\n",
      "Avg. Cor: 0.000\n",
      "Avg. Q-statistic: -0.058\n",
      "Avg. Cohen's k: -0.071\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n",
      "###Conditional Accuracy Table###\n",
      "        3grams  soac  lsi\n",
      "3grams    1.00  0.31 0.35\n",
      "soac      0.21  1.00 0.26\n",
      "lsi       0.33  0.38 1.00\n",
      "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n",
      "\n"
     ]
    }
   ],
   "source": [
    "import EnsembleDiversityTests\n",
    "reload(EnsembleDiversityTests)\n",
    "a = EnsembleDiversityTests.BaseClassifiers(base_predictions_cv, base_model_names, y_cv)\n",
    "b = EnsembleDiversityTests.DiversityTests(base_predictions_cv, base_model_names, y_cv)\n",
    "a.get_classification_report()\n",
    "a.vis_flag = True\n",
    "a.get_comparison_report()\n",
    "b.print_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('LSI', LSI_Model(max_df=1.0, max_features=None, min_df=2, num_topics=25)), ('svm', SVC(C=10, cache_size=200, class_weight='balanced', coef0=0.0,\n",
       "  decision_function_shape=None, degree=3, gamma=1, kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class XGBoostClassifier(BaseEstimator, TransformerMixin):\n",
    "\n",
    "    def __init__(self, **params):\n",
    "\n",
    "        self.clf = None\n",
    "        # self.num_boost_round = 100\n",
    "        self.labels = None\n",
    "        self.params = params.copy()\n",
    "        for name, label in params.iteritems():\n",
    "            setattr(self, name, label)\n",
    "\n",
    "    def fit(self, X, y, num_boost_round=None):\n",
    "\n",
    "        import xgboost as xgb\n",
    "\n",
    "        num_boost_round = num_boost_round or self.num_boost_round\n",
    "        #print self.params\n",
    "        #print self.__dict__\n",
    "        #print num_boost_round\n",
    "        self.label2num = dict((label, i) for i, label in enumerate(sorted(set(y))))\n",
    "        self.params.update({'num_class': len(set(y))})\n",
    "        #print type(self.params)\n",
    "        #print self.params\n",
    "        # self.params.update({'objective': 'multi:softprob', \n",
    "        #                     'num_class': len(set(y)), \n",
    "        #                     'eval_metric': 'merror', \n",
    "        #                     'nthread': -1, \n",
    "        #                     'learning_rate': 0.1, \n",
    "        #                     'n_estimators': 140, \n",
    "        #                     'max_depth': 5,\n",
    "        #                     'min_child_weight': 1, \n",
    "        #                     'gamma': 0, \n",
    "        #                     'subsample': 0.8, \n",
    "        #                     'colsample_bytree': 0.8,\n",
    "        #                     'scale_pos_weight': 1, \n",
    "        #                     'seed': 27})\n",
    "        if y is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            dtrain = xgb.DMatrix(X, label=[self.label2num[label] for label in y])\n",
    "            self.clf = xgb.train(params=self.params, dtrain=dtrain, num_boost_round=self.num_boost_round)\n",
    "\n",
    "    def predict(self, X):\n",
    "\n",
    "        import numpy\n",
    "        from xgboost import DMatrix\n",
    "\n",
    "        num2label = dict((i, label) for label, i in self.label2num.items())\n",
    "        dtest = DMatrix(X)\n",
    "        Y = self.clf.predict(dtest)\n",
    "        return numpy.array([num2label[i] for i in Y] )\n",
    "        #Y = self.predict_proba(X)\n",
    "        #y = numpy.argmax(Y, axis=1)\n",
    "        #return numpy.array([num2label[i] for i in y])\n",
    "\n",
    "    def predict_proba(self, X):\n",
    "\n",
    "        from xgboost import DMatrix\n",
    "\n",
    "        dtest = DMatrix(X)\n",
    "        return self.clf.predict(dtest)\n",
    "\n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "\n",
    "        # Y = self.predict_proba(X)\n",
    "        # return 1 / logloss(y, Y)\n",
    "\n",
    "    def get_params(self, deep=True):\n",
    "        return self.params\n",
    "\n",
    "    def set_params(self, **params):\n",
    "        if 'num_boost_round' in params:\n",
    "            self.num_boost_round = params.pop('num_boost_round')\n",
    "        if 'objective' in params:\n",
    "            del params['objective']\n",
    "        self.params.update(params)\n",
    "        return self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split \n",
    "params = {\n",
    "    'svm__C':[0.01, 0.1, 1, 10, 100],\n",
    "    'svm__kernel':['rbf', 'linear'],\n",
    "    'LSI__num_topics': [25,50,100, 150],\n",
    "    'LSI__max_df': [1.0, 0.9, 0.8],\n",
    "    'LSI__max_features': [None, 5000, 10000]\n",
    "}\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.3, stratify=y, )\n",
    "grid1 = GridSearchCV(pipe2, param_grid =params, cv=4, verbose=2, n_jobs=-1)\n",
    "grid1.fit(X_train, y_train)\n",
    "print(grid1.best_score_)\n",
    "print(grid1.best_params_)\n",
    "pipe2 = grid1.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.424242424242\n",
      "Confusion matrix :\n",
      " [[ 0  0  4  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  0 28  0  0]\n",
      " [ 0  0 12  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.42      1.00      0.60        28\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.18      0.42      0.25        66\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predict = trained_base_models[2].predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  49 tasks       | elapsed:  4.6min\n",
      "[Parallel(n_jobs=1)]: Done 199 tasks       | elapsed: 18.7min\n",
      "[Parallel(n_jobs=1)]: Done 449 tasks       | elapsed: 42.3min\n",
      "[Parallel(n_jobs=1)]: Done 799 tasks       | elapsed: 75.9min\n",
      "[Parallel(n_jobs=1)]: Done 1249 tasks       | elapsed: 120.6min\n",
      "[Parallel(n_jobs=1)]: Done 1799 tasks       | elapsed: 177.9min\n",
      "[Parallel(n_jobs=1)]: Done 2449 tasks       | elapsed: 248.4min\n",
      "[Parallel(n_jobs=1)]: Done 3199 tasks       | elapsed: 328.3min\n",
      "[Parallel(n_jobs=1)]: Done 4049 tasks       | elapsed: 418.8min\n",
      "[Parallel(n_jobs=1)]: Done 4999 tasks       | elapsed: 519.6min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 4 folds for each of 1260 candidates, totalling 5040 fits\n",
      "0.40625\n",
      "{'XGBOOST__max_depth': 3, 'XGBOOST__learning_rate': 0.25000000000000006, 'XGBOOST__min_child_weight': 3, 'XGBOOST__reg_alpha': 0.1, 'XGBOOST__gamma': 0}\n",
      "Accuracy : 0.369230769231\n",
      "Confusion matrix :\n",
      " [[ 1  0  3  0  0]\n",
      " [ 0  5 12  4  0]\n",
      " [ 1  9 16  1  0]\n",
      " [ 0  1  9  2  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.50      0.25      0.33         4\n",
      "      25-34       0.33      0.24      0.28        21\n",
      "      35-49       0.39      0.59      0.47        27\n",
      "      50-64       0.29      0.17      0.21        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.35      0.37      0.34        65\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 5040 out of 5040 | elapsed: 524.0min finished\n"
     ]
    }
   ],
   "source": [
    "#import pan.features\n",
    "#reload(pan.features)\n",
    "from sklearn.pipeline import Pipeline, FeatureUnion\n",
    "#from pan.features import XGBoostClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "estimator_list = []\n",
    "for i, model in enumerate(trained_base_models):\n",
    "    estimator_list.append((base_model_names[i], trained_base_models[i].steps[0][1]))\n",
    "FU = FeatureUnion(estimator_list)\n",
    "\n",
    "\n",
    "xgb_params = {'objective': 'multi:softmax',\n",
    "                            'num_boost_round': 100,\n",
    "                            'num_class': len(set(y)), \n",
    "                            'eval_metric': 'merror', \n",
    "                            'nthread': -1, \n",
    "                            'learning_rate': 0.1, \n",
    "                            'n_estimators': 140, \n",
    "                            'max_depth': 5,\n",
    "                            'min_child_weight': 1, \n",
    "                            'gamma': 0, \n",
    "                            'subsample': 0.8, \n",
    "                            'colsample_bytree': 0.8,\n",
    "                            'scale_pos_weight': 1, \n",
    "                            'seed': 27}\n",
    "#XGB = XGBoostClassifier()\n",
    "#XGB.set_params\n",
    "params = {\n",
    "    'XGBOOST__max_depth':list(numpy.arange(3,10,1)),\n",
    "    'XGBOOST__gamma':[0, 0.1, 0.2],\n",
    "    'XGBOOST__min_child_weight':range(1,6,2),\n",
    "    'XGBOOST__learning_rate': list(numpy.arange(0.1, 0.3, 0.05)),\n",
    "    'XGBOOST__reg_alpha':[1e-5, 1e-2, 0.1, 1, 100],\n",
    "}\n",
    "pipe_fu = Pipeline([('FU',FU), ('XGBOOST', XGBoostClassifier(**xgb_params))])\n",
    "grid1 = GridSearchCV(pipe_fu, param_grid =params, cv=4, verbose=1)\n",
    "grid1.fit(X_meta, y_meta)\n",
    "print(grid1.best_score_)\n",
    "print(grid1.best_params_)\n",
    "pipe_fu = grid1.best_estimator_\n",
    "predict = pipe_fu.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(301, 102406)\n",
      "Fitting 5 folds for each of 38 candidates, totalling 190 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  76 tasks      | elapsed:    1.6s\n",
      "[Parallel(n_jobs=-1)]: Done 190 out of 190 | elapsed:    3.6s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.511627906977\n",
      "{'n_neighbors': 15, 'weights': 'uniform'}\n",
      "(65, 102406)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Found arrays with inconsistent numbers of samples: [65 66]",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-53-411d8051c4d4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     29\u001b[0m \u001b[0mpredict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_estimator_\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcounter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     30\u001b[0m \u001b[1;32mprint\u001b[0m \u001b[0mcounter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 31\u001b[1;33m \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     32\u001b[0m \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m \u001b[0mrep\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mclassification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_names\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.pyc\u001b[0m in \u001b[0;36maccuracy_score\u001b[1;34m(y_true, y_pred, normalize, sample_weight)\u001b[0m\n\u001b[0;32m    170\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    171\u001b[0m     \u001b[1;31m# Compute accuracy for each possible representation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 172\u001b[1;33m     \u001b[0my_type\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_check_targets\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    173\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0my_type\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'multilabel'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m         \u001b[0mdiffering_labels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcount_nonzero\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/metrics/classification.pyc\u001b[0m in \u001b[0;36m_check_targets\u001b[1;34m(y_true, y_pred)\u001b[0m\n\u001b[0;32m     70\u001b[0m     \u001b[0my_pred\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0marray\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mindicator\u001b[0m \u001b[0mmatrix\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     71\u001b[0m     \"\"\"\n\u001b[1;32m---> 72\u001b[1;33m     \u001b[0mcheck_consistent_length\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     73\u001b[0m     \u001b[0mtype_true\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[0mtype_pred\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtype_of_target\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_consistent_length\u001b[1;34m(*arrays)\u001b[0m\n\u001b[0;32m    174\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0muniques\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    175\u001b[0m         raise ValueError(\"Found arrays with inconsistent numbers of samples: \"\n\u001b[1;32m--> 176\u001b[1;33m                          \"%s\" % str(uniques))\n\u001b[0m\u001b[0;32m    177\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    178\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found arrays with inconsistent numbers of samples: [65 66]"
     ]
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "counter = CountVectorizer()\n",
    "parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "counter.set_params(**parameters)\n",
    "X_train_truth = counter.fit_transform(X_train)\n",
    "print X_train_truth.shape\n",
    "num_folds = 5\n",
    "params = {'n_neighbors':[i for i in xrange(1,20)], 'weights':['uniform', 'distance']}\n",
    "gg = GridSearchCV(KNeighborsClassifier(), param_grid=params, n_jobs=-1, cv=num_folds, refit=True, verbose=1)\n",
    "gg.fit(X_train_truth, y_train)\n",
    "print(gg.best_score_)\n",
    "print(gg.best_params_)\n",
    "predict = gg.best_estimator_.predict(counter.transform(X_meta))\n",
    "print counter.transform(X_meta).shape\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[ 0.5,  0.5]]), array([[1, 2]]))"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> from sklearn.neighbors import KNeighborsClassifier\n",
    ">>> X = [[3], [0], [1], [4]]\n",
    ">>> y = [0, 0, 1, 1]\n",
    "neigh = KNeighborsClassifier(n_neighbors=2)\n",
    "neigh.fit(X, y) \n",
    "neigh.kneighbors([0.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         5\n",
      "      25-34       0.32      0.36      0.34        28\n",
      "      35-49       0.46      0.51      0.49        35\n",
      "      50-64       0.14      0.12      0.13        16\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.32      0.35      0.34        85\n",
      "\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         5\n",
      "      25-34       0.32      0.36      0.34        28\n",
      "      35-49       0.46      0.51      0.49        35\n",
      "      50-64       0.14      0.12      0.13        16\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.32      0.35      0.34        85\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print results['LCA']['rep'][-1]\n",
    "print results['OLA']['rep'][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"../EnsembleDiversityTests/\")\n",
    "\n",
    "from EnsembleDiversityTests import DiversityTests, BaseClassifiers\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "\n",
    "class Neigbors_DS(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Best model base on the predictions of the k-nearest neighbors. Many different schemes.\n",
    "        Also, implements a common neighborhoud instead a per transformation one.\n",
    "        \n",
    "        Args:\n",
    "            - scheme: String flag. Can be one of the following:\n",
    "                - 'LCA': Local Class Accuracy\n",
    "                - 'OLA': Overall Local Accuracy\n",
    "                - 'KNE': K_Neighbors Elimination. Start from a k \n",
    "                - 'optimal': The optimal weights are found, this\n",
    "                             is done by optimizing over the classification\n",
    "                             error\n",
    "                - weights: list or numpy.array(not sure?) containing as many\n",
    "                             weights as the models in the ensemble\n",
    "        Returns:\n",
    "            - The  ensemble Model. Needs to be fitted for the encoding part\n",
    "        \n",
    "        \"\"\"\n",
    "\n",
    "    def __init__(self, models, models_tr, k= 5, scheme='LCA', common_neigh=False):\n",
    "\n",
    "        if (not models) or (not models_tr):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.models_tr = models_tr\n",
    "            self.k = k\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.predictions = {}\n",
    "            self.true = []\n",
    "            self.trees = {}\n",
    "            self.scheme = scheme\n",
    "            self.common_neigh = common_neigh\n",
    "            if common_neigh:\n",
    "                from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "                self.counter = CountVectorizer()\n",
    "                parameters = {\n",
    "                        'input': 'content',\n",
    "                        'encoding': 'utf-8',\n",
    "                        'decode_error': 'ignore',\n",
    "                        'analyzer': 'word',\n",
    "                        'stop_words': 'english',\n",
    "                        # 'vocabulary':list(voc),\n",
    "                        #'tokenizer': tokenization,\n",
    "                        #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                        #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                        'max_df': 1.0,\n",
    "                        'min_df': 1,\n",
    "                        'max_features':None\n",
    "                    }\n",
    "                self.counter.set_params(**parameters)\n",
    "                self.gt_tree = None\n",
    "            else:\n",
    "                self.counter = None\n",
    "            if self.scheme == 'LCA':\n",
    "                self.predictor = self.predict_lca\n",
    "            elif self.scheme == 'KNE':\n",
    "                self.predictor = self.predict_kne\n",
    "            elif self.scheme == 'OLA':\n",
    "                self.predictor = self.predict_ola\n",
    "            elif self.scheme == 'KNU':\n",
    "                self.predictor = self.predict_knu\n",
    "            else:\n",
    "                self.predictor = self.predict_ola\n",
    "                \n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        from sklearn.neighbors import BallTree\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        import random\n",
    "        import time\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            t0 = time.time()\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                #predictions.append(model.predict(X_cv))\n",
    "                # print len(predictions[-1])\n",
    "                if self.common_neigh:\n",
    "                    X_tr = self.counter.fit_transform(X_cv)\n",
    "                    self.gt_tree = BallTree(X_tr.toarray(), leaf_size=20)\n",
    "                else:\n",
    "                    X_tr = self.models_tr[name].transform(X_cv)\n",
    "                    if hasattr(X_tr, \"toarray\"):\n",
    "                        self.trees[name] = BallTree(X_tr.toarray(), leaf_size=20)\n",
    "                    else:\n",
    "                        self.trees[name] = BallTree(X_tr, leaf_size=20)    \n",
    "                self.predictions[name] = model.predict(X_cv)\n",
    "            self.true = y_true\n",
    "            print 'Fitting time %0.2f' % (time.time() - t0)\n",
    "\n",
    "    def predict(self, X):\n",
    "        # import time\n",
    "\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        y_pred = []\n",
    "        # t0 = time.time()\n",
    "        for i, x in enumerate(X):\n",
    "            # print 'True Sample: ' + y_real[i]\n",
    "            y_pred.append(self.predictor(x))\n",
    "        # print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        # return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def predict_lca(self, sample):\n",
    "        preds = []\n",
    "        for name, model in self.models.iteritems():\n",
    "            preds.append(model.predict([sample])[0])\n",
    "#         print 'Preds: ' + str(preds)\n",
    "        if len(set(preds))==1:\n",
    "#             print 'Unanimous Decision: ' + str(preds[0])\n",
    "#             print '='*50\n",
    "            return preds[0]\n",
    "        else:\n",
    "            lca = [0 for pred in preds]\n",
    "            model_ind = 0\n",
    "            for name, model in self.models.iteritems():\n",
    "                # print 'Model: ' + name\n",
    "                sample_trans = self.models_tr[name].transform([sample])\n",
    "                step = 50\n",
    "                found_k_class_n = self.k\n",
    "                neigh_indexes = []\n",
    "                while found_k_class_n>=0:\n",
    "                    if self.common_neigh:\n",
    "                        _, model_neig = self.gt_tree.query(self.counter.transform([sample]).toarray(), step)\n",
    "                    else:\n",
    "                        if hasattr(sample_trans, \"toarray\"):\n",
    "                            _, model_neig = self.trees[name].query(sample_trans.toarray(), step)\n",
    "                        else:\n",
    "                            _, model_neig = self.trees[name].query(sample_trans, step)\n",
    "                    for model_n_i in model_neig[0].tolist():\n",
    "                        if name == 'lsi':\n",
    "                            if self.true[model_n_i] != '35-49':\n",
    "                                pass\n",
    "                                # print 'GG'\n",
    "                        if preds[model_ind] == self.true[model_n_i]:\n",
    "                            neigh_indexes.append(model_n_i)\n",
    "                            found_k_class_n -= 1\n",
    "                    step *= 2\n",
    "                    if step >= len(self.predictions[name]):\n",
    "                        step = len(self.predictions[name])-1\n",
    "                neigh_indexes = neigh_indexes[:self.k] \n",
    "                model_neig_pred = []\n",
    "                neigh_true = []\n",
    "                for model_n_i in neigh_indexes:\n",
    "                    model_neig_pred.append(self.predictions[name][model_n_i])\n",
    "                    neigh_true.append(self.true[model_n_i])\n",
    "                lca[model_ind] = accuracy_score(neigh_true, model_neig_pred, normalize=True)\n",
    "#                 print 'True Neigh: ' + str(neigh_true)\n",
    "#                 print 'Predicted Neigh: ' + str(model_neig_pred)\n",
    "                \n",
    "                model_ind += 1\n",
    "#             print 'LCA: %s' % str(['%0.2f' % (100*k) for k in lca])\n",
    "#             print \"Total Predicted: %s from model %s\" % (str(preds[lca.index(max(lca))]), self.models.keys()[lca.index(max(lca))])\n",
    "#             print '='*50\n",
    "            return preds[lca.index(max(lca))]\n",
    "\n",
    "\n",
    "    def predict_ola(self, sample):\n",
    "        preds = []\n",
    "        for name, model in self.models.iteritems():\n",
    "            preds.append(model.predict([sample])[0])\n",
    "#         print 'Preds: ' + str(preds)\n",
    "        if len(set(preds))==1:\n",
    "#             print 'Unanimous Decision: ' + str(preds[0])\n",
    "#             print '='*50\n",
    "            return preds[0]\n",
    "        else:\n",
    "            ola = [0 for pred in preds]\n",
    "            model_ind = 0\n",
    "            for name, model in self.models.iteritems():\n",
    "#                 print 'Model: ' + name\n",
    "                if self.common_neigh:\n",
    "                    _, model_neig = self.gt_tree.query(self.counter.transform([sample]).toarray(), self.k)\n",
    "                else:\n",
    "                    sample_trans = self.models_tr[name].transform([sample])\n",
    "                    if hasattr(sample_trans, \"toarray\"):\n",
    "                        _, model_neig = self.trees[name].query(sample_trans.toarray(), self.k)\n",
    "                    else:\n",
    "                        _, model_neig = self.trees[name].query(sample_trans, self.k)\n",
    "                model_neig_pred = []\n",
    "                neigh_true = []\n",
    "                for model_n_i in model_neig[0].tolist():\n",
    "                    model_neig_pred.append(self.predictions[name][model_n_i])\n",
    "                    neigh_true.append(self.true[model_n_i])\n",
    "                ola[model_ind] = accuracy_score(neigh_true, model_neig_pred, normalize=True)\n",
    "#                 print 'True Neigh: ' + str(neigh_true)\n",
    "#                 print 'Predicted Neigh: ' + str(model_neig_pred)\n",
    "#                 print 'OLA: %s' % str(['%0.2f' % (100*k) for k in ola])\n",
    "                model_ind += 1\n",
    "            \n",
    "#             print \"Total Predicted: %s from model %s\" % (str(preds[ola.index(max(ola))]), self.models.keys()[ola.index(max(ola))])\n",
    "#             print '='*50\n",
    "            return preds[ola.index(max(ola))]\n",
    "\n",
    "    def predict_kne(self, sample):\n",
    "        preds = []\n",
    "        for name, model in self.models.iteritems():\n",
    "            preds.append(model.predict([sample])[0])\n",
    "#         print 'Preds: ' + str(preds)\n",
    "        if len(set(preds))==1:\n",
    "#             print 'Unanimous Decision: ' + str(preds[0])\n",
    "#             print '='*50\n",
    "            return preds[0]\n",
    "        else:\n",
    "            k = self.k\n",
    "            possible_experts = []\n",
    "            neigh_radius = []\n",
    "            ola_scores = []\n",
    "            while k>0 :\n",
    "                model_ind = 0\n",
    "                # print k\n",
    "                for name, model in self.models.iteritems():\n",
    "#                     print 'Model: ' + name\n",
    "                    if self.common_neigh:\n",
    "                        _, model_neig = self.gt_tree.query(self.counter.transform([sample]).toarray(), k)\n",
    "                    else:\n",
    "                        sample_trans = self.models_tr[name].transform([sample])\n",
    "                        if hasattr(sample_trans, \"toarray\"):\n",
    "                            _, model_neig = self.trees[name].query(sample_trans.toarray(), k)\n",
    "                        else:\n",
    "                            _, model_neig = self.trees[name].query(sample_trans, k)\n",
    "                    model_neig_pred = []\n",
    "                    neigh_true = []\n",
    "                    for model_n_i in model_neig[0].tolist():\n",
    "                        model_neig_pred.append(self.predictions[name][model_n_i])\n",
    "                        neigh_true.append(self.true[model_n_i])\n",
    "#                     print 'True Neigh: ' + str(neigh_true)\n",
    "#                     print 'Predicted Neigh: ' + str(model_neig_pred)\n",
    "                    if k == self.k:\n",
    "                        ola_scores.append(accuracy_score(neigh_true, model_neig_pred, normalize=True))\n",
    "                    if neigh_true == model_neig_pred:\n",
    "                        possible_experts.append(preds[model_ind])\n",
    "                        neigh_radius.append(k)\n",
    "                    model_ind += 1\n",
    "                if not(possible_experts):\n",
    "                    k -= 1\n",
    "                else:\n",
    "                    break\n",
    "            if not(possible_experts):\n",
    "#                 print 'No experts'\n",
    "#                 print 'OLA_Scores: %s' % str(['%0.2f' % (100*k) for k in ola_scores])\n",
    "#                 print preds[ola_scores.index(max(ola_scores))]\n",
    "                return preds[ola_scores.index(max(ola_scores))]\n",
    "            else:\n",
    "#                 print 'Experts:'\n",
    "#                 print possible_experts\n",
    "#                 print neigh_radius\n",
    "                return possible_experts[0]\n",
    "            \n",
    "     \n",
    "    def predict_knu(self, sample):\n",
    "        \n",
    "\n",
    "        preds = []\n",
    "        for name, model in self.models.iteritems():\n",
    "            preds.append(model.predict([sample])[0])\n",
    "        #print 'Preds: ' + str(preds)\n",
    "        if len(set(preds))==1:\n",
    "#             print 'Unanimous Decision: ' + str(preds[0])\n",
    "#             print '='*50\n",
    "            return preds[0]\n",
    "        else:\n",
    "            possible_experts = []\n",
    "            neigh_radius = []\n",
    "            ola_scores = []\n",
    "            model_ind = 0\n",
    "            for name, model in self.models.iteritems():\n",
    "#                 print 'Model: ' + name\n",
    "                if self.common_neigh:\n",
    "                    _, model_neig = self.gt_tree.query(self.counter.transform([sample]).toarray(), self.k)\n",
    "                else:\n",
    "                    sample_trans = self.models_tr[name].transform([sample])\n",
    "                    if hasattr(sample_trans, \"toarray\"):\n",
    "                        _, model_neig = self.trees[name].query(sample_trans.toarray(), self.k)\n",
    "                    else:\n",
    "                        _, model_neig = self.trees[name].query(sample_trans, self.k)\n",
    "                model_neig_pred = []\n",
    "                neigh_true = []\n",
    "                for model_n_i in model_neig[0].tolist():\n",
    "                    model_neig_pred.append(self.predictions[name][model_n_i])\n",
    "                    neigh_true.append(self.true[model_n_i])\n",
    "                    if model_neig_pred[-1] == neigh_true[-1]:\n",
    "                        possible_experts.append(preds[model_ind])\n",
    "                ola_scores.append(accuracy_score(neigh_true, model_neig_pred, normalize=True))\n",
    "#                 print 'True Neigh: ' + str(neigh_true)\n",
    "#                 print 'Predicted Neigh: ' + str(model_neig_pred)\n",
    "        if not(possible_experts):\n",
    "#             print 'No experts'\n",
    "#             print 'OLA_Scores: %s' % str(['%0.2f' % (100*k) for k in ola_scores])\n",
    "#             print preds[ola_scores.index(max(ola_scores))]\n",
    "            return preds[ola_scores.index(max(ola_scores))]\n",
    "        else:\n",
    "#             print 'Experts:'\n",
    "#             print possible_experts\n",
    "#             print most_common(possible_experts)\n",
    "            return most_common(possible_experts)\n",
    "                        \n",
    "def most_common(lst):\n",
    "    return max(set(lst), key=lst.count)\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting time 14.78\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "Number of predictions of classifier 3grams is different                                      then the number of true labels. 65 != 131 .                                      ",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-16-3b7657bf6d95>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mLCA\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNeigbors_DS\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodels11\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodels_tr11\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscheme\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'OLA'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcommon_neigh\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mLCA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBaseClassifiers\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbase_predictions_cv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbase_model_names\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_classification_report\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[0mpredict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLCA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/media/kostas/DATA/GIT/EnsembleDiversityTests/EnsembleDiversityTests.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, predictions, names, true, vis_flag)\u001b[0m\n\u001b[0;32m    253\u001b[0m                 raise AttributeError('Number of predictions of classifier %s is different\\\n\u001b[0;32m    254\u001b[0m                                       \u001b[0mthen\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mnumber\u001b[0m \u001b[0mof\u001b[0m \u001b[0mtrue\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m.\u001b[0m \u001b[1;33m%\u001b[0m\u001b[0md\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;33m%\u001b[0m\u001b[0md\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 255\u001b[1;33m                                       ' % (names[i], len(predict), N))\n\u001b[0m\u001b[0;32m    256\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misdisjoint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    257\u001b[0m                 raise AttributeError('Label in predictions of %s not in truth set. \\\n",
      "\u001b[1;31mAttributeError\u001b[0m: Number of predictions of classifier 3grams is different                                      then the number of true labels. 65 != 131 .                                      "
     ]
    }
   ],
   "source": [
    "models11 = {}\n",
    "models_tr11 = {}\n",
    "for i, model in enumerate(trained_base_models[:]):\n",
    "    models11[base_model_names[i]] = model\n",
    "    models_tr11[base_model_names[i]] = trained_base_models[i].steps[0][1]        \n",
    "LCA = Neigbors_DS(models11, models_tr11, k=3, scheme='OLA', common_neigh=False)\n",
    "LCA.fit(X_meta, y_meta)\n",
    "a = BaseClassifiers(base_predictions_cv[:], base_model_names[:], y_cv)\n",
    "a.get_classification_report()\n",
    "predict = LCA.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 4 candidates, totalling 20 fits\n",
      "Fitting time 12.72\n",
      "Fitting time 11.71\n"
     ]
    },
    {
     "ename": "JoblibValueError",
     "evalue": "JoblibValueError\n___________________________________________________________________________\nMultiprocessing exception:\n...........................................................................\n/usr/lib/python2.7/runpy.py in _run_module_as_main(mod_name='ipykernel.__main__', alter_argv=1)\n    157     pkg_name = mod_name.rpartition('.')[0]\n    158     main_globals = sys.modules[\"__main__\"].__dict__\n    159     if alter_argv:\n    160         sys.argv[0] = fname\n    161     return _run_code(code, main_globals, None,\n--> 162                      \"__main__\", fname, loader, pkg_name)\n        fname = '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py'\n        loader = <pkgutil.ImpLoader instance>\n        pkg_name = 'ipykernel'\n    163 \n    164 def run_module(mod_name, init_globals=None,\n    165                run_name=None, alter_sys=False):\n    166     \"\"\"Execute a module's code without importing it\n\n...........................................................................\n/usr/lib/python2.7/runpy.py in _run_code(code=<code object <module> at 0x7f3abb4ad930, file \"/...2.7/dist-packages/ipykernel/__main__.py\", line 1>, run_globals={'__builtins__': <module '__builtin__' (built-in)>, '__doc__': None, '__file__': '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', '__loader__': <pkgutil.ImpLoader instance>, '__name__': '__main__', '__package__': 'ipykernel', 'app': <module 'ipykernel.kernelapp' from '/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.pyc'>}, init_globals=None, mod_name='__main__', mod_fname='/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', mod_loader=<pkgutil.ImpLoader instance>, pkg_name='ipykernel')\n     67         run_globals.update(init_globals)\n     68     run_globals.update(__name__ = mod_name,\n     69                        __file__ = mod_fname,\n     70                        __loader__ = mod_loader,\n     71                        __package__ = pkg_name)\n---> 72     exec code in run_globals\n        code = <code object <module> at 0x7f3abb4ad930, file \"/...2.7/dist-packages/ipykernel/__main__.py\", line 1>\n        run_globals = {'__builtins__': <module '__builtin__' (built-in)>, '__doc__': None, '__file__': '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', '__loader__': <pkgutil.ImpLoader instance>, '__name__': '__main__', '__package__': 'ipykernel', 'app': <module 'ipykernel.kernelapp' from '/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.pyc'>}\n     73     return run_globals\n     74 \n     75 def _run_module_code(code, init_globals=None,\n     76                     mod_name=None, mod_fname=None,\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py in <module>()\n      1 \n      2 \n----> 3 \n      4 if __name__ == '__main__':\n      5     from ipykernel import kernelapp as app\n      6     app.launch_new_instance()\n      7 \n      8 \n      9 \n     10 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py in launch_instance(cls=<class 'ipykernel.kernelapp.IPKernelApp'>, argv=None, **kwargs={})\n    584         \n    585         If a global instance already exists, this reinitializes and starts it\n    586         \"\"\"\n    587         app = cls.instance(**kwargs)\n    588         app.initialize(argv)\n--> 589         app.start()\n        app.start = <bound method IPKernelApp.start of <ipykernel.kernelapp.IPKernelApp object>>\n    590 \n    591 #-----------------------------------------------------------------------------\n    592 # utility functions, for convenience\n    593 #-----------------------------------------------------------------------------\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py in start(self=<ipykernel.kernelapp.IPKernelApp object>)\n    400         \n    401         if self.poller is not None:\n    402             self.poller.start()\n    403         self.kernel.start()\n    404         try:\n--> 405             ioloop.IOLoop.instance().start()\n    406         except KeyboardInterrupt:\n    407             pass\n    408 \n    409 launch_new_instance = IPKernelApp.launch_instance\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py in start(self=<zmq.eventloop.ioloop.ZMQIOLoop object>)\n    157             PollIOLoop.configure(ZMQIOLoop)\n    158         return PollIOLoop.current(*args, **kwargs)\n    159     \n    160     def start(self):\n    161         try:\n--> 162             super(ZMQIOLoop, self).start()\n        self.start = <bound method ZMQIOLoop.start of <zmq.eventloop.ioloop.ZMQIOLoop object>>\n    163         except ZMQError as e:\n    164             if e.errno == ETERM:\n    165                 # quietly return on ETERM\n    166                 pass\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/ioloop.py in start(self=<zmq.eventloop.ioloop.ZMQIOLoop object>)\n    878                 self._events.update(event_pairs)\n    879                 while self._events:\n    880                     fd, events = self._events.popitem()\n    881                     try:\n    882                         fd_obj, handler_func = self._handlers[fd]\n--> 883                         handler_func(fd_obj, events)\n        handler_func = <function null_wrapper>\n        fd_obj = <zmq.sugar.socket.Socket object>\n        events = 1\n    884                     except (OSError, IOError) as e:\n    885                         if errno_from_exception(e) == errno.EPIPE:\n    886                             # Happens when the client closes the connection\n    887                             pass\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/stack_context.py in null_wrapper(*args=(<zmq.sugar.socket.Socket object>, 1), **kwargs={})\n    270         # Fast path when there are no active contexts.\n    271         def null_wrapper(*args, **kwargs):\n    272             try:\n    273                 current_state = _state.contexts\n    274                 _state.contexts = cap_contexts[0]\n--> 275                 return fn(*args, **kwargs)\n        args = (<zmq.sugar.socket.Socket object>, 1)\n        kwargs = {}\n    276             finally:\n    277                 _state.contexts = current_state\n    278         null_wrapper._wrapped = True\n    279         return null_wrapper\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _handle_events(self=<zmq.eventloop.zmqstream.ZMQStream object>, fd=<zmq.sugar.socket.Socket object>, events=1)\n    435             # dispatch events:\n    436             if events & IOLoop.ERROR:\n    437                 gen_log.error(\"got POLLERR event on ZMQStream, which doesn't make sense\")\n    438                 return\n    439             if events & IOLoop.READ:\n--> 440                 self._handle_recv()\n        self._handle_recv = <bound method ZMQStream._handle_recv of <zmq.eventloop.zmqstream.ZMQStream object>>\n    441                 if not self.socket:\n    442                     return\n    443             if events & IOLoop.WRITE:\n    444                 self._handle_send()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _handle_recv(self=<zmq.eventloop.zmqstream.ZMQStream object>)\n    467                 gen_log.error(\"RECV Error: %s\"%zmq.strerror(e.errno))\n    468         else:\n    469             if self._recv_callback:\n    470                 callback = self._recv_callback\n    471                 # self._recv_callback = None\n--> 472                 self._run_callback(callback, msg)\n        self._run_callback = <bound method ZMQStream._run_callback of <zmq.eventloop.zmqstream.ZMQStream object>>\n        callback = <function null_wrapper>\n        msg = [<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>]\n    473                 \n    474         # self.update_state()\n    475         \n    476 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _run_callback(self=<zmq.eventloop.zmqstream.ZMQStream object>, callback=<function null_wrapper>, *args=([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],), **kwargs={})\n    409         close our socket.\"\"\"\n    410         try:\n    411             # Use a NullContext to ensure that all StackContexts are run\n    412             # inside our blanket exception handler rather than outside.\n    413             with stack_context.NullContext():\n--> 414                 callback(*args, **kwargs)\n        callback = <function null_wrapper>\n        args = ([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],)\n        kwargs = {}\n    415         except:\n    416             gen_log.error(\"Uncaught exception, closing connection.\",\n    417                           exc_info=True)\n    418             # Close the socket on an uncaught exception from a user callback\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/stack_context.py in null_wrapper(*args=([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],), **kwargs={})\n    270         # Fast path when there are no active contexts.\n    271         def null_wrapper(*args, **kwargs):\n    272             try:\n    273                 current_state = _state.contexts\n    274                 _state.contexts = cap_contexts[0]\n--> 275                 return fn(*args, **kwargs)\n        args = ([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],)\n        kwargs = {}\n    276             finally:\n    277                 _state.contexts = current_state\n    278         null_wrapper._wrapped = True\n    279         return null_wrapper\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in dispatcher(msg=[<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>])\n    255         if self.control_stream:\n    256             self.control_stream.on_recv(self.dispatch_control, copy=False)\n    257 \n    258         def make_dispatcher(stream):\n    259             def dispatcher(msg):\n--> 260                 return self.dispatch_shell(stream, msg)\n        msg = [<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>]\n    261             return dispatcher\n    262 \n    263         for s in self.shell_streams:\n    264             s.on_recv(make_dispatcher(s), copy=False)\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in dispatch_shell(self=<ipykernel.ipkernel.IPythonKernel object>, stream=<zmq.eventloop.zmqstream.ZMQStream object>, msg={'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}})\n    207             self.log.error(\"UNKNOWN MESSAGE TYPE: %r\", msg_type)\n    208         else:\n    209             self.log.debug(\"%s: %s\", msg_type, msg)\n    210             self.pre_handler_hook()\n    211             try:\n--> 212                 handler(stream, idents, msg)\n        handler = <bound method IPythonKernel.execute_request of <ipykernel.ipkernel.IPythonKernel object>>\n        stream = <zmq.eventloop.zmqstream.ZMQStream object>\n        idents = ['FF0D07E40949469D89802050F88E6BE5']\n        msg = {'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}}\n    213             except Exception:\n    214                 self.log.error(\"Exception in message handler:\", exc_info=True)\n    215             finally:\n    216                 self.post_handler_hook()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in execute_request(self=<ipykernel.ipkernel.IPythonKernel object>, stream=<zmq.eventloop.zmqstream.ZMQStream object>, ident=['FF0D07E40949469D89802050F88E6BE5'], parent={'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}})\n    365         if not silent:\n    366             self.execution_count += 1\n    367             self._publish_execute_input(code, parent, self.execution_count)\n    368 \n    369         reply_content = self.do_execute(code, silent, store_history,\n--> 370                                         user_expressions, allow_stdin)\n        user_expressions = {}\n        allow_stdin = True\n    371 \n    372         # Flush output before sending the reply.\n    373         sys.stdout.flush()\n    374         sys.stderr.flush()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py in do_execute(self=<ipykernel.ipkernel.IPythonKernel object>, code=u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\", silent=False, store_history=True, user_expressions={}, allow_stdin=True)\n    170 \n    171         reply_content = {}\n    172         # FIXME: the shell calls the exception handler itself.\n    173         shell._reply_content = None\n    174         try:\n--> 175             shell.run_cell(code, store_history=store_history, silent=silent)\n        shell.run_cell = <bound method ZMQInteractiveShell.run_cell of <ipykernel.zmqshell.ZMQInteractiveShell object>>\n        code = u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"\n        store_history = True\n        silent = False\n    176         except:\n    177             status = u'error'\n    178             # FIXME: this code right now isn't being used yet by default,\n    179             # because the run_cell() call above directly fires off exception\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_cell(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, raw_cell=u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\", store_history=True, silent=False, shell_futures=True)\n   2897                 self.displayhook.exec_result = result\n   2898 \n   2899                 # Execute the user code\n   2900                 interactivity = \"none\" if silent else self.ast_node_interactivity\n   2901                 self.run_ast_nodes(code_ast.body, cell_name,\n-> 2902                    interactivity=interactivity, compiler=compiler, result=result)\n        interactivity = 'last_expr'\n        compiler = <IPython.core.compilerop.CachingCompiler instance>\n   2903 \n   2904                 # Reset this so later displayed values do not modify the\n   2905                 # ExecutionResult\n   2906                 self.displayhook.exec_result = None\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_ast_nodes(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, nodelist=[<_ast.ImportFrom object>, <_ast.ImportFrom object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.For object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Expr object>, <_ast.Print object>, <_ast.Print object>], cell_name='<ipython-input-15-546b13bf5a7a>', interactivity='none', compiler=<IPython.core.compilerop.CachingCompiler instance>, result=<IPython.core.interactiveshell.ExecutionResult object>)\n   3001 \n   3002         try:\n   3003             for i, node in enumerate(to_run_exec):\n   3004                 mod = ast.Module([node])\n   3005                 code = compiler(mod, cell_name, \"exec\")\n-> 3006                 if self.run_code(code, result):\n        self.run_code = <bound method ZMQInteractiveShell.run_code of <ipykernel.zmqshell.ZMQInteractiveShell object>>\n        code = <code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>\n        result = <IPython.core.interactiveshell.ExecutionResult object>\n   3007                     return True\n   3008 \n   3009             for i, node in enumerate(to_run_interactive):\n   3010                 mod = ast.Interactive([node])\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_code(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, code_obj=<code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>, result=<IPython.core.interactiveshell.ExecutionResult object>)\n   3061         outflag = 1  # happens in more places, so it's easier as default\n   3062         try:\n   3063             try:\n   3064                 self.hooks.pre_run_code_hook()\n   3065                 #rprint('Running code', repr(code_obj)) # dbg\n-> 3066                 exec(code_obj, self.user_global_ns, self.user_ns)\n        code_obj = <code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>\n        self.user_global_ns = {'ArgumentParser': <class 'argparse.ArgumentParser'>, 'BaseClassifiers': <class 'EnsembleDiversityTests.BaseClassifiers'>, 'BaseEstimator': <class 'sklearn.base.BaseEstimator'>, 'DiversityTests': <class 'EnsembleDiversityTests.DiversityTests'>, 'FeatureUnion': <class 'sklearn.pipeline.FeatureUnion'>, 'GridSearchCV': <class 'sklearn.grid_search.GridSearchCV'>, 'In': ['', u'import os\\nfrom argparse import ArgumentParser...  # load data\\n    X, y = dataset.get_data(task)', u'#reload(preprocess)\\n#reload(features)\\nfrom p...$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\\n#print X[0]', u\"from sklearn.pipeline import Pipeline, Feature...e([('3grams',grams3), ('svm', svm)])\\npipe.steps\", u\"soac = features.SOAC_Model2(max_df=1.0, min_df...line([('soac',soac), ('svm', svm)])\\npipe1.steps\", u\"from pan.features import LDA\\nLDAmodel = LDA(l...peline([('LDA', LDAmodel), ('svm', svm)])\\npipe2\", u'def print_overlaps(predictions, names, verbose...es[i,j], 100*temp[i,j])\\n    return  [res, temp]', u'from sklearn.base import BaseEstimator, Transf...el_ind\\n        return data.most_common(1)[0][0]', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...'Total time: %0.3f seconds\\') % (time.time()-t0)', u\"for name in all_models_names:\\n    print '%%%%...~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"], 'LCA': Neigbors_DS(common_neigh=False, k=3,\n      model...e,\n        vocabulary=None)},\n      scheme='LCA'), 'LDA': <class 'pan.features.LDA'>, 'LDAmodel': LDA(lib='mallet', num_topics=90), ...}\n        self.user_ns = {'ArgumentParser': <class 'argparse.ArgumentParser'>, 'BaseClassifiers': <class 'EnsembleDiversityTests.BaseClassifiers'>, 'BaseEstimator': <class 'sklearn.base.BaseEstimator'>, 'DiversityTests': <class 'EnsembleDiversityTests.DiversityTests'>, 'FeatureUnion': <class 'sklearn.pipeline.FeatureUnion'>, 'GridSearchCV': <class 'sklearn.grid_search.GridSearchCV'>, 'In': ['', u'import os\\nfrom argparse import ArgumentParser...  # load data\\n    X, y = dataset.get_data(task)', u'#reload(preprocess)\\n#reload(features)\\nfrom p...$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\\n#print X[0]', u\"from sklearn.pipeline import Pipeline, Feature...e([('3grams',grams3), ('svm', svm)])\\npipe.steps\", u\"soac = features.SOAC_Model2(max_df=1.0, min_df...line([('soac',soac), ('svm', svm)])\\npipe1.steps\", u\"from pan.features import LDA\\nLDAmodel = LDA(l...peline([('LDA', LDAmodel), ('svm', svm)])\\npipe2\", u'def print_overlaps(predictions, names, verbose...es[i,j], 100*temp[i,j])\\n    return  [res, temp]', u'from sklearn.base import BaseEstimator, Transf...el_ind\\n        return data.most_common(1)[0][0]', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...'Total time: %0.3f seconds\\') % (time.time()-t0)', u\"for name in all_models_names:\\n    print '%%%%...~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"], 'LCA': Neigbors_DS(common_neigh=False, k=3,\n      model...e,\n        vocabulary=None)},\n      scheme='LCA'), 'LDA': <class 'pan.features.LDA'>, 'LDAmodel': LDA(lib='mallet', num_topics=90), ...}\n   3067             finally:\n   3068                 # Reset our crash handler in place\n   3069                 sys.excepthook = old_excepthook\n   3070         except SystemExit as e:\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-15-546b13bf5a7a> in <module>()\n     13 LCA = Neigbors_DS(models11, models_tr11, k=3, scheme='LCA', common_neigh=False)\n     14 #LCA.fit(X_train + X_meta, y_train + y_meta)\n     15 \n     16 X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.3, stratify=y, )\n     17 grid1 = GridSearchCV(LCA, param_grid =params, cv=5, verbose=1, n_jobs=3)\n---> 18 grid1.fit(X_meta, y_meta)\n     19 print(grid1.best_score_)\n     20 print(grid1.best_params_)\n     21 #pipe2 = grid1.best_estimator_\n     22 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.py in fit(self=GridSearchCV(cv=5, error_score='raise',\n       e...='2*n_jobs', refit=True, scoring=None, verbose=1), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...])\n    799         y : array-like, shape = [n_samples] or [n_samples, n_output], optional\n    800             Target relative to X for classification or regression;\n    801             None for unsupervised learning.\n    802 \n    803         \"\"\"\n--> 804         return self._fit(X, y, ParameterGrid(self.param_grid))\n        self._fit = <bound method GridSearchCV._fit of GridSearchCV(...'2*n_jobs', refit=True, scoring=None, verbose=1)>\n        X = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...]\n        y = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...]\n        self.param_grid = {'scheme': ['OLA', 'LCA', 'KNE', 'KNU']}\n    805 \n    806 \n    807 class RandomizedSearchCV(BaseSearchCV):\n    808     \"\"\"Randomized search on hyper parameters.\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.py in _fit(self=GridSearchCV(cv=5, error_score='raise',\n       e...='2*n_jobs', refit=True, scoring=None, verbose=1), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], parameter_iterable=<sklearn.grid_search.ParameterGrid object>)\n    548         )(\n    549             delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,\n    550                                     train, test, self.verbose, parameters,\n    551                                     self.fit_params, return_parameters=True,\n    552                                     error_score=self.error_score)\n--> 553                 for parameters in parameter_iterable\n        parameters = undefined\n        parameter_iterable = <sklearn.grid_search.ParameterGrid object>\n    554                 for train, test in cv)\n    555 \n    556         # Out is a list of triplet: score, estimator, n_test_samples\n    557         n_fits = len(out)\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.py in __call__(self=Parallel(n_jobs=3), iterable=<generator object <genexpr>>)\n    805             if pre_dispatch == \"all\" or n_jobs == 1:\n    806                 # The iterable was consumed all at once by the above for loop.\n    807                 # No need to wait for async callbacks to trigger to\n    808                 # consumption.\n    809                 self._iterating = False\n--> 810             self.retrieve()\n        self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=3)>\n    811             # Make sure that we get a last message telling us we are done\n    812             elapsed_time = time.time() - self._start_time\n    813             self._print('Done %3i out of %3i | elapsed: %s finished',\n    814                         (len(self._output), len(self._output),\n\n---------------------------------------------------------------------------\nSub-process traceback:\n---------------------------------------------------------------------------\nValueError                                         Tue Oct 18 15:28:29 2016\nPID: 12175                                    Python 2.7.6: /usr/bin/python\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.py in __call__(self=<sklearn.externals.joblib.parallel.BatchedCalls object>)\n     67     def __init__(self, iterator_slice):\n     68         self.items = list(iterator_slice)\n     69         self._size = len(self.items)\n     70 \n     71     def __call__(self):\n---> 72         return [func(*args, **kwargs) for func, args, kwargs in self.items]\n        func = <function _fit_and_score>\n        args = (Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], <function _passthrough_scorer>, array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), 1, {'scheme': 'OLA'}, {})\n        kwargs = {'error_score': 'raise', 'return_parameters': True}\n        self.items = [(<function _fit_and_score>, (Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], <function _passthrough_scorer>, array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), 1, {'scheme': 'OLA'}, {}), {'error_score': 'raise', 'return_parameters': True})]\n     73 \n     74     def __len__(self):\n     75         return self._size\n     76 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py in _fit_and_score(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], scorer=<function _passthrough_scorer>, train=array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), test=array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), verbose=1, parameters={'scheme': 'OLA'}, fit_params={}, return_train_score=False, return_parameters=True, error_score='raise')\n   1545                              \" numeric value. (Hint: if using 'raise', please\"\n   1546                              \" make sure that it has been spelled correctly.)\"\n   1547                              )\n   1548 \n   1549     else:\n-> 1550         test_score = _score(estimator, X_test, y_test, scorer)\n        test_score = undefined\n        estimator = Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA')\n        X_test = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ']\n        y_test = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']\n        scorer = <function _passthrough_scorer>\n   1551         if return_train_score:\n   1552             train_score = _score(estimator, X_train, y_train, scorer)\n   1553 \n   1554     scoring_time = time.time() - start_time\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py in _score(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X_test=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], y_test=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'], scorer=<function _passthrough_scorer>)\n   1601 def _score(estimator, X_test, y_test, scorer):\n   1602     \"\"\"Compute the score of an estimator on a given test set.\"\"\"\n   1603     if y_test is None:\n   1604         score = scorer(estimator, X_test)\n   1605     else:\n-> 1606         score = scorer(estimator, X_test, y_test)\n        score = undefined\n        scorer = <function _passthrough_scorer>\n        estimator = Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA')\n        X_test = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ']\n        y_test = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']\n   1607     if not isinstance(score, numbers.Number):\n   1608         raise ValueError(\"scoring must return a number, got %s (%s) instead.\"\n   1609                          % (str(score), type(score)))\n   1610     return score\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/metrics/scorer.py in _passthrough_scorer(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), *args=([u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']), **kwargs={})\n    200     return scorer\n    201 \n    202 \n    203 def _passthrough_scorer(estimator, *args, **kwargs):\n    204     \"\"\"Function that wraps estimator.score\"\"\"\n--> 205     return estimator.score(*args, **kwargs)\n        estimator.score = <bound method Neigbors_DS.score of Neigbors_DS(c...b='mallet', num_topics=90)},\n      scheme='OLA')>\n        args = ([u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'])\n        kwargs = {}\n    206 \n    207 \n    208 def check_scoring(estimator, scoring=None, allow_none=False):\n    209     \"\"\"Determine scorer from user options.\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in score(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'], sample_weight=None)\n     99         return y_pred\n    100 \n    101     def score(self, X, y, sample_weight=None):\n    102 \n    103         from sklearn.metrics import accuracy_score\n--> 104         return accuracy_score(y, self.predict(X), normalize=True)\n    105         # return self.svc.score(self.transform_to_y(X), y, sample_weight)\n    106 \n    107 \n    108     def predict_lca(self, sample):\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in predict(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '])\n     92         # print X.shape\n     93         y_pred = []\n     94         # t0 = time.time()\n     95         for i, x in enumerate(X):\n     96             # print 'True Sample: ' + y_real[i]\n---> 97             y_pred.append(self.predictor(x))\n     98         # print('Predict took: %0.3f seconds') % (time.time()-t0)\n     99         return y_pred\n    100 \n    101     def score(self, X, y, sample_weight=None):\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in predict_lca(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), sample=u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ')\n    106 \n    107 \n    108     def predict_lca(self, sample):\n    109         preds = []\n    110         for name, model in self.models.iteritems():\n--> 111             preds.append(model.predict([sample])[0])\n    112 #         print 'Preds: ' + str(preds)\n    113         if len(set(preds))==1:\n    114 #             print 'Unanimous Decision: ' + str(preds[0])\n    115 #             print '='*50\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/utils/metaestimators.py in <lambda>(*args=([u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '],), **kwargs={})\n     32         if obj is not None:\n     33             # delegate only on instances, not the classes.\n     34             # this is to allow access to the docstrings.\n     35             self.get_attribute(obj)\n     36         # lambda, but not partial, allows help() to work with update_wrapper\n---> 37         out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)\n        args = ([u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '],)\n        kwargs = {}\n     38         # update the docstring of the returned function\n     39         update_wrapper(out, self.fn)\n     40         return out\n     41 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.py in predict(self=Pipeline(steps=[('LDA', LDA(lib='mallet', num_to...e, shrinking=True,\n  tol=0.001, verbose=False))]), X=[u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '])\n    199             the pipeline.\n    200         \"\"\"\n    201         Xt = X\n    202         for name, transform in self.steps[:-1]:\n    203             Xt = transform.transform(Xt)\n--> 204         return self.steps[-1][-1].predict(Xt)\n        self.steps.predict = undefined\n        Xt = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    205 \n    206     @if_delegate_has_method(delegate='_final_estimator')\n    207     def fit_predict(self, X, y=None, **fit_params):\n    208         \"\"\"Applies fit_predict of last step in pipeline after transforms.\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    563         Returns\n    564         -------\n    565         y_pred : array, shape (n_samples,)\n    566             Class labels for samples in X.\n    567         \"\"\"\n--> 568         y = super(BaseSVC, self).predict(X)\n        y = undefined\n        self.predict = <bound method SVC.predict of SVC(C=100, cache_si...one, shrinking=True,\n  tol=0.001, verbose=False)>\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    569         return self.classes_.take(np.asarray(y, dtype=np.intp))\n    570 \n    571     # Hacky way of getting predict_proba to raise an AttributeError when\n    572     # probability=False using properties. Do not use this in new code; when\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    300 \n    301         Returns\n    302         -------\n    303         y_pred : array, shape (n_samples,)\n    304         \"\"\"\n--> 305         X = self._validate_for_predict(X)\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n        self._validate_for_predict = <bound method SVC._validate_for_predict of SVC(C...one, shrinking=True,\n  tol=0.001, verbose=False)>\n    306         predict = self._sparse_predict if self._sparse else self._dense_predict\n    307         return predict(X)\n    308 \n    309     def _dense_predict(self, X):\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in _validate_for_predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    449             self.probA_, self.probB_)\n    450 \n    451     def _validate_for_predict(self, X):\n    452         check_is_fitted(self, 'support_')\n    453 \n--> 454         X = check_array(X, accept_sparse='csr', dtype=np.float64, order=\"C\")\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    455         if self._sparse and not sp.isspmatrix(X):\n    456             X = sp.csr_matrix(X)\n    457         if self._sparse:\n    458             X.sort_indices()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/utils/validation.py in check_array(array=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object), accept_sparse=['csr'], dtype=<type 'numpy.float64'>, order='C', copy=False, force_all_finite=True, ensure_2d=True, allow_nd=False, ensure_min_samples=1, ensure_min_features=1, warn_on_dtype=False, estimator=None)\n    368 \n    369     if sp.issparse(array):\n    370         array = _ensure_sparse_format(array, accept_sparse, dtype, copy,\n    371                                       force_all_finite)\n    372     else:\n--> 373         array = np.array(array, dtype=dtype, order=order, copy=copy)\n        array = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n        dtype = <type 'numpy.float64'>\n        order = 'C'\n        copy = False\n    374 \n    375         if ensure_2d:\n    376             if array.ndim == 1:\n    377                 if ensure_min_samples >= 2:\n\nValueError: setting an array element with a sequence.\n___________________________________________________________________________",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mJoblibValueError\u001b[0m                          Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-546b13bf5a7a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     16\u001b[0m \u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_size\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mstratify\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     17\u001b[0m \u001b[0mgrid1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mGridSearchCV\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mLCA\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mparam_grid\u001b[0m \u001b[1;33m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 18\u001b[1;33m \u001b[0mgrid1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     19\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrid1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_score_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     20\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mgrid1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbest_params_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    802\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    803\u001b[0m         \"\"\"\n\u001b[1;32m--> 804\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_fit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mParameterGrid\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    805\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    806\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.pyc\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, parameter_iterable)\u001b[0m\n\u001b[0;32m    551\u001b[0m                                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreturn_parameters\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    552\u001b[0m                                     error_score=self.error_score)\n\u001b[1;32m--> 553\u001b[1;33m                 \u001b[1;32mfor\u001b[0m \u001b[0mparameters\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mparameter_iterable\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    554\u001b[0m                 for train, test in cv)\n\u001b[0;32m    555\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m    808\u001b[0m                 \u001b[1;31m# consumption.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    809\u001b[0m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 810\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    811\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    812\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.pyc\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    755\u001b[0m                     \u001b[1;31m# a working pool as they expect.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    756\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_initialize_pool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 757\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    758\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    759\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miterable\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mJoblibValueError\u001b[0m: JoblibValueError\n___________________________________________________________________________\nMultiprocessing exception:\n...........................................................................\n/usr/lib/python2.7/runpy.py in _run_module_as_main(mod_name='ipykernel.__main__', alter_argv=1)\n    157     pkg_name = mod_name.rpartition('.')[0]\n    158     main_globals = sys.modules[\"__main__\"].__dict__\n    159     if alter_argv:\n    160         sys.argv[0] = fname\n    161     return _run_code(code, main_globals, None,\n--> 162                      \"__main__\", fname, loader, pkg_name)\n        fname = '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py'\n        loader = <pkgutil.ImpLoader instance>\n        pkg_name = 'ipykernel'\n    163 \n    164 def run_module(mod_name, init_globals=None,\n    165                run_name=None, alter_sys=False):\n    166     \"\"\"Execute a module's code without importing it\n\n...........................................................................\n/usr/lib/python2.7/runpy.py in _run_code(code=<code object <module> at 0x7f3abb4ad930, file \"/...2.7/dist-packages/ipykernel/__main__.py\", line 1>, run_globals={'__builtins__': <module '__builtin__' (built-in)>, '__doc__': None, '__file__': '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', '__loader__': <pkgutil.ImpLoader instance>, '__name__': '__main__', '__package__': 'ipykernel', 'app': <module 'ipykernel.kernelapp' from '/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.pyc'>}, init_globals=None, mod_name='__main__', mod_fname='/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', mod_loader=<pkgutil.ImpLoader instance>, pkg_name='ipykernel')\n     67         run_globals.update(init_globals)\n     68     run_globals.update(__name__ = mod_name,\n     69                        __file__ = mod_fname,\n     70                        __loader__ = mod_loader,\n     71                        __package__ = pkg_name)\n---> 72     exec code in run_globals\n        code = <code object <module> at 0x7f3abb4ad930, file \"/...2.7/dist-packages/ipykernel/__main__.py\", line 1>\n        run_globals = {'__builtins__': <module '__builtin__' (built-in)>, '__doc__': None, '__file__': '/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py', '__loader__': <pkgutil.ImpLoader instance>, '__name__': '__main__', '__package__': 'ipykernel', 'app': <module 'ipykernel.kernelapp' from '/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.pyc'>}\n     73     return run_globals\n     74 \n     75 def _run_module_code(code, init_globals=None,\n     76                     mod_name=None, mod_fname=None,\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/__main__.py in <module>()\n      1 \n      2 \n----> 3 \n      4 if __name__ == '__main__':\n      5     from ipykernel import kernelapp as app\n      6     app.launch_new_instance()\n      7 \n      8 \n      9 \n     10 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/traitlets/config/application.py in launch_instance(cls=<class 'ipykernel.kernelapp.IPKernelApp'>, argv=None, **kwargs={})\n    584         \n    585         If a global instance already exists, this reinitializes and starts it\n    586         \"\"\"\n    587         app = cls.instance(**kwargs)\n    588         app.initialize(argv)\n--> 589         app.start()\n        app.start = <bound method IPKernelApp.start of <ipykernel.kernelapp.IPKernelApp object>>\n    590 \n    591 #-----------------------------------------------------------------------------\n    592 # utility functions, for convenience\n    593 #-----------------------------------------------------------------------------\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelapp.py in start(self=<ipykernel.kernelapp.IPKernelApp object>)\n    400         \n    401         if self.poller is not None:\n    402             self.poller.start()\n    403         self.kernel.start()\n    404         try:\n--> 405             ioloop.IOLoop.instance().start()\n    406         except KeyboardInterrupt:\n    407             pass\n    408 \n    409 launch_new_instance = IPKernelApp.launch_instance\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/ioloop.py in start(self=<zmq.eventloop.ioloop.ZMQIOLoop object>)\n    157             PollIOLoop.configure(ZMQIOLoop)\n    158         return PollIOLoop.current(*args, **kwargs)\n    159     \n    160     def start(self):\n    161         try:\n--> 162             super(ZMQIOLoop, self).start()\n        self.start = <bound method ZMQIOLoop.start of <zmq.eventloop.ioloop.ZMQIOLoop object>>\n    163         except ZMQError as e:\n    164             if e.errno == ETERM:\n    165                 # quietly return on ETERM\n    166                 pass\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/ioloop.py in start(self=<zmq.eventloop.ioloop.ZMQIOLoop object>)\n    878                 self._events.update(event_pairs)\n    879                 while self._events:\n    880                     fd, events = self._events.popitem()\n    881                     try:\n    882                         fd_obj, handler_func = self._handlers[fd]\n--> 883                         handler_func(fd_obj, events)\n        handler_func = <function null_wrapper>\n        fd_obj = <zmq.sugar.socket.Socket object>\n        events = 1\n    884                     except (OSError, IOError) as e:\n    885                         if errno_from_exception(e) == errno.EPIPE:\n    886                             # Happens when the client closes the connection\n    887                             pass\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/stack_context.py in null_wrapper(*args=(<zmq.sugar.socket.Socket object>, 1), **kwargs={})\n    270         # Fast path when there are no active contexts.\n    271         def null_wrapper(*args, **kwargs):\n    272             try:\n    273                 current_state = _state.contexts\n    274                 _state.contexts = cap_contexts[0]\n--> 275                 return fn(*args, **kwargs)\n        args = (<zmq.sugar.socket.Socket object>, 1)\n        kwargs = {}\n    276             finally:\n    277                 _state.contexts = current_state\n    278         null_wrapper._wrapped = True\n    279         return null_wrapper\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _handle_events(self=<zmq.eventloop.zmqstream.ZMQStream object>, fd=<zmq.sugar.socket.Socket object>, events=1)\n    435             # dispatch events:\n    436             if events & IOLoop.ERROR:\n    437                 gen_log.error(\"got POLLERR event on ZMQStream, which doesn't make sense\")\n    438                 return\n    439             if events & IOLoop.READ:\n--> 440                 self._handle_recv()\n        self._handle_recv = <bound method ZMQStream._handle_recv of <zmq.eventloop.zmqstream.ZMQStream object>>\n    441                 if not self.socket:\n    442                     return\n    443             if events & IOLoop.WRITE:\n    444                 self._handle_send()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _handle_recv(self=<zmq.eventloop.zmqstream.ZMQStream object>)\n    467                 gen_log.error(\"RECV Error: %s\"%zmq.strerror(e.errno))\n    468         else:\n    469             if self._recv_callback:\n    470                 callback = self._recv_callback\n    471                 # self._recv_callback = None\n--> 472                 self._run_callback(callback, msg)\n        self._run_callback = <bound method ZMQStream._run_callback of <zmq.eventloop.zmqstream.ZMQStream object>>\n        callback = <function null_wrapper>\n        msg = [<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>]\n    473                 \n    474         # self.update_state()\n    475         \n    476 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/zmq/eventloop/zmqstream.py in _run_callback(self=<zmq.eventloop.zmqstream.ZMQStream object>, callback=<function null_wrapper>, *args=([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],), **kwargs={})\n    409         close our socket.\"\"\"\n    410         try:\n    411             # Use a NullContext to ensure that all StackContexts are run\n    412             # inside our blanket exception handler rather than outside.\n    413             with stack_context.NullContext():\n--> 414                 callback(*args, **kwargs)\n        callback = <function null_wrapper>\n        args = ([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],)\n        kwargs = {}\n    415         except:\n    416             gen_log.error(\"Uncaught exception, closing connection.\",\n    417                           exc_info=True)\n    418             # Close the socket on an uncaught exception from a user callback\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/tornado-4.3-py2.7-linux-x86_64.egg/tornado/stack_context.py in null_wrapper(*args=([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],), **kwargs={})\n    270         # Fast path when there are no active contexts.\n    271         def null_wrapper(*args, **kwargs):\n    272             try:\n    273                 current_state = _state.contexts\n    274                 _state.contexts = cap_contexts[0]\n--> 275                 return fn(*args, **kwargs)\n        args = ([<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>],)\n        kwargs = {}\n    276             finally:\n    277                 _state.contexts = current_state\n    278         null_wrapper._wrapped = True\n    279         return null_wrapper\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in dispatcher(msg=[<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>])\n    255         if self.control_stream:\n    256             self.control_stream.on_recv(self.dispatch_control, copy=False)\n    257 \n    258         def make_dispatcher(stream):\n    259             def dispatcher(msg):\n--> 260                 return self.dispatch_shell(stream, msg)\n        msg = [<zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>, <zmq.sugar.frame.Frame object>]\n    261             return dispatcher\n    262 \n    263         for s in self.shell_streams:\n    264             s.on_recv(make_dispatcher(s), copy=False)\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in dispatch_shell(self=<ipykernel.ipkernel.IPythonKernel object>, stream=<zmq.eventloop.zmqstream.ZMQStream object>, msg={'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}})\n    207             self.log.error(\"UNKNOWN MESSAGE TYPE: %r\", msg_type)\n    208         else:\n    209             self.log.debug(\"%s: %s\", msg_type, msg)\n    210             self.pre_handler_hook()\n    211             try:\n--> 212                 handler(stream, idents, msg)\n        handler = <bound method IPythonKernel.execute_request of <ipykernel.ipkernel.IPythonKernel object>>\n        stream = <zmq.eventloop.zmqstream.ZMQStream object>\n        idents = ['FF0D07E40949469D89802050F88E6BE5']\n        msg = {'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}}\n    213             except Exception:\n    214                 self.log.error(\"Exception in message handler:\", exc_info=True)\n    215             finally:\n    216                 self.post_handler_hook()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/kernelbase.py in execute_request(self=<ipykernel.ipkernel.IPythonKernel object>, stream=<zmq.eventloop.zmqstream.ZMQStream object>, ident=['FF0D07E40949469D89802050F88E6BE5'], parent={'buffers': [], 'content': {'allow_stdin': True, 'code': 'from sklearn.grid_search import GridSearchCV\\nfro...rid1.best_params_)\\n#pipe2 = grid1.best_estimator_', 'silent': False, 'stop_on_error': True, 'store_history': True, 'user_expressions': {}}, 'header': {'date': '2016-10-18T15:26:48.830567', 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'session': 'FF0D07E40949469D89802050F88E6BE5', 'username': 'username', 'version': '5.0'}, 'metadata': {}, 'msg_id': 'A4BAC7538F9E45B69F501D122F9E388E', 'msg_type': 'execute_request', 'parent_header': {}})\n    365         if not silent:\n    366             self.execution_count += 1\n    367             self._publish_execute_input(code, parent, self.execution_count)\n    368 \n    369         reply_content = self.do_execute(code, silent, store_history,\n--> 370                                         user_expressions, allow_stdin)\n        user_expressions = {}\n        allow_stdin = True\n    371 \n    372         # Flush output before sending the reply.\n    373         sys.stdout.flush()\n    374         sys.stderr.flush()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/ipykernel/ipkernel.py in do_execute(self=<ipykernel.ipkernel.IPythonKernel object>, code=u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\", silent=False, store_history=True, user_expressions={}, allow_stdin=True)\n    170 \n    171         reply_content = {}\n    172         # FIXME: the shell calls the exception handler itself.\n    173         shell._reply_content = None\n    174         try:\n--> 175             shell.run_cell(code, store_history=store_history, silent=silent)\n        shell.run_cell = <bound method ZMQInteractiveShell.run_cell of <ipykernel.zmqshell.ZMQInteractiveShell object>>\n        code = u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"\n        store_history = True\n        silent = False\n    176         except:\n    177             status = u'error'\n    178             # FIXME: this code right now isn't being used yet by default,\n    179             # because the run_cell() call above directly fires off exception\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_cell(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, raw_cell=u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\", store_history=True, silent=False, shell_futures=True)\n   2897                 self.displayhook.exec_result = result\n   2898 \n   2899                 # Execute the user code\n   2900                 interactivity = \"none\" if silent else self.ast_node_interactivity\n   2901                 self.run_ast_nodes(code_ast.body, cell_name,\n-> 2902                    interactivity=interactivity, compiler=compiler, result=result)\n        interactivity = 'last_expr'\n        compiler = <IPython.core.compilerop.CachingCompiler instance>\n   2903 \n   2904                 # Reset this so later displayed values do not modify the\n   2905                 # ExecutionResult\n   2906                 self.displayhook.exec_result = None\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_ast_nodes(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, nodelist=[<_ast.ImportFrom object>, <_ast.ImportFrom object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.For object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Assign object>, <_ast.Expr object>, <_ast.Print object>, <_ast.Print object>], cell_name='<ipython-input-15-546b13bf5a7a>', interactivity='none', compiler=<IPython.core.compilerop.CachingCompiler instance>, result=<IPython.core.interactiveshell.ExecutionResult object>)\n   3001 \n   3002         try:\n   3003             for i, node in enumerate(to_run_exec):\n   3004                 mod = ast.Module([node])\n   3005                 code = compiler(mod, cell_name, \"exec\")\n-> 3006                 if self.run_code(code, result):\n        self.run_code = <bound method ZMQInteractiveShell.run_code of <ipykernel.zmqshell.ZMQInteractiveShell object>>\n        code = <code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>\n        result = <IPython.core.interactiveshell.ExecutionResult object>\n   3007                     return True\n   3008 \n   3009             for i, node in enumerate(to_run_interactive):\n   3010                 mod = ast.Interactive([node])\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/IPython/core/interactiveshell.py in run_code(self=<ipykernel.zmqshell.ZMQInteractiveShell object>, code_obj=<code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>, result=<IPython.core.interactiveshell.ExecutionResult object>)\n   3061         outflag = 1  # happens in more places, so it's easier as default\n   3062         try:\n   3063             try:\n   3064                 self.hooks.pre_run_code_hook()\n   3065                 #rprint('Running code', repr(code_obj)) # dbg\n-> 3066                 exec(code_obj, self.user_global_ns, self.user_ns)\n        code_obj = <code object <module> at 0x7f3a35332330, file \"<ipython-input-15-546b13bf5a7a>\", line 18>\n        self.user_global_ns = {'ArgumentParser': <class 'argparse.ArgumentParser'>, 'BaseClassifiers': <class 'EnsembleDiversityTests.BaseClassifiers'>, 'BaseEstimator': <class 'sklearn.base.BaseEstimator'>, 'DiversityTests': <class 'EnsembleDiversityTests.DiversityTests'>, 'FeatureUnion': <class 'sklearn.pipeline.FeatureUnion'>, 'GridSearchCV': <class 'sklearn.grid_search.GridSearchCV'>, 'In': ['', u'import os\\nfrom argparse import ArgumentParser...  # load data\\n    X, y = dataset.get_data(task)', u'#reload(preprocess)\\n#reload(features)\\nfrom p...$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\\n#print X[0]', u\"from sklearn.pipeline import Pipeline, Feature...e([('3grams',grams3), ('svm', svm)])\\npipe.steps\", u\"soac = features.SOAC_Model2(max_df=1.0, min_df...line([('soac',soac), ('svm', svm)])\\npipe1.steps\", u\"from pan.features import LDA\\nLDAmodel = LDA(l...peline([('LDA', LDAmodel), ('svm', svm)])\\npipe2\", u'def print_overlaps(predictions, names, verbose...es[i,j], 100*temp[i,j])\\n    return  [res, temp]', u'from sklearn.base import BaseEstimator, Transf...el_ind\\n        return data.most_common(1)[0][0]', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...'Total time: %0.3f seconds\\') % (time.time()-t0)', u\"for name in all_models_names:\\n    print '%%%%...~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"], 'LCA': Neigbors_DS(common_neigh=False, k=3,\n      model...e,\n        vocabulary=None)},\n      scheme='LCA'), 'LDA': <class 'pan.features.LDA'>, 'LDAmodel': LDA(lib='mallet', num_topics=90), ...}\n        self.user_ns = {'ArgumentParser': <class 'argparse.ArgumentParser'>, 'BaseClassifiers': <class 'EnsembleDiversityTests.BaseClassifiers'>, 'BaseEstimator': <class 'sklearn.base.BaseEstimator'>, 'DiversityTests': <class 'EnsembleDiversityTests.DiversityTests'>, 'FeatureUnion': <class 'sklearn.pipeline.FeatureUnion'>, 'GridSearchCV': <class 'sklearn.grid_search.GridSearchCV'>, 'In': ['', u'import os\\nfrom argparse import ArgumentParser...  # load data\\n    X, y = dataset.get_data(task)', u'#reload(preprocess)\\n#reload(features)\\nfrom p...$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\"\\n#print X[0]', u\"from sklearn.pipeline import Pipeline, Feature...e([('3grams',grams3), ('svm', svm)])\\npipe.steps\", u\"soac = features.SOAC_Model2(max_df=1.0, min_df...line([('soac',soac), ('svm', svm)])\\npipe1.steps\", u\"from pan.features import LDA\\nLDAmodel = LDA(l...peline([('LDA', LDAmodel), ('svm', svm)])\\npipe2\", u'def print_overlaps(predictions, names, verbose...es[i,j], 100*temp[i,j])\\n    return  [res, temp]', u'from sklearn.base import BaseEstimator, Transf...el_ind\\n        return data.most_common(1)[0][0]', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...'Total time: %0.3f seconds\\') % (time.time()-t0)', u\"for name in all_models_names:\\n    print '%%%%...~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~'\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u\"models11 = {}\\nmodels_tr11 = {}\\nfor i, model ...int('Classification report :\\\\n {}'.format(rep))\", u'import sys\\nsys.path.insert(0, \"../EnsembleDiv...eturn max(set(lst), key=lst.count)\\n            ', u\"from sklearn.grid_search import GridSearchCV\\n...d1.best_params_)\\n#pipe2 = grid1.best_estimator_\"], 'LCA': Neigbors_DS(common_neigh=False, k=3,\n      model...e,\n        vocabulary=None)},\n      scheme='LCA'), 'LDA': <class 'pan.features.LDA'>, 'LDAmodel': LDA(lib='mallet', num_topics=90), ...}\n   3067             finally:\n   3068                 # Reset our crash handler in place\n   3069                 sys.excepthook = old_excepthook\n   3070         except SystemExit as e:\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-15-546b13bf5a7a> in <module>()\n     13 LCA = Neigbors_DS(models11, models_tr11, k=3, scheme='LCA', common_neigh=False)\n     14 #LCA.fit(X_train + X_meta, y_train + y_meta)\n     15 \n     16 X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.3, stratify=y, )\n     17 grid1 = GridSearchCV(LCA, param_grid =params, cv=5, verbose=1, n_jobs=3)\n---> 18 grid1.fit(X_meta, y_meta)\n     19 print(grid1.best_score_)\n     20 print(grid1.best_params_)\n     21 #pipe2 = grid1.best_estimator_\n     22 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.py in fit(self=GridSearchCV(cv=5, error_score='raise',\n       e...='2*n_jobs', refit=True, scoring=None, verbose=1), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...])\n    799         y : array-like, shape = [n_samples] or [n_samples, n_output], optional\n    800             Target relative to X for classification or regression;\n    801             None for unsupervised learning.\n    802 \n    803         \"\"\"\n--> 804         return self._fit(X, y, ParameterGrid(self.param_grid))\n        self._fit = <bound method GridSearchCV._fit of GridSearchCV(...'2*n_jobs', refit=True, scoring=None, verbose=1)>\n        X = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...]\n        y = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...]\n        self.param_grid = {'scheme': ['OLA', 'LCA', 'KNE', 'KNU']}\n    805 \n    806 \n    807 class RandomizedSearchCV(BaseSearchCV):\n    808     \"\"\"Randomized search on hyper parameters.\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/grid_search.py in _fit(self=GridSearchCV(cv=5, error_score='raise',\n       e...='2*n_jobs', refit=True, scoring=None, verbose=1), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], parameter_iterable=<sklearn.grid_search.ParameterGrid object>)\n    548         )(\n    549             delayed(_fit_and_score)(clone(base_estimator), X, y, self.scorer_,\n    550                                     train, test, self.verbose, parameters,\n    551                                     self.fit_params, return_parameters=True,\n    552                                     error_score=self.error_score)\n--> 553                 for parameters in parameter_iterable\n        parameters = undefined\n        parameter_iterable = <sklearn.grid_search.ParameterGrid object>\n    554                 for train, test in cv)\n    555 \n    556         # Out is a list of triplet: score, estimator, n_test_samples\n    557         n_fits = len(out)\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.py in __call__(self=Parallel(n_jobs=3), iterable=<generator object <genexpr>>)\n    805             if pre_dispatch == \"all\" or n_jobs == 1:\n    806                 # The iterable was consumed all at once by the above for loop.\n    807                 # No need to wait for async callbacks to trigger to\n    808                 # consumption.\n    809                 self._iterating = False\n--> 810             self.retrieve()\n        self.retrieve = <bound method Parallel.retrieve of Parallel(n_jobs=3)>\n    811             # Make sure that we get a last message telling us we are done\n    812             elapsed_time = time.time() - self._start_time\n    813             self._print('Done %3i out of %3i | elapsed: %s finished',\n    814                         (len(self._output), len(self._output),\n\n---------------------------------------------------------------------------\nSub-process traceback:\n---------------------------------------------------------------------------\nValueError                                         Tue Oct 18 15:28:29 2016\nPID: 12175                                    Python 2.7.6: /usr/bin/python\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/externals/joblib/parallel.py in __call__(self=<sklearn.externals.joblib.parallel.BatchedCalls object>)\n     67     def __init__(self, iterator_slice):\n     68         self.items = list(iterator_slice)\n     69         self._size = len(self.items)\n     70 \n     71     def __call__(self):\n---> 72         return [func(*args, **kwargs) for func, args, kwargs in self.items]\n        func = <function _fit_and_score>\n        args = (Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], <function _passthrough_scorer>, array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), 1, {'scheme': 'OLA'}, {})\n        kwargs = {'error_score': 'raise', 'return_parameters': True}\n        self.items = [(<function _fit_and_score>, (Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], <function _passthrough_scorer>, array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), 1, {'scheme': 'OLA'}, {}), {'error_score': 'raise', 'return_parameters': True})]\n     73 \n     74     def __len__(self):\n     75         return self._size\n     76 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py in _fit_and_score(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ', u'The wnba Daily is out  \\u25b8 Top stories toda...r shrinking in size  \\nWord of the day  fourteen', u'Infographic  Doctors Prescribing More Mobile H...e they evidence based \\nWhat is Social Learning ', u'Police Credit Social Media With Solving A Cred...fee With Tim Cook Rejected Because Bidder Used  ', u'Come see our website  IHCCSTEM\\nOur newly reno...lege education students\\nHappy New Year   s Eve ', u'Oh  honey  Don   t force us to stage an interv...me column  at the front edge of the fire escape ', u'Sooooo many slutty cats knocking around last n... t going to but a dump valve is rather tempting ', u'Made the front page again  Thanks to everyone ...elp build league  USSF  college soccer  etc  mls', ...], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34', '50-64', '25-34', '50-64', '50-64', '35-49', '50-64', '18-24', ...], scorer=<function _passthrough_scorer>, train=array([13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 2..., 55, 56, 57, 58, 59, 60, 61, 62, 63,\n       64]), test=array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12]), verbose=1, parameters={'scheme': 'OLA'}, fit_params={}, return_train_score=False, return_parameters=True, error_score='raise')\n   1545                              \" numeric value. (Hint: if using 'raise', please\"\n   1546                              \" make sure that it has been spelled correctly.)\"\n   1547                              )\n   1548 \n   1549     else:\n-> 1550         test_score = _score(estimator, X_test, y_test, scorer)\n        test_score = undefined\n        estimator = Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA')\n        X_test = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ']\n        y_test = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']\n        scorer = <function _passthrough_scorer>\n   1551         if return_train_score:\n   1552             train_score = _score(estimator, X_train, y_train, scorer)\n   1553 \n   1554     scoring_time = time.time() - start_time\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py in _score(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X_test=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], y_test=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'], scorer=<function _passthrough_scorer>)\n   1601 def _score(estimator, X_test, y_test, scorer):\n   1602     \"\"\"Compute the score of an estimator on a given test set.\"\"\"\n   1603     if y_test is None:\n   1604         score = scorer(estimator, X_test)\n   1605     else:\n-> 1606         score = scorer(estimator, X_test, y_test)\n        score = undefined\n        scorer = <function _passthrough_scorer>\n        estimator = Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA')\n        X_test = [u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out ']\n        y_test = ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']\n   1607     if not isinstance(score, numbers.Number):\n   1608         raise ValueError(\"scoring must return a number, got %s (%s) instead.\"\n   1609                          % (str(score), type(score)))\n   1610     return score\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/metrics/scorer.py in _passthrough_scorer(estimator=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), *args=([u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34']), **kwargs={})\n    200     return scorer\n    201 \n    202 \n    203 def _passthrough_scorer(estimator, *args, **kwargs):\n    204     \"\"\"Function that wraps estimator.score\"\"\"\n--> 205     return estimator.score(*args, **kwargs)\n        estimator.score = <bound method Neigbors_DS.score of Neigbors_DS(c...b='mallet', num_topics=90)},\n      scheme='OLA')>\n        args = ([u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], ['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'])\n        kwargs = {}\n    206 \n    207 \n    208 def check_scoring(estimator, scoring=None, allow_none=False):\n    209     \"\"\"Determine scorer from user options.\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in score(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '], y=['25-34', '25-34', '35-49', '50-64', '35-49', '50-64', '25-34', '35-49', '35-49', '35-49', '35-49', '18-24', '25-34'], sample_weight=None)\n     99         return y_pred\n    100 \n    101     def score(self, X, y, sample_weight=None):\n    102 \n    103         from sklearn.metrics import accuracy_score\n--> 104         return accuracy_score(y, self.predict(X), normalize=True)\n    105         # return self.svc.score(self.transform_to_y(X), y, sample_weight)\n    106 \n    107 \n    108     def predict_lca(self, sample):\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in predict(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), X=[u'Thank you  \\nGreat news  Elsevier withdraws su...con for  quot link to PDF broken quot  can help ', u'Chelsea King   s tragic rape and murder should...rave enough to run in desolate areas by th\\u2026', u'awesome \\nA Nonie and art admin day  + both we...t    Could do before that with her  if possible ', u'half moon     photos   flickr  \\n  barcelona  ...le coding is not a good idea   overcookedpasta  ', u'Wonder how this approach compares to what Chen...efensive tactics to MS+Apple   s offensive ones ', u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ', u'L   Oreal Paris Advanced Hair Care\\nDeal  $  o...Diapers\\nFree   in   Stainless Steel Pocket Tool', u'transfering technology to a new spin off UPC c...PI  Download here     graphdb DEXgdb NET J\\u2026', u'We must have the same condition   Did yours la...13 a retelling for adult audiences\\nLight play  ', u'In Search Of  Google Plus Federation | eWEEK L...ance implications of cloud computing in govtech ', u'Getting ready for AOM    AOM   Annual Meeting ...riments  Great w  Twitter  Failed on FB  amp  LI', u'New London Calling Undisguised blog post  foll...Up Search Tools for Recipe Pins via CookFromAPin', u'Photographic Challenge  Toys   Archive Photogr... Stories via\\nThe Yoleidy Carvajal Daily is out '])\n     92         # print X.shape\n     93         y_pred = []\n     94         # t0 = time.time()\n     95         for i, x in enumerate(X):\n     96             # print 'True Sample: ' + y_real[i]\n---> 97             y_pred.append(self.predictor(x))\n     98         # print('Predict took: %0.3f seconds') % (time.time()-t0)\n     99         return y_pred\n    100 \n    101     def score(self, X, y, sample_weight=None):\n\n...........................................................................\n/media/kostas/DATA/GIT/PAN16/<ipython-input-14-8f37d702af73> in predict_lca(self=Neigbors_DS(common_neigh=False, k=3,\n      model...ib='mallet', num_topics=90)},\n      scheme='OLA'), sample=u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way ')\n    106 \n    107 \n    108     def predict_lca(self, sample):\n    109         preds = []\n    110         for name, model in self.models.iteritems():\n--> 111             preds.append(model.predict([sample])[0])\n    112 #         print 'Preds: ' + str(preds)\n    113         if len(set(preds))==1:\n    114 #             print 'Unanimous Decision: ' + str(preds[0])\n    115 #             print '='*50\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/utils/metaestimators.py in <lambda>(*args=([u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '],), **kwargs={})\n     32         if obj is not None:\n     33             # delegate only on instances, not the classes.\n     34             # this is to allow access to the docstrings.\n     35             self.get_attribute(obj)\n     36         # lambda, but not partial, allows help() to work with update_wrapper\n---> 37         out = lambda *args, **kwargs: self.fn(obj, *args, **kwargs)\n        args = ([u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '],)\n        kwargs = {}\n     38         # update the docstring of the returned function\n     39         update_wrapper(out, self.fn)\n     40         return out\n     41 \n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.py in predict(self=Pipeline(steps=[('LDA', LDA(lib='mallet', num_to...e, shrinking=True,\n  tol=0.001, verbose=False))]), X=[u'Thank you Emmanuelle  Ouch  That is not fun  S...nal tweet anymore  Anyone else feeling this way '])\n    199             the pipeline.\n    200         \"\"\"\n    201         Xt = X\n    202         for name, transform in self.steps[:-1]:\n    203             Xt = transform.transform(Xt)\n--> 204         return self.steps[-1][-1].predict(Xt)\n        self.steps.predict = undefined\n        Xt = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    205 \n    206     @if_delegate_has_method(delegate='_final_estimator')\n    207     def fit_predict(self, X, y=None, **fit_params):\n    208         \"\"\"Applies fit_predict of last step in pipeline after transforms.\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    563         Returns\n    564         -------\n    565         y_pred : array, shape (n_samples,)\n    566             Class labels for samples in X.\n    567         \"\"\"\n--> 568         y = super(BaseSVC, self).predict(X)\n        y = undefined\n        self.predict = <bound method SVC.predict of SVC(C=100, cache_si...one, shrinking=True,\n  tol=0.001, verbose=False)>\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    569         return self.classes_.take(np.asarray(y, dtype=np.intp))\n    570 \n    571     # Hacky way of getting predict_proba to raise an AttributeError when\n    572     # probability=False using properties. Do not use this in new code; when\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    300 \n    301         Returns\n    302         -------\n    303         y_pred : array, shape (n_samples,)\n    304         \"\"\"\n--> 305         X = self._validate_for_predict(X)\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n        self._validate_for_predict = <bound method SVC._validate_for_predict of SVC(C...one, shrinking=True,\n  tol=0.001, verbose=False)>\n    306         predict = self._sparse_predict if self._sparse else self._dense_predict\n    307         return predict(X)\n    308 \n    309     def _dense_predict(self, X):\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/svm/base.py in _validate_for_predict(self=SVC(C=100, cache_size=200, class_weight='balance...None, shrinking=True,\n  tol=0.001, verbose=False), X=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object))\n    449             self.probA_, self.probB_)\n    450 \n    451     def _validate_for_predict(self, X):\n    452         check_is_fitted(self, 'support_')\n    453 \n--> 454         X = check_array(X, accept_sparse='csr', dtype=np.float64, order=\"C\")\n        X = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n    455         if self._sparse and not sp.isspmatrix(X):\n    456             X = sp.csr_matrix(X)\n    457         if self._sparse:\n    458             X.sort_indices()\n\n...........................................................................\n/usr/local/lib/python2.7/dist-packages/sklearn/utils/validation.py in check_array(array=array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object), accept_sparse=['csr'], dtype=<type 'numpy.float64'>, order='C', copy=False, force_all_finite=True, ensure_2d=True, allow_nd=False, ensure_min_samples=1, ensure_min_features=1, warn_on_dtype=False, estimator=None)\n    368 \n    369     if sp.issparse(array):\n    370         array = _ensure_sparse_format(array, accept_sparse, dtype, copy,\n    371                                       force_all_finite)\n    372     else:\n--> 373         array = np.array(array, dtype=dtype, order=order, copy=copy)\n        array = array([ [0.0033784948460716378, 0.00120660530216...60588e-05, 0.0037152444873722957]], dtype=object)\n        dtype = <type 'numpy.float64'>\n        order = 'C'\n        copy = False\n    374 \n    375         if ensure_2d:\n    376             if array.ndim == 1:\n    377                 if ensure_min_samples >= 2:\n\nValueError: setting an array element with a sequence.\n___________________________________________________________________________"
     ]
    }
   ],
   "source": [
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.cross_validation import train_test_split \n",
    "params = {\n",
    "    #'common_neigh': [True, False],\n",
    "    'scheme':['OLA', 'LCA', 'KNE', 'KNU']\n",
    "}\n",
    "\n",
    "models11 = {}\n",
    "models_tr11 = {}\n",
    "for i, model in enumerate(trained_base_models[:]):\n",
    "    models11[base_model_names[i]] = model\n",
    "    models_tr11[base_model_names[i]] = trained_base_models[i].steps[0][1]        \n",
    "LCA = Neigbors_DS(models11, models_tr11, k=3, scheme='LCA', common_neigh=False)\n",
    "#LCA.fit(X_train + X_meta, y_train + y_meta)\n",
    "\n",
    "#X_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.3, stratify=y, )\n",
    "grid1 = GridSearchCV(LCA, param_grid =params, cv=5, verbose=1, n_jobs=3)\n",
    "grid1.fit(X_meta, y_meta)\n",
    "print(grid1.best_score_)\n",
    "print(grid1.best_params_)\n",
    "#pipe2 = grid1.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "lower not found",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-80-db3ca9fa9328>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    158\u001b[0m \u001b[1;31m#aa = Neigbors_Combinator(models11, models_tr11, names11, knn_params, scheme='accuracy')\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    159\u001b[0m \u001b[1;31m#aa.fit(X_train, X_meta, y_train, y_meta)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 160\u001b[1;33m \u001b[0mpredict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maa\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    161\u001b[0m \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    162\u001b[0m \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-77-5369274355ba>\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    134\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    135\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 136\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    137\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    138\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-77-5369274355ba>\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    111\u001b[0m         \u001b[0mpredictions\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    112\u001b[0m         \u001b[0mX_truth\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcounter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 113\u001b[1;33m         \u001b[0mpredictions\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgt_knn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_truth\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    114\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    115\u001b[0m             \u001b[1;31m#print name\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/utils/metaestimators.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     35\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_attribute\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[1;31m# lambda, but not partial, allows help() to work with update_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m         \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m         \u001b[1;31m# update the docstring of the returned function\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m         \u001b[0mupdate_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/pipeline.pyc\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    201\u001b[0m         \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    202\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtransform\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 203\u001b[1;33m             \u001b[0mXt\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtransform\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    204\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    205\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, raw_documents)\u001b[0m\n\u001b[0;32m    864\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    865\u001b[0m         \u001b[1;31m# use the same matrix-building strategy as fit_transform\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 866\u001b[1;33m         \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_count_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfixed_vocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    867\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    868\u001b[0m             \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfill\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m_count_vocab\u001b[1;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[0;32m    750\u001b[0m         \u001b[0mindptr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    751\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mraw_documents\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 752\u001b[1;33m             \u001b[1;32mfor\u001b[0m \u001b[0mfeature\u001b[0m \u001b[1;32min\u001b[0m \u001b[0manalyze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    753\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    754\u001b[0m                     \u001b[0mj_indices\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocabulary\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(doc)\u001b[0m\n\u001b[0;32m    236\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m             return lambda doc: self._word_ngrams(\n\u001b[1;32m--> 238\u001b[1;33m                 tokenize(preprocess(self.decode(doc))), stop_words)\n\u001b[0m\u001b[0;32m    239\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    202\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlowercase\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mstrip_accents\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    205\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mstrip_accents\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/sparse/base.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, attr)\u001b[0m\n\u001b[0;32m    523\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetnnz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    524\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 525\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattr\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\" not found\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    526\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    527\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: lower not found"
     ]
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "class Neigbors_Combinator(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\"GIving untrained Models and their transformers create a combinator of information:\n",
    "       - Based on the count vectorizer predictions of a kNN classifier (fitted in X_train).\n",
    "       - Based on the transformed kNN classifiers predictions (fitted in X_train)\n",
    "       - Based on the samples predictions of the now trained models (fitted in X_train)\n",
    "       - Combining the above predictions using weights (fitted in X_meta)\n",
    "       \n",
    "       Args:\n",
    "           - models: dic of trained models in the form of: {\"name\": Model, :\"name2\":Model2...}\n",
    "           - models_tr: dic of transformers of the models. Form like models.\\\n",
    "           - names: list of names for the models to be used\n",
    "           - knn_params: dictionary containing params for fitting the knn classifiers.\n",
    "                         Expects a dictionary like: {\"general\":{\"num_folds\":5, \"n_jobs\":-1},\n",
    "                                                     \"name\":  parameter_grid_for_grid_search1,\n",
    "                                                     \"name2\": parameter_grid_for_grid_search2,...}\n",
    "            - scheme: for wegithing \"\"\"\n",
    "    \n",
    "    def __init__(self, models, models_tr, names, knn_params, scheme='majority', weights=None):\n",
    "        \n",
    "        from combinations import Combinator\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        self.counter = CountVectorizer()\n",
    "        self.models = models\n",
    "        self.models_tr = models_tr\n",
    "        self.names = names\n",
    "        self.knn_params = knn_params\n",
    "        self.scheme = scheme\n",
    "        self.weights = weights\n",
    "        self.ind2names = {}\n",
    "        for i, name in enumerate(names):\n",
    "            self.ind2names[i] = name\n",
    "        self.counter = CountVectorizer()\n",
    "        self.gt_knn = None\n",
    "        self.tr_knn = {}\n",
    "        self.comb = Combinator(scheme=self.scheme)\n",
    "            \n",
    "        \n",
    "\n",
    "    def fit(self, X_train, X_meta, y_train, y_meta, weights=None):\n",
    "        \n",
    "        from sklearn.grid_search import GridSearchCV\n",
    "        from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "        import random, time\n",
    "        #print len(y_train), len(y_meta)\n",
    "        if (y_train is None)  or (y_meta is None):\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "            self.counter = CountVectorizer()\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            \n",
    "            # Fit the Meta-Classifier\n",
    "            predictions_meta = []\n",
    "            # Fitting the ground truth neighbors\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            X_train_truth = self.counter.fit_transform(X_train)\n",
    "            #print X_train_truth.toarray().shape\n",
    "            grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params['true'],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "            grid_.fit(X_train_truth, y_train)\n",
    "            self.gt_knn = grid_.best_estimator_\n",
    "            # Add the gt predictions\n",
    "            #print self.counter.transform(X_meta).toarray().shape\n",
    "            predictions_meta.append(self.gt_knn.predict(self.counter.transform(X_meta)))\n",
    "            # Fit the transformation Classifiers\n",
    "            for name in self.names:\n",
    "                X_train_tr = self.models_tr[name].transform(X_train)\n",
    "                grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params[name],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "                grid_.fit(X_train_tr, y_train)\n",
    "                self.tr_knn[name] = grid_.best_estimator_\n",
    "                # Add the transformed knn predictions\n",
    "                predictions_meta.append(self.tr_knn[name].predict(self.models_tr[name].transform(X_meta)))\n",
    "                # Add the per sample predictions\n",
    "                predictions_meta.append(self.models[name].predict(X_meta))\n",
    "            #print len(predictions_meta)\n",
    "            #print predictions_meta\n",
    "            self.comb.fit(predictions_meta, y_meta)\n",
    "            print \"Best Weights Found:\"\n",
    "            print self.comb.weights\n",
    "            return\n",
    "        \n",
    "        \n",
    "    def transform(self, X, y):\n",
    "\n",
    "        import numpy\n",
    "        \n",
    "        predictions = []\n",
    "        X_truth = self.counter.transform(X)   \n",
    "        predictions.append(self.gt_knn.predict(X_truth))\n",
    "        for name in self.names:\n",
    "            #print name\n",
    "            # Add the transformed knn predictions\n",
    "            predictions.append(self.tr_knn[name].predict(self.models_tr[name].transform(X)))\n",
    "            # Add the per sample predictions\n",
    "            predictions.append(self.models[name].predict(X))\n",
    "            #print len(predictions)\n",
    "        #print len(predictions)\n",
    "        #for pred in predictions:\n",
    "        #    print len(pred)\n",
    "        final_predict =self.comb.predict(predictions)\n",
    "        names = ['Ground_Truth', '3grams-Neighbors', '3grams-Pred',\n",
    "                     'SOAC-Neigbors', 'SOAC-Pred', 'LSI-Neighbors', 'LSI-Pred', 'Comb']\n",
    "        for i in xrange(len(y)):\n",
    "            print 'True: ' + str(y[i])\n",
    "            for  j, pred in enumerate(predictions):\n",
    "                print '%s: %s' % (names[j], pred[i])\n",
    "            print \"Comb: %s\" % final_predict[i]\n",
    "            print '*'*100\n",
    "        return final_predict\n",
    "\n",
    "    def predict(self, X):\n",
    "        return self.transform(X)\n",
    "\n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "\n",
    "models11 = {}\n",
    "models_tr11 = {}\n",
    "tr_params = {'n_neighbors':[i for i in xrange(1,20)], 'weights':['uniform', 'distance']}\n",
    "knn_params = {\"general\":{\"n_jobs\":-1, \"cv\":5, \"verbose\":1}}\n",
    "knn_params['true'] = tr_params\n",
    "names11 = []\n",
    "for i, model in enumerate(trained_base_models):\n",
    "    names11.append(base_model_names[i])\n",
    "    models11[names11[-1]] = model\n",
    "    models_tr11[names11[-1]] = trained_base_models[i].steps[0][1]\n",
    "    knn_params[names11[-1]] = tr_params\n",
    "    #print base_model_names[i]\n",
    "    #print trained_base_models[i].steps[0][1]\n",
    "aa = Neigbors_Combinator(models11, models_tr11, names11, knn_params, scheme='accuracy')            \n",
    "aa.fit(X_train, X_meta, y_train, y_meta)\n",
    "predict = aa.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in xrange(len(y_meta)):\n",
    "    print \"True Label: %s\" % y_meta[i]\n",
    "    prediction_string = ''\n",
    "    for name, model in zip(base_model_names, pred_cv):\n",
    "        prediction_string += ' %s:%s |' % (name, g.lab.inverse_transform(g.transformation(model).argmax(axis=1))[0])\n",
    "    prediction_string += ' %s:%s' % ('Bernoulli', predict[i])\n",
    "    print prediction_string[1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.292307692308\n",
      "Confusion matrix :\n",
      " [[ 0  0  3  1  0]\n",
      " [ 0  0 20  1  0]\n",
      " [ 0  7 18  1  1]\n",
      " [ 0  4  7  1  0]\n",
      " [ 0  0  0  1  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.38      0.67      0.48        27\n",
      "      50-64       0.20      0.08      0.12        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.19      0.29      0.22        65\n",
      "\n"
     ]
    }
   ],
   "source": [
    "g = combinations.Combinator(scheme='majority')\n",
    "g.fit(base_predictions_meta, y_meta)\n",
    "pred_meta = g.lab.transform(base_predictions_meta)\n",
    "pred_meta = g.ohe.transform(pred_meta.reshape(-1, 1)).todense().reshape(len(base_predictions_meta[0]), -1)\n",
    "pred_cv = g.lab.transform(base_predictions_cv)\n",
    "pred_cv = g.ohe.transform(pred_cv.reshape(-1, 1)).todense().reshape(len(base_predictions_cv[0]), -1)\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "gg =MultinomialNB()\n",
    "gg.fit(pred_meta, y_meta)\n",
    "gg.predict(pred_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Per Class Accuracy of the models\n",
      "        18-24  25-34       35-49  50-64  65-xx\n",
      "3grams      0     25   57.142857      0      0\n",
      "soac       50     10   67.857143      0      0\n",
      "lsi         0      0  100.000000      0      0\n"
     ]
    }
   ],
   "source": [
    "import EnsembleDiversityTests\n",
    "reload(EnsembleDiversityTests)\n",
    "from EnsembleDiversityTests import BaseClassifiers\n",
    "\n",
    "a = BaseClassifiers(base_predictions_meta, base_model_names, y_meta)\n",
    "g = a.get_per_class_accuracy()\n",
    "a1 = g.T['3grams']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 38 candidates, totalling 190 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  76 tasks      | elapsed:    1.7s\n",
      "[Parallel(n_jobs=-1)]: Done 190 out of 190 | elapsed:    3.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 38 candidates, totalling 190 fits\n",
      "Fitting 5 folds for each of 38 candidates, totalling 190 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 190 out of 190 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 38 candidates, totalling 190 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 190 out of 190 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Weights Found:\n",
      "[21, 17, 21, 26, 23, 22, 28]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done 190 out of 190 | elapsed:    0.3s finished\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "transform() takes exactly 3 arguments (2 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-71-96384b6fa6ea>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    157\u001b[0m \u001b[0maa\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNeigbors_Combinator\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodels11\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmodels_tr11\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames11\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mknn_params\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscheme\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'accuracy'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    158\u001b[0m \u001b[0maa\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 159\u001b[1;33m \u001b[0mpredict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maa\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    160\u001b[0m \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    161\u001b[0m \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-71-96384b6fa6ea>\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    133\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    134\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 135\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    136\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    137\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_true\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: transform() takes exactly 3 arguments (2 given)"
     ]
    }
   ],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "class Neigbors_Combinator(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\"GIving untrained Models and their transformers create a combinator of information:\n",
    "       - Based on the count vectorizer predictions of a kNN classifier (fitted in X_train).\n",
    "       - Based on the transformed kNN classifiers predictions (fitted in X_train)\n",
    "       - Based on the samples predictions of the now trained models (fitted in X_train)\n",
    "       - Combining the above predictions using weights (fitted in X_meta) based ond LCA\n",
    "       \n",
    "       Args:\n",
    "           - models: dic of trained models in the form of: {\"name\": Model, :\"name2\":Model2...}\n",
    "           - models_tr: dic of transformers of the models. Form like models.\\\n",
    "           - names: list of names for the models to be used\n",
    "           - knn_params: dictionary containing params for fitting the knn classifiers.\n",
    "                         Expects a dictionary like: {\"general\":{\"num_folds\":5, \"n_jobs\":-1},\n",
    "                                                     \"name\":  parameter_grid_for_grid_search1,\n",
    "                                                     \"name2\": parameter_grid_for_grid_search2,...}\n",
    "            - scheme: for weigting \"\"\"\n",
    "    \n",
    "    def __init__(self, models, models_tr, names, knn_params, scheme='majority', weights=None):\n",
    "        \n",
    "        from combinations import Combinator\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        self.counter = CountVectorizer()\n",
    "        self.models = models\n",
    "        self.models_tr = models_tr\n",
    "        self.names = names\n",
    "        self.knn_params = knn_params\n",
    "        self.scheme = scheme\n",
    "        self.weights = weights\n",
    "        self.ind2names = {}\n",
    "        for i, name in enumerate(names):\n",
    "            self.ind2names[i] = name\n",
    "        self.counter = CountVectorizer()\n",
    "        self.gt_knn = None\n",
    "        self.tr_knn = {}\n",
    "        self.comb = Combinator(scheme=self.scheme)\n",
    "            \n",
    "        \n",
    "\n",
    "    def fit(self, X_train, X_meta, y_train, y_meta, weights=None):\n",
    "        \n",
    "        from sklearn.grid_search import GridSearchCV\n",
    "        from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.e        \n",
    "        from EnsembleDiversityTests import BaseClassifiers\n",
    "\n",
    "        import random, time\n",
    "        #print len(y_train), len(y_meta)\n",
    "        if (y_train is None)  or (y_meta is None):\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            \n",
    "            # Fit the Meta-Classifier\n",
    "            predictions_meta = []\n",
    "            # Fitting the ground truth neighbors\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            X_train_truth = self.counter.fit_transform(X_train)\n",
    "            #print X_train_truth.toarray().shape\n",
    "            grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params['true'],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "            grid_.fit(X_train_truth, y_train)\n",
    "            self.gt_knn = grid_.best_estimator_\n",
    "            # Add the gt predictions\n",
    "            #print self.counter.transform(X_meta).toarray().shape\n",
    "            predictions_meta.append(self.gt_knn.predict(self.counter.transform(X_meta)))\n",
    "            # Fit the transformation Classifiers\n",
    "            for name in self.names:\n",
    "                X_train_tr = self.models_tr[name].transform(X_train)\n",
    "                grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params[name],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "                grid_.fit(X_train_tr, y_train)\n",
    "                self.tr_knn[name] = grid_.best_estimator_\n",
    "                # Add the transformed knn predictions\n",
    "                predictions_meta.append(self.tr_knn[name].predict(self.models_tr[name].transform(X_meta)))\n",
    "                # Add the per sample predictions\n",
    "                predictions_meta.append(self.models[name].predict(X_meta))\n",
    "            #print len(predictions_meta)\n",
    "            #print predictions_meta\n",
    "            \n",
    "            self.comb.fit(predictions_meta, y_meta)\n",
    "            print \"Best Weights Found:\"\n",
    "            print self.comb.weights\n",
    "            return\n",
    "        \n",
    "        \n",
    "    def transform(self, X, y):\n",
    "\n",
    "        import numpy\n",
    "        \n",
    "        predictions = []\n",
    "        X_truth = self.counter.transform(X)   \n",
    "        predictions.append(self.gt_knn.predict(X_truth))\n",
    "        for name in self.names:\n",
    "            #print name\n",
    "            # Add the transformed knn predictions\n",
    "            predictions.append(self.tr_knn[name].predict(self.models_tr[name].transform(X)))\n",
    "            # Add the per sample predictions\n",
    "            predictions.append(self.models[name].predict(X))\n",
    "            #print len(predictions)\n",
    "        #print len(predictions)\n",
    "        #for pred in predictions:\n",
    "        #    print len(pred)\n",
    "        final_predict =self.comb.predict(predictions)\n",
    "        names = ['Ground_Truth', '3grams-Neighbors', '3grams-Pred',\n",
    "                     'SOAC-Neigbors', 'SOAC-Pred', 'LSI-Neighbors', 'LSI-Pred', 'Comb']\n",
    "        for i in xrange(len(y)):\n",
    "            print 'True: ' + str(y[i])\n",
    "            for  j, pred in enumerate(predictions):\n",
    "                print '%s: %s' % (names[j], pred[i])\n",
    "            print \"Comb: %s\" % final_predict[i]\n",
    "            print '*'*100\n",
    "        return final_predict\n",
    "\n",
    "    def predict(self, X, y):\n",
    "        return self.transform(X)\n",
    "\n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "\n",
    "models11 = {}\n",
    "models_tr11 = {}\n",
    "tr_params = {'n_neighbors':[i for i in xrange(1,20)], 'weights':['uniform', 'distance']}\n",
    "knn_params = {\"general\":{\"n_jobs\":-1, \"cv\":5, \"verbose\":1}}\n",
    "knn_params['true'] = tr_params\n",
    "names11 = []\n",
    "for i, model in enumerate(trained_base_models):\n",
    "    names11.append(base_model_names[i])\n",
    "    models11[names11[-1]] = model\n",
    "    models_tr11[names11[-1]] = trained_base_models[i].steps[0][1]\n",
    "    knn_params[names11[-1]] = tr_params\n",
    "    #print base_model_names[i]\n",
    "    #print trained_base_models[i].steps[0][1]\n",
    "aa = Neigbors_Combinator(models11, models_tr11, names11, knn_params, scheme='accuracy')            \n",
    "aa.fit(X_train, X_meta, y_train, y_meta)\n",
    "predict = aa.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble3(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Utilizing the neighborhood in all representations and also ground truth model.\n",
    "        Implementing a weighted voting scheme.\"\"\"\n",
    "\n",
    "    def __init__(self, models, k=3, weights= [2,1,3,0.7]):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models):\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\ ')\n",
    "        else:\n",
    "            self.models = models\n",
    "            # self.cv_scores = cv_scores\n",
    "            self.k = k\n",
    "            self.weights = weights\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.representations = []\n",
    "            self.meta = None\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.doc_terms = None\n",
    "            self.tree = None\n",
    "            self.experts = []\n",
    "        \n",
    "\n",
    "    def fit(self, X_cv, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X_cv).toarray()\n",
    "            self.tree = BallTree(self.doc_terms, leaf_size=20)\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "            self.predictions = predictions\n",
    "            self.true = y_true\n",
    "            count = 0\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            return self\n",
    "\n",
    "    def predict(self, X):\n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), self.k)  \n",
    "#             print 'Sample ' + y_real[i]\n",
    "            #print dist\n",
    "            #print type(dist)\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            #pass\n",
    "            y_pred.append(self.expert_decision(neigbors_indexes[0],  dist, X[i]))\n",
    "            \n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, neigbors_indexes, dist, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "#         print 'True'\n",
    "#         print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = self.weights[2]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_neig_pred, normalize=True))\n",
    "            if acc[-1] >self.weights[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(self.weights[1]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(self.weights[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "#                 print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_neig_pred, normalize=True))\n",
    "#                 print 'Predictions'\n",
    "#                 print model_pred\n",
    "#                 print 'Representations'\n",
    "#                 print model_neig_pred\n",
    "#                 print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "        total_pred.extend([n for i, n in enumerate(neigbors_true) for j in xrange(int(weights['true']*(self.k-i)))])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "#         print data\n",
    "#         best_model_ind = acc.index(max(acc))\n",
    "#         print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "#         print '='*50\n",
    "#         #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fit took: 16.118 seconds\n",
      "Predict took: 1215.613 seconds\n",
      "Accuracy : 0.369230769231\n",
      "Confusion matrix :\n",
      " [[ 0  2  2  0  0]\n",
      " [ 0  6 15  0  0]\n",
      " [ 1  7 18  1  0]\n",
      " [ 0  3  9  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.33      0.29      0.31        21\n",
      "      35-49       0.40      0.67      0.50        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.27      0.37      0.31        65\n",
      "\n"
     ]
    }
   ],
   "source": [
    "space2 = SubSpaceEnsemble3(models_for_space, k=3, weights= [2,1,3,-1])\n",
    "#2,1,3,0.7\n",
    "space2.fit(X_meta, y_meta)\n",
    "predict = space2.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "class SubSpaceEnsemble3_w(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Utilizing the neighborhood in all representations and also ground truth model.\n",
    "        Implementing a weighted voting scheme. Finding Optimal weights also!\"\"\"\n",
    "\n",
    "    def __init__(self, models, k=3, scheme = 'weights', weights=[6,3,2,0.7]):\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        if (not models) :\n",
    "            raise AttributeError('Models expexts a dictonary of models \\\n",
    "              containg the predictions of y_true for each classifier.\\\n",
    "              cv_score expects a list len(models.keys()) with the\\\n",
    "              cross validation scores of each model')\n",
    "        else:\n",
    "            self.models = models\n",
    "            self.k = k\n",
    "            self.scheme = scheme\n",
    "            self.weights = None\n",
    "            self.ind2names = {}\n",
    "            for i, name in enumerate(models.keys()):\n",
    "                self.ind2names[i] = name\n",
    "            self.counter = CountVectorizer()\n",
    "            self.representations = []\n",
    "            self.meta = None\n",
    "            self.predictions = []\n",
    "            self.true = []\n",
    "            self.doc_terms = None\n",
    "            self.tree = None\n",
    "            self.experts = []\n",
    "            if self.scheme == 'weights':\n",
    "                if type(self.weights) in (numpy.array, numpy.ndarray):\n",
    "                    pass  # It is from the optimization part\n",
    "                else:\n",
    "                    if not(self.weights):\n",
    "                        print \"Need weights for this scheme!\"\n",
    "                self.weights = weights\n",
    "                weights_string = \" %.2f |\" * len(self.weights) % tuple(self.weights)\n",
    "                # print \"Using given weights: | %s\" % weights_string\n",
    "            elif self.scheme == 'optimal':\n",
    "                # print \"Will find the weights after fitting\"\n",
    "                pass\n",
    "            else:\n",
    "                self.weights = [6,3,2,0.7]\n",
    "        \n",
    "\n",
    "    def fit(self, X, X_cv, y = None, y_true=None, weights=None):\n",
    "        \n",
    "        from sklearn.neighbors import BallTree\n",
    "        import random\n",
    "\n",
    "        if y_true is None:\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            self.doc_terms = self.counter.fit_transform(X + X_cv).toarray()\n",
    "            self.tree = BallTree(self.doc_terms, leaf_size=20)\n",
    "            predictions = []\n",
    "            for name, model in self.models.iteritems():\n",
    "                predictions.append(model.predict(X + X_cv))\n",
    "                #print len(predictions[-1])\n",
    "                transf = model.steps[0][1].transform(X + X_cv)\n",
    "                if hasattr(transf, \"toarray\"):\n",
    "                    #print 'Exei'\n",
    "                    self.representations.append(transf.toarray())\n",
    "                else:\n",
    "                    self.representations.append(transf)\n",
    "            self.predictions = predictions\n",
    "            self.true = y + y_true\n",
    "            count = 0\n",
    "            #print self.expert_scores\n",
    "            #print self.experts\n",
    "            if self.scheme=='optimal':\n",
    "                self.find_weights(X, X_cv, y, y_cv)\n",
    "                weights_string = \" %.2f |\" * len(self.weights) % tuple(self.weights)\n",
    "            print('Fit took: %0.3f seconds') % (time.time()-t0)\n",
    "            return self\n",
    "        \n",
    "    def find_weights(self, X, X_cv, y, y_cv):\n",
    "\n",
    "        import numpy\n",
    "\n",
    "        from scipy.optimize import minimize\n",
    "\n",
    "        w = [6,3,2,0.7]\n",
    "        bnds = tuple([(0, None) for i in xrange(len(w))])\n",
    "        a = minimize(self.f, w, args=(SubSpaceEnsemble3_w, X, X_cv, y, y_cv), method='L-BFGS-B', bounds=bnds)\n",
    "        self.weights = list(a.x)\n",
    "        return\n",
    "    \n",
    "    def find_weigths2(self, X, X_cv, y):\n",
    "        \n",
    "        N_rand = 100\n",
    "        N_rand1 = 50\n",
    "        poss_w = []\n",
    "        acc_ = []\n",
    "        pred = []\n",
    "        for i in xrange(N_rand1):\n",
    "            tmp_w = [2,1,3,6]\n",
    "            tmp_w[0] = round(random.random(), 3)\n",
    "            tmp_w[1] = round(1 - tmp_w[0], 3)\n",
    "            tmp_w[2] = round(random.uniform(0.2, 0.8), 3)\n",
    "            # tmp_w[3] = random.randint(1,10)\n",
    "            poss_w.append(tmp_w)\n",
    "            pred = self.find_weights(X_cv, tmp_w)\n",
    "            acc = accuracy_score(self.true, pred)\n",
    "            # print('Accuracy : {}'.format(acc))\n",
    "            acc_.append(acc)\n",
    "        print('First search took: %0.3f seconds') % (time.time() - t0)\n",
    "        tmp_w = poss_w[acc_.index(max(acc_))]\n",
    "        poss_w = []\n",
    "        acc_ = []\n",
    "        for i in xrange(self.N_rand  -N_rand1):\n",
    "            tmp_w2 = tmp_w\n",
    "            tmp_w2[0] = round(random.uniform(tmp_w[0] - 0.1, tmp_w[0] + 0.1), 3)\n",
    "            tmp_w2[1] = round(1 - tmp_w2[0], 3)\n",
    "            tmp_w2[2] = round(random.uniform(tmp_w[2] - 0.1, tmp_w[1] + 0.1), 3)\n",
    "            poss_w.append(tmp_w2)\n",
    "            pred = self.find_weights(X_cv, tmp_w2)\n",
    "            acc = accuracy_score(self.true, pred)\n",
    "            # print('Accuracy : {}'.format(acc))\n",
    "            acc_.append(acc)\n",
    "        self.weights = poss_w[acc_.index(max(acc_))]\n",
    "        self.k = self.weights[3]\n",
    "\n",
    "    def f(self, w, SubSpaceEnsemble3_w, X, X_cv, y, y_cv):\n",
    "        gg = SubSpaceEnsemble3_w(self.models, self.k, scheme= 'weights', weights= w)\n",
    "        gg.fit(X, X_cv, y, y_cv)\n",
    "        score = 1 - gg.score(x, y)\n",
    "        # print 'Weights'\n",
    "        # print w\n",
    "        # print 'Score: ' + str(score)\n",
    "        return score\n",
    "\n",
    "    def predict(self, X):\n",
    "        \n",
    "        # print \"PRedict\"\n",
    "        # print X.shape\n",
    "        X_transformed = self.counter.transform(X).toarray()\n",
    "        #print type((X_transformed)[0])\n",
    "        #print X_transformed.shape\n",
    "        #return 0\n",
    "        y_pred = []\n",
    "        t0 = time.time()\n",
    "        for i in range(0, X_transformed.shape[0]):\n",
    "            #print X_transformed[i,:].shape\n",
    "            dist, neigbors_indexes = self.tree.query(X_transformed[i,:].reshape(1,-1), self.k)  \n",
    "            # print 'Sample ' + y_real[i]\n",
    "            #print neigbors_indexes[0]\n",
    "            #print dist\n",
    "            #best_model_ind = self.expert_decision(neigbors_indexes[0])\n",
    "            y_pred.append(self.expert_decision(neigbors_indexes[0],  X[i]))\n",
    "            \n",
    "            #y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\n",
    "        #print y_pred\n",
    "        print('Predict took: %0.3f seconds') % (time.time()-t0)\n",
    "        return y_pred\n",
    "\n",
    "    def score(self, X, y, sample_weight=None):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        return accuracy_score(y, self.predict(X), normalize=True)\n",
    "        #return self.svc.score(self.transform_to_y(X), y, sample_weight)\n",
    "\n",
    "\n",
    "    def expert_decision(self, neigbors_indexes, x_sample):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "        from collections import Counter\n",
    "        from sklearn.neighbors import BallTree\n",
    "        \n",
    "        models_pred = []\n",
    "        models_neig_pred = []\n",
    "        acc = []\n",
    "        t0 = time.time()\n",
    "        neigbors_true = [self.true[n_i] for n_i in neigbors_indexes]\n",
    "        #print('Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "#         print 'True'\n",
    "#         print neigbors_true\n",
    "        sample_predictions = []\n",
    "        total_pred = []\n",
    "        weights = {}\n",
    "        weights['true'] = self.weights[2]\n",
    "        weights['models_n'] = []\n",
    "        weights['models'] = []\n",
    "        for model_i in xrange(len(self.models.values())):\n",
    "            ModelTree = BallTree(self.representations[model_i])\n",
    "            temp_trans = self.models[self.ind2names[model_i]].steps[0][1].transform([x_sample])\n",
    "            if hasattr(temp_trans, 'toarray'):\n",
    "                temp_trans = temp_trans.toarray()\n",
    "            _, model_neig = ModelTree.query(temp_trans, self.k)\n",
    "            model_neig_pred = []\n",
    "            for model_n_i in model_neig[0].tolist():\n",
    "                model_neig_pred.append(self.predictions[model_i][model_n_i])\n",
    "            models_neig_pred.append(model_neig_pred)\n",
    "            model_pred = []\n",
    "            for n_i in neigbors_indexes:\n",
    "                model_pred.append(self.predictions[model_i][n_i])\n",
    "            models_pred.append(model_pred)\n",
    "            acc.append(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "            if acc[-1] >self.weights[3]:\n",
    "                # Adding neighbors predictions\n",
    "                weights['models_n'].append(int(self.weights[1]/float((1-acc[-1])+0.01)))\n",
    "                total_pred.extend([pred for j in xrange(weights['models_n'][-1]) for pred in model_pred])\n",
    "                #print('Predicting Neighbors per sample: %0.4f seconds') % (time.time()-t0)\n",
    "                # Adding sample prediction\n",
    "                sample_predictions.append(self.models[self.ind2names[model_i]].predict(x_sample)[0])\n",
    "                weights['models'].append(int(self.weights[0]/float((1-acc[-1])+0.01))) \n",
    "                total_pred.extend([sample_predictions[-1] for j in xrange(weights['models'][-1])])\n",
    "                total_pred.extend([pred for j in xrange(weights['models'][-1]) for pred in model_neig_pred])\n",
    "            #print len(x_sample)\n",
    "            #print self.ind2names[model_i]\n",
    "            \n",
    "#                 print 'Model: ' + self.ind2names[model_i] + ' Accuracy: ' + str(accuracy_score(neigbors_true, model_pred, normalize=True))\n",
    "#                 print 'Predictions'\n",
    "#                 print model_pred\n",
    "#                 print 'Representations'\n",
    "#                 print model_neig_pred\n",
    "#                 print 'Sample prediction: ' + str(sample_predictions[-1])\n",
    "        total_pred.extend([n for j in xrange(int(weights['true'])) for n in neigbors_true])\n",
    "        #print('creating votes: %0.4f seconds') % (time.time()-t0)\n",
    "        data = Counter(total_pred)\n",
    "        #data = Counter([k for pred in models_pred for k in pred])\n",
    "#         print data\n",
    "#         best_model_ind = acc.index(max(acc))\n",
    "#         print 'Total pred: ' + str(data.most_common(1)[0][0])\n",
    "        #print len(total_pred)\n",
    "        #return best_model_ind\n",
    "        return data.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Need weights for this scheme!\n",
      "Fit took: 100.444 seconds\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-b3d864b07b8d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mspace_k\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSubSpaceEnsemble3_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodels_for_space\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscheme\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'optimal'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0.5\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mspace_k\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX_meta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0my_meta\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mpredict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mspace_k\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0macc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mconf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mconfusion_matrix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mpredict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0msorted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mset\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, X_cv, y, y_true, weights)\u001b[0m\n\u001b[0;32m     85\u001b[0m             \u001b[1;31m#print self.experts\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscheme\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'optimal'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 87\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_weights\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     88\u001b[0m                 \u001b[0mweights_string\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m\" %.2f |\"\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     89\u001b[0m             \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Fit took: %0.3f seconds'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0mt0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mfind_weights\u001b[1;34m(self, X, X_cv, y, y_cv)\u001b[0m\n\u001b[0;32m     98\u001b[0m         \u001b[0mw\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m0.7\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     99\u001b[0m         \u001b[0mbnds\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 100\u001b[1;33m         \u001b[0ma\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mminimize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSubSpaceEnsemble3_w\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'L-BFGS-B'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbounds\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mbnds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    101\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mweights\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    102\u001b[0m         \u001b[1;32mreturn\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/_minimize.pyc\u001b[0m in \u001b[0;36mminimize\u001b[1;34m(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)\u001b[0m\n\u001b[0;32m    445\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'l-bfgs-b'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    446\u001b[0m         return _minimize_lbfgsb(fun, x0, args, jac, bounds,\n\u001b[1;32m--> 447\u001b[1;33m                                 callback=callback, **options)\n\u001b[0m\u001b[0;32m    448\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'tnc'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    449\u001b[0m         return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/lbfgsb.pyc\u001b[0m in \u001b[0;36m_minimize_lbfgsb\u001b[1;34m(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)\u001b[0m\n\u001b[0;32m    328\u001b[0m                 \u001b[1;31m# minimization routine wants f and g at the current x\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m                 \u001b[1;31m# Overwrite f and g:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 330\u001b[1;33m                 \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    331\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mtask_str\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb'NEW_X'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    332\u001b[0m             \u001b[1;31m# new iteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/lbfgsb.pyc\u001b[0m in \u001b[0;36mfunc_and_grad\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    271\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mjac\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    272\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 273\u001b[1;33m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    274\u001b[0m             \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_approx_fprime_helper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mepsilon\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf0\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    275\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/scipy/optimize/optimize.pyc\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[1;34m(*wrapper_args)\u001b[0m\n\u001b[0;32m    287\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    288\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 289\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    290\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    291\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mf\u001b[1;34m(self, w, SubSpaceEnsemble3_w, X, X_cv, y, y_cv)\u001b[0m\n\u001b[0;32m    140\u001b[0m         \u001b[0mgg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSubSpaceEnsemble3_w\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mscheme\u001b[0m\u001b[1;33m=\u001b[0m \u001b[1;34m'weights'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mweights\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mw\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    141\u001b[0m         \u001b[0mgg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_cv\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 142\u001b[1;33m         \u001b[0mscore\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m1\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mgg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscore\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    143\u001b[0m         \u001b[1;31m# print 'Weights'\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    144\u001b[0m         \u001b[1;31m# print w\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mscore\u001b[1;34m(self, X, y, sample_weight)\u001b[0m\n\u001b[0;32m    173\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    174\u001b[0m         \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 175\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0maccuracy_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    176\u001b[0m         \u001b[1;31m#return self.svc.score(self.transform_to_y(X), y, sample_weight)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    177\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    163\u001b[0m             \u001b[1;31m#print dist\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m             \u001b[1;31m#best_model_ind = self.expert_decision(neigbors_indexes[0])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 165\u001b[1;33m             \u001b[0my_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexpert_decision\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mneigbors_indexes\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m  \u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    166\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m             \u001b[1;31m#y_pred.append(self.models[self.ind2names[best_model_ind]].predict([X[i]])[0])\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-18-26b1335a0965>\u001b[0m in \u001b[0;36mexpert_decision\u001b[1;34m(self, neigbors_indexes, x_sample)\u001b[0m\n\u001b[0;32m    199\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mmodel_i\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mxrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    200\u001b[0m             \u001b[0mModelTree\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBallTree\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrepresentations\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmodel_i\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 201\u001b[1;33m             \u001b[0mtemp_trans\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mind2names\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mmodel_i\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msteps\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx_sample\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    202\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtemp_trans\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'toarray'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m                 \u001b[0mtemp_trans\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtemp_trans\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/media/kostas/DATA/GIT/PAN16/pan/features.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m   1050\u001b[0m                     \u001b[1;32mprint\u001b[0m \u001b[1;34m'dic'\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1051\u001b[0m                     \u001b[1;31m#print self.__dict__\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1052\u001b[1;33m                 \u001b[0mdoc_topics1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mLDA\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtest_corpus\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1053\u001b[0m                 \u001b[0mdoc_topics1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtopic\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtopic\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mdoc_topics1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1054\u001b[0m                 \u001b[0mdoc_topics\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc_topics1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/gensim/models/wrappers/ldamallet.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, bow, iterations)\u001b[0m\n\u001b[0;32m    169\u001b[0m             \u001b[0mbow\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mbow\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    170\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 171\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconvert_input\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbow\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minfer\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    172\u001b[0m         \u001b[0mcmd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmallet_path\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\" infer-topics --input %s --inferencer %s --output-doc-topics %s --num-iterations %s\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    173\u001b[0m         \u001b[0mcmd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcmd\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfcorpusmallet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'.infer'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfinferencer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfdoctopics\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'.infer'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0miterations\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/local/lib/python2.7/dist-packages/gensim/models/wrappers/ldamallet.pyc\u001b[0m in \u001b[0;36mconvert_input\u001b[1;34m(self, corpus, infer)\u001b[0m\n\u001b[0;32m    147\u001b[0m             \u001b[0mcmd\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcmd\u001b[0m \u001b[1;33m%\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfcorpustxt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfcorpusmallet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    148\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"converting temporary corpus to MALLET format with %s\"\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mcmd\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 149\u001b[1;33m         \u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcmd\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mshell\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    150\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    151\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/subprocess.pyc\u001b[0m in \u001b[0;36mcall\u001b[1;34m(*popenargs, **kwargs)\u001b[0m\n\u001b[0;32m    520\u001b[0m     \u001b[0mretcode\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcall\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"ls\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"-l\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    521\u001b[0m     \"\"\"\n\u001b[1;32m--> 522\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mPopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mpopenargs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    523\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    524\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/subprocess.pyc\u001b[0m in \u001b[0;36mwait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1374\u001b[0m             \u001b[1;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreturncode\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1375\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1376\u001b[1;33m                     \u001b[0mpid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msts\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_eintr_retry_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwaitpid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpid\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1377\u001b[0m                 \u001b[1;32mexcept\u001b[0m \u001b[0mOSError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1378\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0merrno\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mECHILD\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/subprocess.pyc\u001b[0m in \u001b[0;36m_eintr_retry_call\u001b[1;34m(func, *args)\u001b[0m\n\u001b[0;32m    474\u001b[0m     \u001b[1;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    475\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 476\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    477\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mOSError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mIOError\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    478\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrno\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mEINTR\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "space_k = SubSpaceEnsemble3_w(models_for_space, k=3, scheme='optimal', weights=[2,1,3,0.5])\n",
    "space_k.fit(X,X_meta, y,y_meta)\n",
    "predict = space_k.predict(X_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#import EnsembleDiversityTests\n",
    "reload(EnsembleDiversityTests)\n",
    "from EnsembleDiversityTests import DiversityTests, BaseClassifiers\n",
    "#gg = DiversityTests(predictions[:-1], print_names[:-1], predictions[-1])\n",
    "gg = DiversityTests(base_predictions_cv, base_model_names, y_cv)\n",
    "#gg.print_report()\n",
    "gg1 = BaseClassifiers(base_predictions_cv, base_model_names, y_cv, True)\n",
    "#gg1.get_comparison_report()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Per Class Accuracy of the models!\n",
      "        18-24      25-34      35-49      50-64  65-xx\n",
      "3grams      0  47.619048  74.074074   0.000000      0\n",
      "soac        0  52.380952  40.740741  16.666667      0\n",
      "lsi       100   0.000000   0.000000   0.000000      0\n"
     ]
    }
   ],
   "source": [
    "gg1.get_per_class_accuracy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 10 folds for each of 810 candidates, totalling 8100 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done  55 tasks      | elapsed:   58.0s\n",
      "[Parallel(n_jobs=-1)]: Done 205 tasks      | elapsed:  3.9min\n",
      "[Parallel(n_jobs=-1)]: Done 455 tasks      | elapsed:  8.3min\n",
      "[Parallel(n_jobs=-1)]: Done 805 tasks      | elapsed: 14.9min\n",
      "[Parallel(n_jobs=-1)]: Done 1255 tasks      | elapsed: 23.1min\n",
      "[Parallel(n_jobs=-1)]: Done 1805 tasks      | elapsed: 32.9min\n",
      "[Parallel(n_jobs=-1)]: Done 2455 tasks      | elapsed: 44.9min\n",
      "[Parallel(n_jobs=-1)]: Done 3205 tasks      | elapsed: 58.6min\n",
      "[Parallel(n_jobs=-1)]: Done 4055 tasks      | elapsed: 74.0min\n",
      "[Parallel(n_jobs=-1)]: Done 5005 tasks      | elapsed: 91.6min\n",
      "[Parallel(n_jobs=-1)]: Done 6055 tasks      | elapsed: 110.8min\n",
      "[Parallel(n_jobs=-1)]: Done 7205 tasks      | elapsed: 131.8min\n",
      "[Parallel(n_jobs=-1)]: Done 8100 out of 8100 | elapsed: 148.5min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy : 0.338461538462\n",
      "Confusion matrix :\n",
      " [[ 0  1  3  0  0]\n",
      " [ 0  0 21  0  0]\n",
      " [ 0  5 22  0  0]\n",
      " [ 0  5  7  0  0]\n",
      " [ 0  0  1  0  0]]\n",
      "Classification report :\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      18-24       0.00      0.00      0.00         4\n",
      "      25-34       0.00      0.00      0.00        21\n",
      "      35-49       0.41      0.81      0.54        27\n",
      "      50-64       0.00      0.00      0.00        12\n",
      "      65-xx       0.00      0.00      0.00         1\n",
      "\n",
      "avg / total       0.17      0.34      0.23        65\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "tree1 = tree.DecisionTreeClassifier()\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "lab = LabelEncoder()\n",
    "y_meta_tr = lab.fit_transform(y_meta)\n",
    "X_meta_trees = []\n",
    "for i in xrange(len(y_meta)):\n",
    "    tmp = []\n",
    "    for j in xrange(len(base_predictions_meta)):\n",
    "        tmp.append(base_predictions_meta[j][i])\n",
    "    X_meta_trees.append(tmp)\n",
    "#\n",
    "X_meta_trees = numpy.array(X_meta_trees, dtype='|S5')\n",
    "\n",
    "\n",
    "X_meta_trees = lab.transform(X_meta_trees.reshape(-1,1)).reshape(len(y_meta), len(base_predictions_meta))\n",
    "#params = {'criterion': ['gini', 'entropy'], 'min_samples_split': list(xrange(1,10)), \n",
    "#          'max_depth': [None]+list(xrange(1,5)), 'min_samples_leaf': list(xrange(1,4)) }\n",
    "params = {'criterion': ['gini', 'entropy'], 'n_estimators': [100, 1000, 10000],\n",
    "          'min_samples_split': list(xrange(1,10)), 'max_depth': [None]+list(xrange(1,5)), \n",
    "          'min_samples_leaf': list(xrange(1,4)) }\n",
    "\n",
    "grid_ = GridSearchCV(RandomForestClassifier(), param_grid = params, n_jobs= -1, cv=10, refit=True, verbose=1)\n",
    "#grid_ = GridSearchCV(tree1, param_grid = params, n_jobs= -1, cv=10, refit=True, verbose=1)\n",
    "grid_.fit(numpy.array(X_meta_trees), y_meta_tr)\n",
    "tree1 = grid_.best_estimator_\n",
    "#tree1.fit(numpy.array(X_meta_trees), y_meta_tr)\n",
    "\n",
    "y_cv_tr = lab.transform(y_cv)\n",
    "X_cv_trees = []\n",
    "for i in xrange(len(y_cv)):\n",
    "    tmp = []\n",
    "    for j in xrange(len(base_predictions_cv)):\n",
    "        tmp.append(base_predictions_cv[j][i])\n",
    "    X_cv_trees.append(tmp)\n",
    "#\n",
    "X_cv_trees = numpy.array(X_cv_trees, dtype='|S5')\n",
    "\n",
    "X_cv_trees = lab.transform(X_cv_trees.reshape(-1,1)).reshape(len(y_cv), len(base_predictions_cv))\n",
    "predict = lab.inverse_transform(tree1.predict(X_cv_trees))\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from pan.features import tokenization2\n",
    "\n",
    "class Freq(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\" Find the frequency of votes corresponding to specific classes.\n",
    "        It does not learn anything new. Just counting on the evaluation\n",
    "        dataset.\n",
    "        \n",
    "        Args:\n",
    "    \"\"\"\n",
    "    \n",
    "    \n",
    "    def __init__(self):\n",
    "        \n",
    "        from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "        from sklearn.naive_bayes import BernoulliNB\n",
    "        \n",
    "        self.rule_book = None\n",
    "        self.rule_index = {}\n",
    "        self.transformed = None\n",
    "        self.labels = None\n",
    "        self.num_models = None\n",
    "        self.num_labels = None\n",
    "        self.lab = LabelEncoder()\n",
    "        self.ohe = OneHotEncoder()\n",
    "        self.Bernoulli = BernoulliNB()\n",
    "        \n",
    "        \n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        \"\"\"Expects a list of string inputs as X and a list of lists containing\n",
    "           the prediciton of each model for y\"\"\"\n",
    "        \n",
    "        import numpy\n",
    "        \n",
    "        X_tr, y_tr = self.fit_encoders(X, y)\n",
    "        self.transformed = X_tr\n",
    "        tmp = numpy.zeros([1, self.num_labels])\n",
    "        j = 0\n",
    "        self.rule_book = numpy.copy(tmp)\n",
    "        for i, x in enumerate(X_tr):\n",
    "            x = str(x)\n",
    "            #print x\n",
    "            #print self.rule_book\n",
    "            if x in self.rule_index.keys():\n",
    "                #print y_tr[i].argmax(axis=1)\n",
    "                #print self.rule_book[self.rule_index[x], y_tr.argmax(axis=1)].shape\n",
    "                self.rule_book[self.rule_index[x], y_tr[i].argmax(axis=1)] += 1\n",
    "                #print self.rule_book[self.rule_index[x], :]\n",
    "            else:\n",
    "                self.rule_index[x] = j\n",
    "                self.rule_book = numpy.vstack((self.rule_book, tmp))\n",
    "                #self.rule_book = numpy.delete(self.rule_book, 0, 0)\n",
    "                self.rule_book[self.rule_index[x], y_tr[i].argmax(axis=1)] += 1\n",
    "                j += 1\n",
    "        self.num_rules = len(list(self.rule_index.keys()))\n",
    "        self.rule_book = numpy.delete(self.rule_book, self.rule_book.shape[0]-1, 0)\n",
    "#         print \"Fitted\"\n",
    "#         import pprint\n",
    "#         pprint.pprint(self.rule_index)\n",
    "#         print self.rule_book\n",
    "        # print X_tr\n",
    "        self.Bernoulli.fit(X_tr, y)\n",
    "        return\n",
    "            \n",
    "        \n",
    "        \n",
    "    def fit_encoders(self, X, y):\n",
    "\n",
    "        import numpy\n",
    "\n",
    "        self.num_labels = len(set(y))\n",
    "        self.labels = sorted(list(set(y)))\n",
    "        N_samples = len(y)\n",
    "        if type(X) is numpy.array:\n",
    "            y = y.reshape(-1, 1)\n",
    "        else:\n",
    "            y = numpy.array(y).reshape(-1, 1)\n",
    "        y = self.lab.fit_transform(y).reshape(-1, 1)\n",
    "        y = self.ohe.fit_transform(y).todense()\n",
    "        X = self.lab.transform(X)\n",
    "        X = self.ohe.transform(X.T.reshape(-1, 1)).todense().reshape(N_samples, -1)\n",
    "        self.num_models = int(X.shape[1] / self.num_labels)\n",
    "        return X, y\n",
    "    \n",
    "    def transformation(self, X):\n",
    "        \n",
    "        import random\n",
    "        #from sklearn.\n",
    "        predictions = []\n",
    "        #print len(X), len(y)\n",
    "        for i, x in enumerate(X):\n",
    "            x1 = str(x)\n",
    "            if x1 in self.rule_index.keys():\n",
    "#                 print \"True\"\n",
    "#                 print y[i]\n",
    "                #print self.rule_book[self.rule_index[x],:]\n",
    "                #print self.rule_book[self.rule_index[x],:].argmax(axis=0)\n",
    "                predictions.append(self.labels[self.rule_book[self.rule_index[x1],:].argmax(axis=0)])\n",
    "#                 print \"Predicted\"\n",
    "#                 print self.rule_book[self.rule_index[x1],:]\n",
    "#                 print predictions[-1]\n",
    "            else:\n",
    "                predictions.append([])\n",
    "                #print \"Random!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\"\n",
    "                #predictions.append(random.choice(self.labels))\n",
    "                #scores = []\n",
    "                #for x_ in self.transformed:\n",
    "                #    pass\n",
    "                #print [x]\n",
    "                #print type(x)\n",
    "                    # print [x[0]]\n",
    "        #predictions = []\n",
    "        #for x in X:\n",
    "        #    predictions.append(self.Bernoulli.predict([x]))\n",
    "        #predictions = self.Bernoulli.predict(X)    \n",
    "        #predictions.append(self.Bernoulli.predict([]))\n",
    "        return predictions\n",
    "    \n",
    "    \n",
    "    def transform(self, X):\n",
    "\n",
    "        import numpy\n",
    "\n",
    "        if type(X[0]) is numpy.array:\n",
    "            N_samples = X[0].shape[0]\n",
    "        else:\n",
    "            N_samples = len(X[0])\n",
    "        X = self.lab.transform(X)\n",
    "        X = self.ohe.transform(X.reshape(-1, 1)).todense().reshape(N_samples, -1)\n",
    "        #print len(X), len(y)\n",
    "        prediction = self.transformation(X)\n",
    "        #prediction = self.lab.inverse_transform(prediction.argmax(axis=1))\n",
    "        return prediction\n",
    "\n",
    "    def predict(self, X):\n",
    "        return self.transform(X)\n",
    "\n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "class Neigbors_Combinator2(BaseEstimator, TransformerMixin):\n",
    "    \n",
    "    \"\"\"GIving untrained Models and their transformers create a combinator of information:\n",
    "       - Based on the count vectorizer predictions of a kNN classifier (fitted in X_train).\n",
    "       - Based on the transformed kNN classifiers predictions (fitted in X_train)\n",
    "       - Based on the samples predictions of the now trained models (fitted in X_train)\n",
    "       - Combining the above predictions using weights (fitted in X_meta)\n",
    "       \n",
    "       Args:\n",
    "           - models: dic of trained models in the form of: {\"name\": Model, :\"name2\":Model2...}\n",
    "           - models_tr: dic of transformers of the models. Form like models.\\\n",
    "           - names: list of names for the models to be used\n",
    "           - knn_params: dictionary containing params for fitting the knn classifiers.\n",
    "                         Expects a dictionary like: {\"general\":{\"num_folds\":5, \"n_jobs\":-1},\n",
    "                                                     \"name\":  parameter_grid_for_grid_search1,\n",
    "                                                     \"name2\": parameter_grid_for_grid_search2,...}\n",
    "            - scheme: for wegithing \"\"\"\n",
    "    \n",
    "    def __init__(self, models, models_tr, names, knn_params, scheme='majority', weights=None):\n",
    "        \n",
    "        from combinations import Combinator\n",
    "        from sklearn.feature_extraction.text import CountVectorizer\n",
    "        \n",
    "        self.counter = CountVectorizer()\n",
    "        self.models = models\n",
    "        self.models_tr = models_tr\n",
    "        self.names = names\n",
    "        self.knn_params = knn_params\n",
    "        self.scheme = scheme\n",
    "        self.weights = weights\n",
    "        self.ind2names = {}\n",
    "        for i, name in enumerate(names):\n",
    "            self.ind2names[i] = name\n",
    "        self.counter = CountVectorizer()\n",
    "        self.gt_knn = None\n",
    "        self.tr_knn = {}\n",
    "        self.comb = Combinator(scheme=self.scheme)\n",
    "        self.freq = Freq()\n",
    "            \n",
    "        \n",
    "\n",
    "    def fit(self, X_train, X_meta, y_train, y_meta, weights=None):\n",
    "        \n",
    "        from sklearn.grid_search import GridSearchCV\n",
    "        from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "        import random, time\n",
    "        #print len(y_train), len(y_meta)\n",
    "        if (y_train is None)  or (y_meta is None):\n",
    "            raise ValueError('we need y labels to supervise-fit!')\n",
    "        else:\n",
    "            parameters = {\n",
    "                    'input': 'content',\n",
    "                    'encoding': 'utf-8',\n",
    "                    'decode_error': 'ignore',\n",
    "                    'analyzer': 'word',\n",
    "                    'stop_words': 'english',\n",
    "                    # 'vocabulary':list(voc),\n",
    "                    #'tokenizer': tokenization,\n",
    "                    #'tokenizer': _twokenize.tokenizeRawTweetText,  # self.tokenization,\n",
    "                    #'tokenizer': lambda text: _twokenize.tokenizeRawTweetText(nonan.sub(po_re.sub('', text))),\n",
    "                    'max_df': 1.0,\n",
    "                    'min_df': 1,\n",
    "                    'max_features':None\n",
    "                }\n",
    "            \n",
    "            # Fit the Meta-Classifier\n",
    "            predictions_meta = []\n",
    "            # Fitting the ground truth neighbors\n",
    "            t0 = time.time()\n",
    "            self.counter.set_params(**parameters)\n",
    "            X_train_truth = self.counter.fit_transform(X_train)\n",
    "            #print X_train_truth.toarray().shape\n",
    "            grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params['true'],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "            grid_.fit(X_train_truth, y_train)\n",
    "            self.gt_knn = grid_.best_estimator_\n",
    "            # Add the gt predictions\n",
    "            #print self.counter.transform(X_meta).toarray().shape\n",
    "            predictions_meta.append(self.gt_knn.predict(self.counter.transform(X_meta)))\n",
    "            # Fit the transformation Classifiers\n",
    "            for name in self.names:\n",
    "                X_train_tr = self.models_tr[name].transform(X_train)\n",
    "                grid_ = GridSearchCV(KNeighborsClassifier(), param_grid=self.knn_params[name],\n",
    "                                    n_jobs=self.knn_params['general']['n_jobs'], \n",
    "                                     cv=self.knn_params['general']['cv'], refit=True, \n",
    "                                      verbose=self.knn_params['general']['verbose'])\n",
    "                grid_.fit(X_train_tr, y_train)\n",
    "                self.tr_knn[name] = grid_.best_estimator_\n",
    "                # Add the transformed knn predictions\n",
    "                predictions_meta.append(self.tr_knn[name].predict(self.models_tr[name].transform(X_meta)))\n",
    "                # Add the per sample predictions\n",
    "                predictions_meta.append(self.models[name].predict(X_meta))\n",
    "            #print len(predictions_meta)\n",
    "            #print predictions_meta\n",
    "            self.comb.fit(predictions_meta, y_meta)\n",
    "            print \"Best Weights Found:\"\n",
    "            print self.comb.weights\n",
    "            print self.freq.fit(predictions_meta, y_meta)\n",
    "            return\n",
    "        \n",
    "        \n",
    "    def transform(self, X, y):\n",
    "\n",
    "        import numpy\n",
    "        \n",
    "        predictions = []\n",
    "        X_truth = self.counter.transform(X)   \n",
    "        predictions.append(self.gt_knn.predict(X_truth))\n",
    "        for name in self.names:\n",
    "            #print name\n",
    "            # Add the transformed knn predictions\n",
    "            predictions.append(self.tr_knn[name].predict(self.models_tr[name].transform(X)))\n",
    "            # Add the per sample predictions\n",
    "            predictions.append(self.models[name].predict(X))\n",
    "            #print len(predictions)\n",
    "        #print len(predictions)\n",
    "        #for pred in predictions:\n",
    "        #    print len(pred)\n",
    "        final_predict =self.comb.predict(predictions)\n",
    "        final_predict2 = self.freq.predict(predictions)\n",
    "        names = ['Ground_Truth', '3grams-Neighbors', '3grams-Pred',\n",
    "                     'SOAC-Neigbors', 'SOAC-Pred', 'LSI-Neighbors', 'LSI-Pred', 'Comb']\n",
    "        gg_pred = []\n",
    "        for i in xrange(len(y)):\n",
    "            print 'True: ' + str(y[i])\n",
    "            for  j, pred in enumerate(predictions):\n",
    "                print '%s: %s' % (names[j], pred[i])\n",
    "            print \"Comb: %s\" % final_predict[i]\n",
    "            #print \"Freq: %s\" % final_predict2[i]\n",
    "            \n",
    "            if final_predict2[i] != []:\n",
    "                print \"Freq: %s\" % final_predict2[i]\n",
    "                print 'Selected from Freq'\n",
    "                gg_pred.append(final_predict2[i])\n",
    "            else:\n",
    "                gg_pred.append(final_predict[i])\n",
    "            print gg_pred[-1]\n",
    "            print '*'*100\n",
    "        return gg_pred\n",
    "\n",
    "    def predict(self, X, y):\n",
    "        return self.transform(X, y)\n",
    "\n",
    "    def score(self, X, y_true):\n",
    "\n",
    "        from sklearn.metrics import accuracy_score\n",
    "\n",
    "        y_pred = self.predict(X)\n",
    "        return accuracy_score(y_true, y_pred, normalize=True)\n",
    "\n",
    "models11 = {}\n",
    "models_tr11 = {}\n",
    "tr_params = {'n_neighbors':[i for i in xrange(1,20)], 'weights':['uniform', 'distance']}\n",
    "knn_params = {\"general\":{\"n_jobs\":-1, \"cv\":5, \"verbose\":1}}\n",
    "knn_params['true'] = tr_params\n",
    "names11 = []\n",
    "for i, model in enumerate(trained_base_models):\n",
    "    names11.append(base_model_names[i])\n",
    "    models11[names11[-1]] = model\n",
    "    models_tr11[names11[-1]] = trained_base_models[i].steps[0][1]\n",
    "    knn_params[names11[-1]] = tr_params\n",
    "    #print base_model_names[i]\n",
    "    #print trained_base_models[i].steps[0][1]\n",
    "aa = Neigbors_Combinator2(models11, models_tr11, names11, knn_params, scheme='accuracy')            \n",
    "aa.fit(X_train, X_meta, y_train, y_meta)\n",
    "predict = aa.predict(X_cv, y_cv)\n",
    "acc = accuracy_score(y_cv, predict)\n",
    "conf = confusion_matrix(y_cv, predict, labels=sorted(list(set(y))))\n",
    "rep = classification_report(y_cv, predict, target_names=sorted(list(set(y))))\n",
    "print('Accuracy : {}'.format(acc))\n",
    "print('Confusion matrix :\\n {}'.format(conf))\n",
    "print('Classification report :\\n {}'.format(rep))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}