task2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/run/media/miron/windows/Datasets/hillary-emails\n"
     ]
    }
   ],
   "source": [
    "cd ~/ML/Datasets/hillary-emails/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emails = pd.read_csv('Emails.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>DocNumber</th>\n",
       "      <th>MetadataSubject</th>\n",
       "      <th>MetadataTo</th>\n",
       "      <th>MetadataFrom</th>\n",
       "      <th>SenderPersonId</th>\n",
       "      <th>MetadataDateSent</th>\n",
       "      <th>MetadataDateReleased</th>\n",
       "      <th>MetadataPdfLink</th>\n",
       "      <th>MetadataCaseNumber</th>\n",
       "      <th>...</th>\n",
       "      <th>ExtractedTo</th>\n",
       "      <th>ExtractedFrom</th>\n",
       "      <th>ExtractedCc</th>\n",
       "      <th>ExtractedDateSent</th>\n",
       "      <th>ExtractedCaseNumber</th>\n",
       "      <th>ExtractedDocNumber</th>\n",
       "      <th>ExtractedDateReleased</th>\n",
       "      <th>ExtractedReleaseInPartOrFull</th>\n",
       "      <th>ExtractedBodyText</th>\n",
       "      <th>RawText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>C05739545</td>\n",
       "      <td>WOW</td>\n",
       "      <td>H</td>\n",
       "      <td>Sullivan, Jacob J</td>\n",
       "      <td>87.0</td>\n",
       "      <td>2012-09-12T04:00:00+00:00</td>\n",
       "      <td>2015-05-22T04:00:00+00:00</td>\n",
       "      <td>DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739545...</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Sullivan, Jacob J &lt;Sullivan11@state.gov&gt;</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Wednesday, September 12, 2012 10:16 AM</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>C05739545</td>\n",
       "      <td>05/13/2015</td>\n",
       "      <td>RELEASE IN FULL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNCLASSIFIED\\nU.S. Department of State\\nCase N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>C05739546</td>\n",
       "      <td>H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...</td>\n",
       "      <td>H</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2011-03-03T05:00:00+00:00</td>\n",
       "      <td>2015-05-22T04:00:00+00:00</td>\n",
       "      <td>DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739546...</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>C05739546</td>\n",
       "      <td>05/13/2015</td>\n",
       "      <td>RELEASE IN PART</td>\n",
       "      <td>B6\\nThursday, March 3, 2011 9:45 PM\\nH: Latest...</td>\n",
       "      <td>UNCLASSIFIED\\nU.S. Department of State\\nCase N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>C05739547</td>\n",
       "      <td>CHRIS STEVENS</td>\n",
       "      <td>;H</td>\n",
       "      <td>Mills, Cheryl D</td>\n",
       "      <td>32.0</td>\n",
       "      <td>2012-09-12T04:00:00+00:00</td>\n",
       "      <td>2015-05-22T04:00:00+00:00</td>\n",
       "      <td>DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739547...</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>...</td>\n",
       "      <td>B6</td>\n",
       "      <td>Mills, Cheryl D &lt;MillsCD@state.gov&gt;</td>\n",
       "      <td>Abedin, Huma</td>\n",
       "      <td>Wednesday, September 12, 2012 11:52 AM</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>C05739547</td>\n",
       "      <td>05/14/2015</td>\n",
       "      <td>RELEASE IN PART</td>\n",
       "      <td>Thx</td>\n",
       "      <td>UNCLASSIFIED\\nU.S. Department of State\\nCase N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>C05739550</td>\n",
       "      <td>CAIRO CONDEMNATION - FINAL</td>\n",
       "      <td>H</td>\n",
       "      <td>Mills, Cheryl D</td>\n",
       "      <td>32.0</td>\n",
       "      <td>2012-09-12T04:00:00+00:00</td>\n",
       "      <td>2015-05-22T04:00:00+00:00</td>\n",
       "      <td>DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739550...</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Mills, Cheryl D &lt;MillsCD@state.gov&gt;</td>\n",
       "      <td>Mitchell, Andrew B</td>\n",
       "      <td>Wednesday, September 12,2012 12:44 PM</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>C05739550</td>\n",
       "      <td>05/13/2015</td>\n",
       "      <td>RELEASE IN PART</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNCLASSIFIED\\nU.S. Department of State\\nCase N...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>C05739554</td>\n",
       "      <td>H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...</td>\n",
       "      <td>Abedin, Huma</td>\n",
       "      <td>H</td>\n",
       "      <td>80.0</td>\n",
       "      <td>2011-03-11T05:00:00+00:00</td>\n",
       "      <td>2015-05-22T04:00:00+00:00</td>\n",
       "      <td>DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739554...</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F-2015-04841</td>\n",
       "      <td>C05739554</td>\n",
       "      <td>05/13/2015</td>\n",
       "      <td>RELEASE IN PART</td>\n",
       "      <td>H &lt;hrod17@clintonemail.com&gt;\\nFriday, March 11,...</td>\n",
       "      <td>B6\\nUNCLASSIFIED\\nU.S. Department of State\\nCa...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Id  DocNumber                                    MetadataSubject  \\\n",
       "0   1  C05739545                                                WOW   \n",
       "1   2  C05739546  H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...   \n",
       "2   3  C05739547                                      CHRIS STEVENS   \n",
       "3   4  C05739550                         CAIRO CONDEMNATION - FINAL   \n",
       "4   5  C05739554  H: LATEST: HOW SYRIA IS AIDING QADDAFI AND MOR...   \n",
       "\n",
       "     MetadataTo       MetadataFrom  SenderPersonId           MetadataDateSent  \\\n",
       "0             H  Sullivan, Jacob J            87.0  2012-09-12T04:00:00+00:00   \n",
       "1             H                NaN             NaN  2011-03-03T05:00:00+00:00   \n",
       "2            ;H    Mills, Cheryl D            32.0  2012-09-12T04:00:00+00:00   \n",
       "3             H    Mills, Cheryl D            32.0  2012-09-12T04:00:00+00:00   \n",
       "4  Abedin, Huma                  H            80.0  2011-03-11T05:00:00+00:00   \n",
       "\n",
       "        MetadataDateReleased  \\\n",
       "0  2015-05-22T04:00:00+00:00   \n",
       "1  2015-05-22T04:00:00+00:00   \n",
       "2  2015-05-22T04:00:00+00:00   \n",
       "3  2015-05-22T04:00:00+00:00   \n",
       "4  2015-05-22T04:00:00+00:00   \n",
       "\n",
       "                                     MetadataPdfLink MetadataCaseNumber  \\\n",
       "0  DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739545...       F-2015-04841   \n",
       "1  DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739546...       F-2015-04841   \n",
       "2  DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739547...       F-2015-04841   \n",
       "3  DOCUMENTS/HRC_Email_1_296/HRCH2/DOC_0C05739550...       F-2015-04841   \n",
       "4  DOCUMENTS/HRC_Email_1_296/HRCH1/DOC_0C05739554...       F-2015-04841   \n",
       "\n",
       "                         ...                         ExtractedTo  \\\n",
       "0                        ...                                 NaN   \n",
       "1                        ...                                 NaN   \n",
       "2                        ...                                  B6   \n",
       "3                        ...                                 NaN   \n",
       "4                        ...                                 NaN   \n",
       "\n",
       "                              ExtractedFrom         ExtractedCc  \\\n",
       "0  Sullivan, Jacob J <Sullivan11@state.gov>                 NaN   \n",
       "1                                       NaN                 NaN   \n",
       "2       Mills, Cheryl D <MillsCD@state.gov>        Abedin, Huma   \n",
       "3       Mills, Cheryl D <MillsCD@state.gov>  Mitchell, Andrew B   \n",
       "4                                       NaN                 NaN   \n",
       "\n",
       "                        ExtractedDateSent ExtractedCaseNumber  \\\n",
       "0  Wednesday, September 12, 2012 10:16 AM        F-2015-04841   \n",
       "1                                     NaN        F-2015-04841   \n",
       "2  Wednesday, September 12, 2012 11:52 AM        F-2015-04841   \n",
       "3   Wednesday, September 12,2012 12:44 PM        F-2015-04841   \n",
       "4                                     NaN        F-2015-04841   \n",
       "\n",
       "  ExtractedDocNumber ExtractedDateReleased ExtractedReleaseInPartOrFull  \\\n",
       "0          C05739545            05/13/2015              RELEASE IN FULL   \n",
       "1          C05739546            05/13/2015              RELEASE IN PART   \n",
       "2          C05739547            05/14/2015              RELEASE IN PART   \n",
       "3          C05739550            05/13/2015              RELEASE IN PART   \n",
       "4          C05739554            05/13/2015              RELEASE IN PART   \n",
       "\n",
       "                                   ExtractedBodyText  \\\n",
       "0                                                NaN   \n",
       "1  B6\\nThursday, March 3, 2011 9:45 PM\\nH: Latest...   \n",
       "2                                                Thx   \n",
       "3                                                NaN   \n",
       "4  H <hrod17@clintonemail.com>\\nFriday, March 11,...   \n",
       "\n",
       "                                             RawText  \n",
       "0  UNCLASSIFIED\\nU.S. Department of State\\nCase N...  \n",
       "1  UNCLASSIFIED\\nU.S. Department of State\\nCase N...  \n",
       "2  UNCLASSIFIED\\nU.S. Department of State\\nCase N...  \n",
       "3  UNCLASSIFIED\\nU.S. Department of State\\nCase N...  \n",
       "4  B6\\nUNCLASSIFIED\\nU.S. Department of State\\nCa...  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emails.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "regexp = re.compile('(\\s)|([!,.\\-\\\\?/\\\"\\'])', re.UNICODE)\n",
    "emails['words'] = emails.RawText.apply(lambda x : re.sub(regexp, ' ', x).split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UNCLASSIFIED\n",
      "U.S. Department of State\n",
      "Case No. F-2015-04841\n",
      "Doc No. C05739545\n",
      "Date: 05/13/2015\n",
      "STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.\n",
      "SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER.\n",
      "RELEASE IN FULL\n",
      "From: Sullivan, Jacob J <Sullivan11@state.gov>\n",
      "Sent: Wednesday, September 12, 2012 10:16 AM\n",
      "To:\n",
      "Subject: FW: Wow\n",
      "From: Brose, Christian (Armed Services) (mailto:Christian_Brose@armed-servic,essenate.govi\n",
      "Sent: Wednesday, September 12, 2012 10:09 AM\n",
      "To: Sullivan, Jacob J\n",
      "Subject: Wow\n",
      "What a wonderful, strong and moving statement by your boss. please tell her how much Sen. McCain appreciated it. Me\n",
      "too\n",
      "UNCLASSIFIED\n",
      "U.S. Department of State\n",
      "Case No. F-2015-04841\n",
      "Doc No. C05739545\n",
      "Date: 05/13/2015\n",
      "STATE DEPT. - PRODUCED TO HOUSE SELECT BENGHAZI COMM.\n",
      "SUBJECT TO AGREEMENT ON SENSITIVE INFORMATION & REDACTIONS. NO FOIA WAIVER. STATE-5CB0045247\n",
      "\f",
      "\n"
     ]
    }
   ],
   "source": [
    "data = emails.RawText.values\n",
    "print(data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer(min_df=10, max_df=500, stop_words='english', ngram_range=(1,3), preprocessor=(\n",
    "        lambda x : x.replace('\\n', ' ')))\n",
    "features = vectorizer.fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('food', 22530),\n",
       " ('Subject Fw Reuters', 14696),\n",
       " ('O_Shift III', 11615),\n",
       " ('2007 come', 1519),\n",
       " ('DOD', 6060),\n",
       " ('Jared', 9227),\n",
       " ('want clear', 32231),\n",
       " ('framework', 22670),\n",
       " ('interview Original', 24322),\n",
       " ('40 year', 2973)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocabulary = vectorizer.vocabulary_\n",
    "list(vocabulary.items())[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "bigrams = list(filter(lambda x : len(x.split()) == 2, vocabulary.keys()))\n",
    "bigrams = [(x, vocabulary[x]) for x in bigrams]\n",
    "bigrams.sort(key=(lambda x:x[1]), reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('zone The', 32858),\n",
       " ('youtube com', 32854),\n",
       " ('young women', 32849),\n",
       " ('young woman', 32848),\n",
       " ('young people', 32847),\n",
       " ('young men', 32846),\n",
       " ('young man', 32845),\n",
       " ('young children', 32844),\n",
       " ('young American', 32843),\n",
       " ('years working', 32837),\n",
       " ('years talks', 32835),\n",
       " ('years prison', 32834),\n",
       " ('years past', 32833),\n",
       " ('years old', 32832),\n",
       " ('years like', 32831),\n",
       " ('years later', 32830),\n",
       " ('years feared', 32828),\n",
       " ('years earlier', 32827),\n",
       " ('years come', 32826),\n",
       " ('years ahead', 32825)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigrams[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.collocations import *\n",
    "bigram_measures = nltk.collocations.BigramAssocMeasures()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "email_words = emails.words.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('#Venezuela', 'FACEBOOK'),\n",
       " ('$500K', '(privately)'),\n",
       " ('$65', '000—plus'),\n",
       " ('$915', '6M'),\n",
       " ('${resulturl}', '>${result'),\n",
       " ('%0D%0A%0D%0AILLUSTRATION%3A%2OSTEVE%20BRODNER%0D%0A%0D%0A%20%20%',\n",
       "  '20%20*%20Print%0D%0A%20%20%20%20*%20E'),\n",
       " ('%0D%0A%20%20%20%20*%20Twitter%0D%0A%20%20%20%20*%20Digg%0D%0A%20%20%20%20*%20More%0D%0A',\n",
       "  '%0D%0ATo%20get%20more%20of%20The%20New%20Yorker%27s%20signature%20mix%200f%2Opolitics%2C%20cultu'),\n",
       " ('%0D%0ATo%20get%20more%20of%20The%20New%20Yorker%27s%20signature%20mix%200f%2Opolitics%2C%20cultu',\n",
       "  're%20and%20the%20arts%3A%20Subscribe%20now%0D%0ARelated%20Links%0D%0A%0D%0A%20%20%20%20Ask%2'),\n",
       " ('%201',\n",
       "  's%20Abandoning%20Afghanistan%E2%80%99s%20Women%20%E2%80%93%20By%20Valerie%20M'),\n",
       " ('%201s%20A',\n",
       "  'bandoning%20Afghanistan%E2%80%99s%20Women%20%E2%80%93%20By%20Valerie%20M')]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finder = BigramCollocationFinder.from_documents(email_words)\n",
    "finder.nbest(bigram_measures.pmi, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering, KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clust = KMeans(n_clusters=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clusters = clust.fit_predict(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Возьмем центры кластеров. Отсортируем n-граммы по встречаемости в этом кластере. Выведем топ сколько-то"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Sullivan Jacob Sullivan' 'com OriginalArrivalTime' 'sbwhoeop To Sent'\n",
      " 'presume' '11 50' 'mail address try' 'Godspeed' 'summary' 'doomed'\n",
      " 'Monday September 20' '11 14' 'Potential' 'think ahead And' 'Marine'\n",
      " '10 04 PM' 'achieve goals' 'Sunday August 22' 'Musharraf' 'figured'\n",
      " 'State Original Message']\n",
      "['calling Saddam removed' 'Clinton Administration' 'really really'\n",
      " 'election First important' 'August 31 2009' '13 51' 'holiday season'\n",
      " 'HDR22 clintonemailcom To' 'attachments confidential' '11 23'\n",
      " 'Friday May 01' 'today The Observer' 'October 19 2009' 'Boyce' 'virtues'\n",
      " 'maneuvers' 'Sent Tue Feb' 'B6 B6' '2012 02' 'Crocker Bathsheba Sent']\n",
      "['List' 'deteriorated' 'Subject Re Original' 'Mashabane update Confirmed'\n",
      " 'palpable progress Middle' 'questions need' 'Cheryl Verma' 'reality'\n",
      " 'He added' 'confidential The' 'The Murdochs' 'Jeffrey Koh Harold'\n",
      " 'opinion source' 'Sunday April' 'effect' 'Can time' 'St NW' 'From Doug'\n",
      " 'Monday October 26' '201']\n",
      "['developing' 'oil gas' 'Dear All' 'outset' 'OFFICE TIME 10' 'potentially'\n",
      " 'wanted share' 'Fri Dec' 'FROM' 'predict' 'Polling' 'DUP http'\n",
      " 'later tell Europeans' '20 2009 Subject' 'people support' 'waiting'\n",
      " 'troubles' 'bunch' 'Soon' 'The majority']\n",
      "['09 07' 'government officials' 'trapped' 'Friday April 23'\n",
      " 'prospective winner Election' 'Oxbridge educated pillars' 'DEPARTMENT OF'\n",
      " 'delete copies immediately' 'moments' '17 08' 'July 25 2009' 'visible'\n",
      " 'COMM Department State' 'slogan' 'Kouchener meeting' 'advice given'\n",
      " 'For additional' 'open' 'address served 50' 'You ve']\n",
      "['We sight' 'He appointed head' 'like layman' 'Saddam hadn ordered'\n",
      " 'time said' 'figured' 'bbc uk' 'mistrust' 'state gov Mills' 'communique'\n",
      " 'senior advisors' 'home abroad' 'considerations' 'countries like'\n",
      " 'wide ranging' 'Message UNCLASSIFIED Department' 'And pls' 'Armed'\n",
      " 'Aug 19' 'just spoke']\n",
      "['reform The' 'Sid Original Message' 'SF' 'know questions' 'second time'\n",
      " 'Higher' 'armed conflict' 'http static guim' 'Feb 06 08' 'questionable'\n",
      " 'explore' 'holbrooke' 'resident' 'arose' 'creating new' 'big issue'\n",
      " 'women men' '2009 11 AM' 'right way' 'makeup person']\n",
      "['STATE FOR' 'From Toiv Nora' 'unhelpful' 'researchers'\n",
      " 'general Anders Fogh' 'Cc Mills Cheryl' 'Jun 08' 'ONE ON ONE' 'GOI'\n",
      " '2010 30 PM' 'Cc Muscatine Lissa' 'Macmanus' 'agriculture' 'look' 'Tech'\n",
      " 'note' 'Rajiv' 'Refining Company half' 'BDBCO' 'General Ban']\n",
      "['2645 rodriguezme state' 'WORLD' 'HRC' '34 53' 'settlement activities'\n",
      " 'agreement tonight' '13 25' 'Muammar' 'Sent Tuesday September'\n",
      " 'IN CEREMONY FOR' 'time period' 'direct UNCLASSIFIED Department'\n",
      " 'late 2010' 'Plaza' 'indicators' 'electoral So' 'April 25 2010'\n",
      " 'The purpose' '1265368562294 Gordon' 'article today Washington']\n",
      "['said memo' 'gov 10 16' 'Traditional Unionist' 'excited meet lunch'\n",
      " 'Bagley' 'rebellion' 'May 28 2010' 'session session appeared' 'washington'\n",
      " 'LP012665' 'The Observer' '46 03 2009' 'observing' 'September 18 2009'\n",
      " 'HUMAN' 'elected General' 'security prosperity' 'As know good'\n",
      " '15 pm OFFICE' 'fm']\n",
      "['Brown told reporters' 'orphans' 'Tech' 'break' 'Memo' 'SES O_OS'\n",
      " 'just having CITGO' 'Intelligence security services' 'months later'\n",
      " 'speak soon' '42 AM To' 'challenges facing' 'The action' 'Islamic state'\n",
      " '14 40' 'copy Original' 'disdainful resentment indifference'\n",
      " 'media interviews' 'adviser Edwidge Belliard' 'Thursday November']\n",
      "['end month' 'GOI' 'diverse' '15 2009 12' 'best wishes' '1st' 'Sunday June'\n",
      " 'indicating' '2010 like discuss' 'Cyprus' 'centralized' 'seeing BN'\n",
      " 'Tehran' 'Blair The' 'endorsed LDs' 'adapt' 'hoped' 'Low' 'Katie'\n",
      " 'suspend']\n",
      "['silicon steel behemoths' 'inherent' 'thinker' '02 09' 'corrected'\n",
      " 'make party political' 'Fri Jan' '14 2009 Subject' 'satisfied' 'map'\n",
      " 'terrorist attacks' 'embracing' 'Messa From' 'gross domestic product'\n",
      " 'surge essential troop' 'Haiti As' 'cabinet' 'There people' 'Saudi FM'\n",
      " '2009 11 38']\n",
      "['politician' 'day Sir' 'Jan 13' 'typos Will' 'February 22 2010'\n",
      " 'form government' 'revise' 'Of' 'NSS OSD NMCC' 'corporate' 'Strobe Also'\n",
      " 'ihdr22' '14 2010 10' '20 UNCLASSIFIED Department' 'inside outside' 'pot'\n",
      " 'Tue Dec 08' 'National Council' 'cuts' 'ambassador Iraq 2006']\n",
      "['And ready' 'weekend Original' 'Subject Thanks' 'operation respect The'\n",
      " 'Confirmed' 'good trip' 'Drive' 'Castle outside Belfast' 'nationals'\n",
      " 'naturally' 'Tue Dec 08' '20 percent' 'E2 80'\n",
      " 'levels European Governments' '10 10' 'morally' 'examples CITGO'\n",
      " 'Information Officer' 'Tony Original' 'laughter']\n",
      "['Roy Spence To' 'fronts' 'following removed' 'North Carolina' 'You heard'\n",
      " '2012' 'Sun Jul 05' 'OF STATE FOR' 'politics 2010 feb'\n",
      " '2009 Subject Turkey' 'press secretary' 'undertaking' 'hurricane' 'Morris'\n",
      " '2010 19 AM' 'Finance Minister' 'convince' 'crushing' 'Webb'\n",
      " 'promote Denis']\n",
      "['Raj article' 'disclaimer 01112008' 'June 08' 'Summers' 'We ve sent'\n",
      " 'fully briefed' 'CONFIDENTIAL Reason Declassify' 'leaves'\n",
      " 'numbers think provide' 'work new' '08 17 2009' 'form government' 'stated'\n",
      " 'unemployment benefits' '2010 10 01' 'fm' '16 01' 'prevents' 'fast moving'\n",
      " 'today He']\n",
      "['standing begin secure' 'com To sbwhoeop' '08 23' 'DS CA'\n",
      " 'plan He mention' 'media We' 'jumping action' 'President Mubarak' 'pool'\n",
      " 'November 23' 'didn miss' 'reform The' 'guardian uk help' 'NEWS Mahogany'\n",
      " 'Jerusalem' 'develop plan' 'Re Oprah' 'office 2001 voices' 'Dems'\n",
      " 'reforms way']\n",
      "['Sent Mon Jan' 'building' 'Breaking' 'Re Tony' 'employed' '2015 B6'\n",
      " 'AMBASSADOR' 'transmitted' 'violence' 'Notice Requirement This'\n",
      " 'Gordon mentioned Can' '31 2010' 'Otero Maria OteroM2' 'critical role'\n",
      " 'com To McHale' 'issues world' 'banned' 'situations' 'Sun May' 'Jake We']\n",
      "['10 43' 'business China' 'support Secretary' '17 12' 'world war' 'loss'\n",
      " 'heads state' 'evening President' 'envoys' 'October 03 2009'\n",
      " 'born Cities Services' 'Amy Sent Sun' 'country owns companies' 'Situation'\n",
      " 'Huma clintonemail' 'late 2010' 'See DC' '647 9598 office'\n",
      " 'scanned MessageLabs Email' 'Crocker Bathsheba Sent']\n",
      "['Sent Thu Jul' '234' 'Sunday January 10' 'Earlier evening' 'Subject Jim'\n",
      " 'Capricia' 'mob' 'transfer policing powers' 'OBL' '11 21 PM' 'Armenian'\n",
      " 'Secretary Gates' 'Latin' 'failed recipients' 'Nigeria South' 'ask Who'\n",
      " 'From OpsNewsTicker' 'forcefully' 'ColemanCL state gov' 'obstacle']\n",
      "['sbwhoeop Saturday' 'contrary turn' 'programme He' 'abused' 'dating'\n",
      " 'trends' 'sealed' 'leaders say' 'Mills Cheryl Burns'\n",
      " 'Klevorick Caitlin Sent' '06 36' 'companies' 'McLarty' 'horror' 'goal'\n",
      " 'Les' 'pm Secretary' '12 2010 12' 'said look' 'Lenovo']\n",
      "['content' 'happens Iraq' 'Room 7th' 'Amy Subject Re' 'Lawrence'\n",
      " 'want Let close' 'tracking' 'fair balanced compromises'\n",
      " 'inspectors It appears' 'BUT UNCLASSIFIED'\n",
      " 'Office UNCLASSIFIED Department' 'WASHINGTON DC' 'realizes' '10 20'\n",
      " 'Brown Labour attempt' 'ethnic' '08 17 2009' 'Czech Republic'\n",
      " 'Jacob Burns' '2009 Notional Timeline']\n",
      "['Sent Tue Mar' '15 WEEKLY' 'Oscar Flores Sent' 'Update'\n",
      " 'Jack Straw foreign' 'haven' 'map' 'surprised' 'sullivan jake sullivan'\n",
      " 'Ignatius' 'December 17' 'helps set Oslo' '736' 'soon We' 'speculative'\n",
      " 'See' 'noon' 'Quadrennial' 'email In' 'Lew']\n",
      "['2015 11' 'Ansar al Sharia' 'police said' 'August 21 2010' 'Jacob Sent'\n",
      " '20 27 52' 'cautious view corporate' 'death' 'partner'\n",
      " 'hear Original Message' 'fly zone' 'spent years' 'Justice Construction'\n",
      " 'built' 'schedule Thanks' 'missed NBC Andrea' 'catalyst' 'Possible'\n",
      " '51 pm' 'parties opens new']\n",
      "['Wednesday February 2010' 'sensitivity' 'Cc ValmoroU state' 'tribe'\n",
      " 'Blair January 2003' 'conceivably allow Venezuelan' 'concluded'\n",
      " 'consumption' 'negotiations going He' 'expulsion' 'police stations'\n",
      " 'Subject Wanted' 'confidential use recipient' 'He 30 He' 'Wed So Jeff'\n",
      " 'Ben Yezza' 'slogan' 'COMM Department State' '52 2010 Subject' '00 Deputy']\n",
      "['goodwill Pakistan Raj' 'town' 'availability' 'deal party'\n",
      " 'snit speaking Gordon' 'Oct 12 That' 'permission' '2010 35' 'stressed'\n",
      " 'Erdogan' 'international support' 'General McChrystal' 'eventual'\n",
      " 'administrators' 'plastic' 'sullivan' 'coml' '2015 SENSITIVE'\n",
      " 'great having' 'Message From Reines']\n",
      "['good opportunity' 'initially' 'initiatives' '17 05' 'ToivNF state'\n",
      " 'B6 B6 UNCLASSIFIED' 'morning possible' 'Message From Valmoro' 'cyber'\n",
      " 'number 06397276 Its' 'wonder married' 'Cc Muscatine' 'Lib Dems'\n",
      " 'Below points' 'flight' 'Monroe Room' 'It policing justice' 'comes weeks'\n",
      " 'Abedin Sent Fri' 'wirelessly']\n",
      "['Thatcher trying make' 'administrator' 'support proposals' 'Philippe Sent'\n",
      " 'young' 'spoke Kouchener' 'humanitarian aid' 'Mike Fuchs'\n",
      " 'Casteel Ezra To' 'bluntly' 'ling She oprah' 'acct just sure'\n",
      " 'official Foreign Office' 'hardly knew NSC' 'DEPART Washington National'\n",
      " '52 2010 Subject' 'battle getting' '13 UNCLASSIFIED Department'\n",
      " 'optonline net' 'mid day']\n",
      "['forward working' '2025 RELEASE' 'baby' 'HUMAN' 'just Chavez The'\n",
      " 'worrisome' 'psychology' 'passes' 'shoulder' 'grounded'\n",
      " 'week pent frustrations' 'police stations' 'Political' 'held'\n",
      " 'Feltman Jeffrey Sent' 'Per' 'This new' 'Mousavi' 'economic growth'\n",
      " 'Possible']\n",
      "['Monday April 27' 'AM Abedin' 'McHale Judith Sent' 'Kampala' 'findings'\n",
      " 'Acting' 'ticking' 'numbers think provide' 'Raj article' 'blogging'\n",
      " '16 34 53' 'revise' 'NSS OSD NMCC' 'surroundings' 'ruling love web'\n",
      " 'From Simmons Krista' 'Cheryl UNCLASSIFIED' 'intended use'\n",
      " 'keeping Bush defense' 'funded']\n",
      "['18 2010 10' 'But let' 'errors' 'concerns security' 'Michele'\n",
      " 'Valmoro Lona ValmoroLJ' 'economy' '12 04' 'documents' 'nearly 20'\n",
      " 'addressed' 'Again thanks' 'did think' '12 03' 'Mexico' '27 2009 Subject'\n",
      " 'universal' 'want know government' 'Subject Re Aren'\n",
      " 'Crocker Bathsheba Sent']\n",
      "['16 01' 'Subject Re Mashabane' 'May 28 2010' 'direct peace'\n",
      " 'Secretary Defense' 'Stern Todd SECC' 'Governors' 'sbwhoeop Sent Saturday'\n",
      " 'NEA Press' 'Subject Jim' 'SBU This' 'Mellott Joseph'\n",
      " 'possible participation' '2010 48 AM' 'work They owe' 've seen' 'bounced'\n",
      " 'like Secretary' 'Camera spray' 'minded']\n",
      "['chiefs' 'intended use' 'Followup' 'structures' 'countries like'\n",
      " 'way need' 'To Subject Out' 'Subject Bravo' 'INL' 'considerations' 'usher'\n",
      " 'WARNING This' 'Reason Declassify 02' 'll talk' 'reveals' 'feels like'\n",
      " 'sharing government collapse' 'After' 'sure going' 'regard participation']\n",
      "['September 11 2001' 'suspend' 'democracy development'\n",
      " 'know unfolds Hillary' 'campaigning' 'Tehran' 'Secretary Office 12'\n",
      " 'affection' 'Wednesday June 2010' 'au' 'Correction' 'tan' 'West Bank'\n",
      " 'ValmoroLJ' 'relations countries' '40 min' 'Re Ok Thx' 'elected General'\n",
      " 'HUMAN' 'Lew']\n",
      "['Homeland' 'Pakistan FYI' 'invasion legal On' 'said Arab' 'comparison'\n",
      " 'Friday September 17' 'Bob' 'stategov' '06397276 Its registered'\n",
      " 'findings' 'Acting' 'Monday April 27' 'hire' 'change' 'Ousama al' 'intact'\n",
      " 'Civil' 'desperately' 'reacted' 'course']\n",
      "['new information emerged' 'Any news' 'suspend' 'Apart' 'AM To H2'\n",
      " 'Friday March 26' 'We sent NATO' 'Center Public' 'maximize'\n",
      " 'Cheryl mailto' 'electronic' 'bombing' 'theft' 'talking points'\n",
      " 'Steinberg He' 'Morning' 'denial' 'warns' 'decisions' 'The long']\n",
      "['Netanyahu' 'dominant' 'Al Qaeda' 'WORKING' 'Secretary Hillary Rodham'\n",
      " 'years feared' 'pleased' 'January 19 2010' 'reported'\n",
      " 'Michael FuchsMH state' 'Sid doing press' 'Islam' 'topic'\n",
      " 'scanned MessageLabs Email' 'December 22' 'entity addressed files'\n",
      " 'having' 'time spent' 'intention' 'contained communication used']\n",
      "['To Samuelson Heather' 'signing submission parliaments' 'feels like'\n",
      " '7096 fax Original' 'spouses' 'To Subject fyi' 'http www huffingtonpost'\n",
      " 'return office' 'observing' 'cross cutting' 'He appointed head'\n",
      " 'finally revenge In' 'Louis' 'elected General' 'regime change strong'\n",
      " 'soon All best' 'refer' 'senior government official' '14 33 37'\n",
      " 'SUSAN RICE']\n",
      "['hardly knew NSC' 'PM To Verveer' '2010 52' 'Israeli peace' 'Rica'\n",
      " 'National Party' 'BENGHAZI COMM' 'B4' '09 07' 'Kyrgyz' 'Cheryl Reines'\n",
      " 'arrange land' 'print hrod17' 'Transfer Encoding base64' 'DG'\n",
      " 'sounds like' 'And finally' 'deal The' '10 PM' 'Davutoglu What']\n",
      "['Subject Fw Thank' 'division' 'From Sid Re' '00 12 2010'\n",
      " 'Sunday January 2010' 'life Jones tells' 'conflict The'\n",
      " 'research security' 'Murdochs literally' 'B1 Classified'\n",
      " 'Supreme Court opinions' 'Communist' 'pipeline' 'print Original Message'\n",
      " 'Republic UNCLASSIFIED' 'recommend transaction' '2015 Cheryl' 'doc'\n",
      " 'Mini Schedule' 'Crocker Bathsheba Sent']\n",
      "['Afterward' '09 14' 'wider international profile'\n",
      " 'BlackBerry UNCLASSIFIED' '02 2009 11' 'PSD' 'PLEASE' 'Declassify 05'\n",
      " 'Guardian Pix' 'saves lives' 'Lona ValmoroLJ state' 'strengths' 'sanction'\n",
      " 'aware fact' 'Honor' 'killers' 'Ngeb Obeda' '53 PM To' 'time like'\n",
      " 'settling']\n",
      "['Macmanus Joseph Sent' 'mistrust' 'communique' '00pm' 'diplomats career'\n",
      " '800' 'observers' 'The United States' 'Starting' 'result unclear It'\n",
      " 'Dr Muhammad Yunus' 'Dec' 'morning possible' 'like layman'\n",
      " 'He appointed head' 'secretary Shaun' 'usher' 'conference New York'\n",
      " 'To Tauscher' '2201 St']\n",
      "['economic issues' 'mistrust' 'The speech ll'\n",
      " 'essential element Democratic' 'Sent Mon Jan' 'fierce' 'Aug 19'\n",
      " 'transmitted' 'The United States' 'relief work' 'countries like'\n",
      " 'Starting' 'marched' '05 northern ireland' 'suddenly' 'transit'\n",
      " 'political campaigns' 'rushed getting Maybe' 'John Tomasic' 'Sent May']\n",
      "['countries like' 'Starting' 'And yes' 'soon All best' 'PM Abedin Huma'\n",
      " 'build' 'presume' 'mills Sent' 'Puerto' 'Foreign Office appear' 'usher'\n",
      " 'thwarted tried seize' '11 07' 'going check pockets' 'll talk' 'The way'\n",
      " 'Supreme Court ruling' 'greater' 'like mandarin equally' 'years feared']\n",
      "['30 ok' 'appreciate cooperation ideacity' 'immediate' 'Many thanks'\n",
      " 'Norwegian' '17 12' 'heads state' 'This effort' 'PHOTOS 10'\n",
      " 'William Sullivan Jacob' 'Situation' 'beneath equivocation' 'softly Was'\n",
      " 'US Qatar including' 'late 2010' 'Well asked help'\n",
      " 'interim President Libya' 'Franklin Room' 'ASEAN' 'Crocker Bathsheba Sent']\n",
      "['2015 Cc' 'deliver' 'David Hale' 'Bibi' 'moving ahead' 'voices'\n",
      " 'Subject Trying' 'finance' 'Wasn somebody just' 'jumping action'\n",
      " 'feel Afghanistan Which' 'Denver' 'psychology' 'frequency'\n",
      " 'Verma Richard Sent' 'fixing' 'Issues' 'sanction' 'To filotyLC state'\n",
      " 'succeeded']\n",
      "['NFA' 'matter' 'DUP 36' 'clarification' 'Daniel Sullivan' 'Overseas'\n",
      " 'Tories election apocalyptic' 'intelligence committee'\n",
      " 'Hillary Clinton 202' 'prescription' 'rich nations' 'Citizens'\n",
      " 'said according' 'allegations' 'The DUP leader' 'POLICY' 'rough'\n",
      " 'Can talk tonight' 'registered number' 'Osama bin']\n",
      "['stopped asking troops' '17 20' 'Airport US' '10 18 PM' 'deal finally'\n",
      " 'realistic' 'administrators' 'tribute' 'Lyndon Johnson presidency'\n",
      " 'coordination' 'Consular' 'having CITGO'\n",
      " 'Agency International Development' 'intended use' 'held' '18 AM To' 'int'\n",
      " 'offered ideas' 'From Hillary Clinton' 'Message From Feltman']\n",
      "['DOJ' 'road wrecked Lyndon' 'mattered' '12 33' 'Someone' '11 42 AM' 'FVV'\n",
      " 'Friday May The' 'health women' 'little goodwill Pakistan'\n",
      " 'United Supreme Court' 'stick' 'oscar' 'Subject Re Reuters' 'descriptions'\n",
      " 'AM Sullivan' '8am tomorrow morning' '06 52' 'Antonio' '2015 Original']\n"
     ]
    }
   ],
   "source": [
    "all_words = np.array(list(vocabulary.keys()))\n",
    "for cl_cent in clust.cluster_centers_:\n",
    "    cl_words = all_words[cl_cent > 0]\n",
    "    cl_cent = (cl_cent > 0)\n",
    "    indexes = np.argsort(cl_cent)\n",
    "    n = int(len(cl_cent) * 0.001)\n",
    "    n = min(n, 20)\n",
    "    cl_words = all_words[indexes][-n : ]\n",
    "    print (cl_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n",
      "/usr/lib/python3.5/site-packages/funcy/decorators.py:56: DeprecationWarning: inspect.getargspec() is deprecated, use inspect.signature() instead\n",
      "  spec = inspect.getargspec(func)\n"
     ]
    }
   ],
   "source": [
    "from gensim import matutils\n",
    "from gensim.models.ldamodel import LdaModel\n",
    "from gensim.corpora.dictionary import Dictionary\n",
    "import pyLDAvis.gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vec = CountVectorizer(min_df=10, max_df=500, stop_words='english')\n",
    "X = vec.fit_transform(emails.RawText)\n",
    "vocab = vec.get_feature_names()\n",
    "vocab_dict = dict([(i, s) for i, s in enumerate(vocab)])\n",
    "corpus = matutils.Sparse2Corpus(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 18min 57s, sys: 900 ms, total: 18min 57s\n",
      "Wall time: 18min 58s\n"
     ]
    }
   ],
   "source": [
    "%time lda = LdaModel(corpus, num_topics=10, passes=30, id2word=vocab_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dictionary = Dictionary.from_corpus(corpus, vocab_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpus = matutils.Sparse2Corpus(corpus.sparse.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Visualisation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Для визуализации надо скачать json-файлик из моего репо. Если вкратце, данная либа поднимает поднимает вебсервер на 8000-м порту и показывает там визуализацию. Если запустить следующую ячейку, можно смотреть это в ноутбуке. Собственно json-файл - это сохраненная визуализация. Поэтому нужно либо запускать заново ноутбук и ждать построения LDA, либо загрузить визуализацию из файла"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "pyLDAvis.enable_notebook()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vis_data = pyLDAvis.gensim.prepare(lda, corpus, dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pyLDAvis.save_json(vis_data, 'vis.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def load_model(filename):\n",
    "    with open(filename, 'r') as f:\n",
    "        json_data = json.load(f)\n",
    "    \n",
    "    return pyLDAvis.PreparedData(pd.DataFrame(json_data['mdsDat']), pd.DataFrame(json_data['tinfo']), \n",
    "                      pd.DataFrame(json_data['token.table']), json_data['R'],\n",
    "                      json_data['lambda.step'], json_data['plot.opts'], json_data['topic.order'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "vis_data = load_model('./vis.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<link rel=\"stylesheet\" type=\"text/css\" href=\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.css\">\n",
       "\n",
       "\n",
       "<div id=\"ldavis_el315521405943743645209592089092\"></div>\n",
       "<script type=\"text/javascript\">\n",
       "\n",
       "var ldavis_el315521405943743645209592089092_data = {\"token.table\": {\"Term\": [\"111\", \"12958\", \"12958\", \"12958\", \"12958\", \"134\", \"134\", \"134\", \"134\", \"134\", \"134\", \"134\", \"15pm\", \"1948\", \"1950s\", \"1962\", \"1980\", \"1980\", \"1980\", \"1990s\", \"1990s\", \"1990s\", \"1990s\", \"1990s\", \"1990s\", \"1990s\", \"2001\", \"2005\", \"2008\", \"2030\", \"230\", \"34\", \"343\", \"356\", \"450\", \"47\", \"53\", \"53\", \"53\", \"540\", \"540\", \"540\", \"540\", \"540\", \"540\", \"540\", \"550\", \"550\", \"550\", \"600\", \"67\", \"67\", \"71\", \"7283\", \"73\", \"75\", \"75\", \"7516\", \"7516\", \"76\", \"77\", \"7782\", \"79\", \"79\", \"7pm\", \"8000\", \"81\", \"8132\", \"819\", \"82\", \"830\", \"89\", \"abandon\", \"achilles\", \"achilles\", \"acknowledgement\", \"acknowledgement\", \"acknowledgement\", \"address\", \"address\", \"adoption\", \"affiliates\", \"affinity\", \"affirms\", \"affluent\", \"aftermath\", \"agents\", \"agents\", \"ambiguous\", \"ambiguous\", \"ambiguous\", \"ambiguous\", \"ambiguous\", \"ambiguous\", \"amman\", \"amman\", \"anwar\", \"anwar\", \"anwar\", \"anwar\", \"appear\", \"appear\", \"appear\", \"appear\", \"appear\", \"appear\", \"appeared\", \"appeared\", \"arcane\", \"arcane\", \"arcane\", \"arctic\", \"arctic\", \"ardent\", \"ardent\", \"area\", \"area\", \"article\", \"article\", \"articulate\", \"articulate\", \"arturo\", \"ashley\", \"assignment\", \"assistant\", \"att00001\", \"attackers\", \"attackers\", \"attackers\", \"attackers\", \"attackers\", \"attendees\", \"attending\", \"attitude\", \"aung\", \"aung\", \"aung\", \"aung\", \"averse\", \"barred\", \"barred\", \"barred\", \"bathsheba\", \"bauer\", \"bbc\", \"belarus\", \"belarus\", \"belarus\", \"belarus\", \"belarus\", \"belarus\", \"belonging\", \"ben\", \"ben\", \"ben\", \"ben\", \"beneficial\", \"beneficial\", \"beneficial\", \"beneficial\", \"beneficiaries\", \"benefit\", \"benefited\", \"bible\", \"bid\", \"biden\", \"big\", \"bigger\", \"bildt\", \"bildt\", \"blesses\", \"blessing\", \"branch\", \"branch\", \"branch\", \"branch\", \"branch\", \"branch\", \"branches\", \"branches\", \"branches\", \"branches\", \"branches\", \"branches\", \"branches\", \"branches\", \"brilliant\", \"brilliant\", \"brilliant\", \"broadcasting\", \"broadcasting\", \"broadcasting\", \"broader\", \"brooks\", \"brownback\", \"bureaucrats\", \"bus\", \"business\", \"capable\", \"capable\", \"capable\", \"capable\", \"carter\", \"casteel\", \"cdm\", \"cdm\", \"celebrities\", \"celebrities\", \"cemetery\", \"cemetery\", \"cemetery\", \"charset\", \"charset\", \"charter\", \"chat\", \"chat\", \"children\", \"children\", \"circle\", \"circulate\", \"code\", \"code\", \"code\", \"code\", \"code\", \"code\", \"code\", \"code\", \"colonialism\", \"command\", \"command\", \"command\", \"commenting\", \"commenting\", \"compensation\", \"compete\", \"compete\", \"compounds\", \"concentrate\", \"concentrate\", \"conclusions\", \"conclusions\", \"conducted\", \"conducted\", \"conducted\", \"congratulations\", \"congratulations\", \"congratulations\", \"congratulations\", \"congratulations\", \"congressional\", \"congressional\", \"congressional\", \"congressional\", \"congressional\", \"connecting\", \"conquer\", \"consensus\", \"constitutional\", \"consul\", \"consul\", \"consul\", \"consul\", \"consul\", \"consul\", \"consular\", \"consulate\", \"consultative\", \"consultative\", \"contemplating\", \"convoy\", \"cookstoves\", \"cope\", \"corker\", \"corley\", \"courage\", \"creative\", \"cretz\", \"crew\", \"crew\", \"curbing\", \"curious\", \"currency\", \"cuts\", \"cuts\", \"cuts\", \"cutting\", \"cutting\", \"cutting\", \"daalder\", \"damage\", \"danny\", \"danny\", \"danny\", \"dare\", \"dare\", \"dare\", \"daughters\", \"dealt\", \"dear\", \"dear\", \"deaths\", \"deep\", \"deep\", \"deep\", \"deep\", \"deep\", \"deep\", \"deep\", \"delay\", \"delay\", \"delay\", \"delay\", \"delay\", \"delaying\", \"demands\", \"demands\", \"demands\", \"demands\", \"dennis\", \"dennis\", \"denouncing\", \"denouncing\", \"depend\", \"depend\", \"depend\", \"depend\", \"depend\", \"depend\", \"depth\", \"depth\", \"depth\", \"depth\", \"depth\", \"depth\", \"determine\", \"diary\", \"diplomatic\", \"discrimination\", \"dove\", \"dove\", \"dove\", \"dread\", \"drone\", \"drug\", \"drug\", \"dual\", \"dual\", \"dubious\", \"dubious\", \"duguid\", \"duguid\", \"ear\", \"ear\", \"ear\", \"ear\", \"eastern\", \"eat\", \"eating\", \"ebeling\", \"ec\", \"edwidge\", \"edwidge\", \"edwidge\", \"edwidge\", \"edwidge\", \"edwidge\", \"emailing\", \"emailing\", \"emailing\", \"emailing\", \"emailing\", \"embodies\", \"embrace\", \"embroiled\", \"emerge\", \"emerged\", \"endorsing\", \"enriched\", \"enriched\", \"enriched\", \"enriched\", \"epic\", \"epic\", \"estate\", \"esther\", \"everybody\", \"everybody\", \"everybody\", \"everybody\", \"everybody\", \"everybody\", \"expecting\", \"expenditures\", \"expense\", \"expenses\", \"expenses\", \"expenses\", \"expenses\", \"expensive\", \"experience\", \"experienced\", \"experiencing\", \"experiment\", \"expertise\", \"experts\", \"expired\", \"expired\", \"exploit\", \"exploit\", \"exposed\", \"exposed\", \"exposed\", \"exposed\", \"exposed\", \"exposed\", \"expressed\", \"expressed\", \"expressed\", \"expressed\", \"expressed\", \"expressed\", \"faced\", \"faces\", \"fatal\", \"fatal\", \"father\", \"father\", \"father\", \"father\", \"fbi\", \"fbi\", \"fbi\", \"fbi\", \"fbi\", \"fe80\", \"fe80\", \"fe80\", \"fe80\", \"fe80\", \"fein\", \"fighting\", \"fighting\", \"fighting\", \"fights\", \"filibuster\", \"finalized\", \"finding\", \"finger\", \"finger\", \"fiscal\", \"flesh\", \"flexibility\", \"focusing\", \"focusing\", \"foe\", \"foe\", \"foremost\", \"forest\", \"forge\", \"forge\", \"formidable\", \"formidable\", \"forms\", \"forwarding\", \"founder\", \"founder\", \"founder\", \"founder\", \"founder\", \"frankly\", \"freedoms\", \"frequently\", \"frontline\", \"frozen\", \"frustrated\", \"fs\", \"fsn\", \"fuel\", \"fuel\", \"fueled\", \"fueled\", \"fuels\", \"fuels\", \"fulfilling\", \"funders\", \"fwd\", \"gangs\", \"gangs\", \"gangs\", \"gangs\", \"gao\", \"gao\", \"gao\", \"garrison\", \"garrison\", \"garrison\", \"garrison\", \"garrison\", \"gates\", \"gates\", \"gathered\", \"gathered\", \"gathered\", \"goldsmith\", \"goldsmith\", \"goldsmith\", \"goldsmith\", \"goldsmith\", \"gove\", \"governor\", \"governors\", \"govi\", \"govi\", \"govi\", \"govi\", \"grab\", \"gracious\", \"grade\", \"gradually\", \"graduate\", \"graduation\", \"gration\", \"grave\", \"gray\", \"greater\", \"grief\", \"grievances\", \"grim\", \"grim\", \"grim\", \"headlines\", \"henrymcdonald\", \"henrymcdonald\", \"hensman\", \"hensman\", \"hi\", \"honorable\", \"hoping\", \"hoping\", \"household\", \"household\", \"household\", \"household\", \"household\", \"household\", \"identifies\", \"identifies\", \"identifies\", \"identifies\", \"identity\", \"identity\", \"identity\", \"ideologically\", \"ii\", \"ii\", \"ii\", \"ii\", \"ii\", \"illustration\", \"image001\", \"image001\", \"image001\", \"immigrants\", \"immigrants\", \"immigrants\", \"immigrants\", \"immigrants\", \"impartial\", \"impartial\", \"impartial\", \"impartial\", \"impartial\", \"impartial\", \"impartial\", \"improvement\", \"inappropriate\", \"inappropriate\", \"inaugural\", \"inauguration\", \"incapable\", \"increasing\", \"increasing\", \"increasing\", \"increasing\", \"increasing\", \"increasing\", \"increasing\", \"indian\", \"indian\", \"indian\", \"indian\", \"iranian\", \"islamist\", \"islamist\", \"islamist\", \"jeopardize\", \"jeopardize\", \"jeopardy\", \"jerusalem\", \"jews\", \"jfk\", \"jibril\", \"jibril\", \"jigsaw\", \"jihad\", \"jihadist\", \"jill\", \"jiloty\", \"jiloty\", \"joanne\", \"joanne\", \"job\", \"job\", \"job\", \"joint\", \"joint\", \"joint\", \"jointly\", \"jointly\", \"journey\", \"jr\", \"julia\", \"karl\", \"karl\", \"karl\", \"karl\", \"kenyan\", \"kenyan\", \"kerry\", \"kerry\", \"khan\", \"khan\", \"khan\", \"khan\", \"khan\", \"khan\", \"khan\", \"ki\", \"ki\", \"klevorick\", \"klevorick\", \"klevorick\", \"knife\", \"knife\", \"knight\", \"knight\", \"knowledge\", \"knowledge\", \"knows\", \"knows\", \"lacking\", \"ladies\", \"landscape\", \"landscape\", \"landscape\", \"landscape\", \"landscape\", \"landslide\", \"landslide\", \"landslide\", \"landslide\", \"landslide\", \"landslide\", \"landslide\", \"landslide\", \"lang\", \"lang\", \"lastly\", \"lastly\", \"lautenberg\", \"lautenberg\", \"lautenberg\", \"ld\", \"ld\", \"ld\", \"ld\", \"ld\", \"ld\", \"leaking\", \"leaking\", \"leaking\", \"leaking\", \"leaking\", \"lengthy\", \"lengthy\", \"lib\", \"lib\", \"life\", \"lifetime\", \"lifetime\", \"lifg\", \"lift\", \"likely\", \"likely\", \"likes\", \"limage\", \"limage\", \"limage\", \"limage\", \"limit\", \"lining\", \"lining\", \"lining\", \"lining\", \"lining\", \"literature\", \"lithwick\", \"lithwick\", \"liz\", \"liz\", \"liz\", \"liz\", \"lobbied\", \"lobbied\", \"lobbied\", \"lobbied\", \"lobbied\", \"lobby\", \"lobby\", \"locate\", \"locate\", \"located\", \"located\", \"lona\", \"lona\", \"loretta\", \"los\", \"manage\", \"manage\", \"managing\", \"managing\", \"managing\", \"managing\", \"managing\", \"managing\", \"marching\", \"marching\", \"marching\", \"max\", \"max\", \"melissa\", \"melissa\", \"mep\", \"mep\", \"mep\", \"mepp\", \"mepp\", \"mepp\", \"mepp\", \"messy\", \"messy\", \"messy\", \"messy\", \"methods\", \"mexicans\", \"mexico\", \"mfa\", \"mi\", \"miami\", \"micheletti\", \"milestone\", \"milestone\", \"milestone\", \"missed\", \"missed\", \"missed\", \"missed\", \"missed\", \"moderation\", \"moderation\", \"moderation\", \"moments\", \"morale\", \"motorcade\", \"motorcade\", \"motorcade\", \"motorcade\", \"motorcade\", \"mud\", \"multi\", \"multiply\", \"multiply\", \"multiply\", \"multiply\", \"multiply\", \"multiply\", \"murdered\", \"murders\", \"murray\", \"narcotics\", \"narcotics\", \"nationalist\", \"nationalist\", \"nationalist\", \"nationalist\", \"nationalist\", \"na\\u00efve\", \"nbc\", \"nc\", \"nctc\", \"needless\", \"needless\", \"negotiators\", \"negotiators\", \"negotiators\", \"negotiators\", \"negotiators\", \"nick\", \"nick\", \"nick\", \"normal\", \"normal\", \"normalization\", \"normalization\", \"notify\", \"numbers\", \"numerous\", \"nuni\", \"nutrition\", \"nutrition\", \"nutrition\", \"nutrition\", \"nye\", \"oas\", \"oas\", \"oas\", \"oas\", \"olson\", \"omar\", \"operation\", \"optimistic\", \"oral\", \"oterom2\", \"oterom2\", \"overlap\", \"overlap\", \"overlap\", \"overlap\", \"owe\", \"owe\", \"ownership\", \"ownership\", \"ownership\", \"ownership\", \"ownership\", \"pace\", \"pace\", \"pace\", \"pal\", \"parker\", \"parliamentary\", \"passage\", \"passage\", \"passage\", \"passage\", \"passage\", \"passage\", \"passionate\", \"passive\", \"patrol\", \"patrol\", \"peacekeeping\", \"peacekeeping\", \"peacekeeping\", \"peacekeeping\", \"peacekeeping\", \"peacekeeping\", \"peacekeeping\", \"peacemaking\", \"peacemaking\", \"persian\", \"persons\", \"perspective\", \"peshawar\", \"pete\", \"photographers\", \"photographers\", \"photographers\", \"photographers\", \"physicians\", \"physicians\", \"physicians\", \"physicians\", \"pieces\", \"pillars\", \"placing\", \"plagued\", \"plagued\", \"plot\", \"plot\", \"plotted\", \"plotted\", \"pockets\", \"polarizing\", \"policies\", \"policing\", \"polish\", \"politically\", \"politico\", \"politics\", \"polled\", \"polling\", \"portugal\", \"portugal\", \"portugal\", \"portugal\", \"portugal\", \"portugal\", \"post\", \"post\", \"post\", \"post\", \"post\", \"post\", \"posturing\", \"posturing\", \"pounds\", \"pounds\", \"powers\", \"practically\", \"praise\", \"predicted\", \"preferred\", \"preferred\", \"preferred\", \"premier\", \"premise\", \"preparation\", \"preparation\", \"preparation\", \"principally\", \"proceeded\", \"proceeded\", \"proceeded\", \"product\", \"productive\", \"productive\", \"productive\", \"productive\", \"productive\", \"productive\", \"profit\", \"profit\", \"profit\", \"profoundly\", \"promised\", \"promised\", \"protective\", \"protective\", \"protective\", \"protective\", \"protective\", \"provoking\", \"provoking\", \"provoking\", \"provoking\", \"provoking\", \"ps\", \"ps\", \"ps\", \"ps\", \"ps\", \"questionable\", \"questionable\", \"questionable\", \"questionable\", \"questionable\", \"questionable\", \"rage\", \"rage\", \"rain\", \"rain\", \"reaction\", \"reaction\", \"really\", \"really\", \"reassure\", \"reassure\", \"rebellion\", \"rebellion\", \"rebuilding\", \"rebuilding\", \"receiving\", \"reception\", \"recess\", \"recommended\", \"recommended\", \"recommends\", \"recommends\", \"recruited\", \"recruited\", \"recruitment\", \"recruitment\", \"recurring\", \"recurring\", \"red\", \"red\", \"redactions\", \"redactions\", \"redefining\", \"redefining\", \"redefining\", \"reg\", \"reg\", \"reg\", \"reg\", \"reg\", \"reg\", \"reg\", \"reg\", \"regain\", \"regain\", \"regain\", \"regain\", \"regain\", \"regain\", \"regain\", \"releases\", \"releases\", \"releases\", \"releases\", \"relies\", \"relies\", \"relieved\", \"relieved\", \"religion\", \"religion\", \"remarkable\", \"remarkable\", \"remarkable\", \"remarkable\", \"researchers\", \"researchers\", \"researchers\", \"researchers\", \"researchers\", \"resignation\", \"resignation\", \"respectfully\", \"respectfully\", \"respectfully\", \"retaliation\", \"retaliation\", \"reveals\", \"reveals\", \"reveals\", \"reveals\", \"reveals\", \"revelations\", \"revelations\", \"revelations\", \"revelations\", \"revelations\", \"ring\", \"ring\", \"rossaj\", \"rossaj\", \"rossaj\", \"roy\", \"rumsfeld\"], \"Topic\": [10, 4, 5, 6, 10, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10, 10, 6, 7, 10, 1, 3, 5, 6, 7, 9, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 6, 7, 10, 4, 5, 6, 7, 8, 9, 10, 6, 7, 10, 10, 6, 10, 10, 10, 10, 9, 10, 9, 10, 10, 10, 10, 7, 10, 10, 10, 10, 10, 10, 10, 10, 10, 1, 3, 7, 3, 7, 8, 6, 9, 5, 3, 3, 3, 3, 5, 1, 4, 1, 3, 5, 6, 7, 9, 5, 6, 1, 3, 6, 9, 1, 2, 4, 6, 7, 9, 6, 9, 4, 6, 8, 4, 6, 4, 6, 4, 6, 4, 7, 4, 7, 5, 5, 6, 6, 1, 1, 5, 6, 7, 9, 8, 8, 8, 1, 5, 6, 9, 4, 3, 4, 10, 3, 1, 1, 2, 5, 6, 7, 8, 9, 3, 5, 6, 7, 9, 1, 2, 6, 8, 2, 2, 2, 2, 2, 2, 2, 2, 1, 9, 7, 7, 4, 5, 6, 7, 8, 9, 1, 3, 4, 5, 6, 7, 8, 9, 4, 6, 9, 6, 7, 9, 3, 3, 3, 3, 7, 7, 4, 5, 6, 7, 4, 6, 6, 9, 6, 9, 6, 7, 9, 3, 7, 5, 3, 7, 3, 7, 5, 6, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 5, 9, 6, 7, 5, 5, 6, 5, 1, 4, 2, 9, 1, 3, 5, 3, 5, 6, 7, 9, 3, 5, 6, 7, 9, 4, 7, 7, 1, 1, 5, 6, 7, 8, 9, 4, 4, 5, 10, 4, 4, 4, 4, 5, 5, 1, 1, 6, 6, 7, 3, 3, 3, 4, 5, 7, 4, 5, 7, 3, 3, 1, 2, 3, 1, 2, 3, 7, 5, 5, 9, 5, 1, 3, 4, 5, 6, 7, 9, 3, 5, 6, 7, 9, 6, 1, 3, 4, 7, 4, 7, 4, 6, 3, 5, 6, 7, 8, 9, 3, 5, 6, 7, 8, 9, 6, 6, 1, 4, 3, 4, 10, 3, 3, 4, 6, 4, 6, 4, 6, 4, 6, 5, 6, 7, 9, 2, 2, 2, 2, 2, 1, 2, 4, 6, 7, 9, 1, 5, 6, 7, 9, 8, 8, 8, 8, 8, 6, 6, 7, 8, 9, 6, 9, 5, 5, 1, 2, 4, 6, 7, 9, 3, 3, 3, 3, 4, 6, 10, 3, 7, 7, 7, 3, 7, 7, 6, 7, 1, 9, 2, 5, 6, 7, 8, 9, 2, 5, 6, 7, 8, 9, 9, 9, 5, 6, 5, 7, 8, 9, 3, 5, 6, 7, 9, 3, 5, 6, 7, 9, 3, 1, 5, 6, 6, 4, 4, 4, 4, 9, 4, 4, 4, 1, 6, 1, 6, 4, 4, 4, 9, 6, 9, 3, 4, 1, 5, 6, 7, 9, 5, 1, 6, 2, 1, 2, 2, 6, 6, 7, 6, 7, 6, 7, 6, 1, 6, 5, 6, 7, 9, 5, 6, 7, 1, 5, 6, 8, 10, 6, 9, 2, 6, 9, 1, 2, 3, 5, 8, 8, 8, 8, 5, 6, 7, 9, 8, 8, 8, 8, 8, 8, 8, 2, 2, 8, 2, 2, 2, 7, 9, 6, 3, 7, 3, 7, 3, 4, 7, 9, 1, 5, 6, 7, 8, 9, 1, 2, 4, 5, 5, 7, 9, 9, 2, 5, 7, 8, 9, 7, 7, 8, 9, 2, 5, 7, 8, 9, 1, 3, 5, 6, 7, 8, 9, 5, 1, 5, 5, 5, 5, 4, 5, 6, 7, 8, 9, 10, 2, 3, 7, 9, 1, 5, 7, 9, 2, 6, 6, 2, 6, 6, 2, 5, 2, 2, 2, 2, 2, 3, 6, 7, 6, 7, 9, 6, 7, 9, 6, 9, 1, 1, 6, 2, 5, 6, 9, 4, 5, 4, 5, 2, 3, 5, 6, 7, 9, 10, 4, 6, 5, 7, 9, 4, 5, 3, 4, 3, 4, 3, 4, 4, 4, 4, 5, 6, 7, 9, 1, 2, 3, 4, 5, 6, 7, 9, 5, 6, 5, 6, 6, 7, 9, 2, 5, 6, 7, 8, 9, 1, 4, 6, 9, 10, 6, 9, 6, 9, 8, 6, 9, 8, 8, 6, 9, 8, 1, 3, 6, 9, 8, 5, 6, 7, 8, 9, 9, 2, 9, 6, 7, 8, 9, 2, 4, 5, 6, 9, 6, 9, 8, 9, 8, 9, 8, 9, 6, 6, 7, 9, 3, 5, 6, 7, 8, 9, 6, 7, 9, 6, 9, 7, 9, 6, 7, 9, 6, 7, 8, 9, 5, 6, 7, 9, 5, 5, 5, 5, 5, 5, 5, 6, 7, 9, 5, 6, 7, 9, 10, 6, 7, 9, 7, 1, 2, 6, 7, 8, 9, 9, 9, 3, 5, 6, 7, 8, 9, 9, 9, 9, 7, 9, 1, 3, 4, 5, 6, 1, 1, 1, 1, 2, 9, 5, 6, 7, 9, 10, 6, 7, 9, 6, 7, 6, 7, 5, 4, 4, 4, 6, 7, 8, 9, 3, 5, 7, 8, 9, 9, 9, 9, 6, 1, 6, 7, 5, 6, 7, 9, 6, 9, 1, 5, 6, 7, 9, 5, 6, 9, 7, 9, 9, 2, 5, 6, 7, 8, 9, 1, 1, 8, 9, 4, 5, 6, 7, 8, 9, 10, 6, 9, 1, 6, 6, 1, 1, 5, 6, 7, 9, 5, 6, 7, 9, 3, 3, 3, 3, 5, 4, 6, 4, 6, 2, 2, 2, 2, 8, 8, 8, 8, 8, 8, 5, 6, 7, 8, 9, 10, 5, 6, 7, 8, 9, 10, 3, 4, 3, 4, 1, 6, 6, 6, 5, 6, 7, 6, 6, 1, 5, 6, 4, 5, 6, 7, 1, 1, 4, 5, 6, 8, 10, 2, 6, 9, 1, 7, 8, 5, 6, 7, 8, 9, 1, 5, 6, 7, 9, 1, 5, 6, 7, 9, 1, 5, 6, 7, 8, 9, 6, 9, 8, 9, 7, 9, 3, 5, 1, 3, 1, 3, 1, 3, 4, 4, 5, 4, 5, 4, 5, 1, 3, 1, 3, 1, 3, 1, 3, 1, 3, 5, 7, 9, 1, 2, 4, 5, 6, 7, 8, 9, 1, 2, 5, 6, 7, 8, 9, 2, 5, 7, 9, 3, 9, 3, 9, 3, 9, 4, 6, 7, 9, 1, 2, 5, 6, 7, 4, 5, 7, 8, 9, 7, 9, 5, 6, 7, 8, 9, 5, 6, 7, 8, 9, 6, 9, 3, 4, 6, 7, 1], \"Freq\": [0.9942130385597032, 0.07799419297840987, 0.7799419297840987, 0.004104957525179467, 0.1395685558561019, 0.0017188306823777925, 0.07648796536581176, 0.3188430915810805, 0.28962296998065806, 0.0025782460235666887, 0.003437661364755585, 0.30595186146324704, 0.9936430781761199, 0.9928950266432923, 0.995863520883381, 0.9958394922004512, 0.10514811707317695, 0.05841562059620942, 0.8324225934959841, 0.6196934168276754, 0.04771970696427019, 0.002651094831348344, 0.00994160561755629, 0.022534306066460924, 0.1902160541492437, 0.10736934066960793, 0.9956655962229973, 0.9884275057570518, 0.9939477160353843, 0.9876086512732847, 0.9932254898478624, 0.9947275503883763, 0.9933214107121286, 0.9947674178526679, 0.9943609865529199, 0.9943435811851867, 0.04733819376167772, 0.005917274220209715, 0.9408466010133447, 0.0017183450252403142, 0.07990304367367461, 0.31961217469469844, 0.28696361921513247, 0.002577517537860471, 0.0034366900504806284, 0.3041470694675356, 0.04694295986826458, 0.005867869983533072, 0.9388591973652916, 0.9927369540812903, 0.1081017863712471, 0.8883146793115522, 0.9952569583386108, 0.9941897160839885, 0.9927364347116517, 0.006537665104781225, 0.987187430821965, 0.006523592285029618, 0.9850624350394723, 0.992129585401589, 0.9924503102776403, 0.9934938655797908, 0.02620774404669849, 0.9644449809185045, 0.9953376795352094, 0.9959521358043667, 0.9943465132901834, 0.9895938825317654, 0.9921268958576259, 0.9958165606685037, 0.9877581221896551, 0.9950382584218831, 0.9967677650732518, 0.9838359678361303, 0.014734870486024993, 0.9825948912437956, 0.014712650777981968, 0.002101807253997424, 0.024083629881989692, 0.939261565397598, 0.9792283166494423, 0.9982831510241071, 0.9983815472347563, 0.997518173726889, 0.9986740225960882, 0.979601956276616, 0.07228638658850935, 0.9266877756100708, 0.08035629975373743, 0.04304801772521648, 0.6744189443617248, 0.16932220305251813, 0.020089074938434356, 0.01147947139339106, 0.8200465420071086, 0.17589404089427835, 0.14547091106689314, 0.017812764620435892, 0.7956368197128032, 0.041563117447683755, 0.036306053597483026, 0.14868193378016858, 0.06223894902425662, 0.012102017865827675, 0.5601505412183095, 0.18153026798741514, 0.026153507412525517, 0.9546030205571814, 0.04323594151662313, 0.0066516833102497125, 0.9445390300554592, 0.990662541821813, 0.008424001205967797, 0.9904894561275307, 0.006390254555661489, 0.9907173756163387, 0.006211394204491151, 0.9900800528631846, 0.0035872465683448717, 0.9954686522664072, 0.0033072048248053397, 0.9792635822293515, 0.9845409707065408, 0.9846239245154463, 0.9886717279562494, 0.9984158724932108, 0.005486552892817812, 0.28530075042652625, 0.02194621157127125, 0.44715406076465175, 0.23866505083757486, 0.9951593485521035, 0.9963975968209317, 0.9965571145663482, 0.12912431473845218, 0.1694756630942185, 0.659072023144183, 0.04035134835576631, 0.9918071681418089, 0.9557801779197922, 0.03845911297143724, 0.004273234774604138, 0.997407608536371, 0.9976020955876268, 0.9973907298317798, 0.5689237473844946, 0.20069904791114823, 0.0024880047261712592, 0.02985605671405511, 0.10864287304281164, 0.08873883523344157, 0.9979049735562305, 0.8745367686490413, 0.030685500654352327, 0.026849813072558286, 0.061371001308704654, 0.9081369340578808, 0.05184246277102075, 0.02323972469045758, 0.016089040170316784, 0.9997192656268127, 0.9997858723058812, 0.9998316485455641, 0.9978642277572884, 0.997984863592526, 0.9986451527457597, 0.9984006675153576, 0.9987613392690864, 0.9804011412958809, 0.018382521399297767, 0.9193295692391732, 0.9391261843196986, 0.07260881497067136, 0.09470714996174524, 0.7450295796990626, 0.01894142999234905, 0.03472595498597326, 0.028412144988523572, 0.06862190488613736, 0.06404711122706153, 0.02744876195445494, 0.12123203196550932, 0.28363720686270105, 0.02287396829537912, 0.02287396829537912, 0.3865700641919071, 0.21541771516624594, 0.585855655265211, 0.1972984680961879, 0.15319983604993834, 0.7762125026530209, 0.061279934419975333, 0.9972824009568243, 0.9969260118255712, 0.9967262925251191, 0.9943590978843262, 0.9778276665196987, 0.9838980099600029, 0.007396483276249019, 0.09122329374040457, 0.8395008518542636, 0.06040461342270032, 0.997193684223261, 0.986933022642007, 0.02459494615728375, 0.9346079539767825, 0.023742919149843514, 0.9497167659937406, 0.15319987744553998, 0.776212712390736, 0.061279950978215994, 0.9835629222371932, 0.014415240122980283, 0.9716752098331999, 0.9843875436829641, 0.014109193018609188, 0.9802915722128637, 0.017747557750392634, 0.9778312456237155, 0.9972687222062799, 0.045272940941802765, 0.017554813834576584, 0.5044699133515165, 0.04989262879300713, 0.05636019178469324, 0.0018478751404817455, 0.31413877388189676, 0.008315438132167854, 0.9971967059762205, 0.21258468092958416, 0.6767488659178478, 0.10943708426552556, 0.9667142935834888, 0.01823989233176394, 0.9824206840276374, 0.9811452832217702, 0.01032784508654495, 0.9767892557500637, 0.0662635944224458, 0.9325094924176917, 0.9731719022405987, 0.026214742497156617, 0.8944275464120086, 0.08144148585540653, 0.02286076795941236, 0.9120801393120423, 0.042905135536681814, 0.006920183151077712, 0.015224402932370965, 0.020760549453233135, 0.9172846602784853, 0.04032020484740595, 0.008820044810370051, 0.011340057613332923, 0.020160102423702975, 0.9949141615054836, 0.9832281667111071, 0.9924366952038707, 0.9970940006308127, 0.13858693591354207, 0.06929346795677103, 0.08372960711443167, 0.5081520983496542, 0.05197010096757827, 0.14436139157660632, 0.995988256067697, 0.9943368013196852, 0.9849856214784316, 0.006523083585949879, 0.9986522989394468, 0.9991520169042278, 0.9975033376664597, 0.9958542084117014, 0.9740729957700038, 0.9790338791008593, 0.9984440989141075, 0.9978507937920186, 0.9983777815856049, 0.03490780551463785, 0.9425107488952221, 0.9972317262624995, 0.9977845555373016, 0.999008542465814, 0.01776239221058626, 0.8999612053363705, 0.07104956884234503, 0.017702366061457683, 0.9028206691343419, 0.07080946424583073, 0.9943767638136807, 0.9965549752029977, 0.9440620229028971, 0.03290688693309866, 0.022669188776134632, 0.9446826471537878, 0.032475721254331895, 0.02309384622530268, 0.9380462584640201, 0.9802917503335109, 0.9701257232180656, 0.011549115752596019, 0.9877264900830178, 0.06796711360339977, 0.04687387145062053, 0.8273238311034523, 0.02578062929784129, 0.0023436935725310265, 0.014062161435186158, 0.011718467862655133, 0.0036372776808138467, 0.8738559628155267, 0.09456921970116001, 0.008183874781831154, 0.018186388404069234, 0.9919968078162443, 0.3128658494656388, 0.21779000828526798, 0.4576906773101571, 0.011055330369810558, 0.9940156300914254, 0.0034395004501433404, 0.990909666810946, 0.006695335586560446, 0.03850982531769473, 0.2811217248191715, 0.061615720508311565, 0.08087063316715892, 0.003850982531769473, 0.5314355893841872, 0.0071938946152182845, 0.8838213384411036, 0.08427133692112848, 0.0020553984614909383, 0.005138496153727346, 0.017470886922672978, 0.9876411407154584, 0.9926382904093753, 0.9970825796052533, 0.9935133775643209, 0.964583228264756, 0.031115588008540515, 0.004243034728437343, 0.9971662518964693, 0.9975459489988107, 0.9874874833619691, 0.009835532702808457, 0.9892199849249659, 0.007977580523588435, 0.991313693222755, 0.006344407636625632, 0.9900274277378367, 0.0076509074786540706, 0.8725191647312287, 0.03130113595448354, 0.03130113595448354, 0.058689629914656635, 0.9981793398437514, 0.9984689337433391, 0.9985771181094137, 0.9982271578598306, 0.9984393157125784, 0.02338428131381013, 0.12591536092051608, 0.06295768046025804, 0.01259153609205161, 0.5846070328452533, 0.18887304138077413, 0.008242073859931698, 0.2829778691909883, 0.02197886362648453, 0.4478193463896223, 0.23902014193801924, 0.9961817261311257, 0.9906678153363283, 0.9968789056002924, 0.9949325155468375, 0.9956634471106142, 0.9701993693221359, 0.04647220215504853, 0.02904512634690533, 0.0174270758081432, 0.9003989167540651, 0.026513512285101893, 0.9544864422636682, 0.9762089938483164, 0.976208993844824, 0.04448654526553883, 0.14714780357062845, 0.059885734011302275, 0.011977146802260455, 0.5560818158192354, 0.1813682230056583, 0.9964043042050934, 0.9949079478375019, 0.9941023213805131, 0.02448066392470759, 0.09092818029177105, 0.8673149504753547, 0.013988950814118623, 0.9940346866862501, 0.9772306156762265, 0.9852242145779814, 0.883109169286617, 0.9949876320537925, 0.9815445618686958, 0.9826587514505515, 0.015077697504345847, 0.9649726402781342, 0.9782022595666127, 0.019642615653948044, 0.5617837287907874, 0.21099450710380424, 0.0026048704580716574, 0.03299502580224099, 0.09898507740672298, 0.09203875618519856, 0.5741905150375101, 0.20246023940628965, 0.0016595101590679482, 0.02904142778368909, 0.1020598747826788, 0.08961354858966919, 0.9808307746954334, 0.9648734434209781, 0.8193474086927369, 0.17676882869755842, 0.9099058459316619, 0.07779395775185895, 0.004409374123587147, 0.0075589270690065375, 0.9022051182302468, 0.048568954162476585, 0.007358932448860089, 0.017661437877264215, 0.023548583836352284, 0.914415519184826, 0.040924890369111094, 0.008952319768243053, 0.01406793106438194, 0.020462445184555547, 0.997417173780205, 0.9382363529708669, 0.03269611533080294, 0.027009834403706773, 0.9939511179764222, 0.992069428351171, 0.9951726772346281, 0.9954383698524444, 0.9921901928729374, 0.0037869854689806773, 0.9974931872349216, 0.9985366488223468, 0.9986167264417635, 0.9900939426884708, 0.009693227410936077, 0.9901071867000717, 0.009664072005903558, 0.9954965933222772, 0.9995883131225299, 0.9926560665221921, 0.0026260742500587094, 0.9271141713205812, 0.0561887376557928, 0.9979891228022242, 0.996636467533184, 0.2992351851766311, 0.46843623229221315, 0.07833381810906574, 0.11906740352577994, 0.034466879967988924, 0.9732077207127295, 0.9977374888046494, 0.9836320630545804, 0.9994886702556034, 0.9978485029762805, 0.9994433484365695, 0.9998581537815983, 0.9677899274720079, 0.03658423599371042, 0.9328980178396157, 0.03321967467645104, 0.9467607282788547, 0.034687971559817184, 0.936575232115064, 0.9733988682363606, 0.9983099698215367, 0.9588488331548382, 0.002773949631838926, 0.14147143122378525, 0.8488285873427115, 0.002773949631838926, 0.0027541828995189988, 0.14597169367450694, 0.8482883330518516, 0.003453459053856567, 0.7321333194175922, 0.255555969985386, 0.003453459053856567, 0.003453459053856567, 0.0050324450292850076, 0.9863592257398616, 0.12116138667325742, 0.007816863656339188, 0.8715802976818195, 0.833612707079658, 0.11729224019211268, 0.02932306004802817, 0.013963361927632463, 0.005585344771052985, 0.9974053248990283, 0.9960430489191197, 0.9975329200097952, 0.7730899478463638, 0.07817763517547499, 0.1274005906563296, 0.01737280781677222, 0.9976286979545609, 0.9973457909666639, 0.9975836563147692, 0.9972002066281871, 0.9956985811842645, 0.996873628824558, 0.9973430842667876, 0.9997276667472573, 0.999771511960018, 0.997366782273647, 0.9997566294855389, 0.9996811245477454, 0.9869149150764687, 0.011938486875925023, 0.0009363519118372568, 0.9928712781907877, 0.9650734509449982, 0.0339583166620828, 0.9627460877555454, 0.036309281023923425, 0.9962638373041132, 0.9937815581097731, 0.8458934217718856, 0.15182702442059487, 0.2522880704231931, 0.14648984734249923, 0.018989424655509162, 0.4394695420274977, 0.037978849311018324, 0.10037267317911985, 0.9530786188988741, 0.021377464349133627, 0.012470187536994614, 0.010688732174566814, 0.17777716876605432, 0.784311038673769, 0.031372441546950756, 0.9800823658577845, 0.06610985438512665, 0.16055250350673617, 0.7177641333242323, 0.028332794736482852, 0.023610662280402376, 0.96822931371038, 0.011949902875468672, 0.9297024437114626, 0.05496955322715589, 0.09848453406345953, 0.1477268010951893, 0.693868308174374, 0.02685941838094351, 0.02685941838094351, 0.08481667580327194, 0.04780576272548055, 0.0570584909949284, 0.0185054565388957, 0.001542121378241308, 0.7263391691516561, 0.06322697650789363, 0.9864390812737206, 0.03356010857274778, 0.9564630943233116, 0.9887893469380538, 0.9860346025991333, 0.9922327961730079, 0.2189138965720217, 0.5321124911725874, 0.08344737641606768, 0.01950717890245738, 0.006502392967485793, 0.13655025231720166, 0.0032511964837428965, 0.25696140062180733, 0.08871286450038586, 0.5139228012436147, 0.13765789319025393, 0.9983257228926833, 0.0396952487443933, 0.1257016210239121, 0.826984348841527, 0.12675775056786137, 0.8682905913898504, 0.9815626373286745, 0.9986324258775992, 0.9919638347868572, 0.9815626373286744, 0.9989115200417295, 0.000798490423694428, 0.9992617713695736, 0.9989685361878152, 0.9990165258573285, 0.9993661272630721, 0.9945674647509983, 0.004644411109577711, 0.9110879322715767, 0.08214727258186347, 0.907302803639978, 0.037804283484999084, 0.05114697177382229, 0.8967112841138107, 0.005639693610778684, 0.08459540416168025, 0.9070027660682782, 0.08192283048358642, 0.9970850666059978, 0.99718300875549, 0.9699663651156178, 0.00810061750812822, 0.024301852524384656, 0.01620123501625644, 0.9477722484510016, 0.2165773038409416, 0.7807478113772216, 0.2603480872060234, 0.7364131609541806, 0.05914978565040217, 0.009339439839537186, 0.24438200913455635, 0.026461746212022026, 0.5946110031172008, 0.06226293226358124, 0.003113146613179062, 0.024501900502331555, 0.9555741195909306, 0.05204006952293331, 0.9193745615718217, 0.023128919787970357, 0.26970054865971893, 0.7284219946706939, 0.14861083356787863, 0.8501922106441429, 0.13341499306316126, 0.8638620800839691, 0.13584126368418564, 0.8635623191351802, 0.9974250641302335, 0.996862095940948, 0.21973224546793665, 0.14004912348505852, 0.14004912348505852, 0.1376344834249713, 0.357366728892908, 0.10802655370220414, 0.01270900631790637, 0.004236335439302123, 0.19487143020789768, 0.12497189545941263, 0.12073556002011052, 0.11861739230045945, 0.3156069902280082, 0.9816359734000745, 0.009719168053466084, 0.9760087157221781, 0.014353069348855559, 0.013234425112827691, 0.03970327533848307, 0.9264097578979384, 0.016552516711015475, 0.7291819203747344, 0.02439318252149649, 0.02874910797176372, 0.010454221080641353, 0.19078953472170468, 0.7647711722615048, 0.22751257095951577, 0.00274111531276525, 0.00274111531276525, 0.001370557656382625, 0.20032918831747848, 0.7946391136593314, 0.1978987371658493, 0.7915949486633972, 0.9954043788290143, 0.020265247052275635, 0.9727318585092305, 0.9943089200558569, 0.9908776504915112, 0.029787597525174243, 0.9631323199806339, 0.9935864083198884, 0.16792386827641298, 0.07346669237093068, 0.015742862650913715, 0.7399145445929447, 0.991132098680197, 0.6840320643543097, 0.10882328296545837, 0.15027977171420442, 0.01295515273398314, 0.044047519295542674, 0.9977053866087444, 0.07393915555651483, 0.9248660474696261, 0.8300457181290294, 0.03557338791981555, 0.04150228590645147, 0.08300457181290294, 0.02588071879838698, 0.05176143759677396, 0.7602461147026175, 0.058231617296370705, 0.10352287519354791, 0.029414152097134202, 0.9608623018397173, 0.9695320237640677, 0.027365823251405137, 0.9684903049422602, 0.027841610409017746, 0.9725617777229911, 0.025214564607633105, 0.9990018527441094, 0.9883998532557435, 0.929503654963365, 0.04979483865875169, 0.8045896135985552, 0.12405056002167926, 0.008115457197679952, 0.0046374041129599725, 0.023187020564799862, 0.0336211798189598, 0.0018703581877791298, 0.9333087357017859, 0.06359217838449041, 0.08756125920570017, 0.8950706496582684, 0.9591636622080115, 0.01598606103680019, 0.08286626485655009, 0.022599890415422753, 0.8889290230066282, 0.1540281425635081, 0.15891792486711154, 0.007334673455405148, 0.6784572946249762, 0.09141256339681779, 0.05762965953277643, 0.07551472628432773, 0.7710450999557674, 0.9861225591789482, 0.9839917421997691, 0.9770077847920314, 0.9820579436549697, 0.9795842784295733, 0.9802291654812059, 0.9835437208285682, 0.11346306639959779, 0.8406581737788381, 0.041259296872581015, 0.8548978149814466, 0.043025491782436046, 0.056120206672742666, 0.043025491782436046, 0.0018706735557580889, 0.1081667513446855, 0.044758655728835375, 0.8429546828930662, 0.9977032154233613, 0.9964751897126315, 0.048982907447083, 0.06465743783014956, 0.04506427485131636, 0.00783726519153328, 0.8287907940046444, 0.9633538330337955, 0.9600186968494187, 0.007966980171838403, 0.31690876683534985, 0.346121027465424, 0.3133678867589772, 0.00442610009546578, 0.009737420210024716, 0.9677218330740015, 0.9693156365720073, 0.9666465552704189, 0.848645330434517, 0.14435719538319103, 0.06718173877513071, 0.8503793776536283, 0.045966452846142064, 0.03182292889348297, 0.0035358809881647743, 0.997339607509577, 0.9982562886084013, 0.9972603284585471, 0.9987707482885633, 0.9682962965103004, 0.03137337043185067, 0.8759530367629382, 0.0106823541068651, 0.00534117705343255, 0.053411770534325505, 0.053411770534325505, 0.09251808522247788, 0.8326627670023009, 0.06167872348165192, 0.12070855163025998, 0.8708259796183041, 0.09990561540947365, 0.8900682100116744, 0.9794190589860808, 0.9950443015040207, 0.9946379399280397, 0.9961872241552725, 0.18536408380566277, 0.04527977619680312, 0.008489958036900586, 0.7598512443026024, 0.9962393861731262, 0.9131758052588272, 0.00767374626267922, 0.01534749252535844, 0.05371622383875454, 0.9892357788723326, 0.9863318765854645, 0.9824571711151996, 0.9743541170004214, 0.9986347272364603, 0.9658724996628545, 0.01991489690026504, 0.20523332670661396, 0.725917877795616, 0.060809874579737466, 0.007601234322467183, 0.08820495957736231, 0.8996905876890955, 0.3326034432681522, 0.108852035978668, 0.060473353321482216, 0.010078892220247037, 0.48580260501590716, 0.019069181279690917, 0.009534590639845459, 0.9534590639845458, 0.9982393124341417, 0.9877277917928124, 0.9813550537553554, 0.021025085495885342, 0.042050170991770684, 0.8883098622011556, 0.010512542747942671, 0.006570339217464169, 0.030223560400335178, 0.9984796748024577, 0.9988902167570911, 0.020649155869866976, 0.970510325883748, 0.015071366261571618, 0.015071366261571618, 0.04144625721932195, 0.06405330661167938, 0.02449097017505388, 0.8345769067345283, 0.0037678415653929044, 0.012522425511617793, 0.9767491899061879, 0.9973802801753263, 0.9926482680324472, 0.9926952230116078, 0.9973768172797106, 0.9981543840882005, 0.2760359837317254, 0.4747115643794003, 0.08439316700078228, 0.16351176106401566, 0.10761818646997393, 0.050874051785805854, 0.07435438337925471, 0.7631107767870878, 0.997227367416887, 0.9983220082337387, 0.9974906021856488, 0.9965239744655712, 0.0015007891181710409, 0.9921932028717233, 0.003688450568296369, 0.9895771187288286, 0.0036115953238278417, 0.9995523453052124, 0.9995642521518253, 0.999491570824548, 0.9992090116518549, 0.9968750728662275, 0.9959495153221942, 0.9954460715748662, 0.9956419158489623, 0.996389304729929, 0.9977179550997282, 0.11677064651590789, 0.031438250985052124, 0.4558546392832558, 0.01347353613645091, 0.3682766543963249, 0.01347353613645091, 0.1454690479375945, 0.045341001954574904, 0.4099582260059481, 0.017002875732965588, 0.36272801563659923, 0.017002875732965588, 0.14165651095300474, 0.8566846138586477, 0.1452196907108683, 0.8533696431661137, 0.9983441867023091, 0.9707061923524268, 0.9808268705421278, 0.9907530363855948, 0.08907828270395558, 0.16495978278510293, 0.7423190225329632, 0.9930475627111262, 0.9936336873786273, 0.9385001333351574, 0.03174338686280679, 0.027602945098092864, 0.9972031552225991, 0.002725729851727693, 0.14173795228984, 0.8531534435907678, 0.9977608356970832, 0.0805672971508764, 0.01193589587420391, 0.6415544032384602, 0.2566217612953841, 0.0029839739685509777, 0.0029839739685509777, 0.07168015021582352, 0.015928922270183006, 0.9119307999679771, 0.9989683839291934, 0.8918367204085348, 0.05246098355344322, 0.9030190928003734, 0.001595440093286879, 0.05743584335832764, 0.006381760373147516, 0.02871792167916382, 0.07067417855199229, 0.17940368401659582, 0.6007305176919345, 0.12775716892090916, 0.021745901092920707, 0.07055410819254947, 0.17909889002724097, 0.5997099196366705, 0.1275401186557625, 0.0217089563669383, 0.07813736533262816, 0.6826738234324355, 0.05963114722753202, 0.02056246456121794, 0.002056246456121794, 0.1583309771213781, 0.020026779471210943, 0.9712988043537307, 0.9715301266055982, 0.025760268508481773, 0.9678194808316823, 0.015362213981455276, 0.02677176135891252, 0.9637834089208507, 0.9632389282094841, 0.035624768326382054, 0.9589559362031571, 0.039956497341798215, 0.9597233539316445, 0.03914483440790111, 0.9953382308809665, 0.9949002442034999, 0.9883585308968111, 0.20499040874088673, 0.791775453761675, 0.25336149022315924, 0.7457432542417517, 0.9634155819383348, 0.03649301446736117, 0.9583665374659042, 0.04012864812049352, 0.9587654395531671, 0.04137705349489624, 0.9636464842423078, 0.03526923377982473, 0.951021213672124, 0.04829853805831827, 0.05219716313522505, 0.9221498820556426, 0.023198739171211135, 0.012056159881997998, 0.029470613044883992, 0.014735306522441996, 0.3764201029823819, 0.2987248504095059, 0.22906703775796194, 0.006697866601109998, 0.03214975968532799, 0.015197989839037092, 0.045593969517111275, 0.3647517561368902, 0.3081047031004792, 0.22520657670573144, 0.004144906319737389, 0.035922521437724036, 0.1314761273137769, 0.042072360740408606, 0.08940376657336829, 0.7257482227720484, 0.9887248454376101, 0.010798654930511251, 0.988481735893912, 0.010934532476702567, 0.9883658041612352, 0.010947782500678281, 0.03046305204584292, 0.7789837594579833, 0.16972271854112486, 0.01958339060089902, 0.8788896113590853, 0.04634446161592164, 0.014896434090831953, 0.014896434090831953, 0.04468930227249586, 0.2563899177166913, 0.7404157436865197, 0.9312534959489518, 0.001855086645316637, 0.0649280325860823, 0.9600226099419303, 0.015484235644224683, 0.6874740968576061, 0.10628711365890621, 0.13794710496155912, 0.024875707452084433, 0.04070570310341089, 0.6945851329694046, 0.10757884214340946, 0.14032022888270798, 0.014032022888270799, 0.039757398183433934, 0.18838825168260268, 0.7990260329986252, 0.8728671592826799, 0.12069706210522026, 0.005247698352400881, 0.9796354006741946, 0.9974902017184262]}, \"R\": 30, \"plot.opts\": {\"ylab\": \"PC2\", \"xlab\": \"PC1\"}, \"topic.order\": [9, 1, 2, 8, 3, 10, 5, 7, 6, 4], \"lambda.step\": 0.01, \"mdsDat\": {\"x\": [0.17294473869461185, 0.21147974938214453, 0.17214403407624987, 0.13093755931815726, -0.2024207022532354, -0.1710857601478677, -0.2315194275930342, 0.055293865749166754, -0.19422940532562058, 0.056455348099427705], \"y\": [0.08328860915779394, -0.05018137950643835, 0.14801130540955776, 0.20197842109070147, 0.05933636829820431, 0.09702993535986285, 0.01156367135268211, -0.19413850251726272, -0.025744234378983766, -0.33114419426611746], \"topics\": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], \"cluster\": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1], \"Freq\": [30.825089297844315, 20.025140819494297, 15.85446167886285, 8.830543988695451, 7.407636512868406, 4.588132316103055, 3.6098616883543952, 3.478686355349965, 3.3310678516724144, 2.049379490754853]}, \"tinfo\": {\"Freq\": [3175.0, 4307.0, 4269.0, 4265.0, 4256.0, 4233.0, 4176.0, 4100.0, 4087.0, 4271.0, 1673.0, 4345.0, 1894.0, 4332.0, 1889.0, 2804.0, 2315.0, 2286.0, 2283.0, 2315.0, 2479.0, 1105.0, 1508.0, 856.0, 1165.0, 1162.0, 1155.0, 1163.0, 1163.0, 797.0, 691.6636815090703, 663.4369729063582, 642.6632892060237, 521.5915406420856, 480.2677336287515, 465.4569273644672, 465.26269002320834, 457.7490923353397, 448.258797107964, 446.24891297418145, 443.685487885352, 441.697325288231, 436.7571180821097, 425.59771855330547, 425.8638853784186, 422.0108524226144, 421.18091476164994, 419.65346268576707, 418.5219215680782, 417.6123036122555, 417.04584677726433, 408.0207500952868, 405.85584803003957, 396.90329975057574, 396.0054133002115, 379.2897137462193, 378.9405451311982, 380.5283498539865, 365.97797398342675, 362.7308677097043, 4303.000592834446, 4289.8254580339335, 1309.0880974905415, 1290.702345313424, 846.6914236181709, 791.6446448306906, 812.2708012982868, 810.5587381527665, 703.1064639658777, 826.8466996831283, 719.824012189749, 710.7585483880138, 497.8921783374513, 680.4181397565222, 660.3536913147917, 479.50668328592985, 534.7928553324272, 625.5166357458065, 597.1300457376411, 934.7271653215456, 508.14006763466665, 530.5817417064479, 558.032036479474, 4306.406521898056, 4268.751820735066, 4264.070990995002, 4256.005674991805, 4232.18583299911, 4175.92739708932, 4099.614658102092, 4086.180411713802, 2314.216832490791, 1529.6999849320896, 1493.9806236736467, 1465.1927355021446, 1413.491909454916, 1404.415178355279, 1354.7222953549287, 1203.5722470689166, 1128.6618165250748, 1035.7983352744036, 778.098433303675, 765.2288443334462, 743.9557589805763, 725.1944323104789, 703.2815803965833, 683.9610312495105, 671.9905855287489, 662.0481779637223, 652.3495670086753, 1250.5087617135832, 642.4071573154225, 631.1839881745108, 4216.458327659976, 1499.400353008341, 2413.496125494359, 2715.515471656894, 692.1016869153907, 686.1949324786173, 633.7623424497502, 623.539835495482, 592.4038159463378, 576.353644567194, 553.1484182046455, 530.7813475112021, 508.7959452929929, 488.27523147126385, 536.646442256289, 442.81314445565573, 423.0732029473711, 474.0770570561737, 512.6860475358333, 392.93820850638514, 494.8552730432884, 491.8548122914464, 473.8354287758282, 664.0699397523449, 317.1171419230118, 310.08305415869427, 294.40169517537765, 269.44663416354007, 269.4417327439714, 264.83643169399323, 260.4809087891347, 260.4631287302312, 244.67605198629192, 242.7187572019399, 235.123390011912, 232.7614452899055, 2288.6610340934562, 2260.014685197478, 2257.242174809225, 906.8849122406592, 887.3798944352307, 934.5892921457768, 867.7230814684833, 939.0168161256951, 881.1294958906667, 875.2032988098359, 681.6980695089524, 671.0000086701482, 727.9705674810505, 715.1932436707683, 962.4351873675491, 659.3630086986132, 613.4233049013476, 694.388559768678, 498.83724481086335, 1893.3920897985486, 1888.5634948582554, 1164.3978536536713, 1161.3522812154872, 1154.4766753595486, 615.3310148305678, 406.4756447017583, 398.9083499893501, 396.12190518639966, 378.73406553390504, 366.4411754935125, 301.7361147204683, 292.60726304364044, 287.2526117135994, 283.51865739956224, 255.76523717410905, 240.8898221738999, 239.0738709550992, 230.09538978074153, 218.87614907525034, 217.84290640386112, 205.84141703910842, 202.04946280722552, 378.4860731830316, 300.53065209493457, 195.3442865353806, 195.05853364288774, 188.54674080342764, 186.66132992926836, 186.3218317354643, 638.4558343537717, 624.8766366504335, 620.388337356651, 647.1044547828488, 587.7307994657243, 495.97674155505166, 502.3114044098826, 288.6775540516246, 276.3089290452194, 295.55309283026673, 274.2566165744564, 261.93145142570677, 268.617650386895, 774.3715849385474, 782.0302382124491, 533.5055723910078, 518.291629795174, 523.1034826733561, 508.3086821589, 491.59613669708745, 353.4949842137139, 414.17348017768643, 127.45554470835418, 125.69561881155663, 117.08771630899791, 115.33832094423325, 109.95687265207889, 151.38926633837218, 85.0397442294977, 81.42501076482588, 80.71613872937574, 78.27794981877759, 75.96056273188654, 75.2601803523714, 70.86488448547523, 70.16732338141846, 71.06447424839827, 65.10177405907069, 65.0755289925997, 64.87283523198194, 100.81325407134614, 62.168797272503596, 61.570742051305174, 60.36158347997023, 60.07003011692191, 59.80595379774217, 58.087973729505414, 57.37118270117011, 55.173744349605926, 55.049961769266304, 55.04996176922423, 53.85522413072535, 94.51523666350549, 84.34583664661288, 2889.0399162529425, 144.11389741353784, 114.43465407072817, 566.0305792941731, 859.8904517690347, 961.1914773686495, 67.71643427980665, 456.70062047751657, 228.23692283498852, 223.14459183765715, 152.89220632038908, 152.33657311663532, 345.02404975943807, 343.1514215631564, 837.2290760988957, 119.4343684173579, 163.75576166743252, 309.0472965550021, 292.25558559430715, 537.9377794664956, 266.7794598759335, 311.6991337589659, 309.4054914976964, 315.66722408037896, 296.94098053868436, 331.82285199004957, 235.06642871681277, 297.0225041507391, 303.95805803942244, 490.67480900041147, 546.4269724601108, 263.8096217816977, 211.5218966327905, 234.93226841095574, 189.71559521872754, 298.92055535022314, 215.10012907057748, 358.0738091504556, 281.47178929104115, 263.93539971612586, 243.58883028391548, 242.7208148929706, 241.54097995973405, 1672.3389322323567, 1104.4614097497472, 306.5106085705084, 242.22023989665036, 239.34299546551418, 237.11205976489026, 236.41762215063534, 234.39167335300615, 232.3355173943453, 205.33051302121373, 147.39507660196247, 133.92040849755728, 117.54480543381152, 114.9873936469042, 113.08376460884747, 118.14885278704666, 107.3407281482824, 102.50078183188879, 93.41670446325557, 93.41670446325557, 59.668260584611645, 55.32938057076783, 54.74476479026794, 53.31074485504874, 52.03841602644733, 97.23406148493532, 39.89841249560525, 39.76288540267087, 38.9677840068074, 38.30066677347279, 52.84157027492127, 78.04946017845229, 407.6346374065496, 675.9949167112001, 681.4677946912326, 154.86701560201917, 159.38194659991592, 248.21336523987915, 99.33464735422032, 121.50911960980692, 357.9865552455374, 572.6769701593956, 267.9273352393015, 236.1609542058694, 137.1371082949448, 245.02285702303055, 291.04209854672325, 140.1923903575676, 221.34425230087157, 220.62231989422412, 391.46320734640165, 269.527738297519, 372.4072424335184, 370.5535796638509, 222.7719311699504, 222.72522241454806, 855.1776287834344, 552.942649899699, 109.50764509026529, 102.4093200029026, 87.20473909201031, 84.0441027532367, 83.23011787489177, 74.34747153080666, 72.76437041920009, 68.25396394805257, 64.42617982549815, 42.56342244900154, 62.11778014581888, 62.543636835370236, 60.090731777542445, 27.58828605706156, 57.23877512279743, 25.320642274008343, 54.18523339733431, 22.095369430298096, 20.424443055611512, 51.3163730555377, 54.069343931962564, 498.6773076788839, 501.9527838219618, 55.97892062744964, 158.60526230536607, 159.0206741466719, 13.390283668355048, 34.446383361037256, 98.06217939238742, 313.048770750475, 308.3999839547685, 305.8700806615239, 194.27495932738523, 194.68961919567587, 162.58416493345916, 101.03322547246883, 135.2076207983681, 224.6821339501446, 150.06512206644715, 152.0950912041201, 152.09505049423436, 381.81928914234146, 325.30568057217556, 323.6523159257936, 324.61762075229984, 151.7271356755643, 154.95418414003092, 354.238555487645, 202.6396271403145, 336.8794779352721, 334.47972185622285, 176.38867453630732, 216.8877647751315, 167.63323332357433, 162.60776940961424, 246.8861754112587, 162.60299724688684, 162.22950028152334, 170.7716595972647, 163.44086555538928, 405.68447329654725, 398.6320472689818, 397.770250653076, 395.77437397010215, 387.12188442610375, 386.93316061103303, 383.37404638104033, 381.79199306967547, 381.6447484520633, 376.35704632170234, 373.4258802549075, 373.2364678445261, 371.34342381152965, 370.63704070478894, 370.1893831349739, 369.704166268649, 367.9907793829669, 362.85593313334545, 361.4753895769388, 358.84486666198075, 355.08010525019256, 346.4817145661872, 345.97037263491643, 307.5207368059445, 187.48098033928673, 180.55332532234456, 173.71427875035423, 170.23780223960932, 169.27499475380608, 162.7692695255781, 539.9419418909312, 527.9126902559278, 496.18240247921295, 487.1527250773333, 389.4422314088509, 283.8277115541005, 470.8748051176513, 719.2678876223407, 196.42321385106152, 105.81260472032956, 102.07271371847408, 101.46272396023176, 93.42618737485903, 86.74027785777125, 156.4155345309085, 62.939305538575304, 50.32694706393977, 50.241986428025946, 49.349937272793404, 47.59739131892189, 45.371085402620764, 95.73163359878649, 96.66985688131042, 93.74040028593569, 47.117831953589494, 38.24191466233365, 97.94794846818716, 73.32794390252003, 96.50365284350067, 72.25537608831951, 100.42041165841742, 116.59481754700342, 39.68744600373595, 39.049613691097036, 38.15979872914319, 70.1414754381683, 738.343589327038, 228.58181728961083, 236.32688904780466, 154.93707114728423, 443.43350906821564, 222.6430155346958, 423.4880676726167, 225.80782316083685, 102.49603844696739, 536.9276959397645, 388.38648526207925, 389.9922526529772, 554.7304500768734, 125.02908977565964, 92.09383864646298, 123.22507719396677, 140.58746294396283, 138.4358097348906, 240.72802197748518, 120.26741651363328, 340.3784111167091, 118.54916891200331, 192.45346420374653, 286.65879595842483, 169.41240165015898, 137.78830870159985, 163.52676453484028, 219.0992686931591, 147.95880715123766, 148.6428161413184, 126.239548253077, 318.1804896698236, 272.94372397835446, 268.75824987763303, 260.76140965225864, 253.7261731028497, 252.7158667372921, 248.0749275843794, 247.4324763152274, 238.18752840938683, 236.84672604148665, 235.3843970306875, 232.9863519572933, 226.95633496404457, 216.60966425359797, 213.9156848791283, 211.20873029984384, 199.00300353901304, 198.7197995492523, 198.71179275602134, 191.78254959115307, 176.804123507392, 162.95799425463204, 169.87253994272876, 158.92655499122984, 158.32638272030633, 145.5539454174962, 145.36266027449477, 140.62665441331478, 134.32162200123335, 133.33360286650412, 150.7480575116205, 151.06003629084057, 184.40270476125352, 285.0377518130463, 188.7419001058195, 159.96275624528255, 158.63242411864945, 356.1937395087258, 353.88620992268164, 162.14174992792067], \"Term\": [\"father\", \"grievances\", \"benefited\", \"grief\", \"gray\", \"grave\", \"benefit\", \"fs\", \"beneficiaries\", \"grim\", \"loretta\", \"foe\", \"convoy\", \"focusing\", \"forest\", \"needless\", \"relies\", \"relieved\", \"religion\", \"frontline\", \"conclusions\", \"cretz\", \"1990s\", \"pal\", \"flexibility\", \"contemplating\", \"flesh\", \"134\", \"540\", \"lithwick\", \"profoundly\", \"product\", \"passive\", \"nctc\", \"nc\", \"abandon\", \"jr\", \"nbc\", \"journey\", \"constitutional\", \"powers\", \"funders\", \"pete\", \"passionate\", \"frozen\", \"rumsfeld\", \"diplomatic\", \"iranian\", \"oral\", \"att00001\", \"peshawar\", \"persian\", \"freedoms\", \"bauer\", \"na\\u00efve\", \"morale\", \"bbc\", \"courage\", \"colonialism\", \"creative\", \"foe\", \"focusing\", \"dare\", \"danny\", \"red\", \"recruited\", \"recruitment\", \"recurring\", \"reassure\", \"redactions\", \"rebellion\", \"rebuilding\", \"exploit\", \"preparation\", \"fighting\", \"bildt\", \"identifies\", \"conducted\", \"goldsmith\", \"1990s\", \"beneficial\", \"researchers\", \"leaking\", \"grievances\", \"benefited\", \"grief\", \"gray\", \"grave\", \"benefit\", \"fs\", \"beneficiaries\", \"frontline\", \"polarizing\", \"jill\", \"policing\", \"jihad\", \"jihadist\", \"policies\", \"pockets\", \"frustrated\", \"jigsaw\", \"jerusalem\", \"ec\", \"bigger\", \"big\", \"ebeling\", \"biden\", \"eating\", \"eat\", \"bid\", \"jibril\", \"bible\", \"eastern\", \"grim\", \"jiloty\", \"conclusions\", \"needless\", \"expressed\", \"belarus\", \"affluent\", \"currency\", \"placing\", \"affirms\", \"curious\", \"affinity\", \"affiliates\", \"curbing\", \"pillars\", \"forms\", \"broader\", \"pieces\", \"belonging\", \"fein\", \"drone\", \"bathsheba\", \"dread\", \"plagued\", \"nye\", \"hi\", \"experiment\", \"bureaucrats\", \"daalder\", \"damage\", \"expensive\", \"expense\", \"brooks\", \"brownback\", \"expenditures\", \"expecting\", \"relies\", \"relieved\", \"religion\", \"chat\", \"charset\", \"acknowledgement\", \"achilles\", \"children\", \"henrymcdonald\", \"hensman\", \"dove\", \"barred\", \"congressional\", \"fe80\", \"nationalist\", \"congratulations\", \"fbi\", \"managing\", \"rossaj\", \"convoy\", \"forest\", \"flexibility\", \"contemplating\", \"flesh\", \"fiscal\", \"cope\", \"carter\", \"forwarding\", \"cookstoves\", \"foremost\", \"ladies\", \"principally\", \"connecting\", \"lacking\", \"nuni\", \"finding\", \"numerous\", \"consulate\", \"numbers\", \"finalized\", \"reception\", \"honorable\", \"forge\", \"articulate\", \"filibuster\", \"discrimination\", \"consular\", \"receiving\", \"averse\", \"area\", \"dubious\", \"ardent\", \"duguid\", \"arctic\", \"dual\", \"drug\", \"dennis\", \"article\", \"denouncing\", \"plotted\", \"finger\", \"plot\", \"concentrate\", \"agents\", \"knows\", \"knowledge\", \"pounds\", \"posturing\", \"knight\", \"deep\", \"demands\", \"improvement\", \"incapable\", \"recess\", \"inauguration\", \"inaugural\", \"consultative\", \"ashley\", \"arturo\", \"deaths\", \"dealt\", \"micheletti\", \"mi\", \"mexicans\", \"aftermath\", \"miami\", \"corley\", \"notify\", \"compensation\", \"lang\", \"mexico\", \"methods\", \"frankly\", \"circle\", \"mfa\", \"compounds\", \"charter\", \"corker\", \"estate\", \"esther\", \"adoption\", \"compete\", \"dear\", \"father\", \"really\", \"inappropriate\", \"protective\", \"depth\", \"delay\", \"lastly\", \"missed\", \"ben\", \"ear\", \"cutting\", \"cuts\", \"amman\", \"fatal\", \"ld\", \"oas\", \"negotiators\", \"recommended\", \"kenyan\", \"command\", \"govi\", \"recommends\", \"resignation\", \"knife\", \"kerry\", \"questionable\", \"lobbied\", \"revelations\", \"reveals\", \"increasing\", \"code\", \"lining\", \"garrison\", \"ambiguous\", \"12958\", \"founder\", \"productive\", \"multiply\", \"reg\", \"regain\", \"expressed\", \"exposed\", \"belarus\", \"loretta\", \"cretz\", \"circulate\", \"premise\", \"premier\", \"fights\", \"persons\", \"perspective\", \"headlines\", \"jews\", \"los\", \"predicted\", \"diary\", \"assistant\", \"determine\", \"casteel\", \"assignment\", \"delaying\", \"jeopardy\", \"jfk\", \"frequently\", \"practically\", \"praise\", \"julia\", \"optimistic\", \"oterom2\", \"endorsing\", \"fulfilling\", \"fsn\", \"fwd\", \"commenting\", \"ki\", \"job\", \"passage\", \"capable\", \"jointly\", \"joint\", \"expenses\", \"formidable\", \"joanne\", \"remarkable\", \"overlap\", \"anwar\", \"branch\", \"jeopardize\", \"aung\", \"brilliant\", \"liz\", \"ps\", \"provoking\", \"multiply\", \"photographers\", \"540\", \"134\", \"regain\", \"reg\", \"pal\", \"moments\", \"consensus\", \"conquer\", \"experts\", \"business\", \"expertise\", \"bus\", \"experienced\", \"experience\", \"expired\", \"roy\", \"retaliation\", \"reaction\", \"melissa\", \"illustration\", \"fueled\", \"daughters\", \"crew\", \"blessing\", \"blesses\", \"fuel\", \"fuels\", \"marching\", \"respectfully\", \"manage\", \"redefining\", \"klevorick\", \"experiencing\", \"promised\", \"normalization\", \"proceeded\", \"gao\", \"gangs\", \"narcotics\", \"hoping\", \"milestone\", \"normal\", \"nick\", \"preferred\", \"identity\", \"broadcasting\", \"cemetery\", \"khan\", \"edwidge\", \"appear\", \"everybody\", \"ii\", \"immigrants\", \"multiply\", \"portugal\", \"134\", \"540\", \"consul\", \"post\", \"indian\", \"emailing\", \"father\", \"attackers\", \"household\", \"reg\", \"regain\", \"grab\", \"polling\", \"greater\", \"gration\", \"polled\", \"graduation\", \"graduate\", \"gradually\", \"grade\", \"emerged\", \"politico\", \"politically\", \"politics\", \"governors\", \"governor\", \"gracious\", \"attitude\", \"embroiled\", \"attendees\", \"polish\", \"embodies\", \"emerge\", \"attending\", \"gove\", \"embrace\", \"life\", \"lifg\", \"limit\", \"lift\", \"likes\", \"lona\", \"rain\", \"locate\", \"located\", \"image001\", \"arcane\", \"impartial\", \"literature\", \"gates\", \"olson\", \"omar\", \"operation\", \"parliamentary\", \"parker\", \"peacemaking\", \"ideologically\", \"murdered\", \"murders\", \"murray\", \"faced\", \"mud\", \"lifetime\", \"rage\", \"patrol\", \"faces\", \"multi\", \"lobby\", \"appeared\", \"likely\", \"epic\", \"pace\", \"karl\", \"celebrities\", \"address\", \"cdm\", \"lautenberg\", \"lithwick\", \"profit\", \"mep\", \"enriched\", \"peacekeeping\", \"gathered\", \"motorcade\", \"moderation\", \"owe\", \"nutrition\", \"messy\", \"physicians\", \"mepp\", \"islamist\", \"max\", \"ring\", \"limage\", \"releases\", \"ownership\", \"lib\", \"code\", \"lengthy\", \"post\", \"1990s\", \"branches\", \"depend\", \"portugal\", \"ld\", \"landscape\", \"landslide\", \"increasing\", \"71\", \"7pm\", \"8000\", \"82\", \"1962\", \"1950s\", \"81\", \"1948\", \"15pm\", \"89\", \"73\", \"450\", \"47\", \"2001\", \"7283\", \"600\", \"343\", \"34\", \"356\", \"111\", \"7782\", \"76\", \"77\", \"819\", \"8132\", \"2008\", \"2005\", \"230\", \"830\", \"2030\", \"75\", \"7516\", \"79\", \"1980\", \"67\", \"550\", \"53\", \"134\", \"540\", \"1990s\"], \"logprob\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, -6.0024, -6.0441, -6.0759, -6.2846, -6.3672, -6.3985, -6.3989, -6.4152, -6.4362, -6.4407, -6.4464, -6.4509, -6.4622, -6.488, -6.4874, -6.4965, -6.4985, -6.5021, -6.5048, -6.507, -6.5083, -6.5302, -6.5355, -6.5578, -6.5601, -6.6032, -6.6042, -6.6, -6.639, -6.6479, -4.1745, -4.1775, -5.3644, -5.3786, -5.8002, -5.8674, -5.8417, -5.8438, -5.986, -5.8239, -5.9625, -5.9752, -6.3311, -6.0188, -6.0488, -6.3688, -6.2596, -6.103, -6.1494, -5.7013, -6.3108, -6.2676, -6.2171, -3.7423, -3.7511, -3.7522, -3.7541, -3.7597, -3.7731, -3.7915, -3.7948, -4.3634, -4.7774, -4.801, -4.8204, -4.8564, -4.8628, -4.8988, -5.0171, -5.0814, -5.1673, -5.4533, -5.47, -5.4982, -5.5237, -5.5544, -5.5823, -5.5999, -5.6149, -5.6296, -4.9789, -5.645, -5.6626, -3.7634, -4.7974, -4.3214, -4.2035, -5.5705, -5.579, -5.425, -5.4412, -5.4925, -5.5199, -5.561, -5.6023, -5.6446, -5.6858, -5.5913, -5.7835, -5.8291, -5.7153, -5.637, -5.903, -5.6724, -5.6785, -5.7158, -5.3783, -6.1174, -6.1398, -6.1917, -6.2803, -6.2803, -6.2975, -6.3141, -6.3142, -6.3767, -6.3847, -6.4165, -6.4266, -4.1409, -4.1535, -4.1548, -5.0666, -5.0884, -5.0365, -5.1108, -5.0318, -5.0954, -5.1022, -5.3521, -5.3679, -5.2864, -5.3041, -5.0072, -5.3854, -5.4576, -5.3336, -5.6644, -3.7453, -3.7478, -4.2315, -4.2341, -4.24, -4.8693, -5.2839, -5.3027, -5.3097, -5.3546, -5.3876, -5.5819, -5.6126, -5.6311, -5.6441, -5.7472, -5.8071, -5.8146, -5.8529, -5.9029, -5.9076, -5.9643, -5.9829, -5.3552, -5.5859, -6.0167, -6.0181, -6.0521, -6.0621, -6.0639, -4.8324, -4.8539, -4.8611, -4.8189, -4.9151, -5.0849, -5.0722, -5.6261, -5.6699, -5.6026, -5.6774, -5.7233, -5.6981, -4.6394, -4.6295, -5.0119, -5.0409, -5.0316, -5.0603, -5.0938, -5.4235, -5.2651, -6.2679, -6.2818, -6.3528, -6.3678, -6.4156, -6.0959, -6.6726, -6.716, -6.7248, -6.7554, -6.7855, -6.7948, -6.8549, -6.8648, -6.8521, -6.9398, -6.9402, -6.9433, -6.5024, -6.9859, -6.9955, -7.0154, -7.0202, -7.0246, -7.0538, -7.0662, -7.1052, -7.1075, -7.1075, -7.1294, -6.567, -6.6808, -3.147, -6.1451, -6.3757, -4.7771, -4.3589, -4.2475, -6.9004, -4.9917, -5.6853, -5.7079, -6.086, -6.0896, -5.2721, -5.2775, -4.3856, -6.3329, -6.0173, -5.3822, -5.4381, -4.828, -5.5293, -5.3737, -5.3811, -5.361, -5.4222, -5.3111, -5.6558, -5.4219, -5.3988, -4.9199, -4.8123, -5.5405, -5.7614, -5.6564, -5.8702, -5.4155, -5.7446, -5.235, -5.4757, -5.54, -5.6202, -5.6238, -5.6287, -3.2147, -3.6296, -4.9114, -5.1468, -5.1588, -5.1681, -5.1711, -5.1797, -5.1885, -5.3121, -5.6436, -5.7394, -5.8699, -5.8919, -5.9085, -5.8647, -5.9607, -6.0068, -6.0996, -6.0996, -6.5479, -6.6234, -6.634, -6.6605, -6.6847, -6.0596, -6.9503, -6.9537, -6.9739, -6.9912, -6.6694, -6.2793, -4.6263, -4.1205, -4.1124, -5.5941, -5.5654, -5.1224, -6.0382, -5.8367, -4.7562, -4.2864, -5.046, -5.1722, -5.7157, -5.1353, -4.9632, -5.6937, -5.237, -5.2402, -4.6668, -5.04, -4.7167, -4.7217, -5.2305, -5.2307, -3.6456, -4.0816, -5.7009, -5.7679, -5.9286, -5.9655, -5.9753, -6.0881, -6.1096, -6.1736, -6.2313, -6.6459, -6.2678, -6.261, -6.301, -7.0795, -6.3496, -7.1652, -6.4045, -7.3015, -7.3801, -6.4589, -6.4066, -4.1849, -4.1784, -6.3719, -5.3305, -5.3278, -7.8023, -6.8575, -5.8113, -4.6505, -4.6655, -4.6737, -5.1276, -5.1255, -5.3057, -5.7814, -5.4901, -4.9822, -5.3858, -5.3724, -5.3724, -4.4519, -4.6121, -4.6172, -4.6142, -5.3748, -5.3537, -4.5269, -5.0854, -4.5771, -4.5843, -5.2242, -5.0175, -5.2751, -5.3055, -4.8879, -5.3056, -5.3079, -5.2565, -5.3004, -4.3543, -4.3718, -4.374, -4.379, -4.4011, -4.4016, -4.4108, -4.415, -4.4154, -4.4293, -4.4371, -4.4376, -4.4427, -4.4446, -4.4458, -4.4472, -4.4518, -4.4658, -4.4697, -4.477, -4.4875, -4.512, -4.5135, -4.6313, -5.1262, -5.1638, -5.2024, -5.2227, -5.2283, -5.2675, -4.0684, -4.0909, -4.1529, -4.1713, -4.3951, -4.7115, -4.2053, -3.7383, -5.0362, -5.6548, -5.6908, -5.6968, -5.7793, -5.8536, -5.264, -6.1743, -6.398, -6.3996, -6.4176, -6.4537, -6.5016, -5.7549, -5.7452, -5.776, -6.4638, -6.6726, -5.7321, -6.0216, -5.7469, -6.0363, -5.7071, -5.5578, -6.6355, -6.6517, -6.6747, -6.066, -3.7121, -4.8846, -4.8513, -5.2735, -4.2219, -4.9109, -4.268, -4.8968, -5.6867, -4.0306, -4.3545, -4.3504, -3.998, -5.4879, -5.7937, -5.5025, -5.3707, -5.3861, -4.8328, -5.5268, -4.4864, -5.5412, -5.0566, -4.6582, -5.1842, -5.3908, -5.2195, -4.927, -5.3196, -5.3149, -5.4783, -4.0681, -4.2215, -4.2369, -4.2671, -4.2945, -4.2985, -4.317, -4.3196, -4.3577, -4.3633, -4.3695, -4.3798, -4.406, -4.4526, -4.4652, -4.4779, -4.5374, -4.5388, -4.5389, -4.5744, -4.6557, -4.7372, -4.6957, -4.7623, -4.7661, -4.8502, -4.8515, -4.8846, -4.9305, -4.9379, -4.8151, -4.8131, -4.6136, -4.1781, -4.5904, -4.7558, -4.7641, -3.9553, -3.9618, -4.7423], \"Category\": [\"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Default\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic1\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic2\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic3\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic4\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic5\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic6\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic7\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic8\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic9\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\", \"Topic10\"], \"loglift\": [30.0, 29.0, 28.0, 27.0, 26.0, 25.0, 24.0, 23.0, 22.0, 21.0, 20.0, 19.0, 18.0, 17.0, 16.0, 15.0, 14.0, 13.0, 12.0, 11.0, 10.0, 9.0, 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 1.1753, 1.1753, 1.1752, 1.1748, 1.1747, 1.1746, 1.1746, 1.1745, 1.1745, 1.1745, 1.1745, 1.1745, 1.1744, 1.1744, 1.1744, 1.1744, 1.1743, 1.1743, 1.1743, 1.1743, 1.1743, 1.1743, 1.1742, 1.1742, 1.1742, 1.1741, 1.1741, 1.174, 1.174, 1.1739, 1.1669, 1.1668, 1.12, 1.119, 1.1394, 1.1391, 1.1346, 1.1342, 1.1395, 1.1264, 1.1347, 1.1354, 1.1546, 1.114, 1.1136, 1.156, 1.1284, 1.0645, 0.9951, 0.698, 1.0808, 1.047, 0.9087, 1.608, 1.608, 1.608, 1.608, 1.608, 1.608, 1.6079, 1.6079, 1.6078, 1.6075, 1.6075, 1.6075, 1.6075, 1.6075, 1.6075, 1.6074, 1.6073, 1.6072, 1.6069, 1.6069, 1.6069, 1.6068, 1.6068, 1.6068, 1.6067, 1.6067, 1.6067, 1.6067, 1.6067, 1.6067, 1.5951, 1.603, 1.5812, 1.5758, 1.0535, 1.0445, 1.84, 1.84, 1.8399, 1.8398, 1.8398, 1.8397, 1.8396, 1.8395, 1.8394, 1.8393, 1.8392, 1.8391, 1.839, 1.839, 1.839, 1.8388, 1.8385, 1.8383, 1.8383, 1.8382, 1.8381, 1.8377, 1.8377, 1.8377, 1.8376, 1.8376, 1.8373, 1.8373, 1.8371, 1.8371, 1.8302, 1.8301, 1.8301, 1.8259, 1.8256, 1.8237, 1.8251, 1.8218, 1.8063, 1.804, 1.8052, 1.7965, 1.7553, 1.7525, 1.6801, 1.7502, 1.7395, 1.6249, 1.7054, 2.4263, 2.4263, 2.4259, 2.4259, 2.4259, 2.425, 2.424, 2.4239, 2.4239, 2.4238, 2.4236, 2.4229, 2.4228, 2.4227, 2.4227, 2.4222, 2.4219, 2.4219, 2.4217, 2.4214, 2.4214, 2.4211, 2.421, 2.4209, 2.4209, 2.4208, 2.4207, 2.4205, 2.4205, 2.4205, 2.4183, 2.418, 2.418, 2.4171, 2.4171, 2.4161, 2.415, 2.4198, 2.4181, 2.4163, 2.4174, 2.4189, 2.4177, 2.3576, 2.3509, 2.2793, 2.2812, 2.2686, 2.2729, 2.2638, 2.2388, 1.6458, 2.5926, 2.5924, 2.5917, 2.5915, 2.591, 2.5901, 2.5875, 2.5869, 2.5868, 2.5863, 2.5855, 2.5855, 2.5846, 2.5844, 2.5836, 2.583, 2.583, 2.583, 2.5823, 2.5821, 2.5817, 2.5815, 2.5814, 2.5813, 2.5807, 2.5804, 2.5795, 2.5795, 2.5795, 2.579, 2.5785, 2.5764, 2.5083, 2.5666, 2.562, 2.5007, 2.479, 2.468, 2.5742, 2.4452, 2.4696, 2.4669, 2.4997, 2.4995, 2.4043, 2.4039, 2.2871, 2.5155, 2.4687, 2.3693, 2.356, 2.2121, 2.3445, 2.3083, 2.3034, 2.2847, 2.2965, 2.2204, 2.3288, 2.2383, 2.2278, 1.9711, 1.9192, 2.2222, 2.2886, 2.2085, 2.3526, 1.844, 2.1593, 1.4537, 1.6273, 1.5939, 1.0038, 1.0456, 0.9948, 3.0809, 3.0805, 3.0774, 3.0762, 3.0762, 3.0761, 3.0761, 3.076, 3.076, 3.0752, 3.0727, 3.0718, 3.0704, 3.0702, 3.07, 3.0698, 3.0694, 3.0688, 3.0676, 3.0676, 3.0596, 3.0579, 3.0577, 3.057, 3.0565, 3.0494, 3.0489, 3.0488, 3.0481, 3.0476, 3.0449, 3.0369, 2.9835, 2.9633, 2.9074, 2.9832, 2.9751, 2.9402, 3.0094, 2.9845, 2.8319, 2.7608, 2.8528, 2.788, 2.9415, 2.6649, 2.5472, 2.8968, 2.5719, 2.5704, 2.0219, 2.3349, 1.9421, 1.9374, 1.9034, 1.8722, 3.3199, 3.3191, 3.3094, 3.3086, 3.3064, 3.3058, 3.3056, 3.3038, 3.3034, 3.3022, 3.2925, 3.2907, 3.2826, 3.2815, 3.2813, 3.2744, 3.271, 3.2703, 3.2657, 3.263, 3.2584, 3.2582, 3.2573, 3.2518, 3.2502, 3.248, 3.238, 3.2376, 3.2268, 3.2201, 3.2057, 3.1628, 3.1583, 3.1572, 3.1588, 3.1525, 3.1454, 3.1835, 3.1399, 3.0221, 3.079, 3.0688, 3.0688, 2.8012, 2.7856, 2.7409, 2.7335, 2.9881, 2.9557, 2.1618, 2.5341, 2.082, 2.0745, 2.6467, 2.4293, 2.6536, 2.5157, 0.7673, 2.5142, 2.5007, 1.8464, 1.8335, 3.3554, 3.3553, 3.3553, 3.3553, 3.3552, 3.3552, 3.3552, 3.3552, 3.3552, 3.3551, 3.3551, 3.3551, 3.3551, 3.3551, 3.3551, 3.3551, 3.355, 3.355, 3.355, 3.355, 3.3549, 3.3548, 3.3548, 3.3544, 3.3517, 3.3514, 3.3512, 3.351, 3.351, 3.3507, 3.3306, 3.3295, 3.3279, 3.3268, 3.2868, 3.3009, 3.0385, 3.4, 3.3903, 3.3893, 3.3888, 3.3887, 3.3876, 3.3865, 3.381, 3.3808, 3.3756, 3.3755, 3.3751, 3.3741, 3.3728, 3.3714, 3.3693, 3.3692, 3.3686, 3.3674, 3.3614, 3.3599, 3.3592, 3.3588, 3.3584, 3.3448, 3.3424, 3.3405, 3.3384, 3.3275, 3.3242, 3.3079, 3.2855, 3.2966, 3.222, 3.2628, 3.2152, 3.2302, 3.301, 3.1271, 3.1429, 3.1315, 3.0135, 3.2121, 3.292, 3.1793, 3.0977, 3.0845, 2.6788, 3.1704, 2.2451, 3.1682, 2.3901, 1.7411, 2.4539, 2.7682, 2.4001, 1.7457, 2.3726, 2.2462, 1.4127, 3.8834, 3.8828, 3.8827, 3.8825, 3.8824, 3.8824, 3.8823, 3.8823, 3.882, 3.882, 3.882, 3.8819, 3.8818, 3.8815, 3.8814, 3.8813, 3.8809, 3.8809, 3.8809, 3.8807, 3.88, 3.8795, 3.8793, 3.8793, 3.8792, 3.8785, 3.8785, 3.8782, 3.8777, 3.8777, 3.8731, 3.873, 3.8536, 3.7044, 3.7678, 3.8243, 3.8243, 2.7038, 2.6971, 1.657], \"Total\": [3175.0, 4307.0, 4269.0, 4265.0, 4256.0, 4233.0, 4176.0, 4100.0, 4087.0, 4271.0, 1673.0, 4345.0, 1894.0, 4332.0, 1889.0, 2804.0, 2315.0, 2286.0, 2283.0, 2315.0, 2479.0, 1105.0, 1508.0, 856.0, 1165.0, 1162.0, 1155.0, 1163.0, 1163.0, 797.0, 692.7146155298632, 664.4878975799813, 643.7143834359566, 522.6424591373641, 481.3186550215329, 466.50786300841844, 466.31360133214855, 458.80001481229385, 449.30970787172464, 447.2998530909198, 444.7364004458254, 442.74825791734236, 437.808025458099, 426.6486446850119, 426.91851391205256, 423.06179977808256, 422.23182774557614, 420.7043757051912, 419.57283135897563, 418.663215916414, 418.0967441546758, 409.0716531193889, 406.92066255464937, 397.9542562670254, 397.05632566707965, 380.3406285602534, 379.9915004863964, 381.59372208656424, 367.02888989359315, 363.7818421935934, 4345.99410831616, 4332.922175395867, 1385.650518662384, 1367.4948983016, 878.9530329330012, 822.0751406226409, 847.2749916196741, 845.8794680563025, 729.8293075704187, 869.5915381390412, 750.8165629077104, 740.8384896410892, 509.0971679217305, 724.5603658930523, 703.4474819805811, 489.59551328708426, 561.3387913560632, 699.8889988475821, 716.1599084680845, 1508.8106063582811, 559.3870053606059, 604.1714376153332, 729.630012530334, 4307.373515677841, 4269.718813372264, 4265.037984488481, 4256.972667341018, 4233.152828279082, 4176.8943887640435, 4100.581652000584, 4087.1474027642394, 2315.1838223521145, 1530.6669848449183, 1494.9476065309157, 1466.1597152512834, 1414.4589632343966, 1405.3821570119933, 1355.6892719787213, 1204.5392176358305, 1129.628809642984, 1036.7653698790793, 779.065429721344, 766.1957897301204, 744.9227065041125, 726.161373473689, 704.2485214559891, 684.9279727832778, 672.9575390955125, 663.0151200780074, 653.3165219089029, 1252.3631722134305, 643.3741005456249, 632.1509320146545, 4271.898150078453, 1507.1878511281207, 2479.520827147176, 2804.926559967596, 1205.174906023646, 1205.7854908566192, 634.8417858631135, 624.6192834946186, 593.4893007541533, 577.4330886102766, 554.2278610458271, 531.8607915688393, 509.8753790223074, 489.3546676748476, 537.9025961273522, 443.8926135348183, 424.15267690892813, 475.31788184654397, 514.07700491944, 394.01767919288216, 496.2177436506137, 493.27877167688456, 475.3470136985873, 666.3161318884461, 318.1966145884859, 311.16255392633644, 295.48105979281695, 270.52601074636397, 270.5212046270254, 265.9160875154124, 261.560289074766, 261.5424935724294, 245.75544934508824, 243.79812374004973, 236.20275675833932, 233.84082045478678, 2315.1031457967333, 2286.3346058247785, 2283.567471170632, 921.3850843810678, 901.8233403740423, 951.5620408085485, 882.2608934587923, 957.8782748079186, 912.8838837472177, 908.8585361482919, 707.0411137324966, 702.044272837294, 793.6467614960235, 781.9202375714282, 1131.2597944864985, 722.52422961108, 679.4463782276611, 862.5515272264835, 571.6792007733193, 1894.6065943651602, 1889.7779968026152, 1165.61236075779, 1162.5667924992151, 1155.6911820522596, 616.5455642908166, 407.69019859597097, 400.1228711258746, 397.3364540634921, 379.9486033667069, 367.65570314866255, 302.95063001160577, 293.82177389380155, 288.46709706666303, 284.73316965184887, 256.97980639841865, 242.1043906874153, 240.28844105553748, 231.309954227525, 220.09070316666205, 219.0574610687417, 207.0559347032024, 203.26398528084488, 380.7965444913234, 302.3701442679346, 196.5588238356381, 196.27314981712516, 189.76127363810375, 187.8758337600352, 187.536454640148, 643.977804066565, 630.4765124025763, 625.9531549421882, 653.5172479800511, 593.5421752382775, 501.4051551310121, 508.3608738927064, 290.73989507933493, 278.76533741069056, 298.7154227212452, 276.88594937600163, 264.06227544072533, 271.11655191894914, 830.0183604493628, 843.8656693028914, 618.3688057797352, 599.6327561335362, 612.8645472548111, 592.9836859236736, 578.6926695402669, 426.6769392212265, 904.54103726358, 128.74591286064384, 126.98632869824065, 118.3780949346764, 116.62876708065446, 111.24715323909264, 153.30173020531393, 86.33464988155957, 82.71521730196349, 82.006507685333, 79.56814894490661, 77.27160307217986, 76.56309074318416, 72.15507707541883, 71.45759515024253, 72.43204191455095, 66.39198232822723, 66.36587209900696, 66.16310207712547, 102.88946486971963, 63.45906446712448, 62.87250953027744, 61.65179203064578, 61.360280997902294, 61.096191306895065, 59.37821250445938, 58.66157685527941, 56.463940832814636, 56.34039467633292, 56.34039467653448, 55.14546411889727, 96.82561963509634, 86.58671550462374, 3175.053784869272, 149.41116299276925, 119.18912572434789, 626.78630442327, 973.0473372784595, 1099.72357103761, 69.67150897795494, 534.5668125376108, 260.7094500465746, 255.58177861765714, 169.46887153868786, 168.89616918896888, 420.7078285527474, 418.6258433980454, 1147.861701740845, 130.31444691668224, 187.22464917303986, 390.2621614903071, 374.0004079997593, 794.977320383584, 345.36731559347726, 418.374552133538, 417.3331032393953, 433.8144678660585, 403.3062087255109, 486.32302661134355, 309.11042549941084, 427.5933732274146, 442.1984790256997, 922.7372184366692, 1082.3242091338461, 385.94681997722154, 289.5647478093809, 348.4481003457069, 243.607879951518, 638.2939221778262, 335.12356694103516, 1129.6626583574418, 746.5063575872621, 723.7799285630351, 1205.174906023646, 1151.6887493210893, 1205.7854908566192, 1673.6705696863974, 1105.7938391283587, 307.84079873759305, 243.55051874140526, 240.67326578749604, 238.4423093989839, 237.74785853178534, 235.72189588068943, 233.66573804285176, 206.66076001051817, 148.72523454529943, 135.25065791254164, 118.87512414147902, 116.31767830331745, 114.41402685811724, 119.56231810352796, 108.67093246048931, 103.83097928181996, 94.7468826371588, 94.74688263715882, 60.998418263914076, 56.65978071769792, 56.07513583880518, 54.64106994440216, 53.36868710534479, 100.42733387052496, 41.228639457833715, 41.09312359533914, 40.298001552747124, 39.630855966076595, 54.824885027338986, 81.62632118311328, 449.68449161973086, 760.9957164326984, 811.1963180213911, 170.89253285511123, 177.31459703569394, 285.939957410024, 106.78296488444829, 133.90584561450902, 459.5731241548557, 789.3454859384653, 336.8371012502143, 316.76594652164914, 157.78127893877974, 371.7347898203868, 496.70938120118893, 168.66540835313026, 368.5114965813657, 367.8854219843927, 1129.6626583574418, 568.7664263097872, 1163.9106062068622, 1163.5817422302725, 723.7799285630351, 746.5063575872621, 856.508043061476, 554.2730457828005, 110.83830387529486, 103.73990844992719, 88.53531286580919, 85.37470261111184, 84.56060297658006, 75.67795689743787, 74.0948089986495, 69.5843938054941, 66.3231239194028, 43.893881305643895, 64.58181230101471, 65.09478394241643, 62.55449655158844, 28.918769142302022, 60.20528555680805, 26.65113769648803, 57.29377629199126, 23.426031951112897, 21.754983924374123, 54.66835498064907, 57.65687384029153, 534.65694781565, 539.0583790383085, 60.247207959830085, 172.4231636245071, 172.94365827151384, 14.720716817493255, 38.123570404709504, 110.10392113511683, 366.874215126695, 363.08409298984606, 360.49681238699094, 228.59961993860213, 230.52549527047412, 193.89569397428156, 115.98184064773744, 162.13046307572793, 303.10418185465363, 191.25065516563754, 195.82266387165677, 195.82261095910147, 642.4368166707233, 555.9289945901629, 578.4159367146381, 584.4463723763397, 211.768731457828, 223.38532856157974, 1129.6626583574418, 445.31739398150836, 1163.5817422302725, 1163.9106062068622, 346.3529926799519, 529.32222415474, 326.89734643698563, 363.9860611519515, 3175.053784869272, 364.52760760187067, 368.62622891363793, 746.5063575872621, 723.7799285630351, 406.9650370247189, 399.9126185517203, 399.0507876076436, 397.05494152107707, 388.40240271837945, 388.21370012197303, 384.65456036350605, 383.0725239133763, 382.92527908002023, 377.63764562326844, 374.7063860625695, 374.5169752699089, 372.62392642806356, 371.91755034646576, 371.4698881755307, 370.98467086463813, 369.27135898290703, 364.1365044046264, 362.75597523676294, 360.12536552629285, 356.3606826825811, 347.76227994702793, 347.2509378825626, 308.8012388856859, 188.76155771398928, 181.83564775245105, 174.99591574641136, 171.52103158234303, 170.55587025922915, 164.05216359151484, 555.2346517917599, 543.4725960014893, 511.58702120467547, 502.8444761034899, 418.41344252799223, 300.6757698338043, 648.4573874077518, 720.6536214502365, 198.71056597354954, 107.15342314127925, 103.41346804394983, 102.80346357017642, 94.76692420762141, 88.08094772962436, 159.7134675023207, 64.28031173161793, 51.66773993428803, 51.58278491908519, 50.690709787190286, 48.93810557168224, 46.71180874246996, 98.6911235200271, 99.86628168922797, 96.85625952964847, 48.71105150677643, 39.58256242790696, 101.99172119913963, 76.4715786855477, 100.71305675003246, 75.43323489147126, 104.88127259716401, 123.44737904197954, 42.11782020942403, 41.52198007111144, 40.658759470544794, 75.5605167186849, 797.9533922983987, 251.11554517957003, 265.4880129819322, 172.14592012035558, 530.8078817245712, 255.85709153031902, 510.38211700699657, 268.1045666943278, 113.37230976484102, 706.7172739749383, 503.2131064995529, 511.06603636500887, 818.0323277484717, 151.1515909280569, 102.78518241562824, 153.93741245000416, 190.56254675676027, 190.14858827059734, 496.0862653095668, 151.59268032548616, 1082.3242091338461, 149.7535144626877, 529.32222415474, 1508.8106063582811, 437.1781874866089, 259.67399014415, 445.31739398150836, 1147.861701740845, 414.1403998589671, 472.10614661087175, 922.7372184366692, 319.5154752103814, 274.27877554829655, 270.0933010026089, 262.0964646589002, 255.06118404558381, 254.05087614372727, 249.410036325662, 248.76748636262155, 239.52262661242563, 238.18179652295854, 236.71942701313026, 234.3213411939304, 228.2913112683165, 217.94466015816712, 215.25066749123505, 212.54371476003527, 200.3379750541505, 200.05477874047367, 200.04676111082009, 193.1175638957085, 178.15912722994517, 164.29305445419382, 171.29321059151275, 160.26175750689166, 159.6614558648793, 146.8890140241566, 146.6976578003485, 141.9617211209488, 135.66074223004085, 134.66872716083276, 152.95980812303534, 153.28977598658435, 190.78330401467244, 342.37417656224983, 212.7624415105645, 170.41959055096433, 168.99673105982214, 1163.5817422302725, 1163.9106062068622, 1508.8106063582811]}};\n",
       "\n",
       "function LDAvis_load_lib(url, callback){\n",
       "  var s = document.createElement('script');\n",
       "  s.src = url;\n",
       "  s.async = true;\n",
       "  s.onreadystatechange = s.onload = callback;\n",
       "  s.onerror = function(){console.warn(\"failed to load library \" + url);};\n",
       "  document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
       "}\n",
       "\n",
       "if(typeof(LDAvis) !== \"undefined\"){\n",
       "   // already loaded: just create the visualization\n",
       "   !function(LDAvis){\n",
       "       new LDAvis(\"#\" + \"ldavis_el315521405943743645209592089092\", ldavis_el315521405943743645209592089092_data);\n",
       "   }(LDAvis);\n",
       "}else if(typeof define === \"function\" && define.amd){\n",
       "   // require.js is available: use it to load d3/LDAvis\n",
       "   require.config({paths: {d3: \"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min\"}});\n",
       "   require([\"d3\"], function(d3){\n",
       "      window.d3 = d3;\n",
       "      LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "        new LDAvis(\"#\" + \"ldavis_el315521405943743645209592089092\", ldavis_el315521405943743645209592089092_data);\n",
       "      });\n",
       "    });\n",
       "}else{\n",
       "    // require.js not available: dynamically load d3 & LDAvis\n",
       "    LDAvis_load_lib(\"https://cdnjs.cloudflare.com/ajax/libs/d3/3.5.5/d3.min.js\", function(){\n",
       "         LDAvis_load_lib(\"https://cdn.rawgit.com/bmabey/pyLDAvis/files/ldavis.v1.0.0.js\", function(){\n",
       "                 new LDAvis(\"#\" + \"ldavis_el315521405943743645209592089092\", ldavis_el315521405943743645209592089092_data);\n",
       "            })\n",
       "         });\n",
       "}\n",
       "</script>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyLDAvis.display(vis_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Можно настроить параметры LDA (например количество топиков) и получить требуемую визуализацию. Кажется, что более интерпретируемых вариантов нет))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Теперь по поводу последнего пункта."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Оценить кластеризацию можно следующим образом: предложу человеку взглянуть на то, что вывел pyLDAvis, и попробовать определить темы по ключевым словам. Если он сможет выделить темы более-менее конкретно (т.е. сказать, например: \"вот это тема про образование, а эта - про армию...\"), это будет норм.\n",
    "Хочется примерно следующего: ассесор должен говорить, насколько хорошо темы отделены друг от друга. Т.е. определять, нет ли слишком общих тем и нет ли тем, в которых наоборот мало информации (например можно посмотреть на тему 10 в pyLDAvis)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}