Aimee WB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['readMe', 'CLEAN DATA', 'GUIDESTAR MATCHING DATA', 'fb_data']\n"
     ]
    }
   ],
   "source": [
    "execfile('load_data.py')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "counting terms for doc: 0\n",
      "counting terms for doc: 25\n",
      "counting terms for doc: 50\n",
      "counting terms for doc: 75\n",
      "counting terms for doc: 100\n",
      "counting terms for doc: 125\n",
      "counting terms for doc: 150\n",
      "counting terms for doc: 175\n",
      "counting terms for doc: 200\n",
      "counting terms for doc: 225\n",
      "counting terms for doc: 250\n",
      "counting terms for doc: 275\n",
      "counting terms for doc: 300\n",
      "counting terms for doc: 325\n",
      "counting terms for doc: 350\n",
      "counting terms for doc: 375\n",
      "counting terms for doc: 400\n",
      "counting terms for doc: 425\n",
      "counting terms for doc: 450\n",
      "counting terms for doc: 475\n",
      "counting terms for doc: 500\n",
      "counting terms for doc: 525\n",
      "counting terms for doc: 550\n",
      "counting terms for doc: 575\n",
      "counting terms for doc: 600\n",
      "counting terms for doc: 625\n",
      "counting terms for doc: 650\n",
      "counting terms for doc: 675\n",
      "counting terms for doc: 700\n",
      "counting terms for doc: 725\n",
      "counting terms for doc: 750\n",
      "counting terms for doc: 775\n",
      "counting terms for doc: 800\n",
      "Sanity checks:\n",
      "\tTime to build corpus: 5.76380109787\n",
      "\tTerms in first document: [u'give' u'stori' u'for' u'www' u'org' u'nfor' u'mani' u'year' u'went'\n",
      " u'life' u'not' u'truli' u'see' u'the' u'world' u'and' u'it' u'problem'\n",
      " u'knew' u'peopl' u'live' u'poverti' u'howev' u'wa' u'easi' u'put'\n",
      " u'thought' u'out' u'head' u'the' u'distract' u'daili' u'routin' u'coupl'\n",
      " u'year' u'ago' u'god' u'remov' u'the' u'blinder' u'eye' u'and' u'allow'\n",
      " u'see' u'how' u'truli' u'bless' u'famili' u'and' u'mani' u'famili'\n",
      " u'america' u'are' u'compar' u'part' u'the' u'world' u'the' u'process'\n",
      " u'wa' u'astonish' u'discov' u'billion' u'peopl' u'world' u'live' u'per'\n",
      " u'day' u'how' u'doe' u'someon' u'live' u'per' u'day' u'mani' u'america'\n",
      " u'are' u'will' u'pay' u'for' u'cup' u'coffe' u'nafter' u'god' u'began'\n",
      " u'reveal' u'thi' u'husband' u'and' u'felt' u'call' u'someth' u'help'\n",
      " u'not' u'sit' u'anymor' u'and' u'pretend' u'didn\\xe2' u'know' u'other'\n",
      " u'live' u'dire' u'circumst' u'and' u'truli' u'want' u'someth' u'give'\n",
      " u'husband' u'and' u'led' u'organ' u'call' u'help' u'orphan' u'and'\n",
      " u'underprivileg' u'children' u'south' u'africa' u'drawn' u'thi' u'organ'\n",
      " u'and' u'foremost' u'becaus' u'the' u'organization\\xe2' u'name' u'drawn'\n",
      " u'matthew' u'\\u0153truli' u'tell' u'you' u'whatev' u'you' u'did' u'for'\n",
      " u'one' u'the' u'brother' u'and' u'sister' u'mine' u'you' u'did' u'for'\n",
      " u'hear' u'stori' u'mani' u'the' u'children' u'serv' u'and' u'how'\n",
      " u'partner' u'the' u'commun' u'help' u'fix' u'the' u'problem' u'sustain'\n",
      " u'solut' u'felt' u'compel' u'join' u'effort' u'and' u'someth' u'help'\n",
      " u'rais' u'money' u'for' u'the' u'thi' u'wonder' u'organ' u'nbe' u'dancer'\n",
      " u'for' u'year' u'much' u'passion' u'and' u'love' u'for' u'danc' u'god'\n",
      " u'open' u'door' u'danc' u'ministri' u'coupl' u'year' u'ago' u'and'\n",
      " u'prayer' u'decid' u'danc' u'workshop' u'one' u'summer' u'help' u'benefit'\n",
      " u'wa' u'bless' u'for' u'involv' u'danc' u'and' u'the' u'time' u'amaz'\n",
      " u'opportun' u'for' u'use' u'time' u'talent' u'and' u'passion' u'help'\n",
      " u'children' u'south' u'africa' u'who' u'desper' u'need' u'help' u'ndure'\n",
      " u'the' u'workshop' u'wa' u'abl' u'talk' u'the' u'children' u'serv' u'how'\n",
      " u'children' u'live' u'small' u'hut' u'floor' u'had' u'walk' u'everi'\n",
      " u'day' u'distanc' u'collect' u'water' u'for' u'famili' u'the' u'children'\n",
      " u'ask' u'wa' u'make' u'stori' u'sinc' u'wa' u'hard' u'for' u'imagin'\n",
      " u'life' u'children' u'south' u'africa' u'know' u'wa' u'veri' u'heartwarm'\n",
      " u'see' u'the' u'children' u'come' u'the' u'workshop' u'empti' u'own'\n",
      " u'piggi' u'bank' u'give' u'hear' u'the' u'stori' u'the' u'children'\n",
      " u'serv' u'addit' u'the' u'money' u'abl' u'rais' u'for' u'dure' u'the'\n",
      " u'workshop' u'know' u'the' u'workshop' u'great' u'impact' u'children\\xe2'\n",
      " u'heart' u'and' u'mine' u'feel' u'veri' u'bless' u'had' u'the' u'opportun'\n",
      " u'give' u'and' u'help' u'children' u'south' u'africa' u'learn' u'thi'\n",
      " u'give' u'experi' u'god' u'call' u'all' u'someth' u'and' u'the' u'process'\n",
      " u'the' u'hand' u'and' u'feet' u'jesu' u'can' u'amaz' u'impact' u'thi'\n",
      " u'world' u'are' u'mind' u'go' u'within' u'and' u'the' u'commit' u'will'\n",
      " u'can' u'help' u'believ' u'give' u'time' u'talent' u'and' u'treasur'\n",
      " u'can' u'part' u'someth' u'much' u'bigger' u'ourselv' u'and' u'join'\n",
      " u'forc' u'god' u'someth' u'help' u'other' u'think' u'will' u'amaz' u'see'\n",
      " u'not' u'onli' u'will' u'other\\xe2' u'live' u'help' u'and' u'impact'\n",
      " u'becaus' u'but' u'own' u'life' u'will' u'chang' u'heart' u'are'\n",
      " u'transform' u'hi' u'know' u'mine' u'ha' u'and' u'beauti' u'thing']\n",
      "\tTotal words in corpus: 191476\n",
      "\tNumber docs in corpus: 823\n",
      "\tNumber of unique words in corpus: 10846\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "execfile('nlp_code/corpus.py')\n",
    "execfile('nlp_code/document.py')\n",
    "\n",
    "t0 = time.time()\n",
    "corpus = Corpus(documents, 'nlp_code/stopwords.txt', 3)\n",
    "t1 = time.time()\n",
    "\n",
    "corpus.generate_document_term_matrix()\n",
    "termlist = list(corpus.token_set)\n",
    "print 'Sanity checks:'\n",
    "print '\\tTime to build corpus: ' + str(t1 - t0)\n",
    "print '\\tTerms in first document: ' + str(corpus.docs[0].tokens)\n",
    "print '\\tTotal words in corpus: ' + str(corpus.ntotal_tokens)\n",
    "print '\\tNumber docs in corpus: ' + str(corpus.N)\n",
    "print '\\tNumber of unique words in corpus: ' + str(len(corpus.token_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(823, 10828)\n"
     ]
    }
   ],
   "source": [
    "print corpus.document_term_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## NOTE: CODE BELOW IS TOO SLOW, USE PYTHON PACKAGE IN OTHER FILE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Most likely terms for topic 0:\n",
      "give with probability 0.0182264446048\n",
      "year with probability 0.0132366335287\n",
      "wa with probability 0.0122720066908\n",
      "hi with probability 0.0120285193603\n",
      "famili with probability 0.00957713110942\n",
      "thi with probability 0.0092954245021\n",
      "help with probability 0.00837344895672\n",
      "time with probability 0.00818746591353\n",
      "\n",
      "Most likely terms for topic 1:\n",
      "wa with probability 0.0249153198853\n",
      "support with probability 0.0126528696829\n",
      "organ with probability 0.0111264930751\n",
      "thi with probability 0.0102592243691\n",
      "rais with probability 0.00846116011126\n",
      "becaus with probability 0.00813873077241\n",
      "org with probability 0.00811304600214\n",
      "famili with probability 0.00799086726258\n",
      "\n",
      "Most likely terms for topic 2:\n",
      "wa with probability 0.0241817189574\n",
      "thi with probability 0.0116434004981\n",
      "work with probability 0.0110723635683\n",
      "help with probability 0.00969250016354\n",
      "live with probability 0.00877253274697\n",
      "start with probability 0.00857670588978\n",
      "year with probability 0.00845124766151\n",
      "can with probability 0.00792966591105\n",
      "\n",
      "Most likely terms for topic 3:\n",
      "wa with probability 0.0175974590357\n",
      "children with probability 0.013910804754\n",
      "life with probability 0.0132251704795\n",
      "give with probability 0.0124580155496\n",
      "famili with probability 0.0123811783529\n",
      "year with probability 0.0119310556055\n",
      "commun with probability 0.0082772595728\n",
      "work with probability 0.00813824103342\n",
      "\n",
      "Most likely terms for topic 4:\n",
      "wa with probability 0.0202970743056\n",
      "know with probability 0.00871274265728\n",
      "year with probability 0.0087109804308\n",
      "run with probability 0.00856129308565\n",
      "give with probability 0.00853842139069\n",
      "can with probability 0.00832794510383\n",
      "thi with probability 0.00786666916852\n",
      "want with probability 0.00742695511199\n",
      "\n",
      "Most likely terms for topic 5:\n",
      "wa with probability 0.018972757437\n",
      "time with probability 0.0128215434924\n",
      "help with probability 0.0110902470678\n",
      "life with probability 0.00978425088584\n",
      "give with probability 0.00905603243685\n",
      "chang with probability 0.00897027577436\n",
      "becaus with probability 0.00841333558907\n",
      "ha with probability 0.00789824016238\n",
      "\n",
      "Most likely terms for topic 6:\n",
      "wa with probability 0.0245474376594\n",
      "thi with probability 0.0156001925762\n",
      "year with probability 0.0133272898771\n",
      "school with probability 0.0123702157719\n",
      "give with probability 0.0119132434714\n",
      "mani with probability 0.00995997003038\n",
      "provid with probability 0.00955672977176\n",
      "commun with probability 0.00801649021413\n",
      "\n",
      "Most likely terms for topic 7:\n",
      "wa with probability 0.0250361854716\n",
      "help with probability 0.011747179001\n",
      "thi with probability 0.0101450470448\n",
      "know with probability 0.00868124692821\n",
      "commun with probability 0.00809609342071\n",
      "need with probability 0.0072784911027\n",
      "volunt with probability 0.00717328197278\n",
      "time with probability 0.00709785777915\n",
      "\n",
      "Most likely terms for topic 8:\n",
      "commun with probability 0.01418781772\n",
      "thi with probability 0.0129562039318\n",
      "girl with probability 0.0118554982869\n",
      "wa with probability 0.0106956427411\n",
      "organ with probability 0.01024656197\n",
      "give with probability 0.0086797732076\n",
      "time with probability 0.00860233257972\n",
      "help with probability 0.00794585406348\n",
      "\n",
      "Most likely terms for topic 9:\n",
      "wa with probability 0.0255692547778\n",
      "thi with probability 0.0148729877388\n",
      "give with probability 0.0102535357892\n",
      "year with probability 0.00984167956239\n",
      "help with probability 0.00949978194524\n",
      "hi with probability 0.00905837905039\n",
      "provid with probability 0.00882259524336\n",
      "children with probability 0.00787393959518\n"
     ]
    }
   ],
   "source": [
    "# HERE BE THE CALL\n",
    "# time0 = time.time()\n",
    "# probs = corpus.lda_gibbs(K = 10, progress_interval = 1, iters = 10)\n",
    "# time1 = time.time()\n",
    "# print 'Time spent running lda: {0}'.format(time1-time0)\n",
    "# Print the most likely words for every topic\n",
    "beta = probs['beta']\n",
    "K = beta.shape[0]\n",
    "sorted_beta = np.zeros(beta.shape)\n",
    "\n",
    "for k in range(0,K):\n",
    "    print ''\n",
    "    print 'Most likely terms for topic ' + str(k) + ':'\n",
    "    sorted_beta[k,:] = np.argsort(beta[k,:])[::-1]\n",
    "    for i in range(8):\n",
    "        idx = int(sorted_beta[k,:][i])\n",
    "        print termlist[idx] + ' with probability ' + str(beta.item((k, idx)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}