Frequency_Based_TextSummarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Importing the required modules\n",
    "\n",
    "import nltk\n",
    "import bs4 as bs  \n",
    "import urllib.request  \n",
    "import re\n",
    "import heapq  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Incase the data is from a wikipedia page use the following code to get the data from the site.\n",
    "#It return a string containing the information.\n",
    "def scrape_data(url_link):\n",
    "    scraped_data = urllib.request.urlopen(url_link)  \n",
    "    data = scraped_data.read()\n",
    "\n",
    "    parsed_data = bs.BeautifulSoup(data,'lxml')\n",
    "\n",
    "    paragraphs = parsed_data.find_all('p')\n",
    "\n",
    "    data_text = \"\"\n",
    "\n",
    "    for p in paragraphs:  \n",
    "        data_text += p.text\n",
    "    return data_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove the brackets and extra spaces present in the text.\n",
    "def remove_brackets_extraSpaces(raw_data):\n",
    "    raw_data = re.sub(r'\\[[0-9]*\\]', ' ', raw_data)  \n",
    "    raw_data = re.sub(r'\\s+', ' ', raw_data)  \n",
    "    return raw_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove special Character from the text. This was not included in the above function because \".\" is a special character and if we remove this then sentence tokenizing would return just one sentence due to absnce of \".\" in the raw data.\n",
    "def remove_specialChar(raw_data):\n",
    "    raw_data = re.sub('[^a-zA-Z]', ' ', raw_data )  \n",
    "    raw_data = re.sub(r'\\s+', ' ', raw_data)  \n",
    "    return raw_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove's the stop words like a,to,the etc. This function returns a dictionary containing the words and its frequency in a dictionary format.\n",
    "def remove_stopWords_Frequency(paragraph):\n",
    "    stop_words = nltk.corpus.stopwords.words('english') #NLTK's inbuilt list of stop words.\n",
    "    word_frequency ={}\n",
    "    for word in nltk.word_tokenize(paragraph):  \n",
    "        if word not in stop_words:\n",
    "            word_frequency[word] = word_frequency.get(word,0)+1\n",
    "    return word_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sentanize tokenize the processed text.\n",
    "def sentenceTokenize(paragraph):\n",
    "    return nltk.sent_tokenize(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assigning weights to each word based on based on its frequency\n",
    "def assignScores_words(word_frequency):\n",
    "    max_frequency = max(word_frequency.values())\n",
    "    for word in word_frequency.keys():\n",
    "        word_frequency[word] = word_frequency[word]/(max_frequency*1.0)\n",
    "    return word_frequency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Assign scores to each sentence based on the words it contains from the scores assigned to words.\n",
    "def assignSenteceScores(sent_list, word_scores):\n",
    "    sentence_scores = {}\n",
    "    for sent in sent_list:\n",
    "        if(len(sent.split(' ')) <30):\n",
    "            for word in nltk.word_tokenize(sent.lower()):\n",
    "                if word in word_scores.keys():\n",
    "                    sentence_scores[sent] = sentence_scores.get(sent,0)+word_scores[word]\n",
    "    return sentence_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def printTopsenteces(sent_scores):\n",
    "    summary_sentences = heapq.nlargest(int(len(sent_scores)/50), sent_scores, key=sent_scores.get)\n",
    "    summary = ' '.join(summary_sentences)  \n",
    "    return summary  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SummayFromURL(urlLink):\n",
    "    scrapped_data = scrape_data(urlLink)\n",
    "    brac_extra_space = remove_brackets_extraSpaces(scrapped_data)\n",
    "    remove_special_char = remove_specialChar(brac_extra_space)\n",
    "    words_frequency = remove_stopWords_Frequency(remove_special_char)\n",
    "    sentenceList = sentenceTokenize(brac_extra_space)\n",
    "    word_scores = assignScores_words(words_frequency)\n",
    "    sentence_scores = assignSenteceScores(sentenceList,word_scores)\n",
    "    return printTopsenteces(sentence_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SummaryFromText(paragraph):\n",
    "    brac_extra_space = remove_brackets_extraSpaces(paragraph)\n",
    "    remove_special_char = remove_specialChar(brac_extra_space)\n",
    "    words_frequency = remove_stopWords_Frequency(remove_special_char)\n",
    "    sentenceList = sentenceTokenize(brac_extra_space)\n",
    "    word_scores = assignScores_words(words_frequency)\n",
    "    sentence_scores = assignSenteceScores(sentenceList,word_scores)\n",
    "    return printTopsenteces(sentence_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Artificial intelligence (AI), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans and other animals. Colloquially, the term \"artificial intelligence\" is applied when a machine mimics \"cognitive\" functions that humans associate with other human minds, such as \"learning\" and \"problem solving\". Neural networks can be applied to the problem of intelligent control (for robotics) or learning, using such techniques as Hebbian learning (\"fire together, wire together\"), GMDH or competitive learning. Deep learning has transformed many important subfields of artificial intelligence, including computer vision, speech recognition, natural language processing and others. Many of the problems in this article may also require general intelligence, if machines are to solve the problems as well as people do. IBM has created its own artificial intelligence computer, the IBM Watson, which has beaten human intelligence (at some levels). \"robotics\" or \"machine learning\"), the use of particular tools (\"logic\" or artificial neural networks), or deep philosophical differences. Musk also funds companies developing artificial intelligence such as Google DeepMind and Vicarious to \"just keep an eye on what\\'s going on with artificial intelligence.'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SummayFromURL('https://en.wikipedia.org/wiki/Artificial_intelligence')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' Artificial intelligence (AI), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans and other animals.'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paragraph = \"\"\" Artificial intelligence (AI), sometimes called machine intelligence, is intelligence demonstrated by machines, in contrast to the natural intelligence displayed by humans and other animals. Colloquially, the term \"artificial intelligence\" is applied when a machine mimics \"cognitive\" functions that humans associate with other human minds, such as \"learning\" and \"problem solving\". Neural networks can be applied to the problem of intelligent control (for robotics) or learning, using such techniques as Hebbian learning (\"fire together, wire together\"), GMDH or competitive learning. Deep learning has transformed many important subfields of artificial intelligence, including computer vision, speech recognition, natural language processing and others. Many of the problems in this article may also require general intelligence, if machines are to solve the problems as well as people do. Musk also funds companies developing artificial intelligence such as Google DeepMind and Vicarious to \"just keep an eye on what\\'s going on with artificial intelligence. IBM has created its own artificial intelligence computer, the IBM Watson, which has beaten human intelligence (at some levels). \"robotics\" or \"machine learning\"), the use of particular tools (\"logic\" or artificial neural networks), or deep philosophical differences. A superintelligence, hyperintelligence, or superhuman intelligence is a hypothetical agent that would possess intelligence far surpassing that of the brightest and most gifted human mind. The traditional problems (or goals) of AI research include reasoning, knowledge representation, planning, learning, natural language processing, perception and the ability to move and manipulate objects.  \"\"\"\n",
    "SummaryFromText(paragraph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}