scraping_cleaning_sec.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping 10-Ks and 10-Qs from EDGAR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Note: We have already scraped S&P 500 SEC filings. For testing purpose we are scraping the SEC filings for just 2 tickers."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Import the required modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.6.8 |Anaconda, Inc.| (default, Dec 29 2018, 19:04:46) \\n[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import unicodedata\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import requests\n",
    "import bs4 as bs\n",
    "from lxml import html\n",
    "from tqdm import tqdm\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Data Scraping from SEC EDGAR "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get the list of S&P500 tickers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "resp = requests.get('http://en.wikipedia.org/wiki/List_of_S%26P_500_companies')\n",
    "soup = bs.BeautifulSoup(resp.text, 'lxml')\n",
    "table = soup.find('table', {'class': 'wikitable sortable'})\n",
    "tickers = []\n",
    "for row in table.findAll('tr')[1:]:\n",
    "    ticker = row.findAll('td')[1].text\n",
    "    tickers.append(ticker)\n",
    "tickers = tickers[:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Each company has an index in SEC called CIK. Map ticker to CIK to get the corresponding filings on EDGAR."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:02<00:00,  1.16it/s]\n"
     ]
    }
   ],
   "source": [
    "url = 'http://www.sec.gov/cgi-bin/browse-edgar?CIK={}&Find=Search&owner=exclude&action=getcompany'\n",
    "cik_re = re.compile(r'.*CIK=(\\d{10}).*')\n",
    "all_cik = {}\n",
    "for ticker in tqdm(tickers):\n",
    "    res = cik_re.findall(requests.get(url.format(ticker)).text)\n",
    "    if len(res):\n",
    "        all_cik[str(ticker).lower()] = str(res[0])\n",
    "        \n",
    "ciks_tick_df = pd.DataFrame.from_dict(data=all_cik, orient='index')\n",
    "ciks_tick_df.reset_index(inplace=True)\n",
    "ciks_tick_df.columns = ['ticker', 'cik']\n",
    "\n",
    "ciks_tick_df.to_csv('AllSecTickers.csv', sep=',', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Functions to save and close log files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#maintaining the logs \n",
    "def SaveLogFile(log_file_name, text):\n",
    "    \n",
    "    with open(log_file_name, \"a\") as log_file:\n",
    "        log_file.write(text)\n",
    "\n",
    "    return\n",
    "#closing log file\n",
    "def CloseLogFile(log_file_name):\n",
    "    \n",
    "    with open(log_file_name, 'a') as log_file:\n",
    "        log_file.close()\n",
    "\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Path to scraped sec 10-K and 10-Q files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "scraped10k_path = \"/Users/kiranrawat/Desktop/portfolio-risk-assessment-using-ml/data/scraped_10k\"\n",
    "scraped10q_path = \"/Users/kiranrawat/Desktop/portfolio-risk-assessment-using-ml/data/scraped_10q\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Defining function for scraping 10K files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Extract10K(browse_url_base, filing_url_base, doc_url_base, cik, log_file_name):\n",
    "    \n",
    "    requestcount=0\n",
    "    if not os.path.exists(cik):\n",
    "        os.makedirs(cik)\n",
    "    \n",
    "    os.chdir(cik)\n",
    "    print('Scraping CIK ', cik)\n",
    "    \n",
    "    SaveLogFile(log_file_name, 'Scraping CIK: '+cik)\n",
    "            \n",
    "    res = requests.get(browse_url_base % cik)\n",
    "    requestcount+=1\n",
    "    if(requestcount==10):\n",
    "        time.sleep(1)\n",
    "        requestcount=0\n",
    "\n",
    "    if res.status_code != 200:\n",
    "        text = \"\\nFailed at step 1.\\n Failed to hit browse base URL for CIK with error \" + str(res.status_code) + \\\n",
    "               \"\\nFailed URL: \" + (browse_url_base % cik) + '\\n'\n",
    "        \n",
    "        SaveLogFile(log_file_name, text)\n",
    "            \n",
    "        return\n",
    "\n",
    "    SaveLogFile(log_file_name,\"\\nSuccessfully reached browse base UrL\\n\")\n",
    "        \n",
    "    #HTML parsing using BeautifulSoup\n",
    "    soup = bs.BeautifulSoup(res.text, \"lxml\")\n",
    "\n",
    " \n",
    "    html_tables = soup.find_all('table')\n",
    "    \n",
    "    if len(html_tables)<3:\n",
    "        os.chdir('..')\n",
    "        return\n",
    "    \n",
    "\n",
    "    sec_filing_tbl = pd.read_html(str(html_tables[2]), header=0)[0]\n",
    "    sec_filing_tbl['Filings'] = [str(x) for x in sec_filing_tbl['Filings']]\n",
    "\n",
    "    sec_filing_tbl = sec_filing_tbl[(sec_filing_tbl['Filings'] == '10-K') | (sec_filing_tbl['Filings'] == '10-K405')]\n",
    "    sec_filing_tbl = sec_filing_tbl[(sec_filing_tbl['Filing Date'] >= '2008-01-01')]\n",
    "\n",
    "    if len(sec_filing_tbl)==0:\n",
    "        os.chdir('..')\n",
    "        return\n",
    "    \n",
    "    sec_filing_tbl['Acc_No'] = [x.replace('\\xa0',' ')\n",
    "                               .split('Acc-no: ')[1]\n",
    "                               .split(' ')[0] for x in sec_filing_tbl['Description']]\n",
    "\n",
    "   \n",
    "    SaveLogFile(log_file_name, \"Getting all documents for this CIK\\n\")\n",
    "    \n",
    "    total_files_to_scrape = len(sec_filing_tbl)\n",
    "    files_scraped = 0\n",
    "    \n",
    "    for index, row in sec_filing_tbl.iterrows():\n",
    "        \n",
    "       \n",
    "        acc_no = str(row['Acc_No'])\n",
    "        \n",
    "        #check if file is already scraped \n",
    "        date = str(row['Filing Date'])\n",
    "        if (os.path.exists(cik + '_' + date + '.html')) or os.path.exists(cik + '_' + date + '.txt'):\n",
    "            SaveLogFile(log_file_name, \"The file for date: \" + date + \" exists, acc no.: \" + acc_no + \"\\n\")\n",
    "            files_scraped+=1\n",
    "            continue\n",
    "            \n",
    "   \n",
    "        docs_page = requests.get(filing_url_base % (cik, acc_no))\n",
    "        requestcount+=1\n",
    "        if(requestcount==10):\n",
    "            time.sleep(1)\n",
    "            requestcount=0\n",
    "        \n",
    "        \n",
    "        #In case of request failure,log it and jump to the next filing \n",
    "        if docs_page.status_code != 200:\n",
    "            text = \"Request failed with error code \" + str(docs_page.status_code) + \\\n",
    "                   \"\\nFailed URL: \" + (filing_url_base % (cik, acc_no)) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            continue\n",
    "            \n",
    "        SaveLogFile(log_file_name, \"Got acc no. \" + acc_no + \"\\n\")\n",
    "       \n",
    "        docs_page_soup = bs.BeautifulSoup(docs_page.text, 'lxml')\n",
    "        docs_html_tables = docs_page_soup.find_all('table')\n",
    "        if len(docs_html_tables)==0:\n",
    "            continue\n",
    "        docs_table = pd.read_html(str(docs_html_tables[0]), header=0)[0]\n",
    "        docs_table['Type'] = [str(x) for x in docs_table['Type']]\n",
    "        \n",
    " \n",
    "        docs_table = docs_table[(docs_table['Type'] == '10-K') | (docs_table['Type'] == '10-K405')]\n",
    "        \n",
    "        if len(docs_table)==0:\n",
    "            continue\n",
    "        elif len(docs_table)>0:\n",
    "            docs_table = docs_table.iloc[0]\n",
    "        \n",
    "        docname = docs_table['Document']\n",
    "        \n",
    "         # check for nan values, if present then log the failure\n",
    "        if str(docname) == 'nan':\n",
    "            text = 'File with CIK: %s and Acc_No: %s is unavailable' % (cik, acc_no) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            continue       \n",
    "        \n",
    "        file = requests.get(doc_url_base % (cik, acc_no.replace('-', ''), docname.replace(' iXBRL','')))\n",
    "        requestcount+=1\n",
    "        if(requestcount==10):\n",
    "            time.sleep(1)\n",
    "            requestcount=0\n",
    "            \n",
    "       \n",
    "        if file.status_code != 200:\n",
    "            os.chdir('..')\n",
    "            text = \"Request failed with error code \" + str(file.status_code) + \\\n",
    "                   \"\\nFailed URL: \" + (doc_url_base % (cik, acc_no.replace('-', ''), docname)) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            os.chdir(cik)\n",
    "            continue\n",
    "        \n",
    "        SaveLogFile(log_file_name, \"Got 10K for acc no. \" + acc_no + \"\\n\")\n",
    "        files_scraped+=1\n",
    "        \n",
    "        #save HTML or Text file with .txt extension\n",
    "        if '.txt' in docname:\n",
    "            filename = cik + '_' + date + '.txt'\n",
    "            html_file = open(filename, 'a')\n",
    "            html_file.write(file.text)\n",
    "            html_file.close()\n",
    "        else:\n",
    "            filename = cik + '_' + date + '.html'\n",
    "            html_file = open(filename, 'a')\n",
    "            html_file.write(file.text)\n",
    "            html_file.close()\n",
    "    \n",
    "    if(total_files_to_scrape!=files_scraped):\n",
    "        print(\"Some files failed to scrape\\n\")\n",
    "        text=\"Some files failed to scrape\\n\" + str(total_files_to_scrape) + \"!=\"+ str(files_scraped) + \"\\n\"\n",
    "        SaveLogFile(log_file_name, text)\n",
    "        \n",
    "    SaveLogFile(log_file_name, \"saving log files=================\\n\")\n",
    "    CloseLogFile(log_file_name)\n",
    "    os.chdir('..')\n",
    "        \n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### define the function for scraping 10-Q filings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Extract10Q(browse_url_base, filing_url_base, doc_url_base, cik, log_file_name):\n",
    "    \n",
    "    requestcount=0\n",
    "\n",
    "    if not os.path.exists(cik):\n",
    "        os.makedirs(cik)\n",
    "    \n",
    "    os.chdir(cik)\n",
    "    print('Scraping CIK ', cik)\n",
    "    \n",
    "    SaveLogFile(log_file_name, 'Scraping CIK: '+cik)\n",
    "            \n",
    "    res = requests.get(browse_url_base % cik)\n",
    "    requestcount+=1\n",
    "    if(requestcount==10):\n",
    "        time.sleep(1)\n",
    "        requestcount=0\n",
    "    \n",
    "     #In case of request failure,log it and jump to the next filing \n",
    "    if res.status_code != 200:\n",
    "        text = \"\\nFailed at step 1.\\n Failed to hit browse base URL for CIK with error \" + str(res.status_code) + \\\n",
    "               \"\\nFailed URL: \" + (browse_url_base % cik) + '\\n'\n",
    "        \n",
    "        SaveLogFile(log_file_name, text)\n",
    "            \n",
    "        return\n",
    "\n",
    "   \n",
    "    SaveLogFile(log_file_name,\"\\nSuccessfully reached browse base UrL\\n\")\n",
    "        \n",
    "    soup = bs.BeautifulSoup(res.text, \"lxml\")\n",
    "\n",
    "    # fetching all the html tables\n",
    "    html_tables = soup.find_all('table')\n",
    "    \n",
    "    \n",
    "    if len(html_tables)<3:\n",
    "        os.chdir('..')\n",
    "        return\n",
    "    \n",
    "    # Parsing the html filing table\n",
    "    sec_filing_tbl = pd.read_html(str(html_tables[2]), header=0)[0]\n",
    "    sec_filing_tbl['Filings'] = [str(x) for x in sec_filing_tbl['Filings']]\n",
    "\n",
    "\n",
    "    sec_filing_tbl = sec_filing_tbl[sec_filing_tbl['Filings'] == '10-Q']\n",
    "    sec_filing_tbl = sec_filing_tbl[(sec_filing_tbl['Filing Date'] >= '2008-01-01')]\n",
    "    \n",
    "    if len(sec_filing_tbl)==0:\n",
    "        os.chdir('..')\n",
    "        return\n",
    "    \n",
    "  \n",
    "    sec_filing_tbl['Acc_No'] = [x.replace('\\xa0',' ')\n",
    "                               .split('Acc-no: ')[1]\n",
    "                               .split(' ')[0] for x in sec_filing_tbl['Description']]\n",
    "\n",
    "   ..\n",
    "    SaveLogFile(log_file_name, \"Getting all documents for this CIK\\n\")\n",
    "    \n",
    "    total_files_to_scrape = len(sec_filing_tbl)\n",
    "    files_scraped = 0\n",
    "    \n",
    "    for index, row in sec_filing_tbl.iterrows():\n",
    "        \n",
    "     \n",
    "        acc_no = str(row['Acc_No'])\n",
    "        \n",
    "        # check if file already exist\n",
    "        date = str(row['Filing Date'])\n",
    "        if (os.path.exists(cik + '_' + date + '.html')) or os.path.exists(cik + '_' + date + '.txt'):\n",
    "            SaveLogFile(log_file_name, \"The file for date: \" + date + \" exists, acc no.: \" + acc_no + \"\\n\")\n",
    "            files_scraped+=1\n",
    "            continue\n",
    "            \n",
    "      \n",
    "        docs_page = requests.get(filing_url_base % (cik, acc_no))\n",
    "        requestcount+=1\n",
    "        if(requestcount==10):\n",
    "            time.sleep(1)\n",
    "            requestcount=0\n",
    "        \n",
    "        #In case of request failure,log it and jump to the next filing \n",
    "        if docs_page.status_code != 200:\n",
    "            text = \"Request failed with error code \" + str(docs_page.status_code) + \\\n",
    "                   \"\\nFailed URL: \" + (filing_url_base % (cik, acc_no)) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            continue\n",
    "            \n",
    "        SaveLogFile(log_file_name, \"Got acc no. \" + acc_no + \"\\n\")\n",
    "\n",
    "        docs_page_soup = bs.BeautifulSoup(docs_page.text, 'lxml')\n",
    "        docs_html_tables = docs_page_soup.find_all('table')\n",
    "        \n",
    "        if len(docs_html_tables)==0:\n",
    "            continue\n",
    "            \n",
    "        docs_table = pd.read_html(str(docs_html_tables[0]), header=0)[0]\n",
    "        docs_table['Type'] = [str(x) for x in docs_table['Type']]\n",
    "        \n",
    "        docs_table = docs_table[docs_table['Type'] == '10-Q']\n",
    "        \n",
    "        if len(docs_table)==0:\n",
    "            continue\n",
    "            \n",
    "        elif len(docs_table)>0:\n",
    "            docs_table = docs_table.iloc[0]\n",
    "        \n",
    "        docname = docs_table['Document']\n",
    "    \n",
    "        # check for nan values, if present then log the failure\n",
    "        if str(docname) == 'nan':\n",
    "            text = 'File with CIK: %s and Acc_No: %s is unavailable' % (cik, acc_no) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            continue       \n",
    "        \n",
    "       \n",
    "        file = requests.get(doc_url_base % (cik, acc_no.replace('-', ''), docname.replace(' iXBRL','')))\n",
    "        requestcount+=1\n",
    "        if(requestcount==10):\n",
    "            time.sleep(1)\n",
    "            requestcount=0\n",
    "            \n",
    "        #In case of request failure,log it and exit\n",
    "        if file.status_code != 200:\n",
    "            os.chdir('..')\n",
    "            text = \"Request failed with error code \" + str(file.status_code) + \\\n",
    "                   \"\\nFailed URL: \" + (doc_url_base % (cik, acc_no.replace('-', ''), docname)) + '\\n'\n",
    "            SaveLogFile(log_file_name, text)\n",
    "            os.chdir(cik)\n",
    "            continue\n",
    "        \n",
    "      \n",
    "        SaveLogFile(log_file_name, \"Got 10Q for acc no. \" + acc_no + \"\\n\")\n",
    "        files_scraped+=1\n",
    "        \n",
    "         #save HTML or Text file with .txt extension\n",
    "        if '.txt' in docname:\n",
    "            filename = cik + '_' + date + '.txt'\n",
    "            html_file = open(filename, 'a')\n",
    "            html_file.write(file.text)\n",
    "            html_file.close()\n",
    "        else:\n",
    "            filename = cik + '_' + date + '.html'\n",
    "            html_file = open(filename, 'a')\n",
    "            html_file.write(file.text)\n",
    "            html_file.close()\n",
    "    \n",
    "    if(total_files_to_scrape!=files_scraped):\n",
    "        print(\"Some files failed to scrape\\n\")\n",
    "        text=\"Some files failed to scrape\\n\" + str(total_files_to_scrape) + \"!=\"+ str(files_scraped) + \"\\n\"\n",
    "        SaveLogFile(log_file_name, text)\n",
    "        \n",
    "    SaveLogFile(log_file_name, \"saving log files==========================\\n\")\n",
    "    CloseLogFile(log_file_name)\n",
    "    os.chdir('..')\n",
    "        \n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now we call the Extract10K function to scrape 10-K sec filings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/2 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping CIK  0000066740\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 1/2 [00:00<00:00,  1.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping CIK  0000001800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.47it/s]\n"
     ]
    }
   ],
   "source": [
    "# To scrape 10-Ks, call the Extract10K method\n",
    "# Define parameters for Extract10K method\n",
    "browse_url_10k = 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=%s&type=10-K'\n",
    "filing_url_10k = 'http://www.sec.gov/Archives/edgar/data/%s/%s-index.html'\n",
    "doc_url_10k = 'http://www.sec.gov/Archives/edgar/data/%s/%s/%s'\n",
    "\n",
    "# moving the directory, where scraped 10-K filings are\n",
    "os.chdir(scraped10k_path)\n",
    "log_file = 'log.txt'\n",
    "\n",
    "for cik in tqdm(ciks_tick_df['cik']):\n",
    "    Extract10K(browse_url_base=browse_url_10k, \n",
    "          filing_url_base=filing_url_10k, \n",
    "          doc_url_base=doc_url_10k, \n",
    "          cik=cik,\n",
    "          log_file_name=log_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Now we call the Extract10Q function to scrape 10-K sec filings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/2 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping CIK  0000066740\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      " 50%|█████     | 1/2 [00:00<00:00,  1.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping CIK  0000001800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:01<00:00,  1.42it/s]\n"
     ]
    }
   ],
   "source": [
    "# To scrape 10-Ks, call the Extract10Q method\n",
    "# Define parameters for Extract10Q method\n",
    "browse_url_10q = 'https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=%s&type=10-Q&count=1000'\n",
    "filing_url_10q = 'http://www.sec.gov/Archives/edgar/data/%s/%s-index.html'\n",
    "doc_url_10q = 'http://www.sec.gov/Archives/edgar/data/%s/%s/%s'\n",
    "\n",
    "# Set correct directory (fill this out yourself!)\n",
    "os.chdir(scraped10q_path)\n",
    "log_file_name = 'log.txt'\n",
    "\n",
    "# moving the directory, where scraped 10-K filings are\n",
    "for cik in tqdm(ciks_tick_df['cik']):\n",
    "    Extract10Q(browse_url_base=browse_url_10q, \n",
    "          filing_url_base=filing_url_10q, \n",
    "          doc_url_base=doc_url_10q, \n",
    "          cik=cik,\n",
    "          log_file_name=log_file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Parsing the HTML files in text files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### remove numerical tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RemoveNumberTables(soup):\n",
    "    \n",
    "    def GetNumPercentage(tablestring):\n",
    "        if len(tablestring)>0.0:\n",
    "            numbers = sum([char.isdigit() for char in tablestring])\n",
    "            length = len(tablestring)\n",
    "            return numbers/length\n",
    "        else:\n",
    "            return 1\n",
    "        \n",
    "    [x.extract() for x in soup.find_all('table') if GetNumPercentage(x.get_text())>0.15]\n",
    "    \n",
    "    return soup"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### remove html tags from files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RemoveHTMLTags(soup):\n",
    "    \n",
    "    text = soup.get_text()\n",
    "    text = text.replace('\\n', ' ')\n",
    "    text = unicodedata.normalize('NFKD', text)\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### parse html data into textual data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ParseHTMLToText(cik):\n",
    "    try: \n",
    "        os.chdir(cik)\n",
    "  \n",
    "    except FileNotFoundError:\n",
    "        print(\"Could not find directory for CIK\", cik)\n",
    "        return\n",
    "        \n",
    "    print(\"Parsing CIK %s...\" % cik)\n",
    "    #setting the flag to check if a flie already parsed or not\n",
    "    parsed = False \n",
    "    \n",
    "    # Creating a new directory 'clean_data' within the CIK directory\n",
    "    # and storing text data of filings\n",
    "    if os.path.exists('rawtext'):\n",
    "        os.makedirs('clean_data')\n",
    "        os.rename(\"rawtext\", \"clean_data\")\n",
    "    elif not os.path.exists('clean_data'):\n",
    "        os.makedirs('clean_data')\n",
    "      \n",
    "    file_list = [fname for fname in os.listdir() if not (fname.startswith('.') | os.path.isdir(fname) | fname.startswith('log'))]\n",
    " \n",
    "    for filename in file_list:\n",
    "        new_filename = filename.replace('.html', '.txt')\n",
    "        text_file_list = os.listdir('clean_data')\n",
    "        if new_filename in text_file_list:\n",
    "            continue\n",
    "               \n",
    "        #Cleaning the file by removing numerical tables, tags etc.\n",
    "        with open(filename, 'r') as file:\n",
    "            parsed = True\n",
    "            soup = bs.BeautifulSoup(file.read(), \"lxml\")\n",
    "            soup = RemoveNumberTables(soup)\n",
    "            text = RemoveHTMLTags(soup)\n",
    "            hex_handle = re.sub(r'[^\\x00-\\x7f]',' ', text)\n",
    "            with open('clean_data/'+new_filename, 'w') as newfile:\n",
    "                newfile.write(hex_handle)\n",
    "    \n",
    "    # Once all the files parsed, log them\n",
    "    if parsed==False:\n",
    "        print(\"Already parsed CIK\", cik)\n",
    "    \n",
    "    os.chdir('..')\n",
    "    return"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We can now apply this function to each of our 10-K and 10-Q files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00, 242.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing CIK 0000066740...\n",
      "Already parsed CIK 0000066740\n",
      "Parsing CIK 0000001800...\n",
      "Already parsed CIK 0000001800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Parsing the 10-K files\n",
    "os.chdir(scraped10k_path)\n",
    "for cik in tqdm(ciks_tick_df['cik']):\n",
    "    ParseHTMLToText(cik)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2/2 [00:00<00:00, 174.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing CIK 0000066740...\n",
      "Already parsed CIK 0000066740\n",
      "Parsing CIK 0000001800...\n",
      "Already parsed CIK 0000001800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Parsing the 10-Q files\n",
    "os.chdir(scraped10q_path)\n",
    "for cik in tqdm(ciks_tick_df['cik']):\n",
    "    ParseHTMLToText(cik)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}