impute_reduce-prices.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of stocks (rows): 2060\n",
      "\n",
      "Number of days (columns): 3635\n"
     ]
    }
   ],
   "source": [
    "#import training dataset\n",
    "trdt = pd.read_csv(\"training_data.csv\")\n",
    "print(\"\\nNumber of stocks (rows):\", len(trdt))\n",
    "print(\"\\nNumber of days (columns):\", len(trdt.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['price_20151231',\n",
       " 'bm_2010-01',\n",
       " 'bm_2010-02',\n",
       " 'bm_2010-03',\n",
       " 'bm_2010-04',\n",
       " 'bm_2010-05',\n",
       " 'bm_2010-06',\n",
       " 'bm_2010-07',\n",
       " 'bm_2010-08',\n",
       " 'bm_2010-09',\n",
       " 'bm_2010-10',\n",
       " 'bm_2010-11',\n",
       " 'bm_2010-12',\n",
       " 'bm_2011-01',\n",
       " 'bm_2011-02',\n",
       " 'bm_2011-03',\n",
       " 'bm_2011-04',\n",
       " 'bm_2011-05',\n",
       " 'bm_2011-06',\n",
       " 'bm_2011-07',\n",
       " 'bm_2011-08',\n",
       " 'bm_2011-09',\n",
       " 'bm_2011-10',\n",
       " 'bm_2011-11',\n",
       " 'bm_2011-12',\n",
       " 'bm_2012-01',\n",
       " 'bm_2012-02',\n",
       " 'bm_2012-03',\n",
       " 'bm_2012-04',\n",
       " 'bm_2012-05',\n",
       " 'bm_2012-06',\n",
       " 'bm_2012-07',\n",
       " 'bm_2012-08',\n",
       " 'bm_2012-09',\n",
       " 'bm_2012-10',\n",
       " 'bm_2012-11',\n",
       " 'bm_2012-12',\n",
       " 'bm_2013-01',\n",
       " 'bm_2013-02',\n",
       " 'bm_2013-03',\n",
       " 'bm_2013-04',\n",
       " 'bm_2013-05',\n",
       " 'bm_2013-06',\n",
       " 'bm_2013-07',\n",
       " 'bm_2013-08',\n",
       " 'bm_2013-09',\n",
       " 'bm_2013-10',\n",
       " 'bm_2013-11',\n",
       " 'bm_2013-12',\n",
       " 'bm_2014-01',\n",
       " 'bm_2014-02',\n",
       " 'bm_2014-03',\n",
       " 'bm_2014-04',\n",
       " 'bm_2014-05',\n",
       " 'bm_2014-06',\n",
       " 'bm_2014-07',\n",
       " 'bm_2014-08',\n",
       " 'bm_2014-09',\n",
       " 'bm_2014-10',\n",
       " 'bm_2014-11',\n",
       " 'bm_2014-12',\n",
       " 'bm_2015-01',\n",
       " 'bm_2015-02',\n",
       " 'bm_2015-03',\n",
       " 'bm_2015-04',\n",
       " 'bm_2015-05',\n",
       " 'bm_2015-06',\n",
       " 'bm_2015-07',\n",
       " 'bm_2015-08',\n",
       " 'bm_2015-09',\n",
       " 'bm_2015-10',\n",
       " 'bm_2015-11',\n",
       " 'bm_2015-12',\n",
       " 'pe_exi_2010-01',\n",
       " 'pe_exi_2010-02',\n",
       " 'pe_exi_2010-03',\n",
       " 'pe_exi_2010-04',\n",
       " 'pe_exi_2010-05',\n",
       " 'pe_exi_2010-06',\n",
       " 'pe_exi_2010-07',\n",
       " 'pe_exi_2010-08',\n",
       " 'pe_exi_2010-09',\n",
       " 'pe_exi_2010-10',\n",
       " 'pe_exi_2010-11',\n",
       " 'pe_exi_2010-12',\n",
       " 'pe_exi_2011-01',\n",
       " 'pe_exi_2011-02',\n",
       " 'pe_exi_2011-03',\n",
       " 'pe_exi_2011-04',\n",
       " 'pe_exi_2011-05',\n",
       " 'pe_exi_2011-06',\n",
       " 'pe_exi_2011-07',\n",
       " 'pe_exi_2011-08',\n",
       " 'pe_exi_2011-09',\n",
       " 'pe_exi_2011-10',\n",
       " 'pe_exi_2011-11',\n",
       " 'pe_exi_2011-12',\n",
       " 'pe_exi_2012-01',\n",
       " 'pe_exi_2012-02',\n",
       " 'pe_exi_2012-03',\n",
       " 'pe_exi_2012-04',\n",
       " 'pe_exi_2012-05',\n",
       " 'pe_exi_2012-06',\n",
       " 'pe_exi_2012-07',\n",
       " 'pe_exi_2012-08',\n",
       " 'pe_exi_2012-09',\n",
       " 'pe_exi_2012-10',\n",
       " 'pe_exi_2012-11',\n",
       " 'pe_exi_2012-12',\n",
       " 'pe_exi_2013-01',\n",
       " 'pe_exi_2013-02',\n",
       " 'pe_exi_2013-03',\n",
       " 'pe_exi_2013-04',\n",
       " 'pe_exi_2013-05',\n",
       " 'pe_exi_2013-06',\n",
       " 'pe_exi_2013-07',\n",
       " 'pe_exi_2013-08',\n",
       " 'pe_exi_2013-09',\n",
       " 'pe_exi_2013-10',\n",
       " 'pe_exi_2013-11',\n",
       " 'pe_exi_2013-12',\n",
       " 'pe_exi_2014-01',\n",
       " 'pe_exi_2014-02',\n",
       " 'pe_exi_2014-03',\n",
       " 'pe_exi_2014-04',\n",
       " 'pe_exi_2014-05',\n",
       " 'pe_exi_2014-06',\n",
       " 'pe_exi_2014-07',\n",
       " 'pe_exi_2014-08',\n",
       " 'pe_exi_2014-09',\n",
       " 'pe_exi_2014-10',\n",
       " 'pe_exi_2014-11',\n",
       " 'pe_exi_2014-12',\n",
       " 'pe_exi_2015-01',\n",
       " 'pe_exi_2015-02',\n",
       " 'pe_exi_2015-03',\n",
       " 'pe_exi_2015-04',\n",
       " 'pe_exi_2015-05',\n",
       " 'pe_exi_2015-06',\n",
       " 'pe_exi_2015-07',\n",
       " 'pe_exi_2015-08',\n",
       " 'pe_exi_2015-09',\n",
       " 'pe_exi_2015-10',\n",
       " 'pe_exi_2015-11',\n",
       " 'pe_exi_2015-12',\n",
       " 'npm_2010-01',\n",
       " 'npm_2010-02',\n",
       " 'npm_2010-03',\n",
       " 'npm_2010-04',\n",
       " 'npm_2010-05',\n",
       " 'npm_2010-06',\n",
       " 'npm_2010-07',\n",
       " 'npm_2010-08',\n",
       " 'npm_2010-09',\n",
       " 'npm_2010-10',\n",
       " 'npm_2010-11',\n",
       " 'npm_2010-12',\n",
       " 'npm_2011-01',\n",
       " 'npm_2011-02',\n",
       " 'npm_2011-03',\n",
       " 'npm_2011-04',\n",
       " 'npm_2011-05',\n",
       " 'npm_2011-06',\n",
       " 'npm_2011-07',\n",
       " 'npm_2011-08',\n",
       " 'npm_2011-09',\n",
       " 'npm_2011-10',\n",
       " 'npm_2011-11',\n",
       " 'npm_2011-12',\n",
       " 'npm_2012-01',\n",
       " 'npm_2012-02',\n",
       " 'npm_2012-03',\n",
       " 'npm_2012-04',\n",
       " 'npm_2012-05',\n",
       " 'npm_2012-06',\n",
       " 'npm_2012-07',\n",
       " 'npm_2012-08',\n",
       " 'npm_2012-09',\n",
       " 'npm_2012-10',\n",
       " 'npm_2012-11',\n",
       " 'npm_2012-12',\n",
       " 'npm_2013-01',\n",
       " 'npm_2013-02',\n",
       " 'npm_2013-03',\n",
       " 'npm_2013-04',\n",
       " 'npm_2013-05',\n",
       " 'npm_2013-06',\n",
       " 'npm_2013-07',\n",
       " 'npm_2013-08',\n",
       " 'npm_2013-09',\n",
       " 'npm_2013-10',\n",
       " 'npm_2013-11',\n",
       " 'npm_2013-12',\n",
       " 'npm_2014-01',\n",
       " 'npm_2014-02',\n",
       " 'npm_2014-03',\n",
       " 'npm_2014-04',\n",
       " 'npm_2014-05',\n",
       " 'npm_2014-06',\n",
       " 'npm_2014-07',\n",
       " 'npm_2014-08',\n",
       " 'npm_2014-09',\n",
       " 'npm_2014-10',\n",
       " 'npm_2014-11',\n",
       " 'npm_2014-12',\n",
       " 'npm_2015-01',\n",
       " 'npm_2015-02',\n",
       " 'npm_2015-03',\n",
       " 'npm_2015-04',\n",
       " 'npm_2015-05',\n",
       " 'npm_2015-06',\n",
       " 'npm_2015-07',\n",
       " 'npm_2015-08',\n",
       " 'npm_2015-09',\n",
       " 'npm_2015-10',\n",
       " 'npm_2015-11',\n",
       " 'npm_2015-12',\n",
       " 'cfm_2010-01',\n",
       " 'cfm_2010-02',\n",
       " 'cfm_2010-03',\n",
       " 'cfm_2010-04',\n",
       " 'cfm_2010-05',\n",
       " 'cfm_2010-06',\n",
       " 'cfm_2010-07',\n",
       " 'cfm_2010-08',\n",
       " 'cfm_2010-09',\n",
       " 'cfm_2010-10',\n",
       " 'cfm_2010-11',\n",
       " 'cfm_2010-12',\n",
       " 'cfm_2011-01',\n",
       " 'cfm_2011-02',\n",
       " 'cfm_2011-03',\n",
       " 'cfm_2011-04',\n",
       " 'cfm_2011-05',\n",
       " 'cfm_2011-06',\n",
       " 'cfm_2011-07',\n",
       " 'cfm_2011-08',\n",
       " 'cfm_2011-09',\n",
       " 'cfm_2011-10',\n",
       " 'cfm_2011-11',\n",
       " 'cfm_2011-12',\n",
       " 'cfm_2012-01',\n",
       " 'cfm_2012-02',\n",
       " 'cfm_2012-03',\n",
       " 'cfm_2012-04',\n",
       " 'cfm_2012-05',\n",
       " 'cfm_2012-06',\n",
       " 'cfm_2012-07',\n",
       " 'cfm_2012-08',\n",
       " 'cfm_2012-09',\n",
       " 'cfm_2012-10',\n",
       " 'cfm_2012-11',\n",
       " 'cfm_2012-12',\n",
       " 'cfm_2013-01',\n",
       " 'cfm_2013-02',\n",
       " 'cfm_2013-03',\n",
       " 'cfm_2013-04',\n",
       " 'cfm_2013-05',\n",
       " 'cfm_2013-06',\n",
       " 'cfm_2013-07',\n",
       " 'cfm_2013-08',\n",
       " 'cfm_2013-09',\n",
       " 'cfm_2013-10',\n",
       " 'cfm_2013-11',\n",
       " 'cfm_2013-12',\n",
       " 'cfm_2014-01',\n",
       " 'cfm_2014-02',\n",
       " 'cfm_2014-03',\n",
       " 'cfm_2014-04',\n",
       " 'cfm_2014-05',\n",
       " 'cfm_2014-06',\n",
       " 'cfm_2014-07',\n",
       " 'cfm_2014-08',\n",
       " 'cfm_2014-09',\n",
       " 'cfm_2014-10',\n",
       " 'cfm_2014-11',\n",
       " 'cfm_2014-12',\n",
       " 'cfm_2015-01',\n",
       " 'cfm_2015-02',\n",
       " 'cfm_2015-03',\n",
       " 'cfm_2015-04',\n",
       " 'cfm_2015-05',\n",
       " 'cfm_2015-06',\n",
       " 'cfm_2015-07',\n",
       " 'cfm_2015-08',\n",
       " 'cfm_2015-09',\n",
       " 'cfm_2015-10',\n",
       " 'cfm_2015-11',\n",
       " 'cfm_2015-12',\n",
       " 'roa_2010-01',\n",
       " 'roa_2010-02',\n",
       " 'roa_2010-03',\n",
       " 'roa_2010-04',\n",
       " 'roa_2010-05',\n",
       " 'roa_2010-06',\n",
       " 'roa_2010-07',\n",
       " 'roa_2010-08',\n",
       " 'roa_2010-09',\n",
       " 'roa_2010-10',\n",
       " 'roa_2010-11',\n",
       " 'roa_2010-12',\n",
       " 'roa_2011-01',\n",
       " 'roa_2011-02',\n",
       " 'roa_2011-03',\n",
       " 'roa_2011-04',\n",
       " 'roa_2011-05',\n",
       " 'roa_2011-06',\n",
       " 'roa_2011-07',\n",
       " 'roa_2011-08',\n",
       " 'roa_2011-09',\n",
       " 'roa_2011-10',\n",
       " 'roa_2011-11',\n",
       " 'roa_2011-12',\n",
       " 'roa_2012-01',\n",
       " 'roa_2012-02',\n",
       " 'roa_2012-03',\n",
       " 'roa_2012-04',\n",
       " 'roa_2012-05',\n",
       " 'roa_2012-06',\n",
       " 'roa_2012-07',\n",
       " 'roa_2012-08',\n",
       " 'roa_2012-09',\n",
       " 'roa_2012-10',\n",
       " 'roa_2012-11',\n",
       " 'roa_2012-12',\n",
       " 'roa_2013-01',\n",
       " 'roa_2013-02',\n",
       " 'roa_2013-03',\n",
       " 'roa_2013-04',\n",
       " 'roa_2013-05',\n",
       " 'roa_2013-06',\n",
       " 'roa_2013-07',\n",
       " 'roa_2013-08',\n",
       " 'roa_2013-09',\n",
       " 'roa_2013-10',\n",
       " 'roa_2013-11',\n",
       " 'roa_2013-12',\n",
       " 'roa_2014-01',\n",
       " 'roa_2014-02',\n",
       " 'roa_2014-03',\n",
       " 'roa_2014-04',\n",
       " 'roa_2014-05',\n",
       " 'roa_2014-06',\n",
       " 'roa_2014-07',\n",
       " 'roa_2014-08',\n",
       " 'roa_2014-09',\n",
       " 'roa_2014-10',\n",
       " 'roa_2014-11',\n",
       " 'roa_2014-12',\n",
       " 'roa_2015-01',\n",
       " 'roa_2015-02',\n",
       " 'roa_2015-03',\n",
       " 'roa_2015-04',\n",
       " 'roa_2015-05',\n",
       " 'roa_2015-06',\n",
       " 'roa_2015-07',\n",
       " 'roa_2015-08',\n",
       " 'roa_2015-09',\n",
       " 'roa_2015-10',\n",
       " 'roa_2015-11',\n",
       " 'roa_2015-12',\n",
       " 'roe_2010-01',\n",
       " 'roe_2010-02',\n",
       " 'roe_2010-03',\n",
       " 'roe_2010-04',\n",
       " 'roe_2010-05',\n",
       " 'roe_2010-06',\n",
       " 'roe_2010-07',\n",
       " 'roe_2010-08',\n",
       " 'roe_2010-09',\n",
       " 'roe_2010-10',\n",
       " 'roe_2010-11',\n",
       " 'roe_2010-12',\n",
       " 'roe_2011-01',\n",
       " 'roe_2011-02',\n",
       " 'roe_2011-03',\n",
       " 'roe_2011-04',\n",
       " 'roe_2011-05',\n",
       " 'roe_2011-06',\n",
       " 'roe_2011-07',\n",
       " 'roe_2011-08',\n",
       " 'roe_2011-09',\n",
       " 'roe_2011-10',\n",
       " 'roe_2011-11',\n",
       " 'roe_2011-12',\n",
       " 'roe_2012-01',\n",
       " 'roe_2012-02',\n",
       " 'roe_2012-03',\n",
       " 'roe_2012-04',\n",
       " 'roe_2012-05',\n",
       " 'roe_2012-06',\n",
       " 'roe_2012-07',\n",
       " 'roe_2012-08',\n",
       " 'roe_2012-09',\n",
       " 'roe_2012-10',\n",
       " 'roe_2012-11',\n",
       " 'roe_2012-12',\n",
       " 'roe_2013-01',\n",
       " 'roe_2013-02',\n",
       " 'roe_2013-03',\n",
       " 'roe_2013-04',\n",
       " 'roe_2013-05',\n",
       " 'roe_2013-06',\n",
       " 'roe_2013-07',\n",
       " 'roe_2013-08',\n",
       " 'roe_2013-09',\n",
       " 'roe_2013-10',\n",
       " 'roe_2013-11',\n",
       " 'roe_2013-12',\n",
       " 'roe_2014-01',\n",
       " 'roe_2014-02',\n",
       " 'roe_2014-03',\n",
       " 'roe_2014-04',\n",
       " 'roe_2014-05',\n",
       " 'roe_2014-06',\n",
       " 'roe_2014-07',\n",
       " 'roe_2014-08',\n",
       " 'roe_2014-09',\n",
       " 'roe_2014-10',\n",
       " 'roe_2014-11',\n",
       " 'roe_2014-12',\n",
       " 'roe_2015-01',\n",
       " 'roe_2015-02',\n",
       " 'roe_2015-03',\n",
       " 'roe_2015-04',\n",
       " 'roe_2015-05',\n",
       " 'roe_2015-06',\n",
       " 'roe_2015-07',\n",
       " 'roe_2015-08',\n",
       " 'roe_2015-09',\n",
       " 'roe_2015-10',\n",
       " 'roe_2015-11',\n",
       " 'roe_2015-12',\n",
       " 'pretret_noa_2010-01',\n",
       " 'pretret_noa_2010-02',\n",
       " 'pretret_noa_2010-03',\n",
       " 'pretret_noa_2010-04',\n",
       " 'pretret_noa_2010-05',\n",
       " 'pretret_noa_2010-06',\n",
       " 'pretret_noa_2010-07',\n",
       " 'pretret_noa_2010-08',\n",
       " 'pretret_noa_2010-09',\n",
       " 'pretret_noa_2010-10',\n",
       " 'pretret_noa_2010-11',\n",
       " 'pretret_noa_2010-12',\n",
       " 'pretret_noa_2011-01',\n",
       " 'pretret_noa_2011-02',\n",
       " 'pretret_noa_2011-03',\n",
       " 'pretret_noa_2011-04',\n",
       " 'pretret_noa_2011-05',\n",
       " 'pretret_noa_2011-06',\n",
       " 'pretret_noa_2011-07',\n",
       " 'pretret_noa_2011-08',\n",
       " 'pretret_noa_2011-09',\n",
       " 'pretret_noa_2011-10',\n",
       " 'pretret_noa_2011-11',\n",
       " 'pretret_noa_2011-12',\n",
       " 'pretret_noa_2012-01',\n",
       " 'pretret_noa_2012-02',\n",
       " 'pretret_noa_2012-03',\n",
       " 'pretret_noa_2012-04',\n",
       " 'pretret_noa_2012-05',\n",
       " 'pretret_noa_2012-06',\n",
       " 'pretret_noa_2012-07',\n",
       " 'pretret_noa_2012-08',\n",
       " 'pretret_noa_2012-09',\n",
       " 'pretret_noa_2012-10',\n",
       " 'pretret_noa_2012-11',\n",
       " 'pretret_noa_2012-12',\n",
       " 'pretret_noa_2013-01',\n",
       " 'pretret_noa_2013-02',\n",
       " 'pretret_noa_2013-03',\n",
       " 'pretret_noa_2013-04',\n",
       " 'pretret_noa_2013-05',\n",
       " 'pretret_noa_2013-06',\n",
       " 'pretret_noa_2013-07',\n",
       " 'pretret_noa_2013-08',\n",
       " 'pretret_noa_2013-09',\n",
       " 'pretret_noa_2013-10',\n",
       " 'pretret_noa_2013-11',\n",
       " 'pretret_noa_2013-12',\n",
       " 'pretret_noa_2014-01',\n",
       " 'pretret_noa_2014-02',\n",
       " 'pretret_noa_2014-03',\n",
       " 'pretret_noa_2014-04',\n",
       " 'pretret_noa_2014-05',\n",
       " 'pretret_noa_2014-06',\n",
       " 'pretret_noa_2014-07',\n",
       " 'pretret_noa_2014-08',\n",
       " 'pretret_noa_2014-09',\n",
       " 'pretret_noa_2014-10',\n",
       " 'pretret_noa_2014-11',\n",
       " 'pretret_noa_2014-12',\n",
       " 'pretret_noa_2015-01',\n",
       " 'pretret_noa_2015-02',\n",
       " 'pretret_noa_2015-03',\n",
       " 'pretret_noa_2015-04',\n",
       " 'pretret_noa_2015-05',\n",
       " 'pretret_noa_2015-06',\n",
       " 'pretret_noa_2015-07',\n",
       " 'pretret_noa_2015-08',\n",
       " 'pretret_noa_2015-09',\n",
       " 'pretret_noa_2015-10',\n",
       " 'pretret_noa_2015-11',\n",
       " 'pretret_noa_2015-12',\n",
       " 'GProf_2010-01',\n",
       " 'GProf_2010-02',\n",
       " 'GProf_2010-03',\n",
       " 'GProf_2010-04',\n",
       " 'GProf_2010-05',\n",
       " 'GProf_2010-06',\n",
       " 'GProf_2010-07',\n",
       " 'GProf_2010-08',\n",
       " 'GProf_2010-09',\n",
       " 'GProf_2010-10',\n",
       " 'GProf_2010-11',\n",
       " 'GProf_2010-12',\n",
       " 'GProf_2011-01',\n",
       " 'GProf_2011-02',\n",
       " 'GProf_2011-03',\n",
       " 'GProf_2011-04',\n",
       " 'GProf_2011-05',\n",
       " 'GProf_2011-06',\n",
       " 'GProf_2011-07',\n",
       " 'GProf_2011-08',\n",
       " 'GProf_2011-09',\n",
       " 'GProf_2011-10',\n",
       " 'GProf_2011-11',\n",
       " 'GProf_2011-12',\n",
       " 'GProf_2012-01',\n",
       " 'GProf_2012-02',\n",
       " 'GProf_2012-03',\n",
       " 'GProf_2012-04',\n",
       " 'GProf_2012-05',\n",
       " 'GProf_2012-06',\n",
       " 'GProf_2012-07',\n",
       " 'GProf_2012-08',\n",
       " 'GProf_2012-09',\n",
       " 'GProf_2012-10',\n",
       " 'GProf_2012-11',\n",
       " 'GProf_2012-12',\n",
       " 'GProf_2013-01',\n",
       " 'GProf_2013-02',\n",
       " 'GProf_2013-03',\n",
       " 'GProf_2013-04',\n",
       " 'GProf_2013-05',\n",
       " 'GProf_2013-06',\n",
       " 'GProf_2013-07',\n",
       " 'GProf_2013-08',\n",
       " 'GProf_2013-09',\n",
       " 'GProf_2013-10',\n",
       " 'GProf_2013-11',\n",
       " 'GProf_2013-12',\n",
       " 'GProf_2014-01',\n",
       " 'GProf_2014-02',\n",
       " 'GProf_2014-03',\n",
       " 'GProf_2014-04',\n",
       " 'GProf_2014-05',\n",
       " 'GProf_2014-06',\n",
       " 'GProf_2014-07',\n",
       " 'GProf_2014-08',\n",
       " 'GProf_2014-09',\n",
       " 'GProf_2014-10',\n",
       " 'GProf_2014-11',\n",
       " 'GProf_2014-12',\n",
       " 'GProf_2015-01',\n",
       " 'GProf_2015-02',\n",
       " 'GProf_2015-03',\n",
       " 'GProf_2015-04',\n",
       " 'GProf_2015-05',\n",
       " 'GProf_2015-06',\n",
       " 'GProf_2015-07',\n",
       " 'GProf_2015-08',\n",
       " 'GProf_2015-09',\n",
       " 'GProf_2015-10',\n",
       " 'GProf_2015-11',\n",
       " 'GProf_2015-12',\n",
       " 'totdebt_invcap_2010-01',\n",
       " 'totdebt_invcap_2010-02',\n",
       " 'totdebt_invcap_2010-03',\n",
       " 'totdebt_invcap_2010-04',\n",
       " 'totdebt_invcap_2010-05',\n",
       " 'totdebt_invcap_2010-06',\n",
       " 'totdebt_invcap_2010-07',\n",
       " 'totdebt_invcap_2010-08',\n",
       " 'totdebt_invcap_2010-09',\n",
       " 'totdebt_invcap_2010-10',\n",
       " 'totdebt_invcap_2010-11',\n",
       " 'totdebt_invcap_2010-12',\n",
       " 'totdebt_invcap_2011-01',\n",
       " 'totdebt_invcap_2011-02',\n",
       " 'totdebt_invcap_2011-03',\n",
       " 'totdebt_invcap_2011-04',\n",
       " 'totdebt_invcap_2011-05',\n",
       " 'totdebt_invcap_2011-06',\n",
       " 'totdebt_invcap_2011-07',\n",
       " 'totdebt_invcap_2011-08',\n",
       " 'totdebt_invcap_2011-09',\n",
       " 'totdebt_invcap_2011-10',\n",
       " 'totdebt_invcap_2011-11',\n",
       " 'totdebt_invcap_2011-12',\n",
       " 'totdebt_invcap_2012-01',\n",
       " 'totdebt_invcap_2012-02',\n",
       " 'totdebt_invcap_2012-03',\n",
       " 'totdebt_invcap_2012-04',\n",
       " 'totdebt_invcap_2012-05',\n",
       " 'totdebt_invcap_2012-06',\n",
       " 'totdebt_invcap_2012-07',\n",
       " 'totdebt_invcap_2012-08',\n",
       " 'totdebt_invcap_2012-09',\n",
       " 'totdebt_invcap_2012-10',\n",
       " 'totdebt_invcap_2012-11',\n",
       " 'totdebt_invcap_2012-12',\n",
       " 'totdebt_invcap_2013-01',\n",
       " 'totdebt_invcap_2013-02',\n",
       " 'totdebt_invcap_2013-03',\n",
       " 'totdebt_invcap_2013-04',\n",
       " 'totdebt_invcap_2013-05',\n",
       " 'totdebt_invcap_2013-06',\n",
       " 'totdebt_invcap_2013-07',\n",
       " 'totdebt_invcap_2013-08',\n",
       " 'totdebt_invcap_2013-09',\n",
       " 'totdebt_invcap_2013-10',\n",
       " 'totdebt_invcap_2013-11',\n",
       " 'totdebt_invcap_2013-12',\n",
       " 'totdebt_invcap_2014-01',\n",
       " 'totdebt_invcap_2014-02',\n",
       " 'totdebt_invcap_2014-03',\n",
       " 'totdebt_invcap_2014-04',\n",
       " 'totdebt_invcap_2014-05',\n",
       " 'totdebt_invcap_2014-06',\n",
       " 'totdebt_invcap_2014-07',\n",
       " 'totdebt_invcap_2014-08',\n",
       " 'totdebt_invcap_2014-09',\n",
       " 'totdebt_invcap_2014-10',\n",
       " 'totdebt_invcap_2014-11',\n",
       " 'totdebt_invcap_2014-12',\n",
       " 'totdebt_invcap_2015-01',\n",
       " 'totdebt_invcap_2015-02',\n",
       " 'totdebt_invcap_2015-03',\n",
       " 'totdebt_invcap_2015-04',\n",
       " 'totdebt_invcap_2015-05',\n",
       " 'totdebt_invcap_2015-06',\n",
       " 'totdebt_invcap_2015-07',\n",
       " 'totdebt_invcap_2015-08',\n",
       " 'totdebt_invcap_2015-09',\n",
       " 'totdebt_invcap_2015-10',\n",
       " 'totdebt_invcap_2015-11',\n",
       " 'totdebt_invcap_2015-12',\n",
       " 'capital_ratio_2010-01',\n",
       " 'capital_ratio_2010-02',\n",
       " 'capital_ratio_2010-03',\n",
       " 'capital_ratio_2010-04',\n",
       " 'capital_ratio_2010-05',\n",
       " 'capital_ratio_2010-06',\n",
       " 'capital_ratio_2010-07',\n",
       " 'capital_ratio_2010-08',\n",
       " 'capital_ratio_2010-09',\n",
       " 'capital_ratio_2010-10',\n",
       " 'capital_ratio_2010-11',\n",
       " 'capital_ratio_2010-12',\n",
       " 'capital_ratio_2011-01',\n",
       " 'capital_ratio_2011-02',\n",
       " 'capital_ratio_2011-03',\n",
       " 'capital_ratio_2011-04',\n",
       " 'capital_ratio_2011-05',\n",
       " 'capital_ratio_2011-06',\n",
       " 'capital_ratio_2011-07',\n",
       " 'capital_ratio_2011-08',\n",
       " 'capital_ratio_2011-09',\n",
       " 'capital_ratio_2011-10',\n",
       " 'capital_ratio_2011-11',\n",
       " 'capital_ratio_2011-12',\n",
       " 'capital_ratio_2012-01',\n",
       " 'capital_ratio_2012-02',\n",
       " 'capital_ratio_2012-03',\n",
       " 'capital_ratio_2012-04',\n",
       " 'capital_ratio_2012-05',\n",
       " 'capital_ratio_2012-06',\n",
       " 'capital_ratio_2012-07',\n",
       " 'capital_ratio_2012-08',\n",
       " 'capital_ratio_2012-09',\n",
       " 'capital_ratio_2012-10',\n",
       " 'capital_ratio_2012-11',\n",
       " 'capital_ratio_2012-12',\n",
       " 'capital_ratio_2013-01',\n",
       " 'capital_ratio_2013-02',\n",
       " 'capital_ratio_2013-03',\n",
       " 'capital_ratio_2013-04',\n",
       " 'capital_ratio_2013-05',\n",
       " 'capital_ratio_2013-06',\n",
       " 'capital_ratio_2013-07',\n",
       " 'capital_ratio_2013-08',\n",
       " 'capital_ratio_2013-09',\n",
       " 'capital_ratio_2013-10',\n",
       " 'capital_ratio_2013-11',\n",
       " 'capital_ratio_2013-12',\n",
       " 'capital_ratio_2014-01',\n",
       " 'capital_ratio_2014-02',\n",
       " 'capital_ratio_2014-03',\n",
       " 'capital_ratio_2014-04',\n",
       " 'capital_ratio_2014-05',\n",
       " 'capital_ratio_2014-06',\n",
       " 'capital_ratio_2014-07',\n",
       " 'capital_ratio_2014-08',\n",
       " 'capital_ratio_2014-09',\n",
       " 'capital_ratio_2014-10',\n",
       " 'capital_ratio_2014-11',\n",
       " 'capital_ratio_2014-12',\n",
       " 'capital_ratio_2015-01',\n",
       " 'capital_ratio_2015-02',\n",
       " 'capital_ratio_2015-03',\n",
       " 'capital_ratio_2015-04',\n",
       " 'capital_ratio_2015-05',\n",
       " 'capital_ratio_2015-06',\n",
       " 'capital_ratio_2015-07',\n",
       " 'capital_ratio_2015-08',\n",
       " 'capital_ratio_2015-09',\n",
       " 'capital_ratio_2015-10',\n",
       " 'capital_ratio_2015-11',\n",
       " 'capital_ratio_2015-12',\n",
       " 'debt_ebitda_2010-01',\n",
       " 'debt_ebitda_2010-02',\n",
       " 'debt_ebitda_2010-03',\n",
       " 'debt_ebitda_2010-04',\n",
       " 'debt_ebitda_2010-05',\n",
       " 'debt_ebitda_2010-06',\n",
       " 'debt_ebitda_2010-07',\n",
       " 'debt_ebitda_2010-08',\n",
       " 'debt_ebitda_2010-09',\n",
       " 'debt_ebitda_2010-10',\n",
       " 'debt_ebitda_2010-11',\n",
       " 'debt_ebitda_2010-12',\n",
       " 'debt_ebitda_2011-01',\n",
       " 'debt_ebitda_2011-02',\n",
       " 'debt_ebitda_2011-03',\n",
       " 'debt_ebitda_2011-04',\n",
       " 'debt_ebitda_2011-05',\n",
       " 'debt_ebitda_2011-06',\n",
       " 'debt_ebitda_2011-07',\n",
       " 'debt_ebitda_2011-08',\n",
       " 'debt_ebitda_2011-09',\n",
       " 'debt_ebitda_2011-10',\n",
       " 'debt_ebitda_2011-11',\n",
       " 'debt_ebitda_2011-12',\n",
       " 'debt_ebitda_2012-01',\n",
       " 'debt_ebitda_2012-02',\n",
       " 'debt_ebitda_2012-03',\n",
       " 'debt_ebitda_2012-04',\n",
       " 'debt_ebitda_2012-05',\n",
       " 'debt_ebitda_2012-06',\n",
       " 'debt_ebitda_2012-07',\n",
       " 'debt_ebitda_2012-08',\n",
       " 'debt_ebitda_2012-09',\n",
       " 'debt_ebitda_2012-10',\n",
       " 'debt_ebitda_2012-11',\n",
       " 'debt_ebitda_2012-12',\n",
       " 'debt_ebitda_2013-01',\n",
       " 'debt_ebitda_2013-02',\n",
       " 'debt_ebitda_2013-03',\n",
       " 'debt_ebitda_2013-04',\n",
       " 'debt_ebitda_2013-05',\n",
       " 'debt_ebitda_2013-06',\n",
       " 'debt_ebitda_2013-07',\n",
       " 'debt_ebitda_2013-08',\n",
       " 'debt_ebitda_2013-09',\n",
       " 'debt_ebitda_2013-10',\n",
       " 'debt_ebitda_2013-11',\n",
       " 'debt_ebitda_2013-12',\n",
       " 'debt_ebitda_2014-01',\n",
       " 'debt_ebitda_2014-02',\n",
       " 'debt_ebitda_2014-03',\n",
       " 'debt_ebitda_2014-04',\n",
       " 'debt_ebitda_2014-05',\n",
       " 'debt_ebitda_2014-06',\n",
       " 'debt_ebitda_2014-07',\n",
       " 'debt_ebitda_2014-08',\n",
       " 'debt_ebitda_2014-09',\n",
       " 'debt_ebitda_2014-10',\n",
       " 'debt_ebitda_2014-11',\n",
       " 'debt_ebitda_2014-12',\n",
       " 'debt_ebitda_2015-01',\n",
       " 'debt_ebitda_2015-02',\n",
       " 'debt_ebitda_2015-03',\n",
       " 'debt_ebitda_2015-04',\n",
       " 'debt_ebitda_2015-05',\n",
       " 'debt_ebitda_2015-06',\n",
       " 'debt_ebitda_2015-07',\n",
       " 'debt_ebitda_2015-08',\n",
       " 'debt_ebitda_2015-09',\n",
       " 'debt_ebitda_2015-10',\n",
       " 'debt_ebitda_2015-11',\n",
       " 'debt_ebitda_2015-12',\n",
       " 'lt_debt_2010-01',\n",
       " 'lt_debt_2010-02',\n",
       " 'lt_debt_2010-03',\n",
       " 'lt_debt_2010-04',\n",
       " 'lt_debt_2010-05',\n",
       " 'lt_debt_2010-06',\n",
       " 'lt_debt_2010-07',\n",
       " 'lt_debt_2010-08',\n",
       " 'lt_debt_2010-09',\n",
       " 'lt_debt_2010-10',\n",
       " 'lt_debt_2010-11',\n",
       " 'lt_debt_2010-12',\n",
       " 'lt_debt_2011-01',\n",
       " 'lt_debt_2011-02',\n",
       " 'lt_debt_2011-03',\n",
       " 'lt_debt_2011-04',\n",
       " 'lt_debt_2011-05',\n",
       " 'lt_debt_2011-06',\n",
       " 'lt_debt_2011-07',\n",
       " 'lt_debt_2011-08',\n",
       " 'lt_debt_2011-09',\n",
       " 'lt_debt_2011-10',\n",
       " 'lt_debt_2011-11',\n",
       " 'lt_debt_2011-12',\n",
       " 'lt_debt_2012-01',\n",
       " 'lt_debt_2012-02',\n",
       " 'lt_debt_2012-03',\n",
       " 'lt_debt_2012-04',\n",
       " 'lt_debt_2012-05',\n",
       " 'lt_debt_2012-06',\n",
       " 'lt_debt_2012-07',\n",
       " 'lt_debt_2012-08',\n",
       " 'lt_debt_2012-09',\n",
       " 'lt_debt_2012-10',\n",
       " 'lt_debt_2012-11',\n",
       " 'lt_debt_2012-12',\n",
       " 'lt_debt_2013-01',\n",
       " 'lt_debt_2013-02',\n",
       " 'lt_debt_2013-03',\n",
       " 'lt_debt_2013-04',\n",
       " 'lt_debt_2013-05',\n",
       " 'lt_debt_2013-06',\n",
       " 'lt_debt_2013-07',\n",
       " 'lt_debt_2013-08',\n",
       " 'lt_debt_2013-09',\n",
       " 'lt_debt_2013-10',\n",
       " 'lt_debt_2013-11',\n",
       " 'lt_debt_2013-12',\n",
       " 'lt_debt_2014-01',\n",
       " 'lt_debt_2014-02',\n",
       " 'lt_debt_2014-03',\n",
       " 'lt_debt_2014-04',\n",
       " 'lt_debt_2014-05',\n",
       " 'lt_debt_2014-06',\n",
       " 'lt_debt_2014-07',\n",
       " 'lt_debt_2014-08',\n",
       " 'lt_debt_2014-09',\n",
       " 'lt_debt_2014-10',\n",
       " 'lt_debt_2014-11',\n",
       " 'lt_debt_2014-12',\n",
       " 'lt_debt_2015-01',\n",
       " 'lt_debt_2015-02',\n",
       " 'lt_debt_2015-03',\n",
       " 'lt_debt_2015-04',\n",
       " 'lt_debt_2015-05',\n",
       " 'lt_debt_2015-06',\n",
       " 'lt_debt_2015-07',\n",
       " 'lt_debt_2015-08',\n",
       " 'lt_debt_2015-09',\n",
       " 'lt_debt_2015-10',\n",
       " 'lt_debt_2015-11',\n",
       " 'lt_debt_2015-12',\n",
       " 'cash_debt_2010-01',\n",
       " 'cash_debt_2010-02',\n",
       " 'cash_debt_2010-03',\n",
       " 'cash_debt_2010-04',\n",
       " 'cash_debt_2010-05',\n",
       " 'cash_debt_2010-06',\n",
       " 'cash_debt_2010-07',\n",
       " 'cash_debt_2010-08',\n",
       " 'cash_debt_2010-09',\n",
       " 'cash_debt_2010-10',\n",
       " 'cash_debt_2010-11',\n",
       " 'cash_debt_2010-12',\n",
       " 'cash_debt_2011-01',\n",
       " 'cash_debt_2011-02',\n",
       " 'cash_debt_2011-03',\n",
       " 'cash_debt_2011-04',\n",
       " 'cash_debt_2011-05',\n",
       " 'cash_debt_2011-06',\n",
       " 'cash_debt_2011-07',\n",
       " 'cash_debt_2011-08',\n",
       " 'cash_debt_2011-09',\n",
       " 'cash_debt_2011-10',\n",
       " 'cash_debt_2011-11',\n",
       " 'cash_debt_2011-12',\n",
       " 'cash_debt_2012-01',\n",
       " 'cash_debt_2012-02',\n",
       " 'cash_debt_2012-03',\n",
       " 'cash_debt_2012-04',\n",
       " 'cash_debt_2012-05',\n",
       " 'cash_debt_2012-06',\n",
       " 'cash_debt_2012-07',\n",
       " 'cash_debt_2012-08',\n",
       " 'cash_debt_2012-09',\n",
       " 'cash_debt_2012-10',\n",
       " 'cash_debt_2012-11',\n",
       " 'cash_debt_2012-12',\n",
       " 'cash_debt_2013-01',\n",
       " 'cash_debt_2013-02',\n",
       " 'cash_debt_2013-03',\n",
       " 'cash_debt_2013-04',\n",
       " 'cash_debt_2013-05',\n",
       " 'cash_debt_2013-06',\n",
       " 'cash_debt_2013-07',\n",
       " 'cash_debt_2013-08',\n",
       " 'cash_debt_2013-09',\n",
       " 'cash_debt_2013-10',\n",
       " 'cash_debt_2013-11',\n",
       " 'cash_debt_2013-12',\n",
       " 'cash_debt_2014-01',\n",
       " 'cash_debt_2014-02',\n",
       " 'cash_debt_2014-03',\n",
       " 'cash_debt_2014-04',\n",
       " 'cash_debt_2014-05',\n",
       " 'cash_debt_2014-06',\n",
       " 'cash_debt_2014-07',\n",
       " 'cash_debt_2014-08',\n",
       " 'cash_debt_2014-09',\n",
       " 'cash_debt_2014-10',\n",
       " 'cash_debt_2014-11',\n",
       " 'cash_debt_2014-12',\n",
       " 'cash_debt_2015-01',\n",
       " 'cash_debt_2015-02',\n",
       " 'cash_debt_2015-03',\n",
       " 'cash_debt_2015-04',\n",
       " 'cash_debt_2015-05',\n",
       " 'cash_debt_2015-06',\n",
       " 'cash_debt_2015-07',\n",
       " 'cash_debt_2015-08',\n",
       " 'cash_debt_2015-09',\n",
       " 'cash_debt_2015-10',\n",
       " 'cash_debt_2015-11',\n",
       " 'cash_debt_2015-12',\n",
       " 'fcf_ocf_2010-01',\n",
       " 'fcf_ocf_2010-02',\n",
       " 'fcf_ocf_2010-03',\n",
       " 'fcf_ocf_2010-04',\n",
       " 'fcf_ocf_2010-05',\n",
       " 'fcf_ocf_2010-06',\n",
       " 'fcf_ocf_2010-07',\n",
       " 'fcf_ocf_2010-08',\n",
       " 'fcf_ocf_2010-09',\n",
       " 'fcf_ocf_2010-10',\n",
       " 'fcf_ocf_2010-11',\n",
       " 'fcf_ocf_2010-12',\n",
       " 'fcf_ocf_2011-01',\n",
       " 'fcf_ocf_2011-02',\n",
       " 'fcf_ocf_2011-03',\n",
       " 'fcf_ocf_2011-04',\n",
       " 'fcf_ocf_2011-05',\n",
       " 'fcf_ocf_2011-06',\n",
       " 'fcf_ocf_2011-07',\n",
       " 'fcf_ocf_2011-08',\n",
       " 'fcf_ocf_2011-09',\n",
       " 'fcf_ocf_2011-10',\n",
       " 'fcf_ocf_2011-11',\n",
       " 'fcf_ocf_2011-12',\n",
       " 'fcf_ocf_2012-01',\n",
       " 'fcf_ocf_2012-02',\n",
       " 'fcf_ocf_2012-03',\n",
       " 'fcf_ocf_2012-04',\n",
       " 'fcf_ocf_2012-05',\n",
       " 'fcf_ocf_2012-06',\n",
       " 'fcf_ocf_2012-07',\n",
       " 'fcf_ocf_2012-08',\n",
       " 'fcf_ocf_2012-09',\n",
       " 'fcf_ocf_2012-10',\n",
       " 'fcf_ocf_2012-11',\n",
       " 'fcf_ocf_2012-12',\n",
       " 'fcf_ocf_2013-01',\n",
       " 'fcf_ocf_2013-02',\n",
       " 'fcf_ocf_2013-03',\n",
       " 'fcf_ocf_2013-04',\n",
       " 'fcf_ocf_2013-05',\n",
       " 'fcf_ocf_2013-06',\n",
       " 'fcf_ocf_2013-07',\n",
       " 'fcf_ocf_2013-08',\n",
       " 'fcf_ocf_2013-09',\n",
       " 'fcf_ocf_2013-10',\n",
       " 'fcf_ocf_2013-11',\n",
       " 'fcf_ocf_2013-12',\n",
       " 'fcf_ocf_2014-01',\n",
       " 'fcf_ocf_2014-02',\n",
       " 'fcf_ocf_2014-03',\n",
       " 'fcf_ocf_2014-04',\n",
       " 'fcf_ocf_2014-05',\n",
       " 'fcf_ocf_2014-06',\n",
       " 'fcf_ocf_2014-07',\n",
       " 'fcf_ocf_2014-08',\n",
       " 'fcf_ocf_2014-09',\n",
       " 'fcf_ocf_2014-10',\n",
       " 'fcf_ocf_2014-11',\n",
       " 'fcf_ocf_2014-12',\n",
       " 'fcf_ocf_2015-01',\n",
       " 'fcf_ocf_2015-02',\n",
       " 'fcf_ocf_2015-03',\n",
       " ...]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "L=list(trdt.columns);L[2266:]\n",
    "#Cutoff  at 2267"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PERMNO</th>\n",
       "      <th>price_20070103</th>\n",
       "      <th>price_20070104</th>\n",
       "      <th>price_20070105</th>\n",
       "      <th>price_20070108</th>\n",
       "      <th>price_20070109</th>\n",
       "      <th>price_20070110</th>\n",
       "      <th>price_20070111</th>\n",
       "      <th>price_20070112</th>\n",
       "      <th>price_20070116</th>\n",
       "      <th>...</th>\n",
       "      <th>at_turn_2015-03</th>\n",
       "      <th>at_turn_2015-04</th>\n",
       "      <th>at_turn_2015-05</th>\n",
       "      <th>at_turn_2015-06</th>\n",
       "      <th>at_turn_2015-07</th>\n",
       "      <th>at_turn_2015-08</th>\n",
       "      <th>at_turn_2015-09</th>\n",
       "      <th>at_turn_2015-10</th>\n",
       "      <th>at_turn_2015-11</th>\n",
       "      <th>at_turn_2015-12</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10025</td>\n",
       "      <td>49.06</td>\n",
       "      <td>50.34</td>\n",
       "      <td>49.63</td>\n",
       "      <td>49.50</td>\n",
       "      <td>50.62</td>\n",
       "      <td>49.68</td>\n",
       "      <td>49.20</td>\n",
       "      <td>48.90</td>\n",
       "      <td>46.83</td>\n",
       "      <td>...</td>\n",
       "      <td>2.628</td>\n",
       "      <td>2.628</td>\n",
       "      <td>2.628</td>\n",
       "      <td>2.674</td>\n",
       "      <td>2.674</td>\n",
       "      <td>2.674</td>\n",
       "      <td>2.730</td>\n",
       "      <td>2.730</td>\n",
       "      <td>2.730</td>\n",
       "      <td>2.573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10026</td>\n",
       "      <td>41.02</td>\n",
       "      <td>41.89</td>\n",
       "      <td>39.66</td>\n",
       "      <td>40.42</td>\n",
       "      <td>40.54</td>\n",
       "      <td>41.54</td>\n",
       "      <td>41.41</td>\n",
       "      <td>42.23</td>\n",
       "      <td>41.71</td>\n",
       "      <td>...</td>\n",
       "      <td>1.340</td>\n",
       "      <td>1.340</td>\n",
       "      <td>1.348</td>\n",
       "      <td>1.348</td>\n",
       "      <td>1.348</td>\n",
       "      <td>1.357</td>\n",
       "      <td>1.357</td>\n",
       "      <td>1.357</td>\n",
       "      <td>1.349</td>\n",
       "      <td>1.349</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10032</td>\n",
       "      <td>24.12</td>\n",
       "      <td>24.30</td>\n",
       "      <td>23.89</td>\n",
       "      <td>23.70</td>\n",
       "      <td>23.13</td>\n",
       "      <td>20.90</td>\n",
       "      <td>20.84</td>\n",
       "      <td>21.58</td>\n",
       "      <td>20.79</td>\n",
       "      <td>...</td>\n",
       "      <td>1.604</td>\n",
       "      <td>1.604</td>\n",
       "      <td>1.623</td>\n",
       "      <td>1.623</td>\n",
       "      <td>1.623</td>\n",
       "      <td>1.627</td>\n",
       "      <td>1.627</td>\n",
       "      <td>1.627</td>\n",
       "      <td>1.603</td>\n",
       "      <td>1.603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10051</td>\n",
       "      <td>7.40</td>\n",
       "      <td>7.33</td>\n",
       "      <td>7.52</td>\n",
       "      <td>7.44</td>\n",
       "      <td>7.51</td>\n",
       "      <td>7.52</td>\n",
       "      <td>7.45</td>\n",
       "      <td>7.60</td>\n",
       "      <td>7.83</td>\n",
       "      <td>...</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.786</td>\n",
       "      <td>0.787</td>\n",
       "      <td>0.787</td>\n",
       "      <td>0.787</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.834</td>\n",
       "      <td>0.834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10104</td>\n",
       "      <td>17.51</td>\n",
       "      <td>17.68</td>\n",
       "      <td>17.64</td>\n",
       "      <td>17.86</td>\n",
       "      <td>17.82</td>\n",
       "      <td>17.77</td>\n",
       "      <td>17.39</td>\n",
       "      <td>17.50</td>\n",
       "      <td>17.30</td>\n",
       "      <td>...</td>\n",
       "      <td>0.414</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.401</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.380</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.365</td>\n",
       "      <td>0.365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>10107</td>\n",
       "      <td>29.86</td>\n",
       "      <td>29.81</td>\n",
       "      <td>29.64</td>\n",
       "      <td>29.93</td>\n",
       "      <td>29.96</td>\n",
       "      <td>29.66</td>\n",
       "      <td>30.70</td>\n",
       "      <td>31.21</td>\n",
       "      <td>31.16</td>\n",
       "      <td>...</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.544</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.537</td>\n",
       "      <td>0.518</td>\n",
       "      <td>0.518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10138</td>\n",
       "      <td>45.75</td>\n",
       "      <td>46.15</td>\n",
       "      <td>46.06</td>\n",
       "      <td>46.90</td>\n",
       "      <td>47.09</td>\n",
       "      <td>47.36</td>\n",
       "      <td>48.29</td>\n",
       "      <td>48.56</td>\n",
       "      <td>48.06</td>\n",
       "      <td>...</td>\n",
       "      <td>0.746</td>\n",
       "      <td>0.746</td>\n",
       "      <td>0.704</td>\n",
       "      <td>0.704</td>\n",
       "      <td>0.704</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.727</td>\n",
       "      <td>0.749</td>\n",
       "      <td>0.749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>10145</td>\n",
       "      <td>45.10</td>\n",
       "      <td>45.08</td>\n",
       "      <td>44.65</td>\n",
       "      <td>44.92</td>\n",
       "      <td>45.24</td>\n",
       "      <td>44.96</td>\n",
       "      <td>45.71</td>\n",
       "      <td>45.56</td>\n",
       "      <td>45.62</td>\n",
       "      <td>...</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.887</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.868</td>\n",
       "      <td>0.858</td>\n",
       "      <td>0.858</td>\n",
       "      <td>0.858</td>\n",
       "      <td>0.846</td>\n",
       "      <td>0.846</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>10147</td>\n",
       "      <td>13.20</td>\n",
       "      <td>13.53</td>\n",
       "      <td>13.61</td>\n",
       "      <td>14.08</td>\n",
       "      <td>14.16</td>\n",
       "      <td>14.36</td>\n",
       "      <td>14.31</td>\n",
       "      <td>14.26</td>\n",
       "      <td>14.17</td>\n",
       "      <td>...</td>\n",
       "      <td>0.533</td>\n",
       "      <td>0.533</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.550</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.553</td>\n",
       "      <td>0.553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10158</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.961</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.987</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.983</td>\n",
       "      <td>0.992</td>\n",
       "      <td>0.992</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 3635 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   PERMNO  price_20070103  price_20070104  price_20070105  price_20070108  \\\n",
       "0   10025           49.06           50.34           49.63           49.50   \n",
       "1   10026           41.02           41.89           39.66           40.42   \n",
       "2   10032           24.12           24.30           23.89           23.70   \n",
       "3   10051            7.40            7.33            7.52            7.44   \n",
       "4   10104           17.51           17.68           17.64           17.86   \n",
       "5   10107           29.86           29.81           29.64           29.93   \n",
       "6   10138           45.75           46.15           46.06           46.90   \n",
       "7   10145           45.10           45.08           44.65           44.92   \n",
       "8   10147           13.20           13.53           13.61           14.08   \n",
       "9   10158             NaN             NaN             NaN             NaN   \n",
       "\n",
       "   price_20070109  price_20070110  price_20070111  price_20070112  \\\n",
       "0           50.62           49.68           49.20           48.90   \n",
       "1           40.54           41.54           41.41           42.23   \n",
       "2           23.13           20.90           20.84           21.58   \n",
       "3            7.51            7.52            7.45            7.60   \n",
       "4           17.82           17.77           17.39           17.50   \n",
       "5           29.96           29.66           30.70           31.21   \n",
       "6           47.09           47.36           48.29           48.56   \n",
       "7           45.24           44.96           45.71           45.56   \n",
       "8           14.16           14.36           14.31           14.26   \n",
       "9             NaN             NaN             NaN             NaN   \n",
       "\n",
       "   price_20070116  ...  at_turn_2015-03  at_turn_2015-04  at_turn_2015-05  \\\n",
       "0           46.83  ...            2.628            2.628            2.628   \n",
       "1           41.71  ...            1.340            1.340            1.348   \n",
       "2           20.79  ...            1.604            1.604            1.623   \n",
       "3            7.83  ...            0.786            0.786            0.787   \n",
       "4           17.30  ...            0.414            0.401            0.401   \n",
       "5           31.16  ...            0.553            0.553            0.544   \n",
       "6           48.06  ...            0.746            0.746            0.704   \n",
       "7           45.62  ...            0.887            0.887            0.868   \n",
       "8           14.17  ...            0.533            0.533            0.550   \n",
       "9             NaN  ...            0.961            0.961            0.987   \n",
       "\n",
       "   at_turn_2015-06  at_turn_2015-07  at_turn_2015-08  at_turn_2015-09  \\\n",
       "0            2.674            2.674            2.674            2.730   \n",
       "1            1.348            1.348            1.357            1.357   \n",
       "2            1.623            1.623            1.627            1.627   \n",
       "3            0.787            0.787            0.822            0.822   \n",
       "4            0.401            0.380            0.380            0.380   \n",
       "5            0.544            0.544            0.537            0.537   \n",
       "6            0.704            0.704            0.727            0.727   \n",
       "7            0.868            0.868            0.858            0.858   \n",
       "8            0.550            0.550            0.553            0.553   \n",
       "9            0.987            0.987            0.983            0.983   \n",
       "\n",
       "   at_turn_2015-10  at_turn_2015-11  at_turn_2015-12  \n",
       "0            2.730            2.730            2.573  \n",
       "1            1.357            1.349            1.349  \n",
       "2            1.627            1.603            1.603  \n",
       "3            0.822            0.834            0.834  \n",
       "4            0.365            0.365            0.365  \n",
       "5            0.537            0.518            0.518  \n",
       "6            0.727            0.749            0.749  \n",
       "7            0.858            0.846            0.846  \n",
       "8            0.553            0.553            0.553  \n",
       "9            0.983            0.992            0.992  \n",
       "\n",
       "[10 rows x 3635 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trdt.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2060"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Permno.size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "X=pd.DataFrame(trdt).to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, ..., nan, nan, nan])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[47,2267:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "Permno=X[:,0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PERMNO</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10025.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10026.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10032.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10051.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10104.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2055</th>\n",
       "      <td>93423.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2056</th>\n",
       "      <td>93426.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2057</th>\n",
       "      <td>93428.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2058</th>\n",
       "      <td>93429.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2059</th>\n",
       "      <td>93436.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2060 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       PERMNO\n",
       "0     10025.0\n",
       "1     10026.0\n",
       "2     10032.0\n",
       "3     10051.0\n",
       "4     10104.0\n",
       "...       ...\n",
       "2055  93423.0\n",
       "2056  93426.0\n",
       "2057  93428.0\n",
       "2058  93429.0\n",
       "2059  93436.0\n",
       "\n",
       "[2060 rows x 1 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perm=pd.DataFrame(data =Permno)\n",
    "perm1=perm.rename(columns={0: 'PERMNO'})\n",
    "perm1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "pricetr=X[:,1:2267]\n",
    "indtr=X[:,2267:]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2060, 1368)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indtr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#transpose the dataset\n",
    "tr1=np.transpose(pricetr)\n",
    "\n",
    "tr2=np.transpose(indtr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train1=pd.DataFrame(data =tr1)\n",
    "train2=pd.DataFrame(data =tr2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of  stocks: 2060\n",
      "\n",
      "Number of days: 2266\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nNumber of  stocks:\", len(train1.columns))\n",
    "print(\"\\nNumber of days:\", len(train1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2050</th>\n",
       "      <th>2051</th>\n",
       "      <th>2052</th>\n",
       "      <th>2053</th>\n",
       "      <th>2054</th>\n",
       "      <th>2055</th>\n",
       "      <th>2056</th>\n",
       "      <th>2057</th>\n",
       "      <th>2058</th>\n",
       "      <th>2059</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49.06</td>\n",
       "      <td>41.02</td>\n",
       "      <td>24.12</td>\n",
       "      <td>7.40</td>\n",
       "      <td>17.51</td>\n",
       "      <td>29.86</td>\n",
       "      <td>45.75</td>\n",
       "      <td>45.10</td>\n",
       "      <td>13.20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50.34</td>\n",
       "      <td>41.89</td>\n",
       "      <td>24.30</td>\n",
       "      <td>7.33</td>\n",
       "      <td>17.68</td>\n",
       "      <td>29.81</td>\n",
       "      <td>46.15</td>\n",
       "      <td>45.08</td>\n",
       "      <td>13.53</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49.63</td>\n",
       "      <td>39.66</td>\n",
       "      <td>23.89</td>\n",
       "      <td>7.52</td>\n",
       "      <td>17.64</td>\n",
       "      <td>29.64</td>\n",
       "      <td>46.06</td>\n",
       "      <td>44.65</td>\n",
       "      <td>13.61</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49.50</td>\n",
       "      <td>40.42</td>\n",
       "      <td>23.70</td>\n",
       "      <td>7.44</td>\n",
       "      <td>17.86</td>\n",
       "      <td>29.93</td>\n",
       "      <td>46.90</td>\n",
       "      <td>44.92</td>\n",
       "      <td>14.08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50.62</td>\n",
       "      <td>40.54</td>\n",
       "      <td>23.13</td>\n",
       "      <td>7.51</td>\n",
       "      <td>17.82</td>\n",
       "      <td>29.96</td>\n",
       "      <td>47.09</td>\n",
       "      <td>45.24</td>\n",
       "      <td>14.16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2060 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1      2     3      4      5      6      7      8     9     ...  \\\n",
       "0  49.06  41.02  24.12  7.40  17.51  29.86  45.75  45.10  13.20   NaN  ...   \n",
       "1  50.34  41.89  24.30  7.33  17.68  29.81  46.15  45.08  13.53   NaN  ...   \n",
       "2  49.63  39.66  23.89  7.52  17.64  29.64  46.06  44.65  13.61   NaN  ...   \n",
       "3  49.50  40.42  23.70  7.44  17.86  29.93  46.90  44.92  14.08   NaN  ...   \n",
       "4  50.62  40.54  23.13  7.51  17.82  29.96  47.09  45.24  14.16   NaN  ...   \n",
       "\n",
       "   2050  2051  2052  2053  2054  2055  2056  2057  2058  2059  \n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "3   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "4   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 2060 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2050</th>\n",
       "      <th>2051</th>\n",
       "      <th>2052</th>\n",
       "      <th>2053</th>\n",
       "      <th>2054</th>\n",
       "      <th>2055</th>\n",
       "      <th>2056</th>\n",
       "      <th>2057</th>\n",
       "      <th>2058</th>\n",
       "      <th>2059</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.365</td>\n",
       "      <td>0.462</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.741</td>\n",
       "      <td>0.252</td>\n",
       "      <td>0.181</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.361</td>\n",
       "      <td>0.443</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.365</td>\n",
       "      <td>0.504</td>\n",
       "      <td>0.487</td>\n",
       "      <td>0.779</td>\n",
       "      <td>0.252</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.313</td>\n",
       "      <td>0.453</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.338</td>\n",
       "      <td>0.504</td>\n",
       "      <td>0.487</td>\n",
       "      <td>0.779</td>\n",
       "      <td>0.252</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.313</td>\n",
       "      <td>0.453</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.338</td>\n",
       "      <td>0.504</td>\n",
       "      <td>0.487</td>\n",
       "      <td>0.779</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.166</td>\n",
       "      <td>0.209</td>\n",
       "      <td>0.313</td>\n",
       "      <td>0.453</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.338</td>\n",
       "      <td>0.472</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.599</td>\n",
       "      <td>0.233</td>\n",
       "      <td>0.181</td>\n",
       "      <td>0.211</td>\n",
       "      <td>0.276</td>\n",
       "      <td>0.448</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2060 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1      2      3      4      5      6      7      8     9     ...  \\\n",
       "0  0.365  0.462  0.506  0.741  0.252  0.181  0.233  0.361  0.443   NaN  ...   \n",
       "1  0.365  0.504  0.487  0.779  0.252  0.166  0.209  0.313  0.453   NaN  ...   \n",
       "2  0.338  0.504  0.487  0.779  0.252  0.166  0.209  0.313  0.453   NaN  ...   \n",
       "3  0.338  0.504  0.487  0.779  0.233  0.166  0.209  0.313  0.453   NaN  ...   \n",
       "4  0.338  0.472  0.405  0.599  0.233  0.181  0.211  0.276  0.448   NaN  ...   \n",
       "\n",
       "   2050  2051  2052  2053  2054  2055  2056  2057  2058  2059  \n",
       "0   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "1   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "3   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "4   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 2060 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Impute the datasets\n",
    "train1.fillna(train1.mean(), inplace=True)\n",
    "#train2.fillna(train2.mean(), inplace=True) #problem have rows with all nan, will deal with it separately"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>2050</th>\n",
       "      <th>2051</th>\n",
       "      <th>2052</th>\n",
       "      <th>2053</th>\n",
       "      <th>2054</th>\n",
       "      <th>2055</th>\n",
       "      <th>2056</th>\n",
       "      <th>2057</th>\n",
       "      <th>2058</th>\n",
       "      <th>2059</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>49.06</td>\n",
       "      <td>41.02</td>\n",
       "      <td>24.12</td>\n",
       "      <td>7.40</td>\n",
       "      <td>17.51</td>\n",
       "      <td>29.86</td>\n",
       "      <td>45.75</td>\n",
       "      <td>45.10</td>\n",
       "      <td>13.20</td>\n",
       "      <td>9.885732</td>\n",
       "      <td>...</td>\n",
       "      <td>19.818606</td>\n",
       "      <td>20.582484</td>\n",
       "      <td>7.538519</td>\n",
       "      <td>29.782328</td>\n",
       "      <td>29.003871</td>\n",
       "      <td>47.904683</td>\n",
       "      <td>15.039911</td>\n",
       "      <td>29.752606</td>\n",
       "      <td>40.833177</td>\n",
       "      <td>113.933877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50.34</td>\n",
       "      <td>41.89</td>\n",
       "      <td>24.30</td>\n",
       "      <td>7.33</td>\n",
       "      <td>17.68</td>\n",
       "      <td>29.81</td>\n",
       "      <td>46.15</td>\n",
       "      <td>45.08</td>\n",
       "      <td>13.53</td>\n",
       "      <td>9.885732</td>\n",
       "      <td>...</td>\n",
       "      <td>19.818606</td>\n",
       "      <td>20.582484</td>\n",
       "      <td>7.538519</td>\n",
       "      <td>29.782328</td>\n",
       "      <td>29.003871</td>\n",
       "      <td>47.904683</td>\n",
       "      <td>15.039911</td>\n",
       "      <td>29.752606</td>\n",
       "      <td>40.833177</td>\n",
       "      <td>113.933877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>49.63</td>\n",
       "      <td>39.66</td>\n",
       "      <td>23.89</td>\n",
       "      <td>7.52</td>\n",
       "      <td>17.64</td>\n",
       "      <td>29.64</td>\n",
       "      <td>46.06</td>\n",
       "      <td>44.65</td>\n",
       "      <td>13.61</td>\n",
       "      <td>9.885732</td>\n",
       "      <td>...</td>\n",
       "      <td>19.818606</td>\n",
       "      <td>20.582484</td>\n",
       "      <td>7.538519</td>\n",
       "      <td>29.782328</td>\n",
       "      <td>29.003871</td>\n",
       "      <td>47.904683</td>\n",
       "      <td>15.039911</td>\n",
       "      <td>29.752606</td>\n",
       "      <td>40.833177</td>\n",
       "      <td>113.933877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>49.50</td>\n",
       "      <td>40.42</td>\n",
       "      <td>23.70</td>\n",
       "      <td>7.44</td>\n",
       "      <td>17.86</td>\n",
       "      <td>29.93</td>\n",
       "      <td>46.90</td>\n",
       "      <td>44.92</td>\n",
       "      <td>14.08</td>\n",
       "      <td>9.885732</td>\n",
       "      <td>...</td>\n",
       "      <td>19.818606</td>\n",
       "      <td>20.582484</td>\n",
       "      <td>7.538519</td>\n",
       "      <td>29.782328</td>\n",
       "      <td>29.003871</td>\n",
       "      <td>47.904683</td>\n",
       "      <td>15.039911</td>\n",
       "      <td>29.752606</td>\n",
       "      <td>40.833177</td>\n",
       "      <td>113.933877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>50.62</td>\n",
       "      <td>40.54</td>\n",
       "      <td>23.13</td>\n",
       "      <td>7.51</td>\n",
       "      <td>17.82</td>\n",
       "      <td>29.96</td>\n",
       "      <td>47.09</td>\n",
       "      <td>45.24</td>\n",
       "      <td>14.16</td>\n",
       "      <td>9.885732</td>\n",
       "      <td>...</td>\n",
       "      <td>19.818606</td>\n",
       "      <td>20.582484</td>\n",
       "      <td>7.538519</td>\n",
       "      <td>29.782328</td>\n",
       "      <td>29.003871</td>\n",
       "      <td>47.904683</td>\n",
       "      <td>15.039911</td>\n",
       "      <td>29.752606</td>\n",
       "      <td>40.833177</td>\n",
       "      <td>113.933877</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 2060 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    0      1      2     3      4      5      6      7      8         9     \\\n",
       "0  49.06  41.02  24.12  7.40  17.51  29.86  45.75  45.10  13.20  9.885732   \n",
       "1  50.34  41.89  24.30  7.33  17.68  29.81  46.15  45.08  13.53  9.885732   \n",
       "2  49.63  39.66  23.89  7.52  17.64  29.64  46.06  44.65  13.61  9.885732   \n",
       "3  49.50  40.42  23.70  7.44  17.86  29.93  46.90  44.92  14.08  9.885732   \n",
       "4  50.62  40.54  23.13  7.51  17.82  29.96  47.09  45.24  14.16  9.885732   \n",
       "\n",
       "   ...       2050       2051      2052       2053       2054       2055  \\\n",
       "0  ...  19.818606  20.582484  7.538519  29.782328  29.003871  47.904683   \n",
       "1  ...  19.818606  20.582484  7.538519  29.782328  29.003871  47.904683   \n",
       "2  ...  19.818606  20.582484  7.538519  29.782328  29.003871  47.904683   \n",
       "3  ...  19.818606  20.582484  7.538519  29.782328  29.003871  47.904683   \n",
       "4  ...  19.818606  20.582484  7.538519  29.782328  29.003871  47.904683   \n",
       "\n",
       "        2056       2057       2058        2059  \n",
       "0  15.039911  29.752606  40.833177  113.933877  \n",
       "1  15.039911  29.752606  40.833177  113.933877  \n",
       "2  15.039911  29.752606  40.833177  113.933877  \n",
       "3  15.039911  29.752606  40.833177  113.933877  \n",
       "4  15.039911  29.752606  40.833177  113.933877  \n",
       "\n",
       "[5 rows x 2060 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method NDFrame.tail of        0      1      2      3      4      5      6      7      8      9     \\\n",
       "0     0.365  0.462  0.506  0.741  0.252  0.181  0.233  0.361  0.443    NaN   \n",
       "1     0.365  0.504  0.487  0.779  0.252  0.166  0.209  0.313  0.453    NaN   \n",
       "2     0.338  0.504  0.487  0.779  0.252  0.166  0.209  0.313  0.453    NaN   \n",
       "3     0.338  0.504  0.487  0.779  0.233  0.166  0.209  0.313  0.453    NaN   \n",
       "4     0.338  0.472  0.405  0.599  0.233  0.181  0.211  0.276  0.448    NaN   \n",
       "...     ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "1363  2.674  1.357  1.627  0.822  0.380  0.537  0.727  0.858  0.553  0.983   \n",
       "1364  2.730  1.357  1.627  0.822  0.380  0.537  0.727  0.858  0.553  0.983   \n",
       "1365  2.730  1.357  1.627  0.822  0.365  0.537  0.727  0.858  0.553  0.983   \n",
       "1366  2.730  1.349  1.603  0.834  0.365  0.518  0.749  0.846  0.553  0.992   \n",
       "1367  2.573  1.349  1.603  0.834  0.365  0.518  0.749  0.846  0.553  0.992   \n",
       "\n",
       "      ...   2050   2051   2052   2053   2054   2055   2056   2057   2058  \\\n",
       "0     ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "1     ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "2     ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "3     ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "4     ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n",
       "...   ...    ...    ...    ...    ...    ...    ...    ...    ...    ...   \n",
       "1363  ...  1.608  0.438  0.691  0.187  0.301  0.458  0.847  0.529  1.631   \n",
       "1364  ...  1.608  0.395  0.691  0.187  0.301  0.458  0.847  0.529  1.631   \n",
       "1365  ...  1.608  0.395  0.691  0.187  0.301  0.458  0.847  0.529  1.631   \n",
       "1366  ...  1.581  0.395  0.742  0.155  0.271  0.472  0.847  0.521  1.698   \n",
       "1367  ...  1.581  0.388  0.742  0.155  0.271  0.472  0.847  0.521  1.698   \n",
       "\n",
       "       2059  \n",
       "0       NaN  \n",
       "1       NaN  \n",
       "2       NaN  \n",
       "3       NaN  \n",
       "4       NaN  \n",
       "...     ...  \n",
       "1363  0.620  \n",
       "1364  0.620  \n",
       "1365  0.620  \n",
       "1366  0.583  \n",
       "1367  0.583  \n",
       "\n",
       "[1368 rows x 2060 columns]>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2.tail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Transpsose again\n",
    "imtr1=train1.T\n",
    "imtr2=train2.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "imtr1.to_csv('imputedtrain1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imtr2.to_csv('imputedtrain2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "x1 = StandardScaler().fit_transform(imtr1)\n",
    "x2 =  StandardScaler().fit_transform(imtr2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([nan, nan, nan, ..., nan, nan, nan])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x2[47,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.isnan(x2)).any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  47,    0],\n",
       "       [  47,    1],\n",
       "       [  47,    2],\n",
       "       ...,\n",
       "       [2002, 1365],\n",
       "       [2002, 1366],\n",
       "       [2002, 1367]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.argwhere(np.isnan(x2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "k=15\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca = PCA(n_components=k)\n",
    "principalComponents1 = pca.fit_transform(x1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "Red1 = pd.DataFrame(data = principalComponents1\n",
    "             , columns = range(1,k+1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "pandas.core.frame.DataFrame"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(Red1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>11</th>\n",
       "      <th>12</th>\n",
       "      <th>13</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.660520</td>\n",
       "      <td>2.943234</td>\n",
       "      <td>0.090046</td>\n",
       "      <td>-1.894355</td>\n",
       "      <td>5.060483</td>\n",
       "      <td>6.981604</td>\n",
       "      <td>3.945201</td>\n",
       "      <td>3.734542</td>\n",
       "      <td>-0.276868</td>\n",
       "      <td>1.422848</td>\n",
       "      <td>0.063031</td>\n",
       "      <td>2.932516</td>\n",
       "      <td>1.326073</td>\n",
       "      <td>-1.871520</td>\n",
       "      <td>0.153259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.929629</td>\n",
       "      <td>10.413003</td>\n",
       "      <td>-3.344149</td>\n",
       "      <td>6.045255</td>\n",
       "      <td>0.540976</td>\n",
       "      <td>2.694806</td>\n",
       "      <td>1.459398</td>\n",
       "      <td>-1.228156</td>\n",
       "      <td>0.965623</td>\n",
       "      <td>0.791613</td>\n",
       "      <td>-0.136641</td>\n",
       "      <td>0.480383</td>\n",
       "      <td>-0.124342</td>\n",
       "      <td>-0.387489</td>\n",
       "      <td>-0.293851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-6.393924</td>\n",
       "      <td>1.332993</td>\n",
       "      <td>2.746831</td>\n",
       "      <td>2.774044</td>\n",
       "      <td>-1.996780</td>\n",
       "      <td>-1.663309</td>\n",
       "      <td>0.074506</td>\n",
       "      <td>1.714567</td>\n",
       "      <td>0.236020</td>\n",
       "      <td>1.796485</td>\n",
       "      <td>0.674637</td>\n",
       "      <td>-0.401746</td>\n",
       "      <td>-1.121315</td>\n",
       "      <td>1.102452</td>\n",
       "      <td>0.071496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-16.796843</td>\n",
       "      <td>4.370821</td>\n",
       "      <td>4.198720</td>\n",
       "      <td>2.511052</td>\n",
       "      <td>1.794202</td>\n",
       "      <td>-0.668591</td>\n",
       "      <td>1.052303</td>\n",
       "      <td>-0.942642</td>\n",
       "      <td>-0.853217</td>\n",
       "      <td>-0.581849</td>\n",
       "      <td>0.243187</td>\n",
       "      <td>-0.459938</td>\n",
       "      <td>0.085672</td>\n",
       "      <td>-0.150270</td>\n",
       "      <td>0.916831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-7.988193</td>\n",
       "      <td>3.467485</td>\n",
       "      <td>2.660182</td>\n",
       "      <td>2.926140</td>\n",
       "      <td>-0.925021</td>\n",
       "      <td>-0.537990</td>\n",
       "      <td>-0.509021</td>\n",
       "      <td>-0.302044</td>\n",
       "      <td>0.209060</td>\n",
       "      <td>0.445765</td>\n",
       "      <td>-0.318472</td>\n",
       "      <td>0.356342</td>\n",
       "      <td>0.595305</td>\n",
       "      <td>-0.577343</td>\n",
       "      <td>0.236504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>-5.186191</td>\n",
       "      <td>-0.260557</td>\n",
       "      <td>-0.690565</td>\n",
       "      <td>1.957905</td>\n",
       "      <td>-0.162047</td>\n",
       "      <td>0.390312</td>\n",
       "      <td>-0.681610</td>\n",
       "      <td>1.090433</td>\n",
       "      <td>0.796172</td>\n",
       "      <td>1.349957</td>\n",
       "      <td>0.195855</td>\n",
       "      <td>-0.575791</td>\n",
       "      <td>-0.955133</td>\n",
       "      <td>-0.099651</td>\n",
       "      <td>-0.114998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>25.911351</td>\n",
       "      <td>-0.943541</td>\n",
       "      <td>3.252015</td>\n",
       "      <td>0.638148</td>\n",
       "      <td>-0.360859</td>\n",
       "      <td>-2.152514</td>\n",
       "      <td>-0.030206</td>\n",
       "      <td>1.887221</td>\n",
       "      <td>-1.339725</td>\n",
       "      <td>0.561290</td>\n",
       "      <td>-0.605146</td>\n",
       "      <td>0.116826</td>\n",
       "      <td>0.221639</td>\n",
       "      <td>-0.437807</td>\n",
       "      <td>0.218522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>25.530868</td>\n",
       "      <td>3.790936</td>\n",
       "      <td>-3.525778</td>\n",
       "      <td>-1.493869</td>\n",
       "      <td>1.075759</td>\n",
       "      <td>-2.476955</td>\n",
       "      <td>-1.074509</td>\n",
       "      <td>1.295786</td>\n",
       "      <td>0.227521</td>\n",
       "      <td>-0.690718</td>\n",
       "      <td>0.426071</td>\n",
       "      <td>0.168987</td>\n",
       "      <td>1.016485</td>\n",
       "      <td>-0.500913</td>\n",
       "      <td>-0.676826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>-15.503851</td>\n",
       "      <td>3.088464</td>\n",
       "      <td>3.317643</td>\n",
       "      <td>0.673269</td>\n",
       "      <td>-1.097601</td>\n",
       "      <td>-0.094587</td>\n",
       "      <td>-0.730571</td>\n",
       "      <td>-0.070618</td>\n",
       "      <td>0.813732</td>\n",
       "      <td>0.951500</td>\n",
       "      <td>0.077168</td>\n",
       "      <td>0.118997</td>\n",
       "      <td>0.537312</td>\n",
       "      <td>-0.506164</td>\n",
       "      <td>-0.091880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>-26.887160</td>\n",
       "      <td>0.366556</td>\n",
       "      <td>3.015344</td>\n",
       "      <td>2.017800</td>\n",
       "      <td>-0.292609</td>\n",
       "      <td>0.289536</td>\n",
       "      <td>-0.558734</td>\n",
       "      <td>-0.581357</td>\n",
       "      <td>0.732116</td>\n",
       "      <td>-0.062321</td>\n",
       "      <td>-0.048807</td>\n",
       "      <td>-0.086184</td>\n",
       "      <td>0.173956</td>\n",
       "      <td>0.102040</td>\n",
       "      <td>0.157293</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-26.560303</td>\n",
       "      <td>9.683084</td>\n",
       "      <td>-4.333199</td>\n",
       "      <td>-0.116285</td>\n",
       "      <td>0.513347</td>\n",
       "      <td>-0.400748</td>\n",
       "      <td>-1.777834</td>\n",
       "      <td>0.203514</td>\n",
       "      <td>0.149195</td>\n",
       "      <td>1.127114</td>\n",
       "      <td>1.047434</td>\n",
       "      <td>0.327308</td>\n",
       "      <td>0.256815</td>\n",
       "      <td>0.346838</td>\n",
       "      <td>0.219210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>9.670910</td>\n",
       "      <td>2.001549</td>\n",
       "      <td>3.655704</td>\n",
       "      <td>1.272686</td>\n",
       "      <td>-3.217738</td>\n",
       "      <td>0.665021</td>\n",
       "      <td>-0.303741</td>\n",
       "      <td>1.565494</td>\n",
       "      <td>1.126763</td>\n",
       "      <td>2.301349</td>\n",
       "      <td>-1.862495</td>\n",
       "      <td>0.504008</td>\n",
       "      <td>-1.068312</td>\n",
       "      <td>0.070018</td>\n",
       "      <td>-0.603528</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            1          2         3         4         5         6         7  \\\n",
       "0    2.660520   2.943234  0.090046 -1.894355  5.060483  6.981604  3.945201   \n",
       "1   20.929629  10.413003 -3.344149  6.045255  0.540976  2.694806  1.459398   \n",
       "2   -6.393924   1.332993  2.746831  2.774044 -1.996780 -1.663309  0.074506   \n",
       "3  -16.796843   4.370821  4.198720  2.511052  1.794202 -0.668591  1.052303   \n",
       "4   -7.988193   3.467485  2.660182  2.926140 -0.925021 -0.537990 -0.509021   \n",
       "5   -5.186191  -0.260557 -0.690565  1.957905 -0.162047  0.390312 -0.681610   \n",
       "6   25.911351  -0.943541  3.252015  0.638148 -0.360859 -2.152514 -0.030206   \n",
       "7   25.530868   3.790936 -3.525778 -1.493869  1.075759 -2.476955 -1.074509   \n",
       "8  -15.503851   3.088464  3.317643  0.673269 -1.097601 -0.094587 -0.730571   \n",
       "9  -26.887160   0.366556  3.015344  2.017800 -0.292609  0.289536 -0.558734   \n",
       "10 -26.560303   9.683084 -4.333199 -0.116285  0.513347 -0.400748 -1.777834   \n",
       "11   9.670910   2.001549  3.655704  1.272686 -3.217738  0.665021 -0.303741   \n",
       "\n",
       "           8         9        10        11        12        13        14  \\\n",
       "0   3.734542 -0.276868  1.422848  0.063031  2.932516  1.326073 -1.871520   \n",
       "1  -1.228156  0.965623  0.791613 -0.136641  0.480383 -0.124342 -0.387489   \n",
       "2   1.714567  0.236020  1.796485  0.674637 -0.401746 -1.121315  1.102452   \n",
       "3  -0.942642 -0.853217 -0.581849  0.243187 -0.459938  0.085672 -0.150270   \n",
       "4  -0.302044  0.209060  0.445765 -0.318472  0.356342  0.595305 -0.577343   \n",
       "5   1.090433  0.796172  1.349957  0.195855 -0.575791 -0.955133 -0.099651   \n",
       "6   1.887221 -1.339725  0.561290 -0.605146  0.116826  0.221639 -0.437807   \n",
       "7   1.295786  0.227521 -0.690718  0.426071  0.168987  1.016485 -0.500913   \n",
       "8  -0.070618  0.813732  0.951500  0.077168  0.118997  0.537312 -0.506164   \n",
       "9  -0.581357  0.732116 -0.062321 -0.048807 -0.086184  0.173956  0.102040   \n",
       "10  0.203514  0.149195  1.127114  1.047434  0.327308  0.256815  0.346838   \n",
       "11  1.565494  1.126763  2.301349 -1.862495  0.504008 -1.068312  0.070018   \n",
       "\n",
       "          15  \n",
       "0   0.153259  \n",
       "1  -0.293851  \n",
       "2   0.071496  \n",
       "3   0.916831  \n",
       "4   0.236504  \n",
       "5  -0.114998  \n",
       "6   0.218522  \n",
       "7  -0.676826  \n",
       "8  -0.091880  \n",
       "9   0.157293  \n",
       "10  0.219210  \n",
       "11 -0.603528  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Red1.head(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "Red1.to_csv('Reducedtrain1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}